commit 9f16ea6becc76fd17acd2caaf2825cf2624a267a Author: Chris Evich Date: Thu Nov 10 10:36:16 2022 -0500 initial commit Signed-off-by: Chris Evich diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..37b52df --- /dev/null +++ b/Containerfile @@ -0,0 +1,144 @@ +# pipglr/Containerfile +# +# Builds a Podman-in-Podman Gitlab-Runner image for +# executing Gitlab CI/CD jobs. Requires configuration +# steps specific to Gitlab projects. For more info. see +# https://docs.gitlab.com/runner/executors/docker.html#use-podman-to-run-docker-commands +# + +ARG FLAVOR="stable" +ARG BASE_TAG="latest" +FROM quay.io/podman/$FLAVOR:$BASE_TAG + +# This is a list of packages to remove and/or exclude from the image. +# Primarily this is done for security reasons, should a runner process +# escape confinement. Having fewer things to poke, lowers the attack +# surface-area. +# +# This list was formed manually by running these commands in the base image: +# for package in $(rpm -qa); do \ +# if dnf erase $package; then echo "$package" >> exclude; fi; \ +# done; \ +# cat exclude +# +# After adding those packages to this file, the container build was run +# and package list adjusted, untill no dependency errors were raised. +ARG EXCLUDE_PACKAGES="\ + fedora-repos-modular \ + findutils \ + libxcrypt-compat \ + openldap-compat \ + podman-gvproxy \ + rootfiles \ + sudo \ + vim-minimal \ + yum" + +# Base-image runs as user 'podman', temporarily switch to root +# for installation/setup. +USER root +# Not a real build-arg. Avoiding addition of an env. layer +# only to help prevent some extra typing. +ARG dnfcmd="dnf --setopt=tsflags=nodocs -y" +# During install, excluding packages is meaningless if already installed +RUN set -x && \ + rm -f /etc/dnf/protected.d/sudo.conf && \ + rm -f /etc/dnf/protected.d/yum.conf && \ + $dnfcmd remove ${EXCLUDE_PACKAGES} && \ + dnf clean all && \ + rm -rf /var/cache/dnf + +# Enable callers to customize the runner version as needed, otherwise +# assume this image will be version-tagged, so it's fine to grab the latest. +ARG RUNNER_VERSION="latest" +# When building a multi-arch manifest-list, this buid-arg is set automatically. +ARG TARGETARCH="amd64" +ENV RUNNER_RPM_URL=https://gitlab-runner-downloads.s3.amazonaws.com/${RUNNER_VERSION}/rpm/gitlab-runner_${TARGETARCH}.rpm +RUN for rpm in ${EXCLUDE_PACKAGES}; do x+="--exclude=$rpm "; done && \ + set -x && \ + $dnfcmd update && \ + $dnfcmd install $x $RUNNER_RPM_URL && \ + $dnfcmd upgrade && \ + dnf clean all && \ + rm -rf /var/cache/dnf + +# In case of a runner escape, prevent easy installation of packages. +RUN rm -f /etc/dnf/protected.d/* && \ + rpm -e dnf && \ + rm -f $(type -P rpm) + +ADD /config.toml /home/podman/.gitlab-runner/config.toml +# The global "listen_address" option is used for metrics and +# debugging. Disable it by default since use requires special/ +# additional host configuration. +# Ref: https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section +ARG RUNNER_LISTEN_ADDRESS="disabled" +ENV RUNNER_LISTEN_ADDRESS=$RUNNER_LISTEN_ADDRESS +RUN if [[ "$RUNNER_LISTEN_ADDRESS" == "disabled" ]]; then \ + sed -i -r \ + -e "s/.*@@RUNNER_LISTEN_ADDRESS@@.*//g" \ + /home/podman/.gitlab-runner/config.toml; \ + else \ + sed -i -r \ + -e "s/@@RUNNER_LISTEN_ADDRESS@@/$RUNNER_LISTEN_ADDRESS/g" \ + /home/podman/.gitlab-runner/config.toml; \ + fi + +# A small wrapper is needed to launch a background podman system service +# process for the gitlab-runner to connect to. +ADD /gitlab-runner-wrapper /usr/local/bin/ +# Base image UTS NS configuration causes runner to break when launching +# nested rootless containers. +RUN sed -i -r \ + -e 's/^utsns.+host.*/utsns="private"/' \ + /etc/containers/containers.conf && \ + chmod +x /usr/local/bin/gitlab-runner-wrapper && \ + chown -R podman.podman /home/podman && \ + rm -f /home/podman/.bash* && \ + echo DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" > /etc/profile.d/podman.sh + +# Runtime rootless-mode configuration +USER podman +VOLUME ["/home/podman/.local/share/containers/storage/",\ + "/home/podman/.gitlab-runner/"] +WORKDIR /home/podman +ENTRYPOINT ["/usr/local/bin/gitlab-runner-wrapper"] + +# Gitlab-runner configuration options. Default to unprivileged (nested) +# runner. Privileged is required to permit nested container image building. +ARG PRIVILEGED_RUNNER="false" +# Tags allow pinning jobs to specific runners, comma-separated list of +# tags to add to runner (no spaces!) +ARG RUNNER_TAGS="podman_in_podman" +# Permit running jobs without any tag at all +ARG RUNNER_UNTAGGED="true" +ENV REGISTER_NON_INTERACTIVE="true" \ + RUNNER_TAG_LIST="$RUNNER_TAGS" \ + REGISTER_RUN_UNTAGGED="$RUNNER_UNTAGGED" \ + REGISTER_ACCESS_LEVEL="ref_protected" \ + REGISTER_MAXIMUM_TIMEOUT="3600" \ + CI_SERVER_URL="https://gitlab.com/" \ + RUNNER_EXECUTOR="docker" \ + RUNNER_SHELL="bash" \ + REGISTER_MAINTENANCE_NOTE="Podman-in-Podman containerized runner" \ + DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" \ + DOCKER_DEVICES="/dev/fuse" \ + DOCKER_IMAGE="registry.fedoraproject.org/fedora-minimal:latest" \ + DOCKER_CACHE_DIR="/home/podman/.cache/gitlab-runner" \ + DOCKER_NETWORK_MODE="host" \ + DOCKER_PRIVILEGED="$PRIVILEGED_RUNNER" + +# Not a real build-arg. Simply here to save lots of typing. +ARG _pm="--systemd=true --device=/dev/fuse --security-opt label=disable --user podman -v gitlab-runner-storage:/home/podman/.local/share/containers/storage:Z,U -v gitlab-runner-cache:/home/podman/.cache/gitlab-runner:Z,U -v gitlab-runner-config:/home/podman/.gitlab-runner:Z,U -e PODMAN_RUNNER_DEBUG" + +# These labels simply make it easier to register and execute the runner. +# Define them last so they are absent should a image-build failure occur. +LABEL register="podman run -it --rm $_pm --secret REGISTRATION_TOKEN,type=env \$IMAGE register" + +# TODO: Figure out what's needed to run w/o --privileged. When unspecified, +# conmon fails with this error (from podman debug output): +# +# DEBU[0019] running conmon: /usr/bin/conmon args="[--api-version 1 -c 289...c08 -u 289...c08 -r /usr/bin/crun -b /home/podman/.local/share/containers/storage/overlay-containers/289...c08/userdata -p /tmp/podman-run-1000/containers/overlay-containers/289...c08/userdata/pidfile -n runner-8pxm3xb-project-19009784-concurrent-0-a71b53d132a29e56-predefined-0 --exit-dir /tmp/podman-run-1000/libpod/tmp/exits --full-attach -l k8s-file:/home/podman/.local/share/containers/storage/overlay-containers/289...c08/userdata/ctr.log --log-level debug --syslog --runtime-arg --cgroup-manager --runtime-arg disabled -i --conmon-pidfile /tmp/podman-run-1000/containers/overlay-containers/289...c08/userdata/conmon.pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /home/podman/.local/share/containers/storage --exit-command-arg --runroot --exit-command-arg /tmp/podman-run-1000/containers --exit-command-arg --log-level --exit-command-arg debug --exit-command-arg --cgroup-manager --exit-command-arg cgroupfs --exit-command-arg --tmpdir --exit-command-arg /tmp/podman-run-1000/libpod/tmp --exit-command-arg --network-config-dir --exit-command-arg --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /home/podman/.local/share/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --events-backend --exit-command-arg file --exit-command-arg --syslog --exit-command-arg container --exit-command-arg cleanup --exit-command-arg 289...c08]" +# [conmon:d]: failed to write to /proc/self/oom_score_adj: Permission denied + +LABEL run="podman run -d --privileged --name gitlab-runner $_pm -p $RUNNER_SESSION_PORT:$RUNNER_SESSION_PORT \$IMAGE run" diff --git a/README.md b/README.md new file mode 100644 index 0000000..e4ad72c --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +## Overview + +This container image is built daily from this `Containerfile`, and +made available as: + +* FIXME + +It's purpose is to provide an easy method to execute a GitLab runner, +to service CI/CD jobs for groups and/or repositories on +[gitlab.com](https://gitlab.com). It comes pre-configured to utilize +the gitlab-runner app to execute with rootless podman containers, +nested inside a rootless podman container. + +This is intended to provide multiple additional layers of security +for the host, when running potentially arbitrary CI/CD code. Though, +the ultimate responsibility still rests with the end-user to review +the setup and configuration relative to their own situation/environment. + +### Quickstart + +Several labels are set on the built image or manifest list to support +easy registration and execution of a runner container. They require +defining several environment variables for use. + +#### Runner registration + +Each time the registration command is run, a new runner is added into +the configuration. If your intent is to simply update or modify the +configuration, please edit the config.toml file within the +`gitlab-runner-config` volume. + +Note: These commands assume you have both `podman` and `jq` available. + +```bash +$ echo '' | podman secret create REGISTRATION_TOKEN - +$ export IMAGE= +$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.register) +``` + +#### Runner Startup + +With one or more runners registered and configured, and `$IMAGE` set, +the GitLab runner container may be launched with the following commands. + +Note: The first time this is run, startup will take an extended amount +of time as the runner downloads and runs several (inner) support containers. + +Debugging: You may `export PODMAN_RUNNER_DEBUG=debug` to enable inner-podman +debugging (or any other supported log level) to stdout. + +```bash +$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.run) +``` + +## Building + +This image may be built simply with: + +`podman build -t runner .` + +This will utilize the latest stable version of podman and the latest +stable version of the gitlab runner. + +### Multi-arch + +Assuming the host supports foreign-architecture emulation. The +`Containerfile` may be used to produce a multi-arch manifest-list. +For example: + +`podman build --jobs 4 --platform linux/s390x,linux/ppc64le,linux/amd64 --manifest runner .` + +### Build-args + +Several build arguments are available to control the output image: + +* `FLAVOR` - Choose from 'stable', 'testing', or 'upstream'. These + select the podman base-image to utilize - which may affect the + podman version, features, and stability. For more information + see [the podmanimage README](https://github.com/containers/podman/blob/main/contrib/podmanimage/README.md). +* `BASE_TAG` - When `FLAVOR="stable"`, allows granular choice over the + exact podman version. Possible values include, `latest`, `vX`, `vX.Y`, + and `vX.Y.Z` (where, `X`, `Y`, and `Z` represent the podman semantic + version numbers). It's also possible to specify an image SHA. +* `EXCLUDE_PACKAGES` - A space-separated list of RPM packages to prevent + their existance in the final image. This is intended as a security measure + to limit the attack-surface should a gitlab-runner process escape it's + inner-container. +* `RUNNER_VERSION` - Allows specifying an exact gitlab runner version. + By default the `latest` is used, assuming the user is building a tagged + image anyway. Valid versions may be found on the [runner + release page](https://gitlab.com/gitlab-org/gitlab-runner/-/releases). +* `TARGETARCH` - Supports inclusion of non-x86_64 gitlab runners. This + value is assumed to match the image's architecture. If using the + `--platform` build argument, it will be set automatically. +* `RUNNER_LISTEN_ADDRESS` - Disabled by default, setting this to the FQDN + and port supports various observability and debugging features of the + gitlab runner. For more information see the [gitlab runner advanced + configuration documentation](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section). +* `PRIVILEGED_RUNNER` - Defaults to 'false', may be set 'true'. When + `true`, this causes inner-containers to be created with the `--privileged` + flag. This is a potential security weakness, but is necessary for + (among other things) allowing nested container image builds. +* `RUNNER_TAGS` - Defaults to `podman_in_podman`, may be set to any comma-separated + list (with no spaces!) of tags. These show up in GitLab (not the runner + configuration), and determines where jobs are run. +* `RUNNER_UNTAGED` - Defaults to `true`, may be set to `false`. Allows + the runner to service jobs without any tags on them at all. diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..0fcf769 --- /dev/null +++ b/config.toml @@ -0,0 +1,7 @@ + +# Ref: https://docs.gitlab.com/runner/configuration/advanced-configuration.html + +concurrent = 8 +# N/B: This is DIVIDED among the number of registered runners +check_interval = 10 +listen_address = "@@RUNNER_LISTEN_ADDRESS@@" # Will be removed if undefined diff --git a/gitlab-runner-wrapper b/gitlab-runner-wrapper new file mode 100644 index 0000000..d321e74 --- /dev/null +++ b/gitlab-runner-wrapper @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +unset _debug_args +if [[ -n "$PODMAN_RUNNER_DEBUG" ]]; then + _debug_args="--log-level=$PODMAN_RUNNER_DEBUG" +fi + +if [[ "$1" == "run" ]] && [[ ! -S "/tmp/podman-run-1000/podman/podman.sock" ]]; then + podman $_debug_args system service -t 0 & + # Prevent SIGHUP propigation to podman process + disown -ar +fi + +exec gitlab-runner "$@"