Merge branch 'fix_config' into 'main'

Fix registration permission denied

Closes #1

See merge request qontainers/pipglr!20

.gitignore

@@ -0,0 +1 @@
+/.pre-commit-config.yaml

.gitlab-ci.yml

@@ -1,26 +1,56 @@
 ---
 
-stages:
-    - build
-
-build:
-  stage: build
+default:
+  image: quay.io/buildah/stable:v1.28.0
   tags:
     - docker
     - linux
-  image:
-    name: gcr.io/kaniko-project/executor:v1.6.0-debug
-    entrypoint: ["/busybox/sh", "-c"]
+
+envars:
+  stage: test
+  script: |
+    echo "Select CI env. vars.:";
+    printenv | egrep '^CI_' | sort
+
+commit_check:
+  stage: test
   variables:
-    BASE_TAG: latest
-    FLAVOR: stable
+    BADRX: '^(squash!)|(fixup!)'
+  script: |
+    dnf install -y git
+    shortlogtmp=$(mktemp -p '' commit_check_tmp_XXXX)
+    git log --oneline --no-show-signature "${CI_MERGE_REQUEST_DIFF_BASE_SHA}..HEAD" > "$shortlogtmp"
+    if egrep -q "$BADRX" "$shortlogtmp"; then
+        egrep "$BADRX" "$shortlogtmp"
+        die "Found the above commits matching '$BADRX'"
+    fi
+
+build:
+  stage: deploy
+  variables:
+    BUILDAH_FORMAT: docker
+    BUILDAH_ISOLATION: chroot
+    STORAGE_DRIVER: vfs
+  before_script:
+    - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
   script:
-    - mkdir -p /kaniko/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+    # N/B: There could be more than one merge-request open with this branch's HEAD
+    - |
+        IMAGE_TAG="${CI_COMMIT_BRANCH}";
+        if [[ -n "$CI_COMMIT_TAG" ]]; then
+            IMAGE_TAG="${CI_COMMIT_TAG}";
+        elif [[ -n "$CI_OPEN_MERGE_REQUESTS" ]]; then
+            IMAGE_TAG=mr$(echo "${CI_OPEN_MERGE_REQUESTS}" | cut -d, -f -1 | cut -d\! -f 2);
+        elif [[ "$CI_COMMIT_BRANCH" == "main" ]]; then
+            IMAGE_TAG="latest";
+        fi
+        echo "Building/Pushing to: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG}";
     - >-
-        /kaniko/executor \
-            --context $CI_PROJECT_DIR \
-            --dockerfile $CI_PROJECT_DIR/Containerfile \
-            --destination "$CI_REGISTRY_IMAGE:${CI_COMMIT_TAG:-latest}" \
-            --build-arg "BASE_TAG=$BASE_TAG" \
-            --build-arg "FLAVOR=$FLAVOR"
+      buildah build \
+        --label "org.opencontainers.image.source=${CI_PROJECT_URL}" \
+        --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" \
+        --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" \
+        --label "org.opencontainers.image.version=${IMAGE_TAG}" \
+        -t "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}" "$CI_PROJECT_DIR"
+    - buildah images
+    - buildah push "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}"

Containerfile

@@ -1,144 +1,80 @@
-# pipglr/Containerfile
-#
-# Builds a Podman-in-Podman Gitlab-Runner image for
-# executing Gitlab CI/CD jobs.  Requires configuration
-# steps specific to Gitlab projects. For more info. see
-# https://docs.gitlab.com/runner/executors/docker.html#use-podman-to-run-docker-commands
-#
+FROM quay.io/centos/centos:stream9
 
-ARG FLAVOR="stable"
-ARG BASE_TAG="latest"
-FROM quay.io/podman/$FLAVOR:$BASE_TAG
+ADD /setup.sh /xpackages.txt /root/
+ADD /containers.conf /home/podman/.config/containers/containers.conf
+ADD /podman.service /podman.socket /prune.service /prune.timer /home/podman/.config/systemd/user/
+ADD /runner.service /home/runner/.config/systemd/user/
+ADD kmsglog.conf /etc/systemd/system.conf.d/
 
-# This is a list of packages to remove and/or exclude from the image.
-# Primarily this is done for security reasons, should a runner process
-# escape confinement.  Having fewer things to poke, lowers the attack
-# surface-area.
-#
-# This list was formed manually by running these commands in the base image:
-#   for package in $(rpm -qa); do \
-#       if dnf erase $package; then echo "$package" >> exclude; fi; \
-#   done; \
-#   cat exclude
-#
-# After adding those packages to this file, the container build was run
-# and package list adjusted, untill no dependency errors were raised.
-ARG EXCLUDE_PACKAGES="\
-    fedora-repos-modular \
-    findutils \
-    libxcrypt-compat \
-    openldap-compat \
-    podman-gvproxy \
-    rootfiles \
-    sudo \
-    vim-minimal \
-    yum"
+# Allow image-builders to choose another version becides "latest" should
+# an incompatible change be introduced.
+ARG RUNNER_VERSION=latest
 
-# Base-image runs as user 'podman', temporarily switch to root
-# for installation/setup.
-USER root
-# Not a real build-arg.  Avoiding addition of an env. layer
-# only to help prevent some extra typing.
-ARG dnfcmd="dnf --setopt=tsflags=nodocs -y"
-# During install, excluding packages is meaningless if already installed
-RUN set -x && \
-    rm -f /etc/dnf/protected.d/sudo.conf && \
-    rm -f /etc/dnf/protected.d/yum.conf && \
-    $dnfcmd remove ${EXCLUDE_PACKAGES} && \
-    dnf clean all && \
-    rm -rf /var/cache/dnf
+# Permit building containers for alternate architectures.  At the time
+# of this commit, only 'arm64' is available.
+ARG TARGETARCH=amd64
 
-# Enable callers to customize the runner version as needed, otherwise
-# assume this image will be version-tagged, so it's fine to grab the latest.
-ARG RUNNER_VERSION="latest"
-# When building a multi-arch manifest-list, this buid-arg is set automatically.
-ARG TARGETARCH="amd64"
-ENV RUNNER_RPM_URL=https://gitlab-runner-downloads.s3.amazonaws.com/${RUNNER_VERSION}/rpm/gitlab-runner_${TARGETARCH}.rpm
-RUN for rpm in ${EXCLUDE_PACKAGES}; do x+="--exclude=$rpm "; done && \
-    set -x && \
-    $dnfcmd update && \
-    $dnfcmd install $x $RUNNER_RPM_URL && \
-    $dnfcmd upgrade && \
-    dnf clean all && \
-    rm -rf /var/cache/dnf
+# Allow image-builders to choose an alternate nested-container pruning cycle.
+# For most people the default is probably fine.  This setting is dependent
+# on the number and frequency of jobs run, along with the amount of disk-space
+# available for both /cache and /home/podman/.local/share/containers volumes.
+ARG PRUNE_INTERVAL=daily  # see systemd.timer for allowable values
 
-# In case of a runner escape, prevent easy installation of packages.
-RUN rm -f /etc/dnf/protected.d/* && \
-    rpm -e dnf && \
-    rm -f $(type -P rpm)
+# All-in-one packaging/image-setup script to keep things simple.
+RUN PRUNE_INTERVAL=${PRUNE_INTERVAL} \
+    RUNNER_VERSION=${RUNNER_VERSION} \
+    bash /root/setup.sh
 
-ADD /config.toml /home/podman/.gitlab-runner/config.toml
-# The global "listen_address" option is used for metrics and
-# debugging.  Disable it by default since use requires special/
-# additional host configuration.
-# Ref: https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section
-ARG RUNNER_LISTEN_ADDRESS="disabled"
-ENV RUNNER_LISTEN_ADDRESS=$RUNNER_LISTEN_ADDRESS
-RUN if [[ "$RUNNER_LISTEN_ADDRESS" == "disabled" ]]; then \
-        sed -i -r \
-            -e "s/.*@@RUNNER_LISTEN_ADDRESS@@.*//g" \
-            /home/podman/.gitlab-runner/config.toml; \
-    else \
-        sed -i -r \
-            -e "s/@@RUNNER_LISTEN_ADDRESS@@/$RUNNER_LISTEN_ADDRESS/g" \
-            /home/podman/.gitlab-runner/config.toml; \
-    fi
+VOLUME /cache /home/podman/.local/share/containers
+ENTRYPOINT /lib/systemd/systemd
 
-# A small wrapper is needed to launch a background podman system service
-# process for the gitlab-runner to connect to.
-ADD /gitlab-runner-wrapper /usr/local/bin/
-# Base image UTS NS configuration causes runner to break when launching
-# nested rootless containers.
-RUN sed -i -r \
-        -e 's/^utsns.+host.*/utsns="private"/' \
-        /etc/containers/containers.conf && \
-    chmod +x /usr/local/bin/gitlab-runner-wrapper && \
-    chown -R podman.podman /home/podman && \
-    rm -f /home/podman/.bash* && \
-    echo DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" > /etc/profile.d/podman.sh
+# Gitlab-runner configuration options, may be freely overridden at
+# container image build time.
+ARG DEFAULT_JOB_IMAGE=registry.fedoraproject.org/fedora-minimal:latest
+# Run nested containers in --privileged mode - required to allow building
+# container images using podman or buildah.  Otherwise may be set 'false'.
+ARG NESTED_PRIVILEGED=true
 
-# Runtime rootless-mode configuration
-USER podman
-VOLUME ["/home/podman/.local/share/containers/storage/",\
-        "/home/podman/.gitlab-runner/"]
-WORKDIR /home/podman
-ENTRYPOINT ["/usr/local/bin/gitlab-runner-wrapper"]
-
-# Gitlab-runner configuration options.  Default to unprivileged (nested)
-# runner.  Privileged is required to permit nested container image building.
-ARG PRIVILEGED_RUNNER="false"
-# Tags allow pinning jobs to specific runners, comma-separated list of
-# tags to add to runner (no spaces!)
-ARG RUNNER_TAGS="podman_in_podman"
-# Permit running jobs without any tag at all
-ARG RUNNER_UNTAGGED="true"
-ENV REGISTER_NON_INTERACTIVE="true" \
-    RUNNER_TAG_LIST="$RUNNER_TAGS" \
-    REGISTER_RUN_UNTAGGED="$RUNNER_UNTAGGED" \
-    REGISTER_ACCESS_LEVEL="ref_protected" \
-    REGISTER_MAXIMUM_TIMEOUT="3600" \
-    CI_SERVER_URL="https://gitlab.com/" \
-    RUNNER_EXECUTOR="docker" \
-    RUNNER_SHELL="bash" \
-    REGISTER_MAINTENANCE_NOTE="Podman-in-Podman containerized runner" \
-    DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" \
-    DOCKER_DEVICES="/dev/fuse" \
-    DOCKER_IMAGE="registry.fedoraproject.org/fedora-minimal:latest" \
-    DOCKER_CACHE_DIR="/home/podman/.cache/gitlab-runner" \
-    DOCKER_NETWORK_MODE="host" \
-    DOCKER_PRIVILEGED="$PRIVILEGED_RUNNER"
-
-# Not a real build-arg.  Simply here to save lots of typing.
-ARG _pm="--systemd=true --device=/dev/fuse --security-opt label=disable --user podman -v gitlab-runner-storage:/home/podman/.local/share/containers/storage:Z,U -v gitlab-runner-cache:/home/podman/.cache/gitlab-runner:Z,U -v gitlab-runner-config:/home/podman/.gitlab-runner:Z,U -e PODMAN_RUNNER_DEBUG"
-
-# These labels simply make it easier to register and execute the runner.
-# Define them last so they are absent should a image-build failure occur.
-LABEL register="podman run -it --rm $_pm --secret REGISTRATION_TOKEN,type=env \$IMAGE register"
-
-# TODO: Figure out what's needed to run w/o --privileged.  When unspecified,
-# conmon fails with this error (from podman debug output):
-#
-# DEBU[0019] running conmon: /usr/bin/conmon args="[--api-version 1 -c 289...c08 -u 289...c08 -r /usr/bin/crun -b /home/podman/.local/share/containers/storage/overlay-containers/289...c08/userdata -p /tmp/podman-run-1000/containers/overlay-containers/289...c08/userdata/pidfile -n runner-8pxm3xb-project-19009784-concurrent-0-a71b53d132a29e56-predefined-0 --exit-dir /tmp/podman-run-1000/libpod/tmp/exits --full-attach -l k8s-file:/home/podman/.local/share/containers/storage/overlay-containers/289...c08/userdata/ctr.log --log-level debug --syslog --runtime-arg --cgroup-manager --runtime-arg disabled -i --conmon-pidfile /tmp/podman-run-1000/containers/overlay-containers/289...c08/userdata/conmon.pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /home/podman/.local/share/containers/storage --exit-command-arg --runroot --exit-command-arg /tmp/podman-run-1000/containers --exit-command-arg --log-level --exit-command-arg debug --exit-command-arg --cgroup-manager --exit-command-arg cgroupfs --exit-command-arg --tmpdir --exit-command-arg /tmp/podman-run-1000/libpod/tmp --exit-command-arg --network-config-dir --exit-command-arg  --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /home/podman/.local/share/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --events-backend --exit-command-arg file --exit-command-arg --syslog --exit-command-arg container --exit-command-arg cleanup --exit-command-arg 289...c08]"
-# [conmon:d]: failed to write to /proc/self/oom_score_adj: Permission denied
-
-LABEL run="podman run -d --privileged --name gitlab-runner $_pm -p $RUNNER_SESSION_PORT:$RUNNER_SESSION_PORT \$IMAGE run"
+# The registration runlabel may be called multiple times to register more than
+# one runner.  Each expects a REGISTRATION_TOKEN secret to be pre-defined and
+# the file './config.toml' to exist (may be empty).  A local-cache volume
+# '/cache' is configured for bind-mounting into all interrior-containers
+# for container-runtime use, as recommended by the docs.  Other settings
+# may be changed if you know what you're doing.
+LABEL register="podman run -it --rm \
+ --secret=REGISTRATION_TOKEN,type=env \
+ -v ./config.toml:/etc/gitlab-runner/config.toml:Z \
+ -e REGISTER_NON_INTERACTIVE=true \
+ -e CI_SERVER_URL=https://gitlab.com/ \
+ -e RUNNER_NAME=pipglr \
+ -e RUNNER_EXECUTOR=docker \
+ -e RUNNER_SHELL=bash \
+ -e REGISTER_MAINTENANCE_NOTE=Podman-In-Podman-GitLab-Runner \
+ -e DOCKER_HOST=unix:///home/runner/podman.sock \
+ -e DOCKER_IMAGE=${DEFAULT_JOB_IMAGE} \
+ -e DOCKER_CACHE_DIR=/cache \
+ -e DOCKER_VOLUMES=/cache \
+ -e DOCKER_NETWORK_MODE=host \
+ -e DOCKER_PRIVILEGED=${NESTED_PRIVILEGED} \
+ --entrypoint=/usr/bin/gitlab-runner \$IMAGE register"
+# Additionally, the nested-podman storage volumes must be pre-created with
+# 'podman' UID/GID values to allow nested containers access.
+LABEL setupstorage="podman volume create --opt o=uid=1000,gid=1000 pipglr-storage"
+# Lastly, the gitlab-runner will manage container-cache in this directory,
+# which will also be bind-mounted into every container.  So it must be
+# writable by both 'podman' user and 'runner' group.
+LABEL setupcache="podman volume create --opt o=uid=1000,gid=1001 pipglr-cache"
+# Helper to extract the current configuration secret to allow editing.
+LABEL dumpconfig="podman run -it --rm \
+ --secret config.toml --entrypoint=/bin/cat \
+ \$IMAGE /var/run/secrets/config.toml"
+# Executing the runner container depends on the config.toml secret being
+# set (see above) and two volumes existing with correct permissions set.
+# Note: The contents of the volumes are not critical, they may be removed
+# and re-created (see above) to quickly free-up disk space.
+LABEL run="podman run -dt --name pipglr \
+ --secret config.toml,uid=1001,gid=1001 \
+ -v pipglr-storage:/home/podman/.local/share/containers \
+ -v pipglr-cache:/cache \
+ --systemd true --privileged \
+ --device /dev/fuse \$IMAGE"

README.md

@@ -3,105 +3,195 @@
 This container image is built daily from this `Containerfile`, and
 made available as:
 
-* FIXME
+* `registry.gitlab.com/qontainers/pipglr:latest`
+
+-or-
+
+* `registry.gitlab.com/qontainers/pipglr:<version>`
 
 It's purpose is to provide an easy method to execute a GitLab runner,
 to service CI/CD jobs for groups and/or repositories on
 [gitlab.com](https://gitlab.com).  It comes pre-configured to utilize
-the gitlab-runner app to execute with rootless podman containers,
+the gitlab-runner app to execute within a rootless podman container,
 nested inside a rootless podman container.
 
-This is intended to provide multiple additional layers of security
-for the host, when running potentially arbitrary CI/CD code.  Though,
-the ultimate responsibility still rests with the end-user to review
-the setup and configuration relative to their own situation/environment.
+This is intended to provide additional layers of security for the host,
+when running potentially arbitrary CI/CD code.  Though, the ultimate
+responsibility still rests with the end-user to review the setup and
+configuration relative to their own security situation/environment.
 
-### Quickstart
+### Operation
 
-Several labels are set on the built image or manifest list to support
-easy registration and execution of a runner container.  They require
-defining several environment variables for use.
+This image leverages the podman `runlabel` feature heavily.  Several
+labels are set on the image to support easy registration and execution
+of the runner container.  While it's possible to use the container
+with your own command-line,  it's highly recommended to base them
+off of one of the labels.  See the examples below for more information.
 
-#### Runner registration
-
-Each time the registration command is run, a new runner is added into
-the configuration.  If your intent is to simply update or modify the
-configuration, please edit the config.toml file within the
-`gitlab-runner-config` volume.
-
-Note: These commands assume you have both `podman` and `jq` available.
+***Note:*** Some older versions of podman don't support the
+`container runlabel` sub-command.  If this is the case, you may simulate
+it with the following, substituting `<label>` with one of the predefined
+values (i.e. `register`, `setupconfig`, etc.):
 
 ```bash
-$ echo '<registration token>' | podman secret create REGISTRATION_TOKEN -
-$ export IMAGE=<image FQIN:TAG>
-$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.register)
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.<label>)
 ```
 
-#### Runner Startup
+#### Runner registration (step 1)
 
-With one or more runners registered and configured, and `$IMAGE` set,
-the GitLab runner container may be launched with the following commands.
+All runners must be connected to a project or group runner configuration
+on your gitlab instance (or `gitlab.com`).  This is done using a special
+registration *runlabel*.  The command can (and probably should) be run
+more than once (using the same `config.toml`) to configure and register
+multiple runners.  This is necessary for the *pipglr* container to execute
+multiple jobs in parallel.  For example, if you want to support running
+four jobs at the same time, you would use the `register` *runlabel*
+four times.
 
-Note: The first time this is run, startup will take an extended amount
-of time as the runner downloads and runs several (inner) support containers.
-
-Debugging: You may `export PODMAN_RUNNER_DEBUG=debug` to enable inner-podman
-debugging (or any other supported log level) to stdout.
+Before using the `register`  *runlabel*, you must set your unique
+*registration* (a.k.a. *activation*) token as a podman *secret*.  This
+secret may be removed once the registration step is complete.  The
+**<actual registration token>** value (below) should be replaced with
+the value obtained from the "runners" settings page of a gitlab
+group or project's *CI/CD Settings*.  Gitlab version 16 and later
+refers to this value as an *activation* token, but the usage is the same.
 
 ```bash
-$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.run)
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ echo '<actual registration token>' | podman secret create REGISTRATION_TOKEN -
 ```
 
+Next, ***a blank `config.toml` file*** needs to be created.  Without this, the
+`reigster` *runlabel* will return a permission-denied error.  Once the empty
+`config.toml` file is created, you may register one or more runners by repeating
+the registration *runlabel* as follows:
+
+```bash
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ touch ./config.toml  # important: file must exist, even if empty.
+$ podman container runlabel register $IMAGE
+...repeat as desired...
+$ podman secret rm REGISTRATION_TOKEN  # if desired
+```
+
+#### Runner Configuration (step 2)
+
+During the registration process (above), a boiler-plate (default) `config.toml` file
+will be created/updated for you.  At this point you may edit the configuration
+if desired before committing it as a *podman secret*.  Please refer to the
+[gitlab runner documentation](https://docs.gitlab.com/runner/configuration/)
+for details.
+
+```bash
+$ $EDITOR ./config.toml  # if desired
+$ podman secret create config.toml ./config.toml
+$ rm ./config.toml  # if desired
+```
+
+This may be necessary, for example, to increase the default `concurrency` value
+to reflect the number of registered runners.  If you need to edit this file
+after commiting it as a secret, there's
+[ a `dumpconfig`  *runlabel* for that](README.md#configuration-editing).
+
+#### Volume setup (step 3)
+
+Since several users are utilized inside the container volumes must be
+specifically configured to permit access.  This is done using several
+*runlabels* as follows:
+
+```bash
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ podman container runlabel setupstorage $IMAGE
+$ podman container runlabel setupcache $IMAGE
+```
+
+Note: These volumes generally do not contain any critical operational data,
+they may be re-created anytime to quickly free up host disk-space if
+it's running low.  Simply remove them with the command
+ `podman volume rm pipglr-storage pipglr-cache`.  The reuse the `setupstorage`
+and `setupcache` *runlabels* as in the above example.
+
+#### Runner Startup (step 4)
+
+With the runner configuration saved as a Podman secret, and the runner volumes
+created, the GitLab runner container may be launched with the following commands:
+
+```bash
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ podman container runlabel run $IMAGE
+```
+
+### Configuration Editing
+
+The gitlab-runner configuration contains some sensitive values which
+should be protected.  The pipglr container assumes the entire configuration
+will be passed in as a Podman secret.  This makes editing it slightly
+convoluted, so a handy *runlabel* `dumpconfig` is available.
+It's intended use is as follows:
+
+```bash
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ podman container runlabel dumpconfig $IMAGE > ./config.toml
+$ $EDITOR ./config.toml
+$ podman secret rm config.toml
+$ podman secret create config.toml ./config.toml
+$ rm ./config.toml  # if desired
+```
+
+### Debugging
+
+The first thing to check is the container output.  This shows three things:
+Systemd, Podman, and GitLab-Runner output.  For example:
+
+```bash
+$ podman logs --since 0 pipglr
+```
+
+Next, try running a pipglr image built with more verbose logging.  Both
+the `runner.service` and `podman.service` files have a `log-level` option.
+Simply increase one or both to the "info", or "debug" level.  Start the
+debug container, and reproduce the problem.
+
 ## Building
 
 This image may be built simply with:
 
-`podman build -t runner .`
+```bash
+$ podman build -t registry.gitlab.com/qontainers/pipglr:latest .
+```
 
 This will utilize the latest stable version of podman and the latest
 stable version of the gitlab runner.
 
-### Multi-arch
-
-Assuming the host supports foreign-architecture emulation.  The
-`Containerfile` may be used to produce a multi-arch manifest-list.
-For example:
-
-`podman build --jobs 4 --platform linux/s390x,linux/ppc64le,linux/amd64 --manifest runner .`
-
 ### Build-args
 
 Several build arguments are available to control the output image:
 
-* `FLAVOR` - Choose from 'stable', 'testing', or 'upstream'.  These
-  select the podman base-image to utilize - which may affect the
-  podman version, features, and stability.  For more information
-  see [the podmanimage README](https://github.com/containers/podman/blob/main/contrib/podmanimage/README.md).
-* `BASE_TAG` - When `FLAVOR="stable"`, allows granular choice over the
-  exact podman version.  Possible values include, `latest`, `vX`, `vX.Y`,
-  and `vX.Y.Z` (where, `X`, `Y`, and `Z` represent the podman semantic
-  version numbers).  It's also possible to specify an image SHA.
-* `EXCLUDE_PACKAGES` - A space-separated list of RPM packages to prevent
-  their existance in the final image.  This is intended as a security measure
-  to limit the attack-surface should a gitlab-runner process escape it's
-  inner-container.
+* `PRUNE_INTERVAL` - A systemd.timer compatible `OnCalendar` value that
+  determines how often to prune Podman's storage of disused containers and
+  images.  Defaults to "daily", but should be adjusted based on desired
+  caching-effect balanced against available storage space and job
+  execution rate.
 * `RUNNER_VERSION` - Allows specifying an exact gitlab runner version.
   By default the `latest` is used, assuming the user is building a tagged
   image anyway.  Valid versions may be found on the [runner
   release page](https://gitlab.com/gitlab-org/gitlab-runner/-/releases).
 * `TARGETARCH` - Supports inclusion of non-x86_64 gitlab runners.  This
-   value is assumed to match the image's architecture.  If using the
-   `--platform` build argument, it will be set automatically.
-* `RUNNER_LISTEN_ADDRESS` - Disabled by default, setting this to the FQDN
-  and port supports various observability and debugging features of the
-  gitlab runner.  For more information see the [gitlab runner advanced
-  configuration documentation](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section).
-* `PRIVILEGED_RUNNER` - Defaults to 'false', may be set 'true'.  When
-  `true`, this causes inner-containers to be created with the `--privileged`
-  flag.  This is a potential security weakness, but is necessary for
-  (among other things) allowing nested container image builds.
-* `RUNNER_TAGS` - Defaults to `podman_in_podman`, may be set to any comma-separated
-  list (with no spaces!) of tags.  These show up in GitLab (not the runner
-  configuration), and determines where jobs are run.
-* `RUNNER_UNTAGED` - Defaults to `true`, may be set to `false`.  Allows
-  the runner to service jobs without any tags on them at all.
+  value is assumed to match the image's architecture.  If using the
+  `--platform` build argument, it will be set automatically.  Note:
+  as of this writing, only `amd64` and `arm64` builds of the gitlab-runner
+  are available.
+* `NESTED_PRIVILEGED` - Defaults to 'true', may be set 'false' to prevent
+  nested containers running in `--privileged` mode.  This will affect
+  the ability to build container images in CI jobs using tools like
+  podman or buildah.
+
+### Environment variables
+
+Nearly every option to every gitlab-runner sub-command may be specified via
+environment variable.  Some of these are set in the `Containerfile` for
+the `register` *runlabel*.  If you need to set additional runtime
+env. vars., please do so via additional `Environment` optionns in the
+`runner.service` file.  See the *systemd.nspawn* man page for important
+value-format details.

config.toml

@@ -1,7 +0,0 @@
-
-# Ref: https://docs.gitlab.com/runner/configuration/advanced-configuration.html
-
-concurrent = 8
-# N/B: This is DIVIDED among the number of registered runners
-check_interval = 10
-listen_address = "@@RUNNER_LISTEN_ADDRESS@@"  # Will be removed if undefined

containers.conf

@@ -0,0 +1,13 @@
+[containers]
+netns="host"
+userns="host"
+ipcns="host"
+utsns="private"
+cgroupns="host"
+cgroups="disabled"
+log_driver = "k8s-file"
+
+[engine]
+cgroup_manager = "cgroupfs"
+events_logger="file"
+runtime="crun"

gitlab-runner-wrapper

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-set -e
-
-unset _debug_args
-if [[ -n "$PODMAN_RUNNER_DEBUG" ]]; then
-    _debug_args="--log-level=$PODMAN_RUNNER_DEBUG"
-fi
-
-if [[ "$1" == "run" ]] && [[ ! -S "/tmp/podman-run-1000/podman/podman.sock" ]]; then
-    podman $_debug_args system service -t 0 &
-    # Prevent SIGHUP propigation to podman process
-    disown -ar
-fi
-
-exec gitlab-runner "$@"

kmsglog.conf

@@ -0,0 +1,3 @@
+[Manager]
+LogTarget=kmsg
+LogColor=yes

podman.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Podman API Service
+Requires=podman.socket
+After=podman.socket
+Documentation=man:podman-system-service(1)
+StartLimitIntervalSec=0
+
+[Service]
+Delegate=true
+Type=exec
+KillMode=process
+Environment=LOGGING="--log-level=warn"
+ExecStart=/usr/bin/podman $LOGGING system service
+StandardOutput=journal+console
+StandardError=inherit
+
+[Install]
+WantedBy=default.target

podman.socket

@@ -0,0 +1,11 @@
+[Unit]
+Description=Podman API Socket
+Documentation=man:podman-system-service(1)
+
+[Socket]
+ListenStream=/home/runner/podman.sock
+SocketGroup=runner
+SocketMode=0660
+
+[Install]
+WantedBy=sockets.target

prune.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Prune all disused podman volumes, images, and containers
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/podman system prune --all --force

prune.timer

@@ -0,0 +1,6 @@
+[Unit]
+Description=Execute the prune service periodically
+
+[Timer]
+OnCalendar=@@@PRUNE_INTERVAL@@@
+RemainAfterElapse=no

runner.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=Gitlab-runner service
+
+[Service]
+ExecStart=/usr/bin/gitlab-runner --log-level=warn run --user runner --working-directory=/home/runner
+StandardOutput=journal+console
+StandardError=inherit

setup.sh

@@ -0,0 +1,109 @@
+
+
+# This script is intended to be run during container-image build.  Any
+# other usage outside this context is likely to cause harm.
+
+set -eo pipefail
+
+for varname in PRUNE_INTERVAL RUNNER_VERSION TARGETARCH; do
+  if [[ -z "${!varname}" ]]; then
+    echo "Error: \$$varname must be non-empty."
+  fi
+done
+
+# Make image smaller by not installing docs.
+DNF="dnf --setopt=tsflags=nodocs -y"
+
+for rpm in $(egrep -v '^(# )+' < /root/xpackages.txt); do
+  x+="--exclude=$rpm ";
+done
+
+set -x  # show what's happening to make debugging easier
+
+# DNF itself or a dependence may need upgrading, take care of it first.
+$DNF upgrade
+
+$DNF $x install \
+  podman \
+  systemd
+
+# Gitlab-runner package contains scriptlets which do not function properly inside a
+# container-build environment where systemd is not active/running.
+$DNF $x --setopt=tsflags=noscripts install \
+  https://gitlab-runner-downloads.s3.amazonaws.com/$RUNNER_VERSION/rpm/gitlab-runner_${TARGETARCH}.rpm
+
+# Allow removing dnf, sudo, etc. packages.  Also don't start unnecessary or broken
+# systemd services, like anything kernel related or login gettys.
+rm -rf \
+  /etc/dnf/protected.d/* \
+  /etc/sytemd/system/getty.target.wants/* \
+  /etc/sytemd/system/multi-user.target.wants/* \
+  /etc/sytemd/system/sysinit.target.wants/* \
+  /etc/sytemd/system/timers.target.wants/* \
+  /lib/systemd/system/graphical.target.wants/* \
+  /lib/systemd/system/multi-user.target.wants/{getty.target,systemd-ask-password-wall.path} \
+  /lib/systemd/system/sys-kernel*.mount
+
+# Remove unnecessary packages, see xpackages.txt to learn how this list was generated.
+# This makes the image smaller and reduces the attack-surface.
+dnf remove -y $(egrep -v '^(# )+' /root/xpackages.txt)
+
+# Wipe out the DNF cache, then remove it entirely, again to make the image smaller.
+$DNF clean all
+rm -rf /var/cache/dnf /var/log/dnf* /var/log/yum.*
+rpm -e dnf
+
+# Workaround https://bugzilla.redhat.com/show_bug.cgi?id=1995337
+rpm --setcaps shadow-utils
+
+# Prevent copying of skel since it can interfere with the gitlab-runner
+mkdir -p /home/podman /home/runner
+# Guarantee uid/gid 1000 for user 'podman' / 1001 for user 'runner'.
+groupadd -g 1000 podman
+groupadd -g 1001 runner
+# Separate users for services to increase process isolation.
+# The 'podman' user's socket service writes /home/runner/podman.socket
+useradd -M -u 1000 -g podman -G runner podman
+useradd -M -u 1001 -g runner runner
+# Allow 'podman' user to create socket file under /home/runner.
+chmod 770 /home/runner
+
+# Overwrite defaults, only user 'podman' permited to have a user-namespace
+# Split the namespaced ID's around the containers root (ID 0) and the user
+# IDs 1000 and 1001 (defined above) to prevent hijacking from a nested container.
+echo -e "podman:1:999\npodman:1002:64533" | tee /etc/subuid > /etc/subgid
+# Host volume mount necessary for nested-podman to use overlayfs2 for container & volume storage.
+mkdir -p /home/podman/.local/share/containers
+# Nested-container's local container-cache volume mount, recommended by gitlab-runner docs.
+mkdir -p /cache
+# Both the gitlab-runner and podman need access to the cache directory / volume mount.
+chown podman:runner /cache
+
+# Setup persistent 'podman' user services to start & run without a login.
+mkdir -p /var/lib/systemd/linger
+touch /var/lib/systemd/linger/podman
+# Setup 'podman' socket and a container-storage pruning service for 'podman' user.
+mkdir -p /home/podman/.config/systemd/user/{sockets.target.wants,default.target.wants}
+cd /home/podman/.config/systemd/user/
+ln -s $PWD/podman.socket ./sockets.target.wants/     # Added from Containerfile
+ln -s $PWD/prune.timer ./default.target.wants/        # also from Containerfile
+# Substitute value from --build-arg if specified, otherwise use default from Containerfile.
+sed -i -e "s/@@@PRUNE_INTERVAL@@@/$PRUNE_INTERVAL/" ./prune.timer
+# Containerfile ADD instruction does not properly set ownership/permissions.
+chown -R 1000:1000 /home/podman
+chmod 700 /home/podman
+
+# Setup persistent 'runner' user services to start & run without a login.
+touch /var/lib/systemd/linger/runner
+mkdir -p /home/runner/.config/systemd/user/default.target.wants
+cd /home/runner/.config/systemd/user/
+# Does not depend on podman.socket file availablility, will retry if not present.
+ln -s $PWD/runner.service ./default.target.wants/
+# gitlab-runner will create side-car '.runner_system_id' file next to 'config.toml'
+# on first startup.  Ensure access is allowed.  Also link to future config file
+# presented as a container-secret.
+mkdir -p /home/runner/.gitlab-runner
+ln -s /var/run/secrets/config.toml /home/runner/.gitlab-runner/config.toml
+# Containerfile ADD instruction does not properly set ownership/permissions.
+chown -R runner:runner /home/runner
+chmod -R 700 /home/runner/.gitlab-runner

xpackages.txt

@@ -0,0 +1,21 @@
+# This list was formed by running the following commands in the base image:
+#   for package in $(rpm -qa); do if dnf erase $package; then echo "$package" >> remove; fi; done
+#   cat remove
+# Including those packages in this file.  Finally, repeatedly running the container build
+# untill no dependency errors were raised.
+
+criu
+criu-libs
+crypto-policies-scripts
+dejavu-sans-fonts
+findutils
+fonts-filesystem
+gdb-gdbserver
+langpacks-core-en
+langpacks-core-font-en
+langpacks-en
+libnet
+protobuf-c
+rootfiles
+vim-minimal
+yum