Update docs and Containerfile to match

Fully tested README.md instructions end-to-end on F36.

Signed-off-by: Chris Evich <cevich@redhat.com>

.gitignore

@@ -0,0 +1 @@
+/.pre-commit-config.yaml

.gitlab-ci.yml

@@ -15,12 +15,26 @@ build:
     BASE_TAG: latest
     FLAVOR: stable
   script:
-    - mkdir -p /kaniko/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
-    - >-
+    - 'mkdir -p /kaniko/.docker'
+    - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json'
+    - |
+        echo "Select CI env. vars.:";
+        printenv | egrep '^CI_' | sort
+    # N/B: There could be more than one merge-request open with this branch's HEAD
+    - |
+        IMAGE_TAG="${CI_COMMIT_BRANCH}";
+        if [[ -n "$CI_COMMIT_TAG" ]]; then
+            IMAGE_TAG="${CI_COMMIT_TAG}";
+        elif [[ -n "$CI_OPEN_MERGE_REQUESTS" ]]; then
+            IMAGE_TAG=mr$(echo "${CI_OPEN_MERGE_REQUESTS}" | cut -d, -f -1 | cut -d\! -f 2);
+        elif [[ "$CI_COMMIT_BRANCH" == "main" ]]; then
+            IMAGE_TAG="latest";
+        fi
+        echo "Building/Pushing to: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG}";
+    - |
         /kaniko/executor \
             --context $CI_PROJECT_DIR \
             --dockerfile $CI_PROJECT_DIR/Containerfile \
-            --destination "$CI_REGISTRY_IMAGE:${CI_COMMIT_TAG:-latest}" \
+            --destination "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}" \
             --build-arg "BASE_TAG=$BASE_TAG" \
             --build-arg "FLAVOR=$FLAVOR"

Containerfile

@@ -37,16 +37,16 @@ ARG EXCLUDE_PACKAGES="\
 # Base-image runs as user 'podman', temporarily switch to root
 # for installation/setup.
 USER root
-# Not a real build-arg.  Avoiding addition of an env. layer
-# only to help prevent some extra typing.
-ARG dnfcmd="dnf --setopt=tsflags=nodocs -y"
+# Helper for comparison in future RUN operations (DO NOT USE)
+ARG _DNFCMD="dnf --setopt=tsflags=nodocs -y"
+# Set this instead, if (for example) you want to volume-mount in /var/cache/dnf
+ARG DNFCMD="${_DNFCMD}"
+# Avoid installing any documentation to keep image small
 # During install, excluding packages is meaningless if already installed
 RUN set -x && \
     rm -f /etc/dnf/protected.d/sudo.conf && \
     rm -f /etc/dnf/protected.d/yum.conf && \
-    $dnfcmd remove ${EXCLUDE_PACKAGES} && \
-    dnf clean all && \
-    rm -rf /var/cache/dnf
+    $DNFCMD remove ${EXCLUDE_PACKAGES}
 
 # Enable callers to customize the runner version as needed, otherwise
 # assume this image will be version-tagged, so it's fine to grab the latest.
@@ -56,11 +56,13 @@ ARG TARGETARCH="amd64"
 ENV RUNNER_RPM_URL=https://gitlab-runner-downloads.s3.amazonaws.com/${RUNNER_VERSION}/rpm/gitlab-runner_${TARGETARCH}.rpm
 RUN for rpm in ${EXCLUDE_PACKAGES}; do x+="--exclude=$rpm "; done && \
     set -x && \
-    $dnfcmd update && \
-    $dnfcmd install $x $RUNNER_RPM_URL && \
-    $dnfcmd upgrade && \
+    $DNFCMD update && \
+    $DNFCMD install $x $RUNNER_RPM_URL && \
+    $DNFCMD upgrade && \
+    if [[ "${DNFCMD}" == "${_DNFCMD}" ]]; then \
       dnf clean all && \
-    rm -rf /var/cache/dnf
+      rm -rf /var/cache/dnf; \
+    fi
 
 # In case of a runner escape, prevent easy installation of packages.
 RUN rm -f /etc/dnf/protected.d/* && \
@@ -86,14 +88,16 @@ RUN if [[ "$RUNNER_LISTEN_ADDRESS" == "disabled" ]]; then \
 
 # A small wrapper is needed to launch a background podman system service
 # process for the gitlab-runner to connect to.
-ADD /gitlab-runner-wrapper /usr/local/bin/
+ADD /gitlab-runner-wrapper /podman-in-podman-maintenance /usr/local/bin/
 # Base image UTS NS configuration causes runner to break when launching
 # nested rootless containers.
 RUN sed -i -r \
         -e 's/^utsns.+host.*/utsns="private"/' \
         /etc/containers/containers.conf && \
     chmod +x /usr/local/bin/gitlab-runner-wrapper && \
-    chown -R podman.podman /home/podman && \
+    chmod +x /usr/local/bin/podman-in-podman-maintenance && \
+    chown -R podman:podman /home/podman && \
+    chmod u+s /usr/bin/new{uid,gid}map && \
     rm -f /home/podman/.bash* && \
     echo DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" > /etc/profile.d/podman.sh
 
@@ -104,12 +108,17 @@ VOLUME ["/home/podman/.local/share/containers/storage/",\
 WORKDIR /home/podman
 ENTRYPOINT ["/usr/local/bin/gitlab-runner-wrapper"]
 
+# Ensure root storage directory exists with correct permissions
+RUN mkdir -p .local/share/containers/storage
+
 # Gitlab-runner configuration options.  Default to unprivileged (nested)
 # runner.  Privileged is required to permit nested container image building.
-ARG PRIVILEGED_RUNNER="false"
+ARG RUNNER_NAME="qontainers-pipglr"
+# Running inner-podman privileged is necessary at the time of this commit.
+ARG PRIVILEGED_RUNNER="true"
 # Tags allow pinning jobs to specific runners, comma-separated list of
 # tags to add to runner (no spaces!)
-ARG RUNNER_TAGS="podman_in_podman"
+ARG RUNNER_TAGS="podman-in-podman"
 # Permit running jobs without any tag at all
 ARG RUNNER_UNTAGGED="true"
 ENV REGISTER_NON_INTERACTIVE="true" \
@@ -118,27 +127,23 @@ ENV REGISTER_NON_INTERACTIVE="true" \
     REGISTER_ACCESS_LEVEL="ref_protected" \
     REGISTER_MAXIMUM_TIMEOUT="3600" \
     CI_SERVER_URL="https://gitlab.com/" \
+    RUNNER_NAME="${RUNNER_NAME}" \
     RUNNER_EXECUTOR="docker" \
     RUNNER_SHELL="bash" \
     REGISTER_MAINTENANCE_NOTE="Podman-in-Podman containerized runner" \
     DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" \
     DOCKER_DEVICES="/dev/fuse" \
     DOCKER_IMAGE="registry.fedoraproject.org/fedora-minimal:latest" \
-    DOCKER_CACHE_DIR="/home/podman/.cache/gitlab-runner" \
+    DOCKER_CACHE_DIR="/cache" \
+    DOCKER_VOLUMES="/cache" \
     DOCKER_NETWORK_MODE="host" \
     DOCKER_PRIVILEGED="$PRIVILEGED_RUNNER"
 
 # Not a real build-arg.  Simply here to save lots of typing.
-ARG _pm="--systemd=true --device=/dev/fuse --security-opt label=disable --user podman -v gitlab-runner-storage:/home/podman/.local/share/containers/storage:Z,U -v gitlab-runner-cache:/home/podman/.cache/gitlab-runner:Z,U -v gitlab-runner-config:/home/podman/.gitlab-runner:Z,U -e PODMAN_RUNNER_DEBUG"
+ARG _pm="--systemd=true --device=/dev/fuse --security-opt label=disable --user podman --volume pipglr-podman-root:/home/podman/.local/share/containers/storage --volume pipglr-config:/home/podman/.gitlab-runner -v pipglr-podman-cache:/cache -e PODMAN_RUNNER_DEBUG -e LOG_LEVEL"
 
 # These labels simply make it easier to register and execute the runner.
 # Define them last so they are absent should a image-build failure occur.
 LABEL register="podman run -it --rm $_pm --secret REGISTRATION_TOKEN,type=env \$IMAGE register"
-
-# TODO: Figure out what's needed to run w/o --privileged.  When unspecified,
-# conmon fails with this error (from podman debug output):
-#
-# DEBU[0019] running conmon: /usr/bin/conmon args="[--api-version 1 -c 289...c08 -u 289...c08 -r /usr/bin/crun -b /home/podman/.local/share/containers/storage/overlay-containers/289...c08/userdata -p /tmp/podman-run-1000/containers/overlay-containers/289...c08/userdata/pidfile -n runner-8pxm3xb-project-19009784-concurrent-0-a71b53d132a29e56-predefined-0 --exit-dir /tmp/podman-run-1000/libpod/tmp/exits --full-attach -l k8s-file:/home/podman/.local/share/containers/storage/overlay-containers/289...c08/userdata/ctr.log --log-level debug --syslog --runtime-arg --cgroup-manager --runtime-arg disabled -i --conmon-pidfile /tmp/podman-run-1000/containers/overlay-containers/289...c08/userdata/conmon.pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /home/podman/.local/share/containers/storage --exit-command-arg --runroot --exit-command-arg /tmp/podman-run-1000/containers --exit-command-arg --log-level --exit-command-arg debug --exit-command-arg --cgroup-manager --exit-command-arg cgroupfs --exit-command-arg --tmpdir --exit-command-arg /tmp/podman-run-1000/libpod/tmp --exit-command-arg --network-config-dir --exit-command-arg  --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /home/podman/.local/share/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --events-backend --exit-command-arg file --exit-command-arg --syslog --exit-command-arg container --exit-command-arg cleanup --exit-command-arg 289...c08]"
-# [conmon:d]: failed to write to /proc/self/oom_score_adj: Permission denied
-
-LABEL run="podman run -d --privileged --name gitlab-runner $_pm -p $RUNNER_SESSION_PORT:$RUNNER_SESSION_PORT \$IMAGE run"
+# Note: Privileged mode is required to permit building container images with inner-podman
+LABEL run="podman run -d --privileged --name pipglr $_pm \$IMAGE run"

README.md

@@ -3,71 +3,144 @@
 This container image is built daily from this `Containerfile`, and
 made available as:
 
-* FIXME
+* `registry.gitlab.com/qontainers/pipglr:latest`
+
+-or-
+
+* `registry.gitlab.com/qontainers/pipglr:<version>`
 
 It's purpose is to provide an easy method to execute a GitLab runner,
 to service CI/CD jobs for groups and/or repositories on
 [gitlab.com](https://gitlab.com).  It comes pre-configured to utilize
-the gitlab-runner app to execute with rootless podman containers,
+the gitlab-runner app to execute within a rootless podman container,
 nested inside a rootless podman container.
 
-This is intended to provide multiple additional layers of security
-for the host, when running potentially arbitrary CI/CD code.  Though,
-the ultimate responsibility still rests with the end-user to review
-the setup and configuration relative to their own situation/environment.
+This is intended to provide additional layers of security for the host,
+when running potentially arbitrary CI/CD code.  Though, the ultimate
+responsibility still rests with the end-user to review the setup and
+configuration relative to their own security situation/environment.
 
-### Quickstart
+### Operation
 
-Several labels are set on the built image or manifest list to support
-easy registration and execution of a runner container.  They require
-defining several environment variables for use.
+This image supports `podman container runlabel`, or if your version
+lacks this feature, Several labels are set on the image to support
+easy registration and execution of a runner container using a special
+bash command.  See the examples below for more information.
+
+#### [Volume Ownership Bug](https://github.com/containers/podman/issues/16576)
+
+Some versions of podman contain a bug where named local volumes aren't owned
+by the namespaced user within a rootless container (i.e. the 'podman' user).
+Since the `podman` user/group inside the `pipglr` container is known, it's
+possible to manually setup ownership ahead of time. This should be be done
+once, prior to registering your runners:
+
+```bash
+$ for VOLUME in pipglr-podman-root pipglr-config pipglr-podman-cache; do \
+    PUPVM="podman unshare podman volume mount $VOLUME"
+    podman volume create $VOLUME && \
+    podman unshare chown 1000:1000 $($PUPVM) && \
+    podman unshare chmod 02770 $($PUPVM) && \
+    podman unshare ls -land $($PUPVM) ; \
+  done
+```
+
+If you get `podman system service` startup permission-denied errors, or
+errors from gitlab-runner, unable to connect to the podman socket, this is
+likely the cause.  You can fix it after-the-fact using the same commands as above, just add a `-R` option to the `chown`/`chmod`, and additionally target `./*`.
 
 #### Runner registration
 
 Each time the registration command is run, a new runner is added into
-the configuration.  If your intent is to simply update or modify the
-configuration, please edit the config.toml file within the
-`gitlab-runner-config` volume.
-
-Note: These commands assume you have both `podman` and `jq` available.
+the configuration.  If however, you simply need to update/modify the
+configuration, please edit the `config.toml` file directly after mounting
+(default) `pipglr-runner-config` (`/home/podman/.gitlab-runner/`) volume.
+For modern versions of podman, registration can be performed with the
+following commands:
+
+```bash
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ echo '<actual registration token>' | podman secret create REGISTRATION_TOKEN -
+$ podman container runlabel register $IMAGE
+```
+
+Where `<actual registration token>` is the value obtained from the "runners"
+settings page of a gitlab group or project.  When you're finished registering
+as many runners as you want, the secret is no-longer needed and may be removed:
+
+```bash
+$ podman secret rm REGISTRATION_TOKEN
+```
+
+##### Note
+
+Some versions of podman don't support the `container runlabel` sub-command.
+If this is the case, you may simulate it with the following command (in addition
+to the other example commands above):
 
 ```bash
-$ echo '<registration token>' | podman secret create REGISTRATION_TOKEN -
-$ export IMAGE=<image FQIN:TAG>
 $ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.register)
 ```
 
 #### Runner Startup
 
-With one or more runners registered and configured, and `$IMAGE` set,
-the GitLab runner container may be launched with the following commands.
+With one or more runners successfully registered and configured, the GitLab
+runner container may be launched with the following commands:
 
-Note: The first time this is run, startup will take an extended amount
-of time as the runner downloads and runs several (inner) support containers.
+```bash
+$ podman container runlabel run $IMAGE
+```
 
-Debugging: You may `export PODMAN_RUNNER_DEBUG=debug` to enable inner-podman
-debugging (or any other supported log level) to stdout.
+##### Note
+
+As above, if you're missing the `container runlabel` sub-command, the following
+may be used instead (assuming `$IMAGE` remains set):
 
 ```bash
 $ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.run)
 ```
 
+#### Runner configuration
+
+You may inspect/modify the gitlab-runner configuration as you see fit, just be
+sure to use the `podman unshare` command-wrapper to enter the usernamespace.
+For example, to display the config:
+
+```bash
+$ podman unshare $(podman unshare podman volume mount pipglr-config)/config.toml
+```
+
+#### Debugging
+
+The first thing to check is the container output:
+
+```bash
+$ podman logs --since 0 pipglr
+```
+
+Before starting the runner, you may `export PODMAN_RUNNER_DEBUG=debug` to enable
+debugging on the inner-podman.  Whereas `export LOG_LEVEL=debug` can be used to
+debug the gitlab-runner itself.
+
 ## Building
 
 This image may be built simply with:
 
-`podman build -t runner .`
+```bash
+$ podman build -t registry.gitlab.com/qontainers/pipglr:latest .
+```
 
 This will utilize the latest stable version of podman and the latest
 stable version of the gitlab runner.
 
-### Multi-arch
+### Notes
 
-Assuming the host supports foreign-architecture emulation.  The
-`Containerfile` may be used to produce a multi-arch manifest-list.
-For example:
+* If you wish to use the `testing` or `upstream` flavors of the podman base image,
+  simply build with `--build-arg FLAVOR=testing` (or `upstream`).
 
-`podman build --jobs 4 --platform linux/s390x,linux/ppc64le,linux/amd64 --manifest runner .`
+* Additionally or alternatively, you may specify a specific podman base image tag
+  with `--build-arg BASE_TAG=<value>`.  Where `<value>` is either `latest`, the
+  podman image version (e.g. `v4`, `v4.2`, `v4.2.0`, etc.)
 
 ### Build-args
 
@@ -82,13 +155,18 @@ Several build arguments are available to control the output image:
   and `vX.Y.Z` (where, `X`, `Y`, and `Z` represent the podman semantic
   version numbers).  It's also possible to specify an image SHA.
 * `EXCLUDE_PACKAGES` - A space-separated list of RPM packages to prevent
-  their existance in the final image.  This is intended as a security measure
+  their existence in the final image.  This is intended as a security measure
   to limit the attack-surface should a gitlab-runner process escape it's
   inner-container.
 * `RUNNER_VERSION` - Allows specifying an exact gitlab runner version.
   By default the `latest` is used, assuming the user is building a tagged
   image anyway.  Valid versions may be found on the [runner
   release page](https://gitlab.com/gitlab-org/gitlab-runner/-/releases).
+* `DNFCMD` - By default this is set to `dnf --setopt=tsflags=nodocs -y`.
+  However, if you'd like to volume-mount in `/var/cache/dnf` then you'll
+  need to use
+  `--build-arg DNFCMD="dnf --setopt=tsflags=nodocs -y --setopt keepcache=true`"
+  Note: Changing `DNFCMD` will cause build-time cache cleanup to be disabled.
 * `TARGETARCH` - Supports inclusion of non-x86_64 gitlab runners.  This
    value is assumed to match the image's architecture.  If using the
    `--platform` build argument, it will be set automatically.
@@ -96,12 +174,24 @@ Several build arguments are available to control the output image:
   and port supports various observability and debugging features of the
   gitlab runner.  For more information see the [gitlab runner advanced
   configuration documentation](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section).
-* `PRIVILEGED_RUNNER` - Defaults to 'false', may be set 'true'.  When
-  `true`, this causes inner-containers to be created with the `--privileged`
-  flag.  This is a potential security weakness, but is necessary for
-  (among other things) allowing nested container image builds.
+* `PRIVILEGED_RUNNER` - Defaults to 'true', may be set 'true' if you're brave.
+  However this may result in the gitlab-runner failing to launch inner-containers.
+  Setting it false will also prevent building container images using the runner.
 * `RUNNER_TAGS` - Defaults to `podman_in_podman`, may be set to any comma-separated
   list (with no spaces!) of tags.  These show up in GitLab (not the runner
   configuration), and determines where jobs are run.
 * `RUNNER_UNTAGED` - Defaults to `true`, may be set to `false`.  Allows
   the runner to service jobs without any tags on them at all.
+
+### Environment variables
+
+Nearly every option to every gitlab-runner sub-command may be specified via
+environment variable.  Many important/required options are set in the
+`Containerfile`.  However it's entirely possible to pass them in via
+either of the `podman container runlabel...` container commands.  To
+discover them, simply append `--help` to the end of the command.
+For example:
+
+```bash
+podman container runlabel $IMAGE register --help
+```

gitlab-runner-wrapper

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+# This script is intended to be called as the entrypoint for
+# a podman-in-podman gitlab runner container.  Any usage
+# outside that context is not supported and may cause harm.
+
 set -e
 
 unset _debug_args
@@ -7,10 +11,18 @@ if [[ -n "$PODMAN_RUNNER_DEBUG" ]]; then
     _debug_args="--log-level=$PODMAN_RUNNER_DEBUG"
 fi
 
-if [[ "$1" == "run" ]] && [[ ! -S "/tmp/podman-run-1000/podman/podman.sock" ]]; then
+SOCKET="/tmp/podman-run-1000/podman/podman.sock"
+if [[ "$1" == "run" ]] && [[ ! -S "$SOCKET" ]]; then
     podman $_debug_args system service -t 0 &
-    # Prevent SIGHUP propigation to podman process
+    /usr/local/bin/podman-in-podman-maintenance &
+    # Prevent SIGHUP propagation to podman process
     disown -ar
+    sleep 1s  # Give podman a chance to get going
+    # Verify podman is listening on it's socket
+    if [[ ! -S "$SOCKET" ]]; then
+        echo "ERROR: Inner-podman system service failed to start, expecting to find socket '$SOCKET'. Are all volume's owned & writeable by $(id -u podman):$(id -g podman)?" > /dev/stderr
+        exit 1
+    fi
 fi
 
 exec gitlab-runner "$@"

podman-in-podman-maintenance

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# This script is intended to be called by the entrypoint for
+# a podman-in-podman gitlab runner container.  Any usage
+# outside that context is not supported and may cause harm.
+
+set -e
+
+maintain_podman() {
+  # Two days seems to be a good happy-medium beween filling up
+  # about 40gig of storage space from moderate CI activity,
+  # and maintaining a useful level of caching.
+  while sleep 2d; do
+    if [[ -n "$PODMAN_RUNNER_DEBUG" ]]; then
+      echo "$(date --iso-8601=second) ${BASH_SOURCE[0] performing podman maintenance}"
+    fi
+    podman system prune --all --force
+  done
+}