Merge branch 'expand_nested_id_range' into 'main'

Fix <65535 IDs available to nested containers

Closes #3

See merge request qontainers/pipglr!22

.gitlab-ci.yml

@@ -1,25 +1,39 @@
 ---
 
-stages:
-    - build
-
-build:
-  stage: build
+default:
+  image: quay.io/buildah/stable:v1.28.0
   tags:
     - docker
     - linux
-  image:
-    name: gcr.io/kaniko-project/executor:v1.6.0-debug
-    entrypoint: ["/busybox/sh", "-c"]
+
+envars:
+  stage: test
+  script: |
+    echo "Select CI env. vars.:";
+    printenv | egrep '^CI_' | sort
+
+commit_check:
+  stage: test
   variables:
-    BASE_TAG: latest
-    FLAVOR: stable
+    BADRX: '^(squash!)|(fixup!)'
+  script: |
+    dnf install -y git
+    shortlogtmp=$(mktemp -p '' commit_check_tmp_XXXX)
+    git log --oneline --no-show-signature "${CI_MERGE_REQUEST_DIFF_BASE_SHA}..HEAD" > "$shortlogtmp"
+    if egrep -q "$BADRX" "$shortlogtmp"; then
+        egrep "$BADRX" "$shortlogtmp"
+        die "Found the above commits matching '$BADRX'"
+    fi
+
+build:
+  stage: deploy
+  variables:
+    BUILDAH_FORMAT: docker
+    BUILDAH_ISOLATION: chroot
+    STORAGE_DRIVER: vfs
+  before_script:
+    - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
   script:
-    - 'mkdir -p /kaniko/.docker'
-    - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json'
-    - |
-        echo "Select CI env. vars.:";
-        printenv | egrep '^CI_' | sort
     # N/B: There could be more than one merge-request open with this branch's HEAD
     - |
         IMAGE_TAG="${CI_COMMIT_BRANCH}";
@@ -31,10 +45,12 @@ build:
             IMAGE_TAG="latest";
         fi
         echo "Building/Pushing to: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG}";
-    - |
-        /kaniko/executor \
-            --context $CI_PROJECT_DIR \
-            --dockerfile $CI_PROJECT_DIR/Containerfile \
-            --destination "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}" \
-            --build-arg "BASE_TAG=$BASE_TAG" \
-            --build-arg "FLAVOR=$FLAVOR"
+    - >-
+      buildah build \
+        --label "org.opencontainers.image.source=${CI_PROJECT_URL}" \
+        --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" \
+        --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" \
+        --label "org.opencontainers.image.version=${IMAGE_TAG}" \
+        -t "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}" "$CI_PROJECT_DIR"
+    - buildah images
+    - buildah push "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}"

Containerfile

@@ -1,153 +1,80 @@
-# pipglr/Containerfile
-#
-# Builds a Podman-in-Podman Gitlab-Runner image for
-# executing Gitlab CI/CD jobs.  Requires configuration
-# steps specific to Gitlab projects. For more info. see
-# https://docs.gitlab.com/runner/executors/docker.html#use-podman-to-run-docker-commands
-#
+FROM quay.io/centos/centos:stream9
 
-ARG FLAVOR="stable"
-ARG BASE_TAG="latest"
-FROM quay.io/podman/$FLAVOR:$BASE_TAG
+ADD /setup.sh /xpackages.txt /root/
+ADD /containers.conf /home/podman/.config/containers/containers.conf
+ADD /podman.service /podman.socket /prune.service /prune.timer /home/podman/.config/systemd/user/
+ADD /runner.service /home/runner/.config/systemd/user/
+ADD kmsglog.conf /etc/systemd/system.conf.d/
 
-# This is a list of packages to remove and/or exclude from the image.
-# Primarily this is done for security reasons, should a runner process
-# escape confinement.  Having fewer things to poke, lowers the attack
-# surface-area.
-#
-# This list was formed manually by running these commands in the base image:
-#   for package in $(rpm -qa); do \
-#       if dnf erase $package; then echo "$package" >> exclude; fi; \
-#   done; \
-#   cat exclude
-#
-# After adding those packages to this file, the container build was run
-# and package list adjusted, untill no dependency errors were raised.
-ARG EXCLUDE_PACKAGES="\
-    fedora-repos-modular \
-    findutils \
-    libxcrypt-compat \
-    openldap-compat \
-    podman-gvproxy \
-    rootfiles \
-    sudo \
-    vim-minimal \
-    yum"
+# Allow image-builders to choose another version becides "latest" should
+# an incompatible change be introduced.
+ARG RUNNER_VERSION=latest
 
-# Base-image runs as user 'podman', temporarily switch to root
-# for installation/setup.
-USER root
-# Helper for comparison in future RUN operations (DO NOT USE)
-ARG _DNFCMD="dnf --setopt=tsflags=nodocs -y"
-# Set this instead, if (for example) you want to volume-mount in /var/cache/dnf
-ARG DNFCMD="${_DNFCMD}"
-# Avoid installing any documentation to keep image small
-# During install, excluding packages is meaningless if already installed
-RUN set -x && \
-    rm -f /etc/dnf/protected.d/sudo.conf && \
-    rm -f /etc/dnf/protected.d/yum.conf && \
-    $DNFCMD remove ${EXCLUDE_PACKAGES}
+# Permit building containers for alternate architectures.  At the time
+# of this commit, only 'arm64' is available.
+ARG TARGETARCH=amd64
 
-# Enable callers to customize the runner version as needed, otherwise
-# assume this image will be version-tagged, so it's fine to grab the latest.
-ARG RUNNER_VERSION="latest"
-# When building a multi-arch manifest-list, this buid-arg is set automatically.
-ARG TARGETARCH="amd64"
-ENV RUNNER_RPM_URL=https://gitlab-runner-downloads.s3.amazonaws.com/${RUNNER_VERSION}/rpm/gitlab-runner_${TARGETARCH}.rpm
-RUN for rpm in ${EXCLUDE_PACKAGES}; do x+="--exclude=$rpm "; done && \
-    set -x && \
-    $DNFCMD update && \
-    $DNFCMD install $x $RUNNER_RPM_URL && \
-    $DNFCMD upgrade && \
-    if [[ "${DNFCMD}" == "${_DNFCMD}" ]]; then \
-      dnf clean all && \
-      rm -rf /var/cache/dnf; \
-    fi
+# Allow image-builders to choose an alternate nested-container pruning cycle.
+# For most people the default is probably fine.  This setting is dependent
+# on the number and frequency of jobs run, along with the amount of disk-space
+# available for both /cache and /home/podman/.local/share/containers volumes.
+ARG PRUNE_INTERVAL=daily  # see systemd.timer for allowable values
 
-# In case of a runner escape, prevent easy installation of packages.
-RUN rm -f /etc/dnf/protected.d/* && \
-    rpm -e dnf && \
-    rm -f $(type -P rpm)
+# All-in-one packaging/image-setup script to keep things simple.
+RUN PRUNE_INTERVAL=${PRUNE_INTERVAL} \
+    RUNNER_VERSION=${RUNNER_VERSION} \
+    bash /root/setup.sh
 
-ADD /config.toml /home/podman/.gitlab-runner/config.toml
-# The global "listen_address" option is used for metrics and
-# debugging.  Disable it by default since use requires special/
-# additional host configuration.
-# Ref: https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section
-ARG RUNNER_LISTEN_ADDRESS="disabled"
-ENV RUNNER_LISTEN_ADDRESS=$RUNNER_LISTEN_ADDRESS
-RUN if [[ "$RUNNER_LISTEN_ADDRESS" == "disabled" ]]; then \
-        sed -i -r \
-            -e "s/.*@@RUNNER_LISTEN_ADDRESS@@.*//g" \
-            /home/podman/.gitlab-runner/config.toml; \
-    else \
-        sed -i -r \
-            -e "s/@@RUNNER_LISTEN_ADDRESS@@/$RUNNER_LISTEN_ADDRESS/g" \
-            /home/podman/.gitlab-runner/config.toml; \
-    fi
+VOLUME /cache /home/podman/.local/share/containers
+ENTRYPOINT /lib/systemd/systemd
 
-# A small wrapper is needed to launch a background podman system service
-# process for the gitlab-runner to connect to.
-ADD /gitlab-runner-wrapper /podman-in-podman-maintenance /usr/local/bin/
-# Base image UTS NS configuration causes runner to break when launching
-# nested rootless containers.
-RUN sed -i -r \
-        -e 's/^utsns.+host.*/utsns="private"/' \
-        /etc/containers/containers.conf && \
-    chmod +x /usr/local/bin/gitlab-runner-wrapper && \
-    chmod +x /usr/local/bin/podman-in-podman-maintenance && \
-    chown -R podman:podman /home/podman && \
-    chmod u+s /usr/bin/new{uid,gid}map && \
-    rm -f /home/podman/.bash* && \
-    echo DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" > /etc/profile.d/podman.sh
+# Gitlab-runner configuration options, may be freely overridden at
+# container image build time.
+ARG DEFAULT_JOB_IMAGE=registry.fedoraproject.org/fedora-minimal:latest
+# Run nested containers in --privileged mode - required to allow building
+# container images using podman or buildah.  Otherwise may be set 'false'.
+ARG NESTED_PRIVILEGED=true
 
-# Runtime rootless-mode configuration
-USER podman
-VOLUME ["/home/podman/.local/share/containers/storage/",\
-        "/home/podman/.gitlab-runner/", \
-        "/cache"]
-WORKDIR /home/podman
-ENTRYPOINT ["/usr/local/bin/gitlab-runner-wrapper"]
-
-# Ensure root storage directory exists with correct permissions
-RUN mkdir -p .local/share/containers/storage
-
-# Gitlab-runner configuration options.  Default to unprivileged (nested)
-# runner.  Privileged is required to permit nested container image building.
-ARG RUNNER_NAME="qontainers-pipglr"
-# Running inner-podman privileged is necessary at the time of this commit.
-ARG PRIVILEGED_RUNNER="true"
-# Tags allow pinning jobs to specific runners, comma-separated list of
-# tags to add to runner (no spaces!)
-ARG RUNNER_TAGS="podman-in-podman"
-# Permit running jobs without any tag at all
-ARG RUNNER_UNTAGGED="true"
-# Adjust based on usage and storage size to prevent ENOSPACE problems
-ARG CLEAN_INTERVAL="24h"
-ENV CLEAN_INTERVAL="$CLEAN_INTERVAL" \
-    REGISTER_NON_INTERACTIVE="true" \
-    RUNNER_TAG_LIST="$RUNNER_TAGS" \
-    REGISTER_RUN_UNTAGGED="$RUNNER_UNTAGGED" \
-    REGISTER_ACCESS_LEVEL="ref_protected" \
-    REGISTER_MAXIMUM_TIMEOUT="3600" \
-    CI_SERVER_URL="https://gitlab.com/" \
-    RUNNER_NAME="${RUNNER_NAME}" \
-    RUNNER_EXECUTOR="docker" \
-    RUNNER_SHELL="bash" \
-    REGISTER_MAINTENANCE_NOTE="Podman-in-Podman containerized runner" \
-    DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" \
-    DOCKER_DEVICES="/dev/fuse" \
-    DOCKER_IMAGE="registry.fedoraproject.org/fedora-minimal:latest" \
-    DOCKER_CACHE_DIR="/cache" \
-    DOCKER_VOLUMES="/cache" \
-    DOCKER_NETWORK_MODE="host" \
-    DOCKER_PRIVILEGED="$PRIVILEGED_RUNNER"
-
-# Not a real build-arg.  Simply here to save lots of typing.
-ARG _pm="--systemd=true --device=/dev/fuse --security-opt label=disable --user podman --volume pipglr-podman-root:/home/podman/.local/share/containers/storage --volume pipglr-config:/home/podman/.gitlab-runner -v pipglr-podman-cache:/cache -e PODMAN_RUNNER_DEBUG -e LOG_LEVEL"
-
-# These labels simply make it easier to register and execute the runner.
-# Define them last so they are absent should a image-build failure occur.
-LABEL register="podman run -it --rm $_pm --secret REGISTRATION_TOKEN,type=env \$IMAGE register"
-# Note: Privileged mode is required to permit building container images with inner-podman
-LABEL run="podman run -d --privileged --name pipglr $_pm \$IMAGE run"
+# The registration runlabel may be called multiple times to register more than
+# one runner.  Each expects a REGISTRATION_TOKEN secret to be pre-defined and
+# the file './config.toml' to exist (may be empty).  A local-cache volume
+# '/cache' is configured for bind-mounting into all interrior-containers
+# for container-runtime use, as recommended by the docs.  Other settings
+# may be changed if you know what you're doing.
+LABEL register="podman run -it --rm \
+ --secret=REGISTRATION_TOKEN,type=env \
+ -v ./config.toml:/etc/gitlab-runner/config.toml:Z \
+ -e REGISTER_NON_INTERACTIVE=true \
+ -e CI_SERVER_URL=https://gitlab.com/ \
+ -e RUNNER_NAME=pipglr \
+ -e RUNNER_EXECUTOR=docker \
+ -e RUNNER_SHELL=bash \
+ -e REGISTER_MAINTENANCE_NOTE=Podman-In-Podman-GitLab-Runner \
+ -e DOCKER_HOST=unix:///home/runner/podman.sock \
+ -e DOCKER_IMAGE=${DEFAULT_JOB_IMAGE} \
+ -e DOCKER_CACHE_DIR=/cache \
+ -e DOCKER_VOLUMES=/cache \
+ -e DOCKER_NETWORK_MODE=host \
+ -e DOCKER_PRIVILEGED=${NESTED_PRIVILEGED} \
+ --entrypoint=/usr/bin/gitlab-runner \$IMAGE register"
+# Additionally, the nested-podman storage volumes must be pre-created with
+# 'podman' UID/GID values to allow nested containers access.
+LABEL setupstorage="podman volume create --opt o=uid=1000,gid=1000 pipglr-storage"
+# Lastly, the gitlab-runner will manage container-cache in this directory,
+# which will also be bind-mounted into every container.  So it must be
+# writable by both 'podman' user and 'runner' group.
+LABEL setupcache="podman volume create --opt o=uid=1000,gid=1001 pipglr-cache"
+# Helper to extract the current configuration secret to allow editing.
+LABEL dumpconfig="podman run -it --rm \
+ --secret config.toml --entrypoint=/bin/cat \
+ \$IMAGE /var/run/secrets/config.toml"
+# Executing the runner container depends on the config.toml secret being
+# set (see above) and two volumes existing with correct permissions set.
+# Note: The contents of the volumes are not critical, they may be removed
+# and re-created (see above) to quickly free-up disk space.
+LABEL run="podman run -dt --name pipglr \
+ --secret config.toml,uid=1001,gid=1001 \
+ -v pipglr-storage:/home/podman/.local/share/containers \
+ -v pipglr-cache:/cache \
+ --systemd true --privileged \
+ --device /dev/fuse \$IMAGE"

README.md

@@ -22,105 +22,167 @@ configuration relative to their own security situation/environment.
 
 ### Operation
 
-This image supports `podman container runlabel`, or if your version
-lacks this feature, Several labels are set on the image to support
-easy registration and execution of a runner container using a special
-bash command.  See the examples below for more information.
+This image leverages the podman `runlabel` feature heavily.  Several
+labels are set on the image to support easy registration and execution
+of the runner container.  While it's possible to use the container
+with your own command-line,  it's highly recommended to base them
+off of one of the labels.  See the examples below for more information.
 
-#### [Volume Ownership Bug](https://github.com/containers/podman/issues/16576)
-
-Some versions of podman contain a bug where named local volumes aren't owned
-by the namespaced user within a rootless container (i.e. the 'podman' user).
-Since the `podman` user/group inside the `pipglr` container is known, it's
-possible to manually setup ownership ahead of time. This should be be done
-once, prior to registering your runners:
+***Note:*** Some older versions of podman don't support the
+`container runlabel` sub-command.  If this is the case, you may simulate
+it with the following, substituting `<label>` with one of the predefined
+values (i.e. `register`, `setupconfig`, etc.):
 
 ```bash
-$ for VOLUME in pipglr-podman-root pipglr-config pipglr-podman-cache; do \
-    PUPVM="podman unshare podman volume mount $VOLUME"
-    podman volume create $VOLUME && \
-    podman unshare chown 1000:1000 $($PUPVM) && \
-    podman unshare chmod 02770 $($PUPVM) && \
-    podman unshare ls -land $($PUPVM) ; \
-  done
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.<label>)
 ```
 
-If you get `podman system service` startup permission-denied errors, or
-errors from gitlab-runner, unable to connect to the podman socket, this is
-likely the cause.  You can fix it after-the-fact using the same commands as above, just add a `-R` option to the `chown`/`chmod`, and additionally target `./*`.
+#### Persistent containers (step 1)
 
-#### Runner registration
+By default on many distributions, regular users aren't permitted to leave
+background processes running after they log out.  Since this is likely
+desired for the user running the pipglr container, `systemd` needs to be
+configured to override this policy.  For this, you (`$USER`) will need
+root access on the system.
 
-Each time the registration command is run, a new runner is added into
-the configuration.  If however, you simply need to update/modify the
-configuration, please edit the `config.toml` file directly after mounting
-(default) `pipglr-runner-config` (`/home/podman/.gitlab-runner/`) volume.
-For modern versions of podman, registration can be performed with the
-following commands:
+```bash
+$ sudo loginctl enable-linger $USER
+```
+
+#### Expanded user-namespace (step 2)
+
+Some nested container images need to utilize high UID/GID values
+like `65535`.  However, many distributions set that as the default max
+number of user-namespace IDs.  Since three IDs are needed inside the
+`pipglr` container, a larger range is required on the host.  As root,
+edit the two files `/etc/subuid` and `/etc/subgid` to expand the range.
+For example assuming a local user called `pipglr` is used to run the
+container, the contents of these files should be edited to allocate
+`65539` IDs like:
+
+`pipglr:<some number>:65539`
+
+Where `<some number>` was set by your OS when the `pipglr` user was created.
+If you don't make this change, you're likely to experience errors starting
+containers for CI jobs, such as:
+
+`E: setgroups 65534 failed - setgroups (22: Invalid argument)`
+
+#### Runner registration (step 3)
+
+All runners must be connected to a project or group runner configuration
+on your gitlab instance (or `gitlab.com`).  This is done using a special
+registration *runlabel*.  The command can (and probably should) be run
+more than once (using the same `config.toml`) to configure and register
+multiple runners.  This is necessary for the *pipglr* container to execute
+multiple jobs in parallel.  For example, if you want to support running
+four jobs at the same time, you would use the `register` *runlabel*
+four times.
+
+Before using the `register`  *runlabel*, you must set your unique
+*registration* (a.k.a. *activation*) token as a podman *secret*.  This
+secret may be removed once the registration step is complete.  The
+**<actual registration token>** value (below) should be replaced with
+the value obtained from the "runners" settings page of a gitlab
+group or project's *CI/CD Settings*.  Gitlab version 16 and later
+refers to this value as an *activation* token, but the usage is the same.
 
 ```bash
 $ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
 $ echo '<actual registration token>' | podman secret create REGISTRATION_TOKEN -
+```
+
+Next, ***a blank `config.toml` file*** needs to be created.  Without this, the
+`reigster` *runlabel* will return a permission-denied error.  Once the empty
+`config.toml` file is created, you may register one or more runners by repeating
+the registration *runlabel* as follows:
+
+```bash
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ touch ./config.toml  # important: file must exist, even if empty.
 $ podman container runlabel register $IMAGE
+...repeat as desired...
+$ podman secret rm REGISTRATION_TOKEN  # if desired
 ```
 
-Where `<actual registration token>` is the value obtained from the "runners"
-settings page of a gitlab group or project.  When you're finished registering
-as many runners as you want, the secret is no-longer needed and may be removed:
+#### Runner Configuration (step 4)
+
+During the registration process (above), a boiler-plate (default) `config.toml` file
+will be created/updated for you.  At this point you may edit the configuration
+if desired before committing it as a *podman secret*.  Please refer to the
+[gitlab runner documentation](https://docs.gitlab.com/runner/configuration/)
+for details.
 
 ```bash
-$ podman secret rm REGISTRATION_TOKEN
+$ $EDITOR ./config.toml  # if desired
+$ podman secret create config.toml ./config.toml
+$ rm ./config.toml  # if desired
 ```
 
-##### Note
+This may be necessary, for example, to increase the default `concurrency` value
+to reflect the number of registered runners.  If you need to edit this file
+after committing it as a secret, there's
+[ a `dumpconfig`  *runlabel* for that](README.md#configuration-editing).
 
-Some versions of podman don't support the `container runlabel` sub-command.
-If this is the case, you may simulate it with the following command (in addition
-to the other example commands above):
+#### Volume setup (step 5)
+
+Since several users are utilized inside the container volumes must be
+specifically configured to permit access.  This is done using several
+*runlabels* as follows:
 
 ```bash
-$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.register)
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ podman container runlabel setupstorage $IMAGE
+$ podman container runlabel setupcache $IMAGE
 ```
 
-#### Runner Startup
+Note: These volumes generally do not contain any critical operational data,
+they may be re-created anytime to quickly free up host disk-space if
+it's running low.  Simply remove them with the command
+ `podman volume rm pipglr-storage pipglr-cache`.  Then reuse the `setupstorage`
+and `setupcache` *runlabels* as in the above example.
 
-With one or more runners successfully registered and configured, the GitLab
-runner container may be launched with the following commands:
+#### Runner Startup (step 6)
+
+With the runner configuration saved as a Podman secret, and the runner volumes
+created, the GitLab runner container may be launched with the following commands:
 
 ```bash
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
 $ podman container runlabel run $IMAGE
 ```
 
-##### Note
+### Configuration Editing
 
-As above, if you're missing the `container runlabel` sub-command, the following
-may be used instead (assuming `$IMAGE` remains set):
+The gitlab-runner configuration contains some sensitive values which
+should be protected.  The pipglr container assumes the entire configuration
+will be passed in as a Podman secret.  This makes editing it slightly
+convoluted, so a handy *runlabel* `dumpconfig` is available.
+It's intended use is as follows:
 
 ```bash
-$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.run)
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ podman container runlabel dumpconfig $IMAGE > ./config.toml
+$ $EDITOR ./config.toml
+$ podman secret rm config.toml
+$ podman secret create config.toml ./config.toml
+$ rm ./config.toml  # if desired
 ```
 
-#### Runner configuration
+### Debugging
 
-You may inspect/modify the gitlab-runner configuration as you see fit, just be
-sure to use the `podman unshare` command-wrapper to enter the usernamespace.
-For example, to display the config:
-
-```bash
-$ podman unshare $(podman unshare podman volume mount pipglr-config)/config.toml
-```
-
-#### Debugging
-
-The first thing to check is the container output:
+The first thing to check is the container output.  This shows three things:
+Systemd, Podman, and GitLab-Runner output.  For example:
 
 ```bash
 $ podman logs --since 0 pipglr
 ```
 
-Before starting the runner, you may `export PODMAN_RUNNER_DEBUG=debug` to enable
-debugging on the inner-podman.  Whereas `export LOG_LEVEL=debug` can be used to
-debug the gitlab-runner itself.
+Next, try running a pipglr image built with more verbose logging.  Both
+the `runner.service` and `podman.service` files have a `log-level` option.
+Simply increase one or both to the "info", or "debug" level.  Start the
+debug container, and reproduce the problem.
 
 ## Building
 
@@ -133,69 +195,34 @@ $ podman build -t registry.gitlab.com/qontainers/pipglr:latest .
 This will utilize the latest stable version of podman and the latest
 stable version of the gitlab runner.
 
-### Notes
-
-* If you wish to use the `testing` or `upstream` flavors of the podman base image,
-  simply build with `--build-arg FLAVOR=testing` (or `upstream`).
-
-* Additionally or alternatively, you may specify a specific podman base image tag
-  with `--build-arg BASE_TAG=<value>`.  Where `<value>` is either `latest`, the
-  podman image version (e.g. `v4`, `v4.2`, `v4.2.0`, etc.)
-
 ### Build-args
 
 Several build arguments are available to control the output image:
 
-* `FLAVOR` - Choose from 'stable', 'testing', or 'upstream'.  These
-  select the podman base-image to utilize - which may affect the
-  podman version, features, and stability.  For more information
-  see [the podmanimage README](https://github.com/containers/podman/blob/main/contrib/podmanimage/README.md).
-* `BASE_TAG` - When `FLAVOR="stable"`, allows granular choice over the
-  exact podman version.  Possible values include, `latest`, `vX`, `vX.Y`,
-  and `vX.Y.Z` (where, `X`, `Y`, and `Z` represent the podman semantic
-  version numbers).  It's also possible to specify an image SHA.
-* `CLEAN_INTERVAL` - A `sleep` (command) compatible time-argument that
-  determines how often to clean out podman storage of disused containers and
-  images.  Defaults to 24-hours, but should be adjusted based on desired caching-effect
-  versus available storage space and rate of job execution.
-* `EXCLUDE_PACKAGES` - A space-separated list of RPM packages to prevent
-  their existence in the final image.  This is intended as a security measure
-  to limit the attack-surface should a gitlab-runner process escape it's
-  inner-container.
+* `PRUNE_INTERVAL` - A systemd.timer compatible `OnCalendar` value that
+  determines how often to prune Podman's storage of disused containers and
+  images.  Defaults to "daily", but should be adjusted based on desired
+  caching-effect balanced against available storage space and job
+  execution rate.
 * `RUNNER_VERSION` - Allows specifying an exact gitlab runner version.
   By default the `latest` is used, assuming the user is building a tagged
   image anyway.  Valid versions may be found on the [runner
   release page](https://gitlab.com/gitlab-org/gitlab-runner/-/releases).
-* `DNFCMD` - By default this is set to `dnf --setopt=tsflags=nodocs -y`.
-  However, if you'd like to volume-mount in `/var/cache/dnf` then you'll
-  need to use
-  `--build-arg DNFCMD="dnf --setopt=tsflags=nodocs -y --setopt keepcache=true`"
-  Note: Changing `DNFCMD` will cause build-time cache cleanup to be disabled.
 * `TARGETARCH` - Supports inclusion of non-x86_64 gitlab runners.  This
-   value is assumed to match the image's architecture.  If using the
-   `--platform` build argument, it will be set automatically.
-* `RUNNER_LISTEN_ADDRESS` - Disabled by default, setting this to the FQDN
-  and port supports various observability and debugging features of the
-  gitlab runner.  For more information see the [gitlab runner advanced
-  configuration documentation](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section).
-* `PRIVILEGED_RUNNER` - Defaults to 'true', may be set 'true' if you're brave.
-  However this may result in the gitlab-runner failing to launch inner-containers.
-  Setting it false will also prevent building container images using the runner.
-* `RUNNER_TAGS` - Defaults to `podman_in_podman`, may be set to any comma-separated
-  list (with no spaces!) of tags.  These show up in GitLab (not the runner
-  configuration), and determines where jobs are run.
-* `RUNNER_UNTAGED` - Defaults to `true`, may be set to `false`.  Allows
-  the runner to service jobs without any tags on them at all.
+  value is assumed to match the image's architecture.  If using the
+  `--platform` build argument, it will be set automatically.  Note:
+  as of this writing, only `amd64` and `arm64` builds of the gitlab-runner
+  are available.
+* `NESTED_PRIVILEGED` - Defaults to 'true', may be set 'false' to prevent
+  nested containers running in `--privileged` mode.  This will affect
+  the ability to build container images in CI jobs using tools like
+  podman or buildah.
 
 ### Environment variables
 
 Nearly every option to every gitlab-runner sub-command may be specified via
-environment variable.  Many important/required options are set in the
-`Containerfile`.  However it's entirely possible to pass them in via
-either of the `podman container runlabel...` container commands.  To
-discover them, simply append `--help` to the end of the command.
-For example:
-
-```bash
-podman container runlabel $IMAGE register --help
-```
+environment variable.  Some of these are set in the `Containerfile` for
+the `register` *runlabel*.  If you need to set additional runtime
+env. vars., please do so via additional `Environment` optionns in the
+`runner.service` file.  See the *systemd.nspawn* man page for important
+value-format details.

config.toml

@@ -1,7 +0,0 @@
-
-# Ref: https://docs.gitlab.com/runner/configuration/advanced-configuration.html
-
-concurrent = 8
-# N/B: This is DIVIDED among the number of registered runners
-check_interval = 10
-listen_address = "@@RUNNER_LISTEN_ADDRESS@@"  # Will be removed if undefined

containers.conf

@@ -0,0 +1,13 @@
+[containers]
+netns="host"
+userns="host"
+ipcns="host"
+utsns="private"
+cgroupns="host"
+cgroups="disabled"
+log_driver = "k8s-file"
+
+[engine]
+cgroup_manager = "cgroupfs"
+events_logger="file"
+runtime="crun"

gitlab-runner-wrapper

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-# This script is intended to be called as the entrypoint for
-# a podman-in-podman gitlab runner container.  Any usage
-# outside that context is not supported and may cause harm.
-
-set -e
-
-unset _debug_args
-if [[ -n "$PODMAN_RUNNER_DEBUG" ]]; then
-    _debug_args="--log-level=$PODMAN_RUNNER_DEBUG"
-fi
-
-SOCKET="/tmp/podman-run-1000/podman/podman.sock"
-if [[ "$1" == "run" ]] && [[ ! -S "$SOCKET" ]]; then
-    podman $_debug_args system service -t 0 &
-    /usr/local/bin/podman-in-podman-maintenance &
-    # Prevent SIGHUP propagation to podman process
-    disown -ar
-    sleep 1s  # Give podman a chance to get going
-    # Verify podman is listening on it's socket
-    if [[ ! -S "$SOCKET" ]]; then
-        echo "ERROR: Inner-podman system service failed to start, expecting to find socket '$SOCKET'. Are all volume's owned & writeable by $(id -u podman):$(id -g podman)?" > /dev/stderr
-        exit 1
-    fi
-fi
-
-exec gitlab-runner "$@"

kmsglog.conf

@@ -0,0 +1,3 @@
+[Manager]
+LogTarget=kmsg
+LogColor=yes

podman-in-podman-maintenance

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-# This script is intended to be called by the entrypoint for
-# a podman-in-podman gitlab runner container.  Any usage
-# outside that context is not supported and may cause harm.
-
-set -eo pipefail
-
-maintain_podman() {
-  # Two days seems to be a good happy-medium beween filling up
-  # about 40gig of storage space from moderate CI activity,
-  # and maintaining a useful level of caching.
-  while sleep "$CLEAN_INTERVAL"; do
-    if [[ -n "$PODMAN_RUNNER_DEBUG" ]]; then
-      echo "$(date --iso-8601=second) ${BASH_SOURCE[0] performing podman maintenance}"
-    fi
-    podman system prune --all --force
-  done
-}
-
-if [[ -z "$CLEAN_INTERVAL" ]]; then
-  echo "ERROR: Empty/unset \$CLEAN_INTERVAL"
-  exit 1
-fi
-
-maintain_podman

podman.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Podman API Service
+Requires=podman.socket
+After=podman.socket
+Documentation=man:podman-system-service(1)
+StartLimitIntervalSec=0
+
+[Service]
+Delegate=true
+Type=exec
+KillMode=process
+Environment=LOGGING="--log-level=warn"
+ExecStart=/usr/bin/podman $LOGGING system service
+StandardOutput=journal+console
+StandardError=inherit
+
+[Install]
+WantedBy=default.target

podman.socket

@@ -0,0 +1,11 @@
+[Unit]
+Description=Podman API Socket
+Documentation=man:podman-system-service(1)
+
+[Socket]
+ListenStream=/home/runner/podman.sock
+SocketGroup=runner
+SocketMode=0660
+
+[Install]
+WantedBy=sockets.target

prune.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Prune all disused podman volumes, images, and containers
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/podman system prune --all --force

prune.timer

@@ -0,0 +1,6 @@
+[Unit]
+Description=Execute the prune service periodically
+
+[Timer]
+OnCalendar=@@@PRUNE_INTERVAL@@@
+RemainAfterElapse=no

runner.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=Gitlab-runner service
+
+[Service]
+ExecStart=/usr/bin/gitlab-runner --log-level=warn run --user runner --working-directory=/home/runner
+StandardOutput=journal+console
+StandardError=inherit

setup.sh

@@ -0,0 +1,112 @@
+
+
+# This script is intended to be run during container-image build.  Any
+# other usage outside this context is likely to cause harm.
+
+set -eo pipefail
+
+for varname in PRUNE_INTERVAL RUNNER_VERSION TARGETARCH; do
+  if [[ -z "${!varname}" ]]; then
+    echo "Error: \$$varname must be non-empty."
+  fi
+done
+
+# Make image smaller by not installing docs.
+DNF="dnf --setopt=tsflags=nodocs -y"
+
+for rpm in $(egrep -v '^(# )+' < /root/xpackages.txt); do
+  x+="--exclude=$rpm ";
+done
+
+set -x  # show what's happening to make debugging easier
+
+# DNF itself or a dependence may need upgrading, take care of it first.
+$DNF upgrade
+
+$DNF $x install \
+  podman \
+  systemd
+
+# Gitlab-runner package contains scriptlets which do not function properly inside a
+# container-build environment where systemd is not active/running.
+$DNF $x --setopt=tsflags=noscripts install \
+  https://gitlab-runner-downloads.s3.amazonaws.com/$RUNNER_VERSION/rpm/gitlab-runner_${TARGETARCH}.rpm
+
+# Allow removing dnf, sudo, etc. packages.  Also don't start unnecessary or broken
+# systemd services, like anything kernel related or login gettys.
+rm -rf \
+  /etc/dnf/protected.d/* \
+  /etc/sytemd/system/getty.target.wants/* \
+  /etc/sytemd/system/multi-user.target.wants/* \
+  /etc/sytemd/system/sysinit.target.wants/* \
+  /etc/sytemd/system/timers.target.wants/* \
+  /lib/systemd/system/graphical.target.wants/* \
+  /lib/systemd/system/multi-user.target.wants/{getty.target,systemd-ask-password-wall.path} \
+  /lib/systemd/system/sys-kernel*.mount
+
+# Remove unnecessary packages, see xpackages.txt to learn how this list was generated.
+# This makes the image smaller and reduces the attack-surface.
+dnf remove -y $(egrep -v '^(# )+' /root/xpackages.txt)
+
+# Wipe out the DNF cache, then remove it entirely, again to make the image smaller.
+$DNF clean all
+rm -rf /var/cache/dnf /var/log/dnf* /var/log/yum.*
+rpm -e dnf
+
+# Workaround https://bugzilla.redhat.com/show_bug.cgi?id=1995337
+rpm --setcaps shadow-utils
+
+# Prevent copying of skel since it can interfere with the gitlab-runner
+mkdir -p /home/podman /home/runner
+# Guarantee uid/gid 1000 for user 'podman' / 1001 for user 'runner'.
+groupadd -g 1000 podman
+groupadd -g 1001 runner
+# Separate users for services to increase process isolation.
+# The 'podman' user's socket service writes /home/runner/podman.socket
+useradd -M -u 1000 -g podman -G runner podman
+useradd -M -u 1001 -g runner runner
+# Allow 'podman' user to create socket file under /home/runner.
+chmod 770 /home/runner
+
+# Overwrite defaults, only user 'podman' permited to have a user-namespace
+# Split the namespaced ID's around the containers root (ID 0), podman (ID 1000), and
+# runner (ID 1001) such that the user-namespace of any nested containers cannot
+# read or write any files owned by these users (and/or hijack nested container processes).
+# N/B: The range-end (999+64536) ensures a total of 65535 IDs are available for nested-containers.
+#      This requires the host provide a sufficiently large range, i.e. `pipglr:<start>:65539`
+echo -e "podman:1:999\npodman:1002:64536" | tee /etc/subuid > /etc/subgid
+# Host volume mount necessary for nested-podman to use overlayfs2 for container & volume storage.
+mkdir -p /home/podman/.local/share/containers
+# Nested-container's local container-cache volume mount, recommended by gitlab-runner docs.
+mkdir -p /cache
+# Both the gitlab-runner and podman need access to the cache directory / volume mount.
+chown podman:runner /cache
+
+# Setup persistent 'podman' user services to start & run without a login.
+mkdir -p /var/lib/systemd/linger
+touch /var/lib/systemd/linger/podman
+# Setup 'podman' socket and a container-storage pruning service for 'podman' user.
+mkdir -p /home/podman/.config/systemd/user/{sockets.target.wants,default.target.wants}
+cd /home/podman/.config/systemd/user/
+ln -s $PWD/podman.socket ./sockets.target.wants/     # Added from Containerfile
+ln -s $PWD/prune.timer ./default.target.wants/        # also from Containerfile
+# Substitute value from --build-arg if specified, otherwise use default from Containerfile.
+sed -i -e "s/@@@PRUNE_INTERVAL@@@/$PRUNE_INTERVAL/" ./prune.timer
+# Containerfile ADD instruction does not properly set ownership/permissions.
+chown -R 1000:1000 /home/podman
+chmod 700 /home/podman
+
+# Setup persistent 'runner' user services to start & run without a login.
+touch /var/lib/systemd/linger/runner
+mkdir -p /home/runner/.config/systemd/user/default.target.wants
+cd /home/runner/.config/systemd/user/
+# Does not depend on podman.socket file availablility, will retry if not present.
+ln -s $PWD/runner.service ./default.target.wants/
+# gitlab-runner will create side-car '.runner_system_id' file next to 'config.toml'
+# on first startup.  Ensure access is allowed.  Also link to future config file
+# presented as a container-secret.
+mkdir -p /home/runner/.gitlab-runner
+ln -s /var/run/secrets/config.toml /home/runner/.gitlab-runner/config.toml
+# Containerfile ADD instruction does not properly set ownership/permissions.
+chown -R runner:runner /home/runner
+chmod -R 700 /home/runner/.gitlab-runner

xpackages.txt

@@ -0,0 +1,21 @@
+# This list was formed by running the following commands in the base image:
+#   for package in $(rpm -qa); do if dnf erase $package; then echo "$package" >> remove; fi; done
+#   cat remove
+# Including those packages in this file.  Finally, repeatedly running the container build
+# untill no dependency errors were raised.
+
+criu
+criu-libs
+crypto-policies-scripts
+dejavu-sans-fonts
+findutils
+fonts-filesystem
+gdb-gdbserver
+langpacks-core-en
+langpacks-core-font-en
+langpacks-en
+libnet
+protobuf-c
+rootfiles
+vim-minimal
+yum