Setup for quadlet/systemd runtime management

Rather than setting up volumes and starting the pipglr container manually, utilize quadlet + systemd. Retain the old setup and execution method, but move them into separate documentation. Signed-off-by: Chris Evich <cevich@redhat.com>
2024-02-16 13:17:39 -05:00
12 changed files with 385 additions and 517 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1 @@
+/.pre-commit-config.yaml
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,21 +1,10 @@
 ---
+
 default:
-  image: quay.io/buildah/stable:v1.32
+  image: quay.io/buildah/stable:v1.31.0
  tags:
-    - saas-linux-small-amd64
-
-workflow:
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
-      when: never
-    - if: $CI_COMMIT_BRANCH
-    - if: $CI_COMMIT_TAG
-
-include:
-  - component: gitlab.com/blue42u/ci.pre-commit/lite@0.2.0
-    inputs:
-      job_stage: test
+    - docker
+    - linux

 envars:
  stage: test
@@ -25,11 +14,8 @@ envars:

 commit_check:
  stage: test
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - when: never
  variables:
-    BADRX: "^(squash!)|(fixup!)"
+    BADRX: '^(squash!)|(fixup!)'
  script: |
    dnf install -y git
    shortlogtmp=$(mktemp -p '' commit_check_tmp_XXXX)
@@ -40,15 +26,31 @@ commit_check:
    fi

 build:
-  tags:
-    - saas-linux-medium-amd64
  stage: deploy
  variables:
-    FF_GITLAB_REGISTRY_HELPER_IMAGE: 0
    BUILDAH_FORMAT: docker
    BUILDAH_ISOLATION: chroot
    STORAGE_DRIVER: vfs
  before_script:
    - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
  script:
-    - scripts/build.sh
+    # N/B: There could be more than one merge-request open with this branch's HEAD
+    - |
+        IMAGE_TAG="${CI_COMMIT_REF_SLUG}";
+        if [[ -n "$CI_COMMIT_TAG" ]]; then
+            IMAGE_TAG="${CI_COMMIT_TAG}";
+        elif [[ -n "$CI_OPEN_MERGE_REQUESTS" ]]; then
+            IMAGE_TAG=mr$(echo "${CI_OPEN_MERGE_REQUESTS}" | cut -d, -f -1 | cut -d\! -f 2);
+        elif [[ "$CI_COMMIT_BRANCH" == "main" ]]; then
+            IMAGE_TAG="latest";
+        fi
+        echo "Building/Pushing to: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG}";
+    - >-
+      buildah build \
+        --label "org.opencontainers.image.source=${CI_PROJECT_URL}" \
+        --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" \
+        --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" \
+        --label "org.opencontainers.image.version=${IMAGE_TAG}" \
+        -t "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}" "$CI_PROJECT_DIR"
+    - buildah images
+    - buildah push "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}"
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,48 +0,0 @@
-default_language_version:
-  python: python3
-default_install_hook_types: [pre-commit, commit-msg]
-default_stages: [pre-commit]
-
-repos:
-  - repo: https://github.com/executablebooks/mdformat
-    rev: '0.7.17'
-    hooks:
-    - id: mdformat
-      additional_dependencies:
-      - mdformat-footnote
-      - mdformat-tables
-
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
-    hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-      - id: fix-byte-order-marker
-      - id: mixed-line-ending
-      - id: check-executables-have-shebangs
-      - id: check-symlinks
-      - id: destroyed-symlinks
-      - id: check-merge-conflict
-      - id: check-case-conflict
-      - id: no-commit-to-branch
-        args: [--branch, main]
-      - id: check-yaml
-
-  - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: '0.28.0'
-    hooks:
-      # Validate the GitLab CI scripts against the schema. Doesn't catch everything but helps.
-      - id: check-gitlab-ci
-        files: '.*\.gitlab-ci\.yml$'
-
-  - repo: https://github.com/jumanjihouse/pre-commit-hooks
-    rev: '3.0.0'
-    hooks:
-      - id: forbid-binary
-      - id: require-ascii
-
-  - repo: meta
-    hooks:
-      # Un-comment (maybe temporarily) to check which hooks don't apply.
-      # - id: check-hooks-apply
-      - id: check-useless-excludes
--- a/29
+++ b/29
@@ -20,15 +20,11 @@ ADD /root/ /root/
 ADD /etc/ /etc/
 ADD /home/ /home/

-# The build type: either `dev` or `prod`
-# In `dev` mode: the package manager will not be deleted.
-ARG BUILD_TYPE=prod
-
 # Allow image-builders to choose another version besides "latest" should
 # an incompatible change be introduced.
 ARG RUNNER_VERSION=latest

-# Permit building containers for alternate architectures. At the time
+# Permit building containers for alternate architectures.  At the time
 # of this commit, only 'arm64' is available.
 ARG TARGETARCH=amd64

@@ -39,11 +35,9 @@ ARG TARGETARCH=amd64
 ARG PRUNE_INTERVAL=daily  # see systemd.timer for allowable values

 # All-in-one packaging/image-setup script to keep things simple.
-RUN \
- PRUNE_INTERVAL=${PRUNE_INTERVAL} \
- RUNNER_VERSION=${RUNNER_VERSION} \
- BUILD_TYPE=${BUILD_TYPE} \
- bash /root/setup.sh
+RUN PRUNE_INTERVAL=${PRUNE_INTERVAL} \
+    RUNNER_VERSION=${RUNNER_VERSION} \
+    bash /root/setup.sh

 VOLUME /cache /home/podman/.local/share/containers
 ENTRYPOINT /lib/systemd/systemd
@@ -51,17 +45,12 @@ ENTRYPOINT /lib/systemd/systemd
 # Gitlab-runner configuration options, may be freely overridden at
 # container image build time.
 ARG DEFAULT_JOB_IMAGE=registry.fedoraproject.org/fedora-minimal:latest
-
 # Allow image-builders to override the Gitlab URL
-ARG GITLAB_URL=https://gitlab.math.ethz.ch/
-
+ARG GITLAB_URL=https://gitlab.com/
 # Run nested containers in --privileged mode - required to allow building
 # container images using podman or buildah.  Otherwise may be set 'false'.
 ARG NESTED_PRIVILEGED=true

-# Download the FIPS version of gitlab-runner when enabled on the host system.
-ARG ENABLE_FIPS=true
-
 # The registration runlabel may be called multiple times to register more than
 # one runner.  Each expects a REGISTRATION_TOKEN secret to be pre-defined and
 # the file './config.toml' to exist (may be empty).  A local-cache volume
@@ -70,7 +59,6 @@ ARG ENABLE_FIPS=true
 # may be changed if you know what you're doing.
 LABEL register="podman run -it --rm \
 --secret=REGISTRATION_TOKEN,type=env \
- --user=root \
 -v ./config.toml:/etc/gitlab-runner/config.toml:Z \
 -e REGISTER_NON_INTERACTIVE=true \
 -e CI_SERVER_URL=${GITLAB_URL} \
@@ -84,31 +72,24 @@ LABEL register="podman run -it --rm \
 -e DOCKER_NETWORK_MODE=host \
 -e DOCKER_PRIVILEGED=${NESTED_PRIVILEGED} \
 --entrypoint=/usr/bin/gitlab-runner \$IMAGE register"
-
 # Additionally, the nested-podman storage volumes must be pre-created with
 # 'podman' UID/GID values to allow nested containers access.
 LABEL setupstorage="podman volume create --opt o=uid=1000,gid=1000 pipglr-storage"
-
 # Lastly, the gitlab-runner will manage container-cache in this directory,
 # which will also be bind-mounted into every container.  So it must be
 # writable by both 'podman' user and 'runner' group.
 LABEL setupcache="podman volume create --opt o=uid=1000,gid=1001 pipglr-cache"
-
 # Helper to extract the current configuration secret to allow editing.
 LABEL dumpconfig="podman run -it --rm \
- --user=root \
 --secret config.toml --entrypoint=/bin/cat \
 \$IMAGE /var/run/secrets/config.toml"
-
 # Executing the runner container depends on the config.toml secret being
 # set (see above) and two volumes existing with correct permissions set.
 # Note: The contents of the volumes are not critical, they may be removed
 # and re-created (see above) to quickly free-up disk space.
 LABEL run="podman run -dt --name pipglr \
- --user=root \
 --secret config.toml,uid=1001,gid=1001 \
 -v pipglr-storage:/home/podman/.local/share/containers \
 -v pipglr-cache:/cache \
 --systemd true --privileged \
 --device /dev/fuse \$IMAGE"
-# ==========================
--- a/README.md
+++ b/README.md
@@ -1,133 +1,152 @@
-# Podmand-In-Podman Gitlab Runner
-
-This project provides a Gitlab Runner which runs inside a container launched
-with `podman`. The Gitlab Runner itself uses an independent `podman` instance
-inside to launch jobs.
-
 ## Overview

-This container image is built weekly from this `Containerfile`, and made
-available as:
+This container image is built daily from this `Containerfile`, and
+made available as:

- `registry.gitlab.com/qontainers/pipglr:latest`
+* `registry.gitlab.com/qontainers/pipglr:latest`

 -or-

- `registry.gitlab.com/qontainers/pipglr:<version>`
+* `registry.gitlab.com/qontainers/pipglr:<version>`

-It's purpose is to provide an easy method to execute a GitLab runner, to service
-CI/CD jobs for groups and/or repositories on [gitlab.com](https://gitlab.com).
-It comes pre-configured to utilize the gitlab-runner app to execute within a
-rootless podman container, nested inside a rootless podman container.
+It's purpose is to provide an easy method to execute a GitLab runner,
+to service CI/CD jobs for groups and/or repositories on
+[gitlab.com](https://gitlab.com).  It comes pre-configured to utilize
+the gitlab-runner app to execute within a rootless podman container,
+nested inside a rootless podman container.

-This is intended to provide additional layers of security for the host, when
-running potentially arbitrary CI/CD code. Though, the ultimate responsibility
-still rests with the end-user to review the setup and configuration relative to
-their own security situation/environment.
+This is intended to provide additional layers of security for the host,
+when running potentially arbitrary CI/CD code.  Though, the ultimate
+responsibility still rests with the end-user to review the setup and
+configuration relative to their own security situation/environment.

-**Note**: While this can run entirely under a regular user, it will require root
-access for the first two setup steps (below).
+**Note**: While this can run entirely under a regular user, it will require
+root access for the first two setup steps (below).

 ### Operation

-This image leverages the podman `runlabel` feature heavily. Several labels are
-set on the image to support easy registration and execution of the runner
-container. While it's possible to use the container with your own command-line,
-it's highly recommended to base them off of one of the labels. See the examples
-below for more information.
+This image leverages the podman `runlabel` feature heavily.  Several
+labels are set on the image to support easy registration and execution
+of the runner container.  While it's possible to use the container
+with your own command-line,  it's highly recommended to base them
+off of one of the labels.  See the examples below for more information.

-*_Only podman versions 4.8.x and later are supported by pipglr._*
-
-#### Persistent Containers (step 1)
-
-By default on many distributions, regular users aren't permitted to leave
-background processes running after they log out. Since this is likely desired
-for running the pipglr container long-term, `systemd` needs to be configured to
-override this policy.  Doing so requires root or `sudo` access on the
-system.  Assuming the user `johndoe` will be executing the pipglr container,
-linger may be enabled (as the admin user or root) with a command like:
+***Note:*** Some older versions of podman don't support the
+`container runlabel` sub-command.  If this is the case, you may simulate
+it with the following, substituting `<label>` with one of the predefined
+values (i.e. `register`, `setupconfig`, etc.):

 ```bash
-$ sudo loginctl enable-linger `johndoe`
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.<label>)
 ```

-Side-effect: This will allow the user (`johndoe` for example) to persist
-other user-level systemd services as well. For example `podman.socket` is
-handy to enable for `podman remote` access. You could also
-[setup quadlet](https://www.redhat.com/sysadmin/quadlet-podman) or a systemd
-unit so pipglr starts up on system boot.
+#### Persistent containers (step 1)

-#### Expanded Host-system User-Namespace (step 2)
+By default on many distributions, regular users aren't permitted to leave
+background processes running after they log out.  Since this is likely
+desired for running the pipglr container long-term, `systemd` needs to be
+configured to override this policy.  For this, you (`$USER`) will need
+root access on the system.

-**_This is really important and frequently the cause of pipglr issues_**
+```bash
+$ sudo loginctl enable-linger $USER
+```

-On the host, as root, edit the two files `/etc/subuid` and `/etc/subgid` to
-expand the pipglr user's ID allocation by 3.  For example if the host user
-running the pipglr container is named `johndoe`, this entry in the files should
-be edited like:
+Side-effect: This will allow your user to persist other user-level systemd
+services as well.  For example `podman.socket` is handy to enable for
+`podman remote` access.  You could also [setup
+quadlet](https://www.redhat.com/sysadmin/quadlet-podman) or a systemd unit
+so pipglr starts up on system boot.

-`johndoe:<some number>:65539`
+#### Expanded user-namespace (step 2) ***This is probably important***

-Where `<some number>` was set by the OS when the `johndoe` user was created
-(you can ignore this).  Only the last number needs to be increased by three.
-This change will be effective on the user's next login, or immediately
-by running (as the host pipglr user):
+As an added protection/safety measure, pipglr excludes three UID/GIDs
+from being used by job-level containers.  One for `root`, another for
+`runner` and a third for `podman`.  However, some container images
+you may want to use for jobs (mainly Debian/Ubuntu), assign one/more
+essential users a high UID/GID value (like `65535`).
+
+At the same time, most distributions also set `65536` as the default maximum
+number (including ID `0`) of IDs to allocate for user-namespaces (via `/etc/login.defs`).  This
+creates a problem you won't realize until the runner actually picks up a job
+😞 The main symptom of this issue will be messages in the pipglr containers log,
+similar to (abbreviated):
+
+```
+...cut...
+running `/usr/bin/newuidmap ...cut...`: newuidmap: write to uid_map failed: Operation not permitted
+Error: cannot set up namespace using "/usr/bin/newuidmap": exit status 1
+...cut...
+```
+
+or
+
+```
+E: setgroups 65534 failed - setgroups (22: Invalid argument)
+```
+
+***The good news is, working around this is relatively simple:***
+
+As root, edit the two files `/etc/subuid` and `/etc/subgid` to expand the
+by 3 IDs.  For example assuming a user running the pipglr container is
+called `johndoe`, the contents of these files should be edited to allocate
+`65539` IDs like:
+
+`jogndoe:<some number>:65539`
+
+Where `<some number>` was set by your OS when the `johndoe` user was created
+(you can ignore this).  Only the last number needs to be increased.  This
+change will be effective on next login, or immediately by running:

 `podman system migrate`

-_Note:_ This will stop any currently running containers.
+*Note:* This will stop any currently running containers.

-_Details:_ As an added protection/safety measure, pipglr excludes three UID/GIDs
-from being used by job-level (nested) containers. One for `root`, another for
-`runner` and a third for `podman`.  This necessitates expanding the number of
-available UIDs/GIDs from the host machine, to the pipglr container.  This way,
-the full set of 65535 UIDs/GIDs may be utilized by job-level (nested)
-sub-containers.
+#### Runner registration (step 3)

-#### Runner Activation/Registration (step 3)
+All runners must be connected to a project or group runner configuration
+on your gitlab instance (or `gitlab.com`).  This is done using a special
+registration *runlabel*.  The command can (and probably should) be run
+more than once (using the same `config.toml`) to configure and register
+multiple runners.  This is necessary for the *pipglr* container to execute
+multiple jobs in parallel.  For example, if you want to support running
+four jobs at the same time, you would use the `register` *runlabel*
+four times.

-All runners must be connected to a project or group runner configuration on your
-gitlab instance (or `gitlab.com`). This is done using a special registration
-_runlabel_. The command can (and probably should) be run more than once (using
-the same `config.toml`) to configure and register multiple runners. This is
-necessary for the _pipglr_ container to execute multiple jobs in parallel. For
-example, if you want to support running four jobs at the same time, you would
-use the `register` _runlabel_ four times.
-
-Before using the `register` _runlabel_, you must set your unique _registration_
-(a.k.a. _activation_) token as a podman _secret_. This secret may be removed
-once the registration step is complete. The **`<actual registration token>`**
-value (below) should be replaced with the value obtained from the `runners`
-settings page of a gitlab group or project's _CI/CD Settings_. Gitlab version 16
-and later refers to this value as an _activation_ token, but the usage is the
-same.
+Before using the `register`  *runlabel*, you must set your unique
+*registration* (a.k.a. *activation*) token as a podman *secret*.  This
+secret may be removed once the registration step is complete.  The
+**<actual registration token>** value (below) should be replaced with
+the value obtained from the "runners" settings page of a gitlab
+group or project's *CI/CD Settings*.  Gitlab version 16 and later
+refers to this value as an *activation* token, but the usage is the same.

 ```bash
 $ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
 $ echo '<actual registration token>' | podman secret create REGISTRATION_TOKEN -
 ```

-Next, **_a blank `config.toml` file_** needs to be created. Without this, the
-`reigster` _runlabel_ will return a permission-denied error. Once the empty
+Next, ***a blank `config.toml` file*** needs to be created.  Without this, the
+`reigster` *runlabel* will return a permission-denied error.  Once the empty
 `config.toml` file is created, you may register one or more runners by repeating
-the registration _runlabel_ as follows:
+the registration *runlabel* as follows:

 ```bash
 $ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
 $ touch ./config.toml  # important: file must exist, even if empty.
 $ podman container runlabel register $IMAGE
-# ...repeat as desired...
+...repeat as desired...
 $ podman secret rm REGISTRATION_TOKEN  # if desired
 ```

 #### Runner Configuration (step 4)

-During the registration process (above), a boiler-plate (default) `config.toml`
-file will be created/updated for you. At this point you may edit the
-configuration if desired before committing it as a _podman secret_. Please refer
-to the
-[gitlab runner documentation](https://docs.gitlab.com/runner/configuration/) for
-details.
+During the registration process (above), a boiler-plate (default) `config.toml` file
+will be created/updated for you.  At this point you may edit the configuration
+if desired before committing it as a *podman secret*.  Please refer to the
+[gitlab runner documentation](https://docs.gitlab.com/runner/configuration/)
+for details.

 ```bash
 $ $EDITOR ./config.toml  # if desired
@@ -136,45 +155,41 @@ $ rm ./config.toml  # if desired
 ```

 This may be necessary, for example, to increase the default `concurrency` value
-to reflect the number of registered runners. If you need to edit this file after
-committing it as a secret, there's
-[a `dumpconfig` _runlabel_ for that](README.md#configuration-editing).
+to reflect the number of registered runners.  If you need to edit this file
+after committing it as a secret, there's
+[ a `dumpconfig`  *runlabel* for that](README.md#configuration-editing).

-#### Volume Setup (step 5)
+#### Quadlet setup and container start (step 5)

-Since several users are utilized inside the container volumes must be
-specifically configured to permit access. This is done using several _runlabels_
-as follows:
+**Note**: If your system is missing or does not support the use of quadlet
+(`man 5 podman-systemd.unit`), you'll find [manual volume setup steps
+here](manual_setup.md).
+
+Create and copy the quadlet configuration files:

 ```bash
-$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
-$ podman container runlabel setupstorage $IMAGE
-$ podman container runlabel setupcache $IMAGE
+$ mkdir -p ~/.config/containers/systemd
+$ cp quadlet/* ~/.config/containers/systemd/
 ```

-Note: These volumes generally do not contain any critical operational data, they
-may be re-created anytime to quickly free up host disk-space if it's running
-low. Simply remove them with the command
-`podman volume rm pipglr-storage pipglr-cache`. Then reuse the `setupstorage`
-and `setupcache` _runlabels_ as in the above example.
-
-#### Runner Startup (step 6)
-
-With the runner configuration saved as a Podman secret, and the runner volumes
-created, the GitLab runner container may be launched with the following
-commands:
+Finally, reload the local systemd user-slice to generate the unit files,
+and fire up pipglr!

 ```bash
-$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
-$ podman container runlabel run $IMAGE
+$ systemd --user daemon-reload
+$ systemd start pipglr
 ```

+If you want the service to start automatically on boot, you may run
+`systemctl --user enable podman-restart.service`
+
 ### Configuration Editing

-The gitlab-runner configuration contains some sensitive values which should be
-protected. The pipglr container assumes the entire configuration will be passed
-in as a Podman secret. This makes editing it slightly convoluted, so a handy
-_runlabel_ `dumpconfig` is available. It's intended use is as follows:
+The gitlab-runner configuration contains some sensitive values which
+should be protected.  The pipglr container assumes the entire configuration
+will be passed in as a Podman secret.  This makes editing it slightly
+convoluted, so a handy *runlabel* `dumpconfig` is available.
+It's intended use is as follows:

 ```bash
 $ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
@@ -187,17 +202,17 @@ $ rm ./config.toml  # if desired

 ### Debugging

-The first thing to check is the container output. This shows three things:
-Systemd, Podman, and GitLab-Runner output. For example:
+The first thing to check is the container output.  This shows three things:
+Systemd, Podman, and GitLab-Runner output.  For example:

 ```bash
 $ podman logs --since 0 pipglr
 ```

-Next, try running a pipglr image built with more verbose logging. Both the
-`runner.service` and `podman.service` files have a `log-level` option. Simply
-increase one or both to the `info`, or `debug` level. Start the debug container,
-and reproduce the problem.
+Next, try running a pipglr image built with more verbose logging.  Both
+the `runner.service` and `podman.service` files have a `log-level` option.
+Simply increase one or both to the "info", or "debug" level.  Start the
+debug container, and reproduce the problem.

 ## Building

@@ -207,42 +222,39 @@ This image may be built simply with:
 $ podman build -t registry.gitlab.com/qontainers/pipglr:latest .
 ```

-This will utilize the latest stable version of podman and the latest stable
-version of the gitlab runner.
+This will utilize the latest stable version of podman and the latest
+stable version of the gitlab runner.

-### Build-Arguments
+### Build-args

 Several build arguments are available to control the output image:

- `BUILD_TYPE`: The build type, either `prod` or `dev`. In `dev` mode, the package
-  manager is not deleted for development and debugging purposes. Please see
-  [`build.sh`](scripts/build.sh) for more details.
- `PRUNE_INTERVAL`: A systemd.timer compatible `OnCalendar` value that
+* `PRUNE_INTERVAL` - A systemd.timer compatible `OnCalendar` value that
  determines how often to prune Podman's storage of disused containers and
-  images. Defaults to `daily`, but should be adjusted based on desired
-  caching-effect balanced against available storage space and job execution
-  rate.
- `RUNNER_VERSION`: Allows specifying an exact gitlab runner version. By default
-  the `latest` is used, assuming the user is building a tagged image anyway.
-  Valid versions may be found on the
-  [runner release page](https://gitlab.com/gitlab-org/gitlab-runner/-/releases).
- `TARGETARCH`: Supports inclusion of non-x86_64 gitlab runners. This value is
-  assumed to match the image's architecture. If using the `--platform` build
-  argument, it will be set automatically. Note: as of this writing, only `amd64`
-  and `arm64` builds of the gitlab-runner are available.
- `GITLAB_URL`: Defaults to `https://gitlab.com/` but can be set to point to a
-  self hosted instance of Gitlab.
- `NESTED_PRIVILEGED`: Defaults to `true`, may be set `false` to prevent nested
-  containers running in `--privileged` mode. This will affect the ability to
-  build container images in CI jobs using tools like podman or buildah.
+  images.  Defaults to "daily", but should be adjusted based on desired
+  caching-effect balanced against available storage space and job
+  execution rate.
+* `RUNNER_VERSION` - Allows specifying an exact gitlab runner version.
+  By default the `latest` is used, assuming the user is building a tagged
+  image anyway.  Valid versions may be found on the [runner
+  release page](https://gitlab.com/gitlab-org/gitlab-runner/-/releases).
+* `TARGETARCH` - Supports inclusion of non-x86_64 gitlab runners.  This
+  value is assumed to match the image's architecture.  If using the
+  `--platform` build argument, it will be set automatically.  Note:
+  as of this writing, only `amd64` and `arm64` builds of the gitlab-runner
+  are available.
+* `GITLAB_URL` - Defaults to 'https://gitlab.com/' but can be set to point
+  to a self hosted instance of Gitlab.
+* `NESTED_PRIVILEGED` - Defaults to 'true', may be set 'false' to prevent
+  nested containers running in `--privileged` mode.  This will affect
+  the ability to build container images in CI jobs using tools like
+  podman or buildah.

-Additional build-args are available as well.  See the `Containerfile` comments
-for more details.
-
-### Environment Variables
+### Environment variables

 Nearly every option to every gitlab-runner sub-command may be specified via
-environment variable. Some of these are set in the `Containerfile` for the
-`register` _runlabel_. If you need to set additional runtime env. vars., please
-do so via additional `Environment` optionns in the `runner.service` file. See
-the _systemd.nspawn_ man page for important value-format details.
+environment variable.  Some of these are set in the `Containerfile` for
+the `register` *runlabel*.  If you need to set additional runtime
+env. vars., please do so via additional `Environment` optionns in the
+`runner.service` file.  See the *systemd.nspawn* man page for important
+value-format details.
--- a/manual_setup.md
+++ b/manual_setup.md
@@ -0,0 +1,32 @@
+### Additional Manual steps
+
+On systems without Quadlet, some additional steps are required
+to get the pipglr container up and running.
+
+### Manual Volume setup
+
+Since several users are utilized inside the container volumes must be
+specifically configured to permit access.  This is done using several
+*runlabels* as follows:
+
+```bash
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ podman container runlabel setupstorage $IMAGE
+$ podman container runlabel setupcache $IMAGE
+```
+
+Note: These volumes generally do not contain any critical operational data,
+they may be re-created anytime to quickly free up host disk-space if
+it's running low.  Simply remove them with the command
+ `podman volume rm pipglr-storage pipglr-cache`.  Then reuse the `setupstorage`
+and `setupcache` *runlabels* as in the above example.
+
+#### Manual Runner Startup
+
+With the runner configuration saved as a Podman secret, and the runner volumes
+created, the GitLab runner container may be launched with the following commands:
+
+```bash
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ podman container runlabel run $IMAGE
+```
--- a/quadlet/pipglr-cache.volume
+++ b/quadlet/pipglr-cache.volume
@@ -0,0 +1,13 @@
+[Unit]
+Description=Podman-in-podman GitLab Runner job cache storage volume
+Documentation=https://gitlab.com/qontainers/pipglr/-/blob/main/README.md
+After=local-fs.target
+Requires=podman.socket
+
+[Volume]
+VolumeName=pipglr-cache
+Copy=false
+# The `podman` user inside the container should own everything
+Options=o=uid=1000,gid=1000
+# Support podman...prune --filters=persistent!=true
+Label=persistent=true
--- a/quadlet/pipglr-storage.volume
+++ b/quadlet/pipglr-storage.volume
@@ -0,0 +1,13 @@
+[Unit]
+Description=Podman-in-podman GitLab Runner nested-container storage volume
+Documentation=https://gitlab.com/qontainers/pipglr/-/blob/main/README.md
+After=local-fs.target
+Requires=podman.socket
+
+[Volume]
+VolumeName=pipglr-storage
+Copy=false
+# The `podman` user inside the container should own everything
+Options=o=uid=1000,gid=1000
+# Support podman...prune --filters=persistent!=true
+Label=persistent=true
--- a/quadlet/pipglr.container
+++ b/quadlet/pipglr.container
@@ -0,0 +1,33 @@
+[Unit]
+Description=Podman-in-podman GitLab Runner
+Documentation=https://gitlab.com/qontainers/pipglr/-/blob/main/README.md
+After=pipglr-storage-volume.service pipglr-cache-volume.service
+Requires=podman.socket pipglr-storage-volume.service pipglr-cache-volume.service
+
+[Container]
+ContainerName=pipglr
+Image=registry.gitlab.com/qontainers/pipglr
+
+# Required to run containers inside a container and ensure
+# container can be managed with podman-restart.service
+# Note: See https://github.com/containers/podman/issues/20418
+PodmanArgs=--privileged --restart=always
+
+# A nested systemd is used to manage nested podman & gitlab runner services
+Systemd=true
+
+# Allow jobs access to utilize fuse-overlayfs, for example to build container images.
+Device=/dev/fuse
+
+# Must be owned by the gitlab-runner user
+Secret config.toml,uid=1001,gid=1001
+
+# Add network isolation from other containers
+Network=pipglr.network
+
+# Storage for nested container images and job cache
+Volume=pipglr-storage:/home/podman/.local/share/containers
+Volume=pipglr-cache:/cache
+
+# No need to preserve this between runs
+VolatileTmp=true
--- a/quadlet/pipglr.network
+++ b/quadlet/pipglr.network
@@ -0,0 +1,11 @@
+[Unit]
+Description=Podman-in-podman GitLab Runner dedicated network
+Documentation=https://gitlab.com/qontainers/pipglr/-/blob/main/README.md
+After=network-online.target
+Requires=podman.socket network.target
+
+[Network]
+Driver=bridge
+# The pipglr container never accesses other containers on/or the host
+Options=isolate
+DisableDNS=true
--- a/root/setup.sh
+++ b/root/setup.sh
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-#
+
 # This script is intended to be run during container-image build.  Any
 # other usage outside this context is likely to cause harm.
 #
@@ -21,202 +20,108 @@

 set -eo pipefail

-function die() {
-  echo -n '!! ERROR:' >&2
-  printf "         %s\n" "$@" >&2
-  exit 1
-}
-
-function check_vars() {
-  for varname in PRUNE_INTERVAL RUNNER_VERSION TARGETARCH; do
-    if [[ -z "${!varname}" ]]; then
-      die "Env. variable '$varname' must be non-empty."
-    fi
-  done
-
-  if [[ ! "$BUILD_TYPE" =~ dev|prod ]]; then
-    die "Build type must be 'dev' or 'prod': '$BUILD_TYPE'."
+for varname in PRUNE_INTERVAL RUNNER_VERSION TARGETARCH; do
+  if [[ -z "${!varname}" ]]; then
+    echo "Error: \$$varname must be non-empty."
  fi
-}
+done

-function main() {
-  # Show what's happening to make debugging easier
-  set -x
+# Make image smaller by not installing docs.
+DNF="dnf --setopt=tsflags=nodocs -y"

-  # Make image smaller by not installing docs.
-  dnf=(dnf --setopt=tsflags=nodocs -y)
+for rpm in $(egrep -v '^(# )+' < /root/xpackages.txt); do
+  x+="--exclude=$rpm ";
+done

-  install_packages
-  setup_user
-  setup_service_podman
-  setup_service_runner
-  setup_gitlab_config
-  setup_volumes
+set -x  # show what's happening to make debugging easier

-  finalize_ownership
-}
+# DNF itself or a dependence may need upgrading, take care of it first.
+$DNF upgrade

-function is_release() {
-  [ "$BUILD_TYPE" = "prod" ] || return 1
-}
+$DNF $x install \
+  podman \
+  systemd

-function install_packages() {
-  readarray xpackages < <(grep -vE '^(# )+' </root/xpackages.txt)
-  local exclude_args=()
-  for rpm in "${xpackages[@]}"; do
-    exclude_args+=("--exclude=$rpm")
-  done
+# Gitlab-runner package contains scriptlets which do not function properly inside a
+# container-build environment where systemd is not active/running.
+$DNF $x --setopt=tsflags=noscripts install \
+  https://gitlab-runner-downloads.s3.amazonaws.com/$RUNNER_VERSION/rpm/gitlab-runner_${TARGETARCH}.rpm

-  # DNF itself or a dependence may need upgrading, take care of it first.
-  "${dnf[@]}" upgrade &&
-    "${dnf[@]}" "${exclude_args[@]}" install \
-      podman \
-      systemd
+# Allow removing dnf, sudo, etc. packages.  Also don't start unnecessary or broken
+# systemd services, like anything kernel related or login gettys.
+rm -rf \
+  /etc/dnf/protected.d/* \
+  /etc/sytemd/system/getty.target.wants/* \
+  /etc/sytemd/system/multi-user.target.wants/* \
+  /etc/sytemd/system/sysinit.target.wants/* \
+  /etc/sytemd/system/timers.target.wants/* \
+  /lib/systemd/system/graphical.target.wants/* \
+  /lib/systemd/system/multi-user.target.wants/{getty.target,systemd-ask-password-wall.path} \
+  /lib/systemd/system/sys-kernel*.mount

-  # Gitlab-runner package contains scriptlets which do not function properly inside a
-  # container-build environment where systemd is not active/running.
-  if [[ ${ENABLE_FIPS} == true && $(cat /proc/sys/crypto/fips_enabled) == 1 ]]; then
-    PACKAGE_FILES=(
-      "https://s3.dualstack.us-east-1.amazonaws.com/gitlab-runner-downloads/$RUNNER_VERSION/rpm/gitlab-runner_${TARGETARCH}-fips.rpm"
-    )
-  else
-    PACKAGE_FILES=(
-      "https://s3.dualstack.us-east-1.amazonaws.com/gitlab-runner-downloads/$RUNNER_VERSION/rpm/gitlab-runner_${TARGETARCH}.rpm"
-      "https://s3.dualstack.us-east-1.amazonaws.com/gitlab-runner-downloads/$RUNNER_VERSION/rpm/gitlab-runner-helper-images.rpm"
-    )
-  fi
+# Remove unnecessary packages, see xpackages.txt to learn how this list was generated.
+# This makes the image smaller and reduces the attack-surface.
+dnf remove -y $(egrep -v '^(# )+' /root/xpackages.txt)

-  "${dnf[@]}" "${exclude_args[@]}" \
-    --setopt=tsflags=noscripts install \
-    ${PACKAGE_FILES[@]}
+# Wipe out the DNF cache, then remove it entirely, again to make the image smaller.
+$DNF clean all
+rm -rf /var/cache/dnf /var/log/dnf* /var/log/yum.*
+rpm -e dnf

-  # Also don't start unnecessary or broken
-  # systemd services, like anything kernel related or login gettys.
-  rm -rf \
-    /etc/systemd/system/getty.target.wants/* \
-    /etc/systemd/system/multi-user.target.wants/* \
-    /etc/systemd/system/sysinit.target.wants/* \
-    /etc/systemd/system/timers.target.wants/* \
-    /lib/systemd/system/graphical.target.wants/* \
-    /lib/systemd/system/multi-user.target.wants/{getty.target,systemd-ask-password-wall.path} \
-    /lib/systemd/system/sys-kernel*.mount
+# Workaround https://bugzilla.redhat.com/show_bug.cgi?id=1995337
+rpm --setcaps shadow-utils

-  # Allow removing dnf, sudo, etc. packages.
-  rm -rf \
-    /etc/dnf/protected.d/*
-  # Remove unnecessary packages, see xpackages.txt to learn how this list was generated.
-  # This makes the image smaller and reduces the attack-surface.
-  dnf remove -y "${xpackages[@]}"
+# Prevent copying of skel since it can interfere with the gitlab-runner
+mkdir -p /home/podman /home/runner
+# Guarantee uid/gid 1000 for user 'podman' / 1001 for user 'runner'.
+groupadd -g 1000 podman
+groupadd -g 1001 runner
+# Separate users for services to increase process isolation.
+# The 'podman' user's socket service writes /home/runner/podman.socket
+useradd -M -u 1000 -g podman -G runner podman
+useradd -M -u 1001 -g runner runner
+# Allow 'podman' user to create socket file under /home/runner.
+chmod 770 /home/runner

-  if is_release; then
-    # Wipe out the DNF cache, then remove it entirely, again to make the image smaller.
-    "${dnf[@]}" clean all
-    rm -rf /var/cache/dnf /var/log/dnf* /var/log/yum.*
-    rpm -e dnf
-  fi
+# Overwrite defaults, only user 'podman' permited to have a user-namespace
+# Split the namespaced ID's around the containers root (ID 0), podman (ID 1000), and
+# runner (ID 1001) such that the user-namespace of any nested containers cannot
+# read or write any files owned by these users (and/or hijack nested container processes).
+# N/B: The range-end (999+64536) ensures a total of 65535 IDs are available for nested-containers.
+#      This requires the host provide a sufficiently large range, i.e. `pipglr:<start>:65539`
+echo -e "podman:1:999\npodman:1002:64536" | tee /etc/subuid > /etc/subgid
+# Host volume mount necessary for nested-podman to use overlayfs2 for container & volume storage.
+mkdir -p /home/podman/.local/share/containers
+# Nested-container's local container-cache volume mount, recommended by gitlab-runner docs.
+mkdir -p /cache
+# Both the gitlab-runner and podman need access to the cache directory / volume mount.
+chown podman:runner /cache

-  # Workaround https://bugzilla.redhat.com/show_bug.cgi?id=1995337
-  # Base-image failing to confer capabilities properly on
-  # /usr/bin/new{u,g}idmap to `cap_set{u,g}id=ep` in new image layers
-  rpm --setcaps shadow-utils
-}
+# Setup persistent 'podman' user services to start & run without a login.
+mkdir -p /var/lib/systemd/linger
+touch /var/lib/systemd/linger/podman
+# Setup 'podman' socket and a container-storage pruning service for 'podman' user.
+mkdir -p /home/podman/.config/systemd/user/{sockets.target.wants,default.target.wants}
+cd /home/podman/.config/systemd/user/
+ln -s $PWD/podman.socket ./sockets.target.wants/     # Added from Containerfile
+ln -s $PWD/prune.timer ./default.target.wants/        # also from Containerfile
+# Substitute value from --build-arg if specified, otherwise use default from Containerfile.
+sed -i -e "s/@@@PRUNE_INTERVAL@@@/$PRUNE_INTERVAL/" ./prune.timer
+# Containerfile ADD instruction does not properly set ownership/permissions.
+chown -R 1000:1000 /home/podman
+chmod 700 /home/podman

-function setup_user() {
-  # Guarantee uid/gid 1000 for user 'podman' / 1001 for user 'runner'.
-  groupadd -g 1000 podman
-  groupadd -g 1001 runner
-
-  # Separate users for services to increase process isolation.
-  # The 'podman' user's socket service writes /home/runner/podman.socket
-  # Prevent copying of skel since it can interfere with the gitlab-runner
-  mkdir -p /home/podman /home/runner
-  useradd -M -u 1000 -g podman -G runner podman
-  useradd -M -u 1001 -g runner runner
-
-  # Allow only podman in `/home/podman`.
-  chmod 700 /home/podman
-  # Allow 'podman' user to create socket file under `/home/runner`.
-  chmod 770 /home/runner
-
-  # Set permissions.
-  chown -R runner:runner /home/runner
-  chown -R podman:podman /home/podman
-
-  # Overwrite defaults, only user 'podman' permitted to have a user-namespace
-  # Split the namespaced ID's around the containers root (ID 0), podman (ID 1000), and
-  # runner (ID 1001) such that the user-namespace of any nested containers cannot
-  # read or write any files owned by these users (and/or hijack nested container processes).
-  # N/B: The range-end (999+64536) ensures a total of 65535 IDs are available for nested-containers.
-  #      This requires the host provide a sufficiently large range, i.e. `pipglr:<start>:65539`
-  echo -e "podman:1:999\npodman:1002:64536" | tee /etc/subuid >/etc/subgid
-}
-
-function setup_volumes() {
-  # Host volume mount necessary for nested-podman to use overlayfs2 for container & volume storage.
-  mkdir -p /home/podman/.local/share/containers
-  touch /home/podman/.local/share/containers/.placeholder
-
-  # Nested-container's local container-cache volume mount, recommended by gitlab-runner docs.
-  mkdir -p /cache
-  touch /cache/.placeholder
-
-  # Both the gitlab-runner and podman need access to the cache directory / volume mount.
-  chown podman:runner /cache
-}
-
-function setup_service_podman() {
-  # Setup persistent 'podman' user services to start & run without a login.
-  mkdir -p /var/lib/systemd/linger
-  touch /var/lib/systemd/linger/podman
-
-  # Setup 'podman' socket and a container-storage pruning service for 'podman' user.
-  mkdir -p /home/podman/.config/systemd/user/{sockets.target.wants,default.target.wants}
-
-  cd /home/podman/.config/systemd/user/
-  ln -s "$(pwd)/podman.socket" ./sockets.target.wants/ # Added from Containerfile
-  ln -s "$(pwd)/prune.timer" ./default.target.wants/   # also from Containerfile
-
-  # Substitute value from --build-arg if specified, otherwise use default from Containerfile.
-  sed -i -e "s/@@@PRUNE_INTERVAL@@@/$PRUNE_INTERVAL/" ./prune.timer
-}
-
-function setup_service_runner() {
-  # Setup persistent 'runner' user services to start & run without a login.
-  touch /var/lib/systemd/linger/runner
-
-  # Setup persistent 'runner' user services to start & run without a login.
-  mkdir -p /home/runner/.config/systemd/user/default.target.wants
-
-  cd /home/runner/.config/systemd/user/
-  # Does not depend on podman.socket file availability, will retry if not present.
-  ln -s "$(pwd)/runner.service" ./default.target.wants/
-}
-
-function setup_gitlab_config() {
-  # gitlab-runner will create side-car '.runner_system_id' file next to 'config.toml'
-  # on first startup.  Ensure access is allowed.  Also link to future config file
-  # presented as a container-secret.
-  mkdir -p /home/runner/.gitlab-runner
-  ln -s /var/run/secrets/config.toml /home/runner/.gitlab-runner/config.toml
-
-  chmod -R 700 /home/runner/.gitlab-runner
-}
-
-function finalize_ownership() {
-  # Adjust ownership to all files created after `setup_user`.
-  # and also to the `ADD` instruction in the `Containerfile`.
-  chown -R runner:runner /home/runner
-  chown -R podman:podman /home/podman
-
-  # Ensure correct permissions of system configuration files.
-  # Somehow these can be set incorrectly during Containerfile
-  # ADD instruction.
-  local path
-  for path in "/etc/systemd/system.conf.d" "/etc/systemd/system/user-.slice.d"; do
-    chown root:root ${path}/*
-    chmod 0644 ${path}/*
-  done
-}
-
-check_vars
-main
+# Setup persistent 'runner' user services to start & run without a login.
+touch /var/lib/systemd/linger/runner
+mkdir -p /home/runner/.config/systemd/user/default.target.wants
+cd /home/runner/.config/systemd/user/
+# Does not depend on podman.socket file availablility, will retry if not present.
+ln -s $PWD/runner.service ./default.target.wants/
+# gitlab-runner will create side-car '.runner_system_id' file next to 'config.toml'
+# on first startup.  Ensure access is allowed.  Also link to future config file
+# presented as a container-secret.
+mkdir -p /home/runner/.gitlab-runner
+ln -s /var/run/secrets/config.toml /home/runner/.gitlab-runner/config.toml
+# Containerfile ADD instruction does not properly set ownership/permissions.
+chown -R runner:runner /home/runner
+chmod -R 700 /home/runner/.gitlab-runner
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -1,87 +0,0 @@
-#!/usr/bin/env bash
-# Usage: build.sh [<build-type>]
-#
-# If the build type (second arg. `<build-type>`) is `prod`
-# the images are build in `release` mode. For all other build types
-# the images are build for development and testing purposes
-# By default the build type is `prod`.
-
-set -eu
-set -o pipefail
-
-ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)/.."
-cd "$ROOT_DIR"
-
-function ci_running() {
-  [ "${CI:-}" = "true" ] && return 0
-  return 1
-}
-
-# Define the image tag depending on the context.
-function get_image_tag() {
-  local build_type="$1"
-  local image_tag="${build_type}-latest"
-
-  # Define image tag.
-  if ci_running; then
-    # The image tag gets adjusted depending on
-    # if it is a merge request or build on the
-    # main branch or on a tag.
-
-    image_tag="${CI_COMMIT_REF_SLUG:-}"
-
-    if [[ -n "${CI_COMMIT_TAG:-}" ]]; then
-      image_tag="${CI_COMMIT_TAG}"
-    elif [[ -n "${CI_OPEN_MERGE_REQUESTS:-}" ]]; then
-      image_tag=mr$(echo "${CI_OPEN_MERGE_REQUESTS}" | cut -d, -f -1 | cut -d\! -f 2)
-    elif [[ "${CI_COMMIT_BRANCH:-}" == "main" ]]; then
-      image_tag="latest"
-    fi
-  fi
-
-  echo "$image_tag"
-}
-
-function main() {
-  # Define common build variables.
-  local container_mgr=${CI_CONTAINER_MGR:-buildah}
-  local project_dir=${CI_PROJECT_DIR:-.}
-  local registry_name=${CI_REGISTRY_IMAGE:-"containers-storage:pipglr"}
-  local build_type=${CI_BUILD_TYPE:-${1:-prod}}
-
-  # Define image name and tag.
-  local image_tag image_name
-  image_tag=$(get_image_tag "$build_type")
-  image_name="${registry_name}:${image_tag}"
-
-  # Define OpenContainers labels.
-  local oc_project_url=${CI_PROJECT_URL:-file://$ROOT_DIR}
-  local oc_commit_sha=${CI_COMMIT_SHA:-$(git rev-parse HEAD)}
-  local oc_job_started_at=${CI_JOB_STARTED_AT:-$(date -u --iso-8601=seconds)}
-  local oc_version="${image_tag}"
-
-  BUILD_CMD=(
-    "${container_mgr}" build
-    --label "org.opencontainers.image.source=${oc_project_url}"
-    --label "org.opencontainers.image.revision=${oc_commit_sha}"
-    --label "org.opencontainers.image.created=${oc_job_started_at}"
-    --label "org.opencontainers.image.version=${oc_version}"
-    --build-arg "BUILD_TYPE=${build_type}"
-    -t "$image_name"
-    "${project_dir}")
-
-  echo "Build image: '$image_name'"
-  echo -e "Build command:\n" "${BUILD_CMD[@]}"
-
-  "${BUILD_CMD[@]}"
-
-  echo "Images are:"
-  "${container_mgr}" images
-
-  if ci_running; then
-    echo "Pushing image: ${image_name}"
-    "${container_mgr}" push "${image_name}"
-  fi
-}
-
-main "$@"