public/pipglr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: public:main
Branches Tags
public:main public:quadlet
public:v4.3.0 public:v4.2.0 public:v4.1.0 public:v4.0.0 public:v3.0.1 public:v3.0.0 public:v2.0.3 public:v2.0.2 public:v2.0.1 public:v2.0.0 public:v1.4.2 public:v1.4.1 public:v1.4.0 public:v1.3.0 public:v1.2.1 public:v1.2.0 public:v1.1.2 public:v1.1.1 public:v1.1.0 public:v1.0.1 public:v1.0.0 public:v0.0.0-f36
..
compare: public:quadlet
Branches Tags
public:main public:quadlet
public:v4.3.0 public:v4.2.0 public:v4.1.0 public:v4.0.0 public:v3.0.1 public:v3.0.0 public:v2.0.3 public:v2.0.2 public:v2.0.1 public:v2.0.0 public:v1.4.2 public:v1.4.1 public:v1.4.0 public:v1.3.0 public:v1.2.1 public:v1.2.0 public:v1.1.2 public:v1.1.1 public:v1.1.0 public:v1.0.1 public:v1.0.0 public:v0.0.0-f36

1 Commits
main ... quadlet

Author SHA1 Message Date
Chris Evich
942d58d41c Setup for quadlet/systemd runtime management 
Rather than setting up volumes and starting the pipglr container
manually, utilize quadlet + systemd.  Retain the old setup and execution
method, but move them into separate documentation.

Signed-off-by: Chris Evich <cevich@redhat.com>
 2024-02-16 13:17:39 -05:00
12 changed files with 385 additions and 517 deletions
Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
/.pre-commit-config.yaml

48
.gitlab-ci.yml
View File

@@ -1,21 +1,10 @@
--- ---
default: default:
image: quay.io/buildah/stable:v1.32 image: quay.io/buildah/stable:v1.31.0
tags: tags:
- saas-linux-small-amd64 - docker
- linux
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
when: never
- if: $CI_COMMIT_BRANCH
- if: $CI_COMMIT_TAG
include:
- component: gitlab.com/blue42u/ci.pre-commit/lite@0.2.0
inputs:
job_stage: test
envars: envars:
stage: test stage: test
@@ -25,11 +14,8 @@ envars:
commit_check: commit_check:
stage: test stage: test
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- when: never
variables: variables:
BADRX: "^(squash!)|(fixup!)" BADRX: '^(squash!)|(fixup!)'
script: | script: |
dnf install -y git dnf install -y git
shortlogtmp=$(mktemp -p '' commit_check_tmp_XXXX) shortlogtmp=$(mktemp -p '' commit_check_tmp_XXXX)
@@ -40,15 +26,31 @@ commit_check:
fi fi
build: build:
tags:
- saas-linux-medium-amd64
stage: deploy stage: deploy
variables: variables:
FF_GITLAB_REGISTRY_HELPER_IMAGE: 0
BUILDAH_FORMAT: docker BUILDAH_FORMAT: docker
BUILDAH_ISOLATION: chroot BUILDAH_ISOLATION: chroot
STORAGE_DRIVER: vfs STORAGE_DRIVER: vfs
before_script: before_script:
- echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
script: script:
- scripts/build.sh # N/B: There could be more than one merge-request open with this branch's HEAD
- |
IMAGE_TAG="${CI_COMMIT_REF_SLUG}";
if [[ -n "$CI_COMMIT_TAG" ]]; then
IMAGE_TAG="${CI_COMMIT_TAG}";
elif [[ -n "$CI_OPEN_MERGE_REQUESTS" ]]; then
IMAGE_TAG=mr$(echo "${CI_OPEN_MERGE_REQUESTS}" | cut -d, -f -1 | cut -d\! -f 2);
elif [[ "$CI_COMMIT_BRANCH" == "main" ]]; then
IMAGE_TAG="latest";
fi
echo "Building/Pushing to: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG}";
- >-
buildah build \
--label "org.opencontainers.image.source=${CI_PROJECT_URL}" \
--label "org.opencontainers.image.revision=$CI_COMMIT_SHA" \
--label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" \
--label "org.opencontainers.image.version=${IMAGE_TAG}" \
-t "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}" "$CI_PROJECT_DIR"
- buildah images
- buildah push "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}"

48
.pre-commit-config.yaml
View File

@@ -1,48 +0,0 @@
default_language_version:
python: python3
default_install_hook_types: [pre-commit, commit-msg]
default_stages: [pre-commit]
repos:
- repo: https://github.com/executablebooks/mdformat
rev: '0.7.17'
hooks:
- id: mdformat
additional_dependencies:
- mdformat-footnote
- mdformat-tables
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: fix-byte-order-marker
- id: mixed-line-ending
- id: check-executables-have-shebangs
- id: check-symlinks
- id: destroyed-symlinks
- id: check-merge-conflict
- id: check-case-conflict
- id: no-commit-to-branch
args: [--branch, main]
- id: check-yaml
- repo: https://github.com/python-jsonschema/check-jsonschema
rev: '0.28.0'
hooks:
# Validate the GitLab CI scripts against the schema. Doesn't catch everything but helps.
- id: check-gitlab-ci
files: '.*\.gitlab-ci\.yml$'
- repo: https://github.com/jumanjihouse/pre-commit-hooks
rev: '3.0.0'
hooks:
- id: forbid-binary
- id: require-ascii
- repo: meta
hooks:
# Un-comment (maybe temporarily) to check which hooks don't apply.
# - id: check-hooks-apply
- id: check-useless-excludes

29
Containerfile
View File

@@ -20,15 +20,11 @@ ADD /root/ /root/
ADD /etc/ /etc/ ADD /etc/ /etc/
ADD /home/ /home/ ADD /home/ /home/
# The build type: either `dev` or `prod`
# In `dev` mode: the package manager will not be deleted.
ARG BUILD_TYPE=prod
# Allow image-builders to choose another version besides "latest" should # Allow image-builders to choose another version besides "latest" should
# an incompatible change be introduced. # an incompatible change be introduced.
ARG RUNNER_VERSION=latest ARG RUNNER_VERSION=latest
# Permit building containers for alternate architectures. At the time # Permit building containers for alternate architectures. At the time
# of this commit, only 'arm64' is available. # of this commit, only 'arm64' is available.
ARG TARGETARCH=amd64 ARG TARGETARCH=amd64
@@ -39,11 +35,9 @@ ARG TARGETARCH=amd64
ARG PRUNE_INTERVAL=daily # see systemd.timer for allowable values ARG PRUNE_INTERVAL=daily # see systemd.timer for allowable values
# All-in-one packaging/image-setup script to keep things simple. # All-in-one packaging/image-setup script to keep things simple.
RUN \ RUN PRUNE_INTERVAL=${PRUNE_INTERVAL} \
PRUNE_INTERVAL=${PRUNE_INTERVAL} \ RUNNER_VERSION=${RUNNER_VERSION} \
RUNNER_VERSION=${RUNNER_VERSION} \ bash /root/setup.sh
BUILD_TYPE=${BUILD_TYPE} \
bash /root/setup.sh
VOLUME /cache /home/podman/.local/share/containers VOLUME /cache /home/podman/.local/share/containers
ENTRYPOINT /lib/systemd/systemd ENTRYPOINT /lib/systemd/systemd
@@ -51,17 +45,12 @@ ENTRYPOINT /lib/systemd/systemd
# Gitlab-runner configuration options, may be freely overridden at # Gitlab-runner configuration options, may be freely overridden at
# container image build time. # container image build time.
ARG DEFAULT_JOB_IMAGE=registry.fedoraproject.org/fedora-minimal:latest ARG DEFAULT_JOB_IMAGE=registry.fedoraproject.org/fedora-minimal:latest
# Allow image-builders to override the Gitlab URL # Allow image-builders to override the Gitlab URL
ARG GITLAB_URL=https://gitlab.math.ethz.ch/ ARG GITLAB_URL=https://gitlab.com/
# Run nested containers in --privileged mode - required to allow building # Run nested containers in --privileged mode - required to allow building
# container images using podman or buildah. Otherwise may be set 'false'. # container images using podman or buildah. Otherwise may be set 'false'.
ARG NESTED_PRIVILEGED=true ARG NESTED_PRIVILEGED=true
# Download the FIPS version of gitlab-runner when enabled on the host system.
ARG ENABLE_FIPS=true
# The registration runlabel may be called multiple times to register more than # The registration runlabel may be called multiple times to register more than
# one runner. Each expects a REGISTRATION_TOKEN secret to be pre-defined and # one runner. Each expects a REGISTRATION_TOKEN secret to be pre-defined and
# the file './config.toml' to exist (may be empty). A local-cache volume # the file './config.toml' to exist (may be empty). A local-cache volume
@@ -70,7 +59,6 @@ ARG ENABLE_FIPS=true
# may be changed if you know what you're doing. # may be changed if you know what you're doing.
LABEL register="podman run -it --rm \ LABEL register="podman run -it --rm \
--secret=REGISTRATION_TOKEN,type=env \ --secret=REGISTRATION_TOKEN,type=env \
--user=root \
-v ./config.toml:/etc/gitlab-runner/config.toml:Z \ -v ./config.toml:/etc/gitlab-runner/config.toml:Z \
-e REGISTER_NON_INTERACTIVE=true \ -e REGISTER_NON_INTERACTIVE=true \
-e CI_SERVER_URL=${GITLAB_URL} \ -e CI_SERVER_URL=${GITLAB_URL} \
@@ -84,31 +72,24 @@ LABEL register="podman run -it --rm \
-e DOCKER_NETWORK_MODE=host \ -e DOCKER_NETWORK_MODE=host \
-e DOCKER_PRIVILEGED=${NESTED_PRIVILEGED} \ -e DOCKER_PRIVILEGED=${NESTED_PRIVILEGED} \
--entrypoint=/usr/bin/gitlab-runner \$IMAGE register" --entrypoint=/usr/bin/gitlab-runner \$IMAGE register"
# Additionally, the nested-podman storage volumes must be pre-created with # Additionally, the nested-podman storage volumes must be pre-created with
# 'podman' UID/GID values to allow nested containers access. # 'podman' UID/GID values to allow nested containers access.
LABEL setupstorage="podman volume create --opt o=uid=1000,gid=1000 pipglr-storage" LABEL setupstorage="podman volume create --opt o=uid=1000,gid=1000 pipglr-storage"
# Lastly, the gitlab-runner will manage container-cache in this directory, # Lastly, the gitlab-runner will manage container-cache in this directory,
# which will also be bind-mounted into every container. So it must be # which will also be bind-mounted into every container. So it must be
# writable by both 'podman' user and 'runner' group. # writable by both 'podman' user and 'runner' group.
LABEL setupcache="podman volume create --opt o=uid=1000,gid=1001 pipglr-cache" LABEL setupcache="podman volume create --opt o=uid=1000,gid=1001 pipglr-cache"
# Helper to extract the current configuration secret to allow editing. # Helper to extract the current configuration secret to allow editing.
LABEL dumpconfig="podman run -it --rm \ LABEL dumpconfig="podman run -it --rm \
--user=root \
--secret config.toml --entrypoint=/bin/cat \ --secret config.toml --entrypoint=/bin/cat \
\$IMAGE /var/run/secrets/config.toml" \$IMAGE /var/run/secrets/config.toml"
# Executing the runner container depends on the config.toml secret being # Executing the runner container depends on the config.toml secret being
# set (see above) and two volumes existing with correct permissions set. # set (see above) and two volumes existing with correct permissions set.
# Note: The contents of the volumes are not critical, they may be removed # Note: The contents of the volumes are not critical, they may be removed
# and re-created (see above) to quickly free-up disk space. # and re-created (see above) to quickly free-up disk space.
LABEL run="podman run -dt --name pipglr \ LABEL run="podman run -dt --name pipglr \
--user=root \
--secret config.toml,uid=1001,gid=1001 \ --secret config.toml,uid=1001,gid=1001 \
-v pipglr-storage:/home/podman/.local/share/containers \ -v pipglr-storage:/home/podman/.local/share/containers \
-v pipglr-cache:/cache \ -v pipglr-cache:/cache \
--systemd true --privileged \ --systemd true --privileged \
--device /dev/fuse \$IMAGE" --device /dev/fuse \$IMAGE"
# ==========================

310
README.md
View File

@@ -1,133 +1,152 @@
# Podmand-In-Podman Gitlab Runner
This project provides a Gitlab Runner which runs inside a container launched
with `podman`. The Gitlab Runner itself uses an independent `podman` instance
inside to launch jobs.
## Overview ## Overview
This container image is built weekly from this `Containerfile`, and made This container image is built daily from this `Containerfile`, and
available as: made available as:
- `registry.gitlab.com/qontainers/pipglr:latest` * `registry.gitlab.com/qontainers/pipglr:latest`
-or- -or-
- `registry.gitlab.com/qontainers/pipglr:<version>` * `registry.gitlab.com/qontainers/pipglr:<version>`
It's purpose is to provide an easy method to execute a GitLab runner, to service It's purpose is to provide an easy method to execute a GitLab runner,
CI/CD jobs for groups and/or repositories on [gitlab.com](https://gitlab.com). to service CI/CD jobs for groups and/or repositories on
It comes pre-configured to utilize the gitlab-runner app to execute within a [gitlab.com](https://gitlab.com). It comes pre-configured to utilize
rootless podman container, nested inside a rootless podman container. the gitlab-runner app to execute within a rootless podman container,
nested inside a rootless podman container.
This is intended to provide additional layers of security for the host, when This is intended to provide additional layers of security for the host,
running potentially arbitrary CI/CD code. Though, the ultimate responsibility when running potentially arbitrary CI/CD code. Though, the ultimate
still rests with the end-user to review the setup and configuration relative to responsibility still rests with the end-user to review the setup and
their own security situation/environment. configuration relative to their own security situation/environment.
**Note**: While this can run entirely under a regular user, it will require root **Note**: While this can run entirely under a regular user, it will require
access for the first two setup steps (below). root access for the first two setup steps (below).
### Operation ### Operation
This image leverages the podman `runlabel` feature heavily. Several labels are This image leverages the podman `runlabel` feature heavily. Several
set on the image to support easy registration and execution of the runner labels are set on the image to support easy registration and execution
container. While it's possible to use the container with your own command-line, of the runner container. While it's possible to use the container
it's highly recommended to base them off of one of the labels. See the examples with your own command-line, it's highly recommended to base them
below for more information. off of one of the labels. See the examples below for more information.
*_Only podman versions 4.8.x and later are supported by pipglr._* ***Note:*** Some older versions of podman don't support the
`container runlabel` sub-command. If this is the case, you may simulate
#### Persistent Containers (step 1) it with the following, substituting `<label>` with one of the predefined
values (i.e. `register`, `setupconfig`, etc.):
By default on many distributions, regular users aren't permitted to leave
background processes running after they log out. Since this is likely desired
for running the pipglr container long-term, `systemd` needs to be configured to
override this policy. Doing so requires root or `sudo` access on the
system. Assuming the user `johndoe` will be executing the pipglr container,
linger may be enabled (as the admin user or root) with a command like:
```bash ```bash
$ sudo loginctl enable-linger `johndoe` $ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.<label>)
``` ```
Side-effect: This will allow the user (`johndoe` for example) to persist #### Persistent containers (step 1)
other user-level systemd services as well. For example `podman.socket` is
handy to enable for `podman remote` access. You could also
[setup quadlet](https://www.redhat.com/sysadmin/quadlet-podman) or a systemd
unit so pipglr starts up on system boot.
#### Expanded Host-system User-Namespace (step 2) By default on many distributions, regular users aren't permitted to leave
background processes running after they log out. Since this is likely
desired for running the pipglr container long-term, `systemd` needs to be
configured to override this policy. For this, you (`$USER`) will need
root access on the system.
**_This is really important and frequently the cause of pipglr issues_** ```bash
$ sudo loginctl enable-linger $USER
```
On the host, as root, edit the two files `/etc/subuid` and `/etc/subgid` to Side-effect: This will allow your user to persist other user-level systemd
expand the pipglr user's ID allocation by 3. For example if the host user services as well. For example `podman.socket` is handy to enable for
running the pipglr container is named `johndoe`, this entry in the files should `podman remote` access. You could also [setup
be edited like: quadlet](https://www.redhat.com/sysadmin/quadlet-podman) or a systemd unit
so pipglr starts up on system boot.
`johndoe:<some number>:65539` #### Expanded user-namespace (step 2) ***This is probably important***
Where `<some number>` was set by the OS when the `johndoe` user was created As an added protection/safety measure, pipglr excludes three UID/GIDs
(you can ignore this). Only the last number needs to be increased by three. from being used by job-level containers. One for `root`, another for
This change will be effective on the user's next login, or immediately `runner` and a third for `podman`. However, some container images
by running (as the host pipglr user): you may want to use for jobs (mainly Debian/Ubuntu), assign one/more
essential users a high UID/GID value (like `65535`).
At the same time, most distributions also set `65536` as the default maximum
number (including ID `0`) of IDs to allocate for user-namespaces (via `/etc/login.defs`). This
creates a problem you won't realize until the runner actually picks up a job
😞 The main symptom of this issue will be messages in the pipglr containers log,
similar to (abbreviated):
```
...cut...
running `/usr/bin/newuidmap ...cut...`: newuidmap: write to uid_map failed: Operation not permitted
Error: cannot set up namespace using "/usr/bin/newuidmap": exit status 1
...cut...
```
or
```
E: setgroups 65534 failed - setgroups (22: Invalid argument)
```
***The good news is, working around this is relatively simple:***
As root, edit the two files `/etc/subuid` and `/etc/subgid` to expand the
by 3 IDs. For example assuming a user running the pipglr container is
called `johndoe`, the contents of these files should be edited to allocate
`65539` IDs like:
`jogndoe:<some number>:65539`
Where `<some number>` was set by your OS when the `johndoe` user was created
(you can ignore this). Only the last number needs to be increased. This
change will be effective on next login, or immediately by running:
`podman system migrate` `podman system migrate`
_Note:_ This will stop any currently running containers. *Note:* This will stop any currently running containers.
_Details:_ As an added protection/safety measure, pipglr excludes three UID/GIDs #### Runner registration (step 3)
from being used by job-level (nested) containers. One for `root`, another for
`runner` and a third for `podman`. This necessitates expanding the number of
available UIDs/GIDs from the host machine, to the pipglr container. This way,
the full set of 65535 UIDs/GIDs may be utilized by job-level (nested)
sub-containers.
#### Runner Activation/Registration (step 3) All runners must be connected to a project or group runner configuration
on your gitlab instance (or `gitlab.com`). This is done using a special
registration *runlabel*. The command can (and probably should) be run
more than once (using the same `config.toml`) to configure and register
multiple runners. This is necessary for the *pipglr* container to execute
multiple jobs in parallel. For example, if you want to support running
four jobs at the same time, you would use the `register` *runlabel*
four times.
All runners must be connected to a project or group runner configuration on your Before using the `register` *runlabel*, you must set your unique
gitlab instance (or `gitlab.com`). This is done using a special registration *registration* (a.k.a. *activation*) token as a podman *secret*. This
_runlabel_. The command can (and probably should) be run more than once (using secret may be removed once the registration step is complete. The
the same `config.toml`) to configure and register multiple runners. This is **<actual registration token>** value (below) should be replaced with
necessary for the _pipglr_ container to execute multiple jobs in parallel. For the value obtained from the "runners" settings page of a gitlab
example, if you want to support running four jobs at the same time, you would group or project's *CI/CD Settings*. Gitlab version 16 and later
use the `register` _runlabel_ four times. refers to this value as an *activation* token, but the usage is the same.
Before using the `register` _runlabel_, you must set your unique _registration_
(a.k.a. _activation_) token as a podman _secret_. This secret may be removed
once the registration step is complete. The **`<actual registration token>`**
value (below) should be replaced with the value obtained from the `runners`
settings page of a gitlab group or project's _CI/CD Settings_. Gitlab version 16
and later refers to this value as an _activation_ token, but the usage is the
same.
```bash ```bash
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest" $ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
$ echo '<actual registration token>' | podman secret create REGISTRATION_TOKEN - $ echo '<actual registration token>' | podman secret create REGISTRATION_TOKEN -
``` ```
Next, **_a blank `config.toml` file_** needs to be created. Without this, the Next, ***a blank `config.toml` file*** needs to be created. Without this, the
`reigster` _runlabel_ will return a permission-denied error. Once the empty `reigster` *runlabel* will return a permission-denied error. Once the empty
`config.toml` file is created, you may register one or more runners by repeating `config.toml` file is created, you may register one or more runners by repeating
the registration _runlabel_ as follows: the registration *runlabel* as follows:
```bash ```bash
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest" $ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
$ touch ./config.toml # important: file must exist, even if empty. $ touch ./config.toml # important: file must exist, even if empty.
$ podman container runlabel register $IMAGE $ podman container runlabel register $IMAGE
# ...repeat as desired... ...repeat as desired...
$ podman secret rm REGISTRATION_TOKEN # if desired $ podman secret rm REGISTRATION_TOKEN # if desired
``` ```
#### Runner Configuration (step 4) #### Runner Configuration (step 4)
During the registration process (above), a boiler-plate (default) `config.toml` During the registration process (above), a boiler-plate (default) `config.toml` file
file will be created/updated for you. At this point you may edit the will be created/updated for you. At this point you may edit the configuration
configuration if desired before committing it as a _podman secret_. Please refer if desired before committing it as a *podman secret*. Please refer to the
to the [gitlab runner documentation](https://docs.gitlab.com/runner/configuration/)
[gitlab runner documentation](https://docs.gitlab.com/runner/configuration/) for for details.
details.
```bash ```bash
$ $EDITOR ./config.toml # if desired $ $EDITOR ./config.toml # if desired
@@ -136,45 +155,41 @@ $ rm ./config.toml # if desired
``` ```
This may be necessary, for example, to increase the default `concurrency` value This may be necessary, for example, to increase the default `concurrency` value
to reflect the number of registered runners. If you need to edit this file after to reflect the number of registered runners. If you need to edit this file
committing it as a secret, there's after committing it as a secret, there's
[a `dumpconfig` _runlabel_ for that](README.md#configuration-editing). [ a `dumpconfig` *runlabel* for that](README.md#configuration-editing).
#### Volume Setup (step 5) #### Quadlet setup and container start (step 5)
Since several users are utilized inside the container volumes must be **Note**: If your system is missing or does not support the use of quadlet
specifically configured to permit access. This is done using several _runlabels_ (`man 5 podman-systemd.unit`), you'll find [manual volume setup steps
as follows: here](manual_setup.md).
Create and copy the quadlet configuration files:
```bash ```bash
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest" $ mkdir -p ~/.config/containers/systemd
$ podman container runlabel setupstorage $IMAGE $ cp quadlet/* ~/.config/containers/systemd/
$ podman container runlabel setupcache $IMAGE
``` ```
Note: These volumes generally do not contain any critical operational data, they Finally, reload the local systemd user-slice to generate the unit files,
may be re-created anytime to quickly free up host disk-space if it's running and fire up pipglr!
low. Simply remove them with the command
`podman volume rm pipglr-storage pipglr-cache`. Then reuse the `setupstorage`
and `setupcache` _runlabels_ as in the above example.
#### Runner Startup (step 6)
With the runner configuration saved as a Podman secret, and the runner volumes
created, the GitLab runner container may be launched with the following
commands:
```bash ```bash
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest" $ systemd --user daemon-reload
$ podman container runlabel run $IMAGE $ systemd start pipglr
``` ```
If you want the service to start automatically on boot, you may run
`systemctl --user enable podman-restart.service`
### Configuration Editing ### Configuration Editing
The gitlab-runner configuration contains some sensitive values which should be The gitlab-runner configuration contains some sensitive values which
protected. The pipglr container assumes the entire configuration will be passed should be protected. The pipglr container assumes the entire configuration
in as a Podman secret. This makes editing it slightly convoluted, so a handy will be passed in as a Podman secret. This makes editing it slightly
_runlabel_ `dumpconfig` is available. It's intended use is as follows: convoluted, so a handy *runlabel* `dumpconfig` is available.
It's intended use is as follows:
```bash ```bash
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest" $ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
@@ -187,17 +202,17 @@ $ rm ./config.toml # if desired
### Debugging ### Debugging
The first thing to check is the container output. This shows three things: The first thing to check is the container output. This shows three things:
Systemd, Podman, and GitLab-Runner output. For example: Systemd, Podman, and GitLab-Runner output. For example:
```bash ```bash
$ podman logs --since 0 pipglr $ podman logs --since 0 pipglr
``` ```
Next, try running a pipglr image built with more verbose logging. Both the Next, try running a pipglr image built with more verbose logging. Both
`runner.service` and `podman.service` files have a `log-level` option. Simply the `runner.service` and `podman.service` files have a `log-level` option.
increase one or both to the `info`, or `debug` level. Start the debug container, Simply increase one or both to the "info", or "debug" level. Start the
and reproduce the problem. debug container, and reproduce the problem.
## Building ## Building
@@ -207,42 +222,39 @@ This image may be built simply with:
$ podman build -t registry.gitlab.com/qontainers/pipglr:latest . $ podman build -t registry.gitlab.com/qontainers/pipglr:latest .
``` ```
This will utilize the latest stable version of podman and the latest stable This will utilize the latest stable version of podman and the latest
version of the gitlab runner. stable version of the gitlab runner.
### Build-Arguments ### Build-args
Several build arguments are available to control the output image: Several build arguments are available to control the output image:
- `BUILD_TYPE`: The build type, either `prod` or `dev`. In `dev` mode, the package * `PRUNE_INTERVAL` - A systemd.timer compatible `OnCalendar` value that
manager is not deleted for development and debugging purposes. Please see
[`build.sh`](scripts/build.sh) for more details.
- `PRUNE_INTERVAL`: A systemd.timer compatible `OnCalendar` value that
determines how often to prune Podman's storage of disused containers and determines how often to prune Podman's storage of disused containers and
images. Defaults to `daily`, but should be adjusted based on desired images. Defaults to "daily", but should be adjusted based on desired
caching-effect balanced against available storage space and job execution caching-effect balanced against available storage space and job
rate. execution rate.
- `RUNNER_VERSION`: Allows specifying an exact gitlab runner version. By default * `RUNNER_VERSION` - Allows specifying an exact gitlab runner version.
the `latest` is used, assuming the user is building a tagged image anyway. By default the `latest` is used, assuming the user is building a tagged
Valid versions may be found on the image anyway. Valid versions may be found on the [runner
[runner release page](https://gitlab.com/gitlab-org/gitlab-runner/-/releases). release page](https://gitlab.com/gitlab-org/gitlab-runner/-/releases).
- `TARGETARCH`: Supports inclusion of non-x86_64 gitlab runners. This value is * `TARGETARCH` - Supports inclusion of non-x86_64 gitlab runners. This
assumed to match the image's architecture. If using the `--platform` build value is assumed to match the image's architecture. If using the
argument, it will be set automatically. Note: as of this writing, only `amd64` `--platform` build argument, it will be set automatically. Note:
and `arm64` builds of the gitlab-runner are available. as of this writing, only `amd64` and `arm64` builds of the gitlab-runner
- `GITLAB_URL`: Defaults to `https://gitlab.com/` but can be set to point to a are available.
self hosted instance of Gitlab. * `GITLAB_URL` - Defaults to 'https://gitlab.com/' but can be set to point
- `NESTED_PRIVILEGED`: Defaults to `true`, may be set `false` to prevent nested to a self hosted instance of Gitlab.
containers running in `--privileged` mode. This will affect the ability to * `NESTED_PRIVILEGED` - Defaults to 'true', may be set 'false' to prevent
build container images in CI jobs using tools like podman or buildah. nested containers running in `--privileged` mode. This will affect
the ability to build container images in CI jobs using tools like
podman or buildah.
Additional build-args are available as well. See the `Containerfile` comments ### Environment variables
for more details.
### Environment Variables
Nearly every option to every gitlab-runner sub-command may be specified via Nearly every option to every gitlab-runner sub-command may be specified via
environment variable. Some of these are set in the `Containerfile` for the environment variable. Some of these are set in the `Containerfile` for
`register` _runlabel_. If you need to set additional runtime env. vars., please the `register` *runlabel*. If you need to set additional runtime
do so via additional `Environment` optionns in the `runner.service` file. See env. vars., please do so via additional `Environment` optionns in the
the _systemd.nspawn_ man page for important value-format details. `runner.service` file. See the *systemd.nspawn* man page for important
value-format details.

32
manual_setup.md Normal file
View File

@@ -0,0 +1,32 @@
### Additional Manual steps
On systems without Quadlet, some additional steps are required
to get the pipglr container up and running.
### Manual Volume setup
Since several users are utilized inside the container volumes must be
specifically configured to permit access. This is done using several
*runlabels* as follows:
```bash
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
$ podman container runlabel setupstorage $IMAGE
$ podman container runlabel setupcache $IMAGE
```
Note: These volumes generally do not contain any critical operational data,
they may be re-created anytime to quickly free up host disk-space if
it's running low. Simply remove them with the command
`podman volume rm pipglr-storage pipglr-cache`. Then reuse the `setupstorage`
and `setupcache` *runlabels* as in the above example.
#### Manual Runner Startup
With the runner configuration saved as a Podman secret, and the runner volumes
created, the GitLab runner container may be launched with the following commands:
```bash
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
$ podman container runlabel run $IMAGE
```

13
quadlet/pipglr-cache.volume Normal file
View File

@@ -0,0 +1,13 @@
[Unit]
Description=Podman-in-podman GitLab Runner job cache storage volume
Documentation=https://gitlab.com/qontainers/pipglr/-/blob/main/README.md
After=local-fs.target
Requires=podman.socket
[Volume]
VolumeName=pipglr-cache
Copy=false
# The `podman` user inside the container should own everything
Options=o=uid=1000,gid=1000
# Support podman...prune --filters=persistent!=true
Label=persistent=true

13
quadlet/pipglr-storage.volume Normal file
View File

@@ -0,0 +1,13 @@
[Unit]
Description=Podman-in-podman GitLab Runner nested-container storage volume
Documentation=https://gitlab.com/qontainers/pipglr/-/blob/main/README.md
After=local-fs.target
Requires=podman.socket
[Volume]
VolumeName=pipglr-storage
Copy=false
# The `podman` user inside the container should own everything
Options=o=uid=1000,gid=1000
# Support podman...prune --filters=persistent!=true
Label=persistent=true

33
quadlet/pipglr.container Normal file
View File

@@ -0,0 +1,33 @@
[Unit]
Description=Podman-in-podman GitLab Runner
Documentation=https://gitlab.com/qontainers/pipglr/-/blob/main/README.md
After=pipglr-storage-volume.service pipglr-cache-volume.service
Requires=podman.socket pipglr-storage-volume.service pipglr-cache-volume.service
[Container]
ContainerName=pipglr
Image=registry.gitlab.com/qontainers/pipglr
# Required to run containers inside a container and ensure
# container can be managed with podman-restart.service
# Note: See https://github.com/containers/podman/issues/20418
PodmanArgs=--privileged --restart=always
# A nested systemd is used to manage nested podman & gitlab runner services
Systemd=true
# Allow jobs access to utilize fuse-overlayfs, for example to build container images.
Device=/dev/fuse
# Must be owned by the gitlab-runner user
Secret config.toml,uid=1001,gid=1001
# Add network isolation from other containers
Network=pipglr.network
# Storage for nested container images and job cache
Volume=pipglr-storage:/home/podman/.local/share/containers
Volume=pipglr-cache:/cache
# No need to preserve this between runs
VolatileTmp=true

11
quadlet/pipglr.network Normal file
View File

@@ -0,0 +1,11 @@
[Unit]
Description=Podman-in-podman GitLab Runner dedicated network
Documentation=https://gitlab.com/qontainers/pipglr/-/blob/main/README.md
After=network-online.target
Requires=podman.socket network.target
[Network]
Driver=bridge
# The pipglr container never accesses other containers on/or the host
Options=isolate
DisableDNS=true

277
root/setup.sh
View File

@@ -1,5 +1,4 @@
#!/usr/bin/env bash
#
# This script is intended to be run during container-image build. Any # This script is intended to be run during container-image build. Any
# other usage outside this context is likely to cause harm. # other usage outside this context is likely to cause harm.
# #
@@ -21,202 +20,108 @@
set -eo pipefail set -eo pipefail
function die() { for varname in PRUNE_INTERVAL RUNNER_VERSION TARGETARCH; do
echo -n '!! ERROR:' >&2 if [[ -z "${!varname}" ]]; then
printf " %s\n" "$@" >&2 echo "Error: \$$varname must be non-empty."
exit 1
}
function check_vars() {
for varname in PRUNE_INTERVAL RUNNER_VERSION TARGETARCH; do
if [[ -z "${!varname}" ]]; then
die "Env. variable '$varname' must be non-empty."
fi
done
if [[ ! "$BUILD_TYPE" =~ dev|prod ]]; then
die "Build type must be 'dev' or 'prod': '$BUILD_TYPE'."
fi fi
} done
function main() { # Make image smaller by not installing docs.
# Show what's happening to make debugging easier DNF="dnf --setopt=tsflags=nodocs -y"
set -x
# Make image smaller by not installing docs. for rpm in $(egrep -v '^(# )+' < /root/xpackages.txt); do
dnf=(dnf --setopt=tsflags=nodocs -y) x+="--exclude=$rpm ";
done
install_packages set -x # show what's happening to make debugging easier
setup_user
setup_service_podman
setup_service_runner
setup_gitlab_config
setup_volumes
finalize_ownership # DNF itself or a dependence may need upgrading, take care of it first.
} $DNF upgrade
function is_release() { $DNF $x install \
[ "$BUILD_TYPE" = "prod" ] || return 1 podman \
} systemd
function install_packages() { # Gitlab-runner package contains scriptlets which do not function properly inside a
readarray xpackages < <(grep -vE '^(# )+' </root/xpackages.txt) # container-build environment where systemd is not active/running.
local exclude_args=() $DNF $x --setopt=tsflags=noscripts install \
for rpm in "${xpackages[@]}"; do https://gitlab-runner-downloads.s3.amazonaws.com/$RUNNER_VERSION/rpm/gitlab-runner_${TARGETARCH}.rpm
exclude_args+=("--exclude=$rpm")
done
# DNF itself or a dependence may need upgrading, take care of it first. # Allow removing dnf, sudo, etc. packages. Also don't start unnecessary or broken
"${dnf[@]}" upgrade && # systemd services, like anything kernel related or login gettys.
"${dnf[@]}" "${exclude_args[@]}" install \ rm -rf \
podman \ /etc/dnf/protected.d/* \
systemd /etc/sytemd/system/getty.target.wants/* \
/etc/sytemd/system/multi-user.target.wants/* \
/etc/sytemd/system/sysinit.target.wants/* \
/etc/sytemd/system/timers.target.wants/* \
/lib/systemd/system/graphical.target.wants/* \
/lib/systemd/system/multi-user.target.wants/{getty.target,systemd-ask-password-wall.path} \
/lib/systemd/system/sys-kernel*.mount
# Gitlab-runner package contains scriptlets which do not function properly inside a # Remove unnecessary packages, see xpackages.txt to learn how this list was generated.
# container-build environment where systemd is not active/running. # This makes the image smaller and reduces the attack-surface.
if [[ ${ENABLE_FIPS} == true && $(cat /proc/sys/crypto/fips_enabled) == 1 ]]; then dnf remove -y $(egrep -v '^(# )+' /root/xpackages.txt)
PACKAGE_FILES=(
"https://s3.dualstack.us-east-1.amazonaws.com/gitlab-runner-downloads/$RUNNER_VERSION/rpm/gitlab-runner_${TARGETARCH}-fips.rpm"
)
else
PACKAGE_FILES=(
"https://s3.dualstack.us-east-1.amazonaws.com/gitlab-runner-downloads/$RUNNER_VERSION/rpm/gitlab-runner_${TARGETARCH}.rpm"
"https://s3.dualstack.us-east-1.amazonaws.com/gitlab-runner-downloads/$RUNNER_VERSION/rpm/gitlab-runner-helper-images.rpm"
)
fi
"${dnf[@]}" "${exclude_args[@]}" \ # Wipe out the DNF cache, then remove it entirely, again to make the image smaller.
--setopt=tsflags=noscripts install \ $DNF clean all
${PACKAGE_FILES[@]} rm -rf /var/cache/dnf /var/log/dnf* /var/log/yum.*
rpm -e dnf
# Also don't start unnecessary or broken # Workaround https://bugzilla.redhat.com/show_bug.cgi?id=1995337
# systemd services, like anything kernel related or login gettys. rpm --setcaps shadow-utils
rm -rf \
/etc/systemd/system/getty.target.wants/* \
/etc/systemd/system/multi-user.target.wants/* \
/etc/systemd/system/sysinit.target.wants/* \
/etc/systemd/system/timers.target.wants/* \
/lib/systemd/system/graphical.target.wants/* \
/lib/systemd/system/multi-user.target.wants/{getty.target,systemd-ask-password-wall.path} \
/lib/systemd/system/sys-kernel*.mount
# Allow removing dnf, sudo, etc. packages. # Prevent copying of skel since it can interfere with the gitlab-runner
rm -rf \ mkdir -p /home/podman /home/runner
/etc/dnf/protected.d/* # Guarantee uid/gid 1000 for user 'podman' / 1001 for user 'runner'.
# Remove unnecessary packages, see xpackages.txt to learn how this list was generated. groupadd -g 1000 podman
# This makes the image smaller and reduces the attack-surface. groupadd -g 1001 runner
dnf remove -y "${xpackages[@]}" # Separate users for services to increase process isolation.
# The 'podman' user's socket service writes /home/runner/podman.socket
useradd -M -u 1000 -g podman -G runner podman
useradd -M -u 1001 -g runner runner
# Allow 'podman' user to create socket file under /home/runner.
chmod 770 /home/runner
if is_release; then # Overwrite defaults, only user 'podman' permited to have a user-namespace
# Wipe out the DNF cache, then remove it entirely, again to make the image smaller. # Split the namespaced ID's around the containers root (ID 0), podman (ID 1000), and
"${dnf[@]}" clean all # runner (ID 1001) such that the user-namespace of any nested containers cannot
rm -rf /var/cache/dnf /var/log/dnf* /var/log/yum.* # read or write any files owned by these users (and/or hijack nested container processes).
rpm -e dnf # N/B: The range-end (999+64536) ensures a total of 65535 IDs are available for nested-containers.
fi # This requires the host provide a sufficiently large range, i.e. `pipglr:<start>:65539`
echo -e "podman:1:999\npodman:1002:64536" | tee /etc/subuid > /etc/subgid
# Host volume mount necessary for nested-podman to use overlayfs2 for container & volume storage.
mkdir -p /home/podman/.local/share/containers
# Nested-container's local container-cache volume mount, recommended by gitlab-runner docs.
mkdir -p /cache
# Both the gitlab-runner and podman need access to the cache directory / volume mount.
chown podman:runner /cache
# Workaround https://bugzilla.redhat.com/show_bug.cgi?id=1995337 # Setup persistent 'podman' user services to start & run without a login.
# Base-image failing to confer capabilities properly on mkdir -p /var/lib/systemd/linger
# /usr/bin/new{u,g}idmap to `cap_set{u,g}id=ep` in new image layers touch /var/lib/systemd/linger/podman
rpm --setcaps shadow-utils # Setup 'podman' socket and a container-storage pruning service for 'podman' user.
} mkdir -p /home/podman/.config/systemd/user/{sockets.target.wants,default.target.wants}
cd /home/podman/.config/systemd/user/
ln -s $PWD/podman.socket ./sockets.target.wants/ # Added from Containerfile
ln -s $PWD/prune.timer ./default.target.wants/ # also from Containerfile
# Substitute value from --build-arg if specified, otherwise use default from Containerfile.
sed -i -e "s/@@@PRUNE_INTERVAL@@@/$PRUNE_INTERVAL/" ./prune.timer
# Containerfile ADD instruction does not properly set ownership/permissions.
chown -R 1000:1000 /home/podman
chmod 700 /home/podman
function setup_user() { # Setup persistent 'runner' user services to start & run without a login.
# Guarantee uid/gid 1000 for user 'podman' / 1001 for user 'runner'. touch /var/lib/systemd/linger/runner
groupadd -g 1000 podman mkdir -p /home/runner/.config/systemd/user/default.target.wants
groupadd -g 1001 runner cd /home/runner/.config/systemd/user/
# Does not depend on podman.socket file availablility, will retry if not present.
# Separate users for services to increase process isolation. ln -s $PWD/runner.service ./default.target.wants/
# The 'podman' user's socket service writes /home/runner/podman.socket # gitlab-runner will create side-car '.runner_system_id' file next to 'config.toml'
# Prevent copying of skel since it can interfere with the gitlab-runner # on first startup. Ensure access is allowed. Also link to future config file
mkdir -p /home/podman /home/runner # presented as a container-secret.
useradd -M -u 1000 -g podman -G runner podman mkdir -p /home/runner/.gitlab-runner
useradd -M -u 1001 -g runner runner ln -s /var/run/secrets/config.toml /home/runner/.gitlab-runner/config.toml
# Containerfile ADD instruction does not properly set ownership/permissions.
# Allow only podman in `/home/podman`. chown -R runner:runner /home/runner
chmod 700 /home/podman chmod -R 700 /home/runner/.gitlab-runner
# Allow 'podman' user to create socket file under `/home/runner`.
chmod 770 /home/runner
# Set permissions.
chown -R runner:runner /home/runner
chown -R podman:podman /home/podman
# Overwrite defaults, only user 'podman' permitted to have a user-namespace
# Split the namespaced ID's around the containers root (ID 0), podman (ID 1000), and
# runner (ID 1001) such that the user-namespace of any nested containers cannot
# read or write any files owned by these users (and/or hijack nested container processes).
# N/B: The range-end (999+64536) ensures a total of 65535 IDs are available for nested-containers.
# This requires the host provide a sufficiently large range, i.e. `pipglr:<start>:65539`
echo -e "podman:1:999\npodman:1002:64536" | tee /etc/subuid >/etc/subgid
}
function setup_volumes() {
# Host volume mount necessary for nested-podman to use overlayfs2 for container & volume storage.
mkdir -p /home/podman/.local/share/containers
touch /home/podman/.local/share/containers/.placeholder
# Nested-container's local container-cache volume mount, recommended by gitlab-runner docs.
mkdir -p /cache
touch /cache/.placeholder
# Both the gitlab-runner and podman need access to the cache directory / volume mount.
chown podman:runner /cache
}
function setup_service_podman() {
# Setup persistent 'podman' user services to start & run without a login.
mkdir -p /var/lib/systemd/linger
touch /var/lib/systemd/linger/podman
# Setup 'podman' socket and a container-storage pruning service for 'podman' user.
mkdir -p /home/podman/.config/systemd/user/{sockets.target.wants,default.target.wants}
cd /home/podman/.config/systemd/user/
ln -s "$(pwd)/podman.socket" ./sockets.target.wants/ # Added from Containerfile
ln -s "$(pwd)/prune.timer" ./default.target.wants/ # also from Containerfile
# Substitute value from --build-arg if specified, otherwise use default from Containerfile.
sed -i -e "s/@@@PRUNE_INTERVAL@@@/$PRUNE_INTERVAL/" ./prune.timer
}
function setup_service_runner() {
# Setup persistent 'runner' user services to start & run without a login.
touch /var/lib/systemd/linger/runner
# Setup persistent 'runner' user services to start & run without a login.
mkdir -p /home/runner/.config/systemd/user/default.target.wants
cd /home/runner/.config/systemd/user/
# Does not depend on podman.socket file availability, will retry if not present.
ln -s "$(pwd)/runner.service" ./default.target.wants/
}
function setup_gitlab_config() {
# gitlab-runner will create side-car '.runner_system_id' file next to 'config.toml'
# on first startup. Ensure access is allowed. Also link to future config file
# presented as a container-secret.
mkdir -p /home/runner/.gitlab-runner
ln -s /var/run/secrets/config.toml /home/runner/.gitlab-runner/config.toml
chmod -R 700 /home/runner/.gitlab-runner
}
function finalize_ownership() {
# Adjust ownership to all files created after `setup_user`.
# and also to the `ADD` instruction in the `Containerfile`.
chown -R runner:runner /home/runner
chown -R podman:podman /home/podman
# Ensure correct permissions of system configuration files.
# Somehow these can be set incorrectly during Containerfile
# ADD instruction.
local path
for path in "/etc/systemd/system.conf.d" "/etc/systemd/system/user-.slice.d"; do
chown root:root ${path}/*
chmod 0644 ${path}/*
done
}
check_vars
main

87
scripts/build.sh
View File

@@ -1,87 +0,0 @@
#!/usr/bin/env bash
# Usage: build.sh [<build-type>]
#
# If the build type (second arg. `<build-type>`) is `prod`
# the images are build in `release` mode. For all other build types
# the images are build for development and testing purposes
# By default the build type is `prod`.
set -eu
set -o pipefail
ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)/.."
cd "$ROOT_DIR"
function ci_running() {
[ "${CI:-}" = "true" ] && return 0
return 1
}
# Define the image tag depending on the context.
function get_image_tag() {
local build_type="$1"
local image_tag="${build_type}-latest"
# Define image tag.
if ci_running; then
# The image tag gets adjusted depending on
# if it is a merge request or build on the
# main branch or on a tag.
image_tag="${CI_COMMIT_REF_SLUG:-}"
if [[ -n "${CI_COMMIT_TAG:-}" ]]; then
image_tag="${CI_COMMIT_TAG}"
elif [[ -n "${CI_OPEN_MERGE_REQUESTS:-}" ]]; then
image_tag=mr$(echo "${CI_OPEN_MERGE_REQUESTS}" | cut -d, -f -1 | cut -d\! -f 2)
elif [[ "${CI_COMMIT_BRANCH:-}" == "main" ]]; then
image_tag="latest"
fi
fi
echo "$image_tag"
}
function main() {
# Define common build variables.
local container_mgr=${CI_CONTAINER_MGR:-buildah}
local project_dir=${CI_PROJECT_DIR:-.}
local registry_name=${CI_REGISTRY_IMAGE:-"containers-storage:pipglr"}
local build_type=${CI_BUILD_TYPE:-${1:-prod}}
# Define image name and tag.
local image_tag image_name
image_tag=$(get_image_tag "$build_type")
image_name="${registry_name}:${image_tag}"
# Define OpenContainers labels.
local oc_project_url=${CI_PROJECT_URL:-file://$ROOT_DIR}
local oc_commit_sha=${CI_COMMIT_SHA:-$(git rev-parse HEAD)}
local oc_job_started_at=${CI_JOB_STARTED_AT:-$(date -u --iso-8601=seconds)}
local oc_version="${image_tag}"
BUILD_CMD=(
"${container_mgr}" build
--label "org.opencontainers.image.source=${oc_project_url}"
--label "org.opencontainers.image.revision=${oc_commit_sha}"
--label "org.opencontainers.image.created=${oc_job_started_at}"
--label "org.opencontainers.image.version=${oc_version}"
--build-arg "BUILD_TYPE=${build_type}"
-t "$image_name"
"${project_dir}")
echo "Build image: '$image_name'"
echo -e "Build command:\n" "${BUILD_CMD[@]}"
"${BUILD_CMD[@]}"
echo "Images are:"
"${container_mgr}" images
if ci_running; then
echo "Pushing image: ${image_name}"
"${container_mgr}" push "${image_name}"
fi
}
main "$@"