Merge branch 'reimplement_systemd' into 'main'
Fix podman leaking conmon processes See merge request qontainers/pipglr!18
This commit is contained in:
Chris Evich
222
Containerfile
222
Containerfile
@@ -1,157 +1,81 @@
|
||||
# pipglr/Containerfile
|
||||
#
|
||||
# Builds a Podman-in-Podman Gitlab-Runner image for
|
||||
# executing Gitlab CI/CD jobs. Requires configuration
|
||||
# steps specific to Gitlab projects. For more info. see
|
||||
# https://docs.gitlab.com/runner/executors/docker.html#use-podman-to-run-docker-commands
|
||||
#
|
||||
FROM quay.io/centos/centos:stream9
|
||||
|
||||
FROM quay.io/podman/stable:v4.3.1
|
||||
ADD /setup.sh /xpackages.txt /root/
|
||||
ADD /containers.conf /home/podman/.config/containers/containers.conf
|
||||
ADD /podman.service /podman.socket /prune.service /prune.timer /home/podman/.config/systemd/user/
|
||||
ADD /runner.service /home/runner/.config/systemd/user/
|
||||
ADD kmsglog.conf /etc/systemd/system.conf.d/
|
||||
|
||||
# This is a list of packages to remove and/or exclude from the image.
|
||||
# Primarily this is done for security reasons, should a runner process
|
||||
# escape confinement. Having fewer things to poke, lowers the attack
|
||||
# surface-area.
|
||||
#
|
||||
# This list was formed manually by running these commands in the base image:
|
||||
# for package in $(rpm -qa); do \
|
||||
# if dnf erase $package; then echo "$package" >> exclude; fi; \
|
||||
# done; \
|
||||
# cat exclude
|
||||
#
|
||||
# After adding those packages to this file, the container build was run
|
||||
# and package list adjusted, untill no dependency errors were raised.
|
||||
ARG EXCLUDE_PACKAGES="\
|
||||
fedora-repos-modular \
|
||||
findutils \
|
||||
libxcrypt-compat \
|
||||
openldap-compat \
|
||||
podman-gvproxy \
|
||||
rootfiles \
|
||||
sudo \
|
||||
vim-minimal \
|
||||
yum"
|
||||
# Allow image-builders to choose another version becides "latest" should
|
||||
# an incompatible change be introduced.
|
||||
ARG RUNNER_VERSION=latest
|
||||
|
||||
# Base-image runs as user 'podman', temporarily switch to root
|
||||
# for installation/setup.
|
||||
USER root
|
||||
# Helper for comparison in future RUN operations (DO NOT USE)
|
||||
ARG _DNFCMD="dnf --setopt=tsflags=nodocs -y"
|
||||
# Set this instead, if (for example) you want to volume-mount in /var/cache/dnf
|
||||
ARG DNFCMD="${_DNFCMD}"
|
||||
# Avoid installing any documentation to keep image small
|
||||
# During install, excluding packages is meaningless if already installed
|
||||
RUN set -x && \
|
||||
rm -f /etc/dnf/protected.d/sudo.conf && \
|
||||
rm -f /etc/dnf/protected.d/yum.conf && \
|
||||
$DNFCMD remove ${EXCLUDE_PACKAGES}
|
||||
# Permit building containers for alternate architectures. At the time
|
||||
# of this commit, only 'arm64' is available.
|
||||
ARG TARGETARCH=amd64
|
||||
|
||||
# Enable callers to customize the runner version as needed, otherwise
|
||||
# assume this image will be version-tagged, so it's fine to grab the latest.
|
||||
ARG RUNNER_VERSION="latest"
|
||||
# When building a multi-arch manifest-list, this buid-arg is set automatically.
|
||||
ARG TARGETARCH="amd64"
|
||||
ENV RUNNER_RPM_URL=https://gitlab-runner-downloads.s3.amazonaws.com/${RUNNER_VERSION}/rpm/gitlab-runner_${TARGETARCH}.rpm
|
||||
RUN for rpm in ${EXCLUDE_PACKAGES}; do x+="--exclude=$rpm "; done && \
|
||||
set -x && \
|
||||
$DNFCMD update && \
|
||||
$DNFCMD install $x $RUNNER_RPM_URL && \
|
||||
$DNFCMD upgrade && \
|
||||
$DNFCMD reinstall shadow-utils && \
|
||||
if [[ "${DNFCMD}" == "${_DNFCMD}" ]]; then \
|
||||
dnf clean all && \
|
||||
rm -rf /var/cache/dnf; \
|
||||
fi
|
||||
# Allow image-builders to choose an alternate nested-container pruning cycle.
|
||||
# For most people the default is probably fine. This setting is dependent
|
||||
# on the number and frequency of jobs run, along with the amount of disk-space
|
||||
# available for both /cache and /home/podman/.local/share/containers volumes.
|
||||
ARG PRUNE_INTERVAL=daily # see systemd.timer for allowable values
|
||||
|
||||
# In case of a runner escape, prevent easy installation of packages.
|
||||
RUN rm -f /etc/dnf/protected.d/* && \
|
||||
rpm -e dnf && \
|
||||
rm -f $(type -P rpm)
|
||||
# All-in-one packaging/image-setup script to keep things simple.
|
||||
RUN PRUNE_INTERVAL=${PRUNE_INTERVAL} \
|
||||
RUNNER_VERSION=${RUNNER_VERSION} \
|
||||
bash /root/setup.sh
|
||||
|
||||
ADD /config.toml /home/podman/.gitlab-runner/config.toml
|
||||
# The global "listen_address" option is used for metrics and
|
||||
# debugging. Disable it by default since use requires special/
|
||||
# additional host configuration.
|
||||
# Ref: https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section
|
||||
ARG RUNNER_LISTEN_ADDRESS="disabled"
|
||||
ENV RUNNER_LISTEN_ADDRESS=$RUNNER_LISTEN_ADDRESS
|
||||
RUN if [[ "$RUNNER_LISTEN_ADDRESS" == "disabled" ]]; then \
|
||||
sed -i -r \
|
||||
-e "s/.*@@RUNNER_LISTEN_ADDRESS@@.*//g" \
|
||||
/home/podman/.gitlab-runner/config.toml; \
|
||||
else \
|
||||
sed -i -r \
|
||||
-e "s/@@RUNNER_LISTEN_ADDRESS@@/$RUNNER_LISTEN_ADDRESS/g" \
|
||||
/home/podman/.gitlab-runner/config.toml; \
|
||||
fi
|
||||
VOLUME /cache /home/podman/.local/share/containers
|
||||
ENTRYPOINT /lib/systemd/systemd
|
||||
|
||||
# A small wrapper is needed to launch a background podman system service
|
||||
# process for the gitlab-runner to connect to.
|
||||
ADD /gitlab-runner-wrapper /podman-in-podman-maintenance /usr/local/bin/
|
||||
# Base image UTS NS configuration causes runner to break when launching
|
||||
# nested rootless containers.
|
||||
RUN sed -i -r \
|
||||
-e 's/^utsns.+host.*/utsns="private"/' \
|
||||
/etc/containers/containers.conf && \
|
||||
chmod +x /usr/local/bin/gitlab-runner-wrapper && \
|
||||
chmod +x /usr/local/bin/podman-in-podman-maintenance && \
|
||||
chown -R podman:podman /home/podman && \
|
||||
chmod u+s /usr/bin/new{uid,gid}map && \
|
||||
rm -f /home/podman/.bash* && \
|
||||
echo DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" > /etc/profile.d/podman.sh && \
|
||||
echo "podman:10000:10000" | tee /etc/subuid > /etc/subgid && \
|
||||
setcap -n 10000 cap_setuid+ep /usr/bin/newuidmap && \
|
||||
setcap -n 10000 cap_setuid+ep /usr/bin/newgidmap
|
||||
# Gitlab-runner configuration options, may be freely overridden at
|
||||
# container image build time.
|
||||
ARG DEFAULT_JOB_IMAGE=registry.fedoraproject.org/fedora-minimal:latest
|
||||
# Run nested containers in --privileged mode - required to allow building
|
||||
# container images using podman or buildah. Otherwise may be set 'false'.
|
||||
ARG NESTED_PRIVILEGED=true
|
||||
|
||||
# Runtime rootless-mode configuration
|
||||
USER podman
|
||||
# N/B: Volumes are cumulative with the base image
|
||||
VOLUME ["/home/podman/.gitlab-runner/", "/cache"]
|
||||
WORKDIR /home/podman
|
||||
ENTRYPOINT ["/usr/local/bin/gitlab-runner-wrapper"]
|
||||
|
||||
# Ensure root storage directory exists with correct permissions
|
||||
RUN mkdir -p .local/share/containers/storage
|
||||
|
||||
# Gitlab-runner configuration options. Default to unprivileged (nested)
|
||||
# runner. Privileged is required to permit nested container image building.
|
||||
ARG RUNNER_NAME="qontainers-pipglr"
|
||||
# Running inner-podman privileged is necessary at the time of this commit.
|
||||
ARG PRIVILEGED_RUNNER="true"
|
||||
# Tags allow pinning jobs to specific runners, comma-separated list of
|
||||
# tags to add to runner (no spaces!)
|
||||
ARG RUNNER_TAGS="podman-in-podman"
|
||||
# Permit running jobs without any tag at all
|
||||
ARG RUNNER_UNTAGGED="true"
|
||||
# Adjust based on usage and storage size to prevent ENOSPACE problems
|
||||
ARG CLEAN_INTERVAL="24h"
|
||||
ENV CLEAN_INTERVAL="$CLEAN_INTERVAL" \
|
||||
REGISTER_NON_INTERACTIVE="true" \
|
||||
RUNNER_TAG_LIST="$RUNNER_TAGS" \
|
||||
REGISTER_RUN_UNTAGGED="$RUNNER_UNTAGGED" \
|
||||
REGISTER_ACCESS_LEVEL="ref_protected" \
|
||||
REGISTER_MAXIMUM_TIMEOUT="3600" \
|
||||
CI_SERVER_URL="https://gitlab.com/" \
|
||||
RUNNER_NAME="${RUNNER_NAME}" \
|
||||
RUNNER_EXECUTOR="docker" \
|
||||
RUNNER_SHELL="bash" \
|
||||
REGISTER_MAINTENANCE_NOTE="Podman-in-Podman containerized runner" \
|
||||
DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" \
|
||||
DOCKER_DEVICES="/dev/fuse" \
|
||||
DOCKER_IMAGE="registry.fedoraproject.org/fedora-minimal:latest" \
|
||||
DOCKER_CACHE_DIR="/cache" \
|
||||
DOCKER_VOLUMES="/cache" \
|
||||
DOCKER_NETWORK_MODE="host" \
|
||||
DOCKER_PRIVILEGED="$PRIVILEGED_RUNNER"
|
||||
|
||||
# Not a real build-arg. Simply here to save lots of typing.
|
||||
ARG _pm="--systemd=true --device=/dev/fuse --security-opt label=disable --user podman --volume pipglr-podman-root:/home/podman/.local/share/containers --volume pipglr-config:/home/podman/.gitlab-runner -v pipglr-podman-cache:/cache --tmpfs /var/lib/containers,ro,size=1k -e PODMAN_RUNNER_DEBUG -e LOG_LEVEL"
|
||||
|
||||
# These labels simply make it easier to register and execute the runner.
|
||||
# Define them last so they are absent should a image-build failure occur.
|
||||
LABEL register="podman run -it --rm $_pm --secret REGISTRATION_TOKEN,type=env \$IMAGE register"
|
||||
# Note: Privileged mode is required to permit building container images with inner-podman
|
||||
LABEL run="podman run -d --privileged --name pipglr $_pm \$IMAGE run"
|
||||
|
||||
# In case it's helpful, include the documentation
|
||||
ADD /README.md /home/podman/
|
||||
# The registration runlabel may be called multiple times to register more than
|
||||
# one runner. Each expects a REGISTRATION_TOKEN secret to be pre-defined and
|
||||
# the file './config.toml' to exist (may be empty). A local-cache volume
|
||||
# '/cache' is configured for bind-mounting into all interrior-containers
|
||||
# for container-runtime use, as recommended by the docs. Other settings
|
||||
# may be changed if you know what you're doing.
|
||||
LABEL register="podman run -it --rm \
|
||||
--secret=REGISTRATION_TOKEN,type=env \
|
||||
-v ./config.toml:/home/runner/.gitlab-runner//config.toml:Z \
|
||||
-e REGISTER_NON_INTERACTIVE=true \
|
||||
-e CI_SERVER_URL=https://gitlab.com/ \
|
||||
-e RUNNER_NAME=pipglr \
|
||||
-e RUNNER_EXECUTOR=docker \
|
||||
-e RUNNER_SHELL=bash \
|
||||
-e REGISTER_MAINTENANCE_NOTE=Podman-In-Podman-GitLab-Runner \
|
||||
-e DOCKER_HOST=unix:///home/runner/podman.sock \
|
||||
-e DOCKER_IMAGE=${DEFAULT_JOB_IMAGE} \
|
||||
-e DOCKER_CACHE_DIR=/cache \
|
||||
-e DOCKER_VOLUMES=/cache \
|
||||
-e DOCKER_NETWORK_MODE=host \
|
||||
-e DOCKER_PRIVILEGED=${NESTED_PRIVILEGED} \
|
||||
--user runner \
|
||||
--entrypoint=/usr/bin/gitlab-runner \$IMAGE register"
|
||||
# Additionally, the nested-podman storage volumes must be pre-created with
|
||||
# 'podman' UID/GID values to allow nested containers access.
|
||||
LABEL setupstorage="podman volume create --opt o=uid=1000,gid=1000 pipglr-storage"
|
||||
# Lastly, the gitlab-runner will manage container-cache in this directory,
|
||||
# which will also be bind-mounted into every container. So it must be
|
||||
# writable by both 'podman' user and 'runner' group.
|
||||
LABEL setupcache="podman volume create --opt o=uid=1000,gid=1001 pipglr-cache"
|
||||
# Helper to extract the current configuration secret to allow editing.
|
||||
LABEL dumpconfig="podman run -it --rm \
|
||||
--secret config.toml --entrypoint=/bin/cat \
|
||||
\$IMAGE /var/run/secrets/config.toml"
|
||||
# Executing the runner container depends on the config.toml secret being
|
||||
# set (see above) and two volumes existing with correct permissions set.
|
||||
# Note: The contents of the volumes are not critical, they may be removed
|
||||
# and re-created (see above) to quickly free-up disk space.
|
||||
LABEL run="podman run -dt --name pipglr \
|
||||
--secret config.toml,uid=1001,gid=1001 \
|
||||
-v pipglr-storage:/home/podman/.local/share/containers \
|
||||
-v pipglr-cache:/cache \
|
||||
--systemd true --privileged \
|
||||
--device /dev/fuse \$IMAGE"
|
||||
|
||||
211
README.md
211
README.md
@@ -22,112 +22,122 @@ configuration relative to their own security situation/environment.
|
||||
|
||||
### Operation
|
||||
|
||||
This image supports `podman container runlabel`, or if your version
|
||||
lacks this feature, Several labels are set on the image to support
|
||||
easy registration and execution of a runner container using a special
|
||||
bash command. See the examples below for more information.
|
||||
This image leverages the podman `runlabel` feature heavily. Several
|
||||
labels are set on the image to support easy registration and execution
|
||||
of the runner container. While it's possible to use the container
|
||||
with your own command-line, it's highly recommended to base them
|
||||
off of one of the labels. See the examples below for more information.
|
||||
|
||||
#### [Volume setup]
|
||||
|
||||
Since podman inside the container runs as user `podman`, the volumes
|
||||
used by it need to be pre-created with ownership information. While,
|
||||
we're at it, might as well add the performance-improving `noatime`,
|
||||
option as well.
|
||||
***Note:*** Some older versions of podman don't support the
|
||||
`container runlabel` sub-command. If this is the case, you may simulate
|
||||
it with the following, substituting `<label>` with one of the predefined
|
||||
values (i.e. `register`, `setupconfig`, etc.):
|
||||
|
||||
```bash
|
||||
$ VOLOPTS="o=uid=1000,gid=1000,noatime"; \
|
||||
for VOLUME in pipglr-podman-root pipglr-config pipglr-podman-cache; do \
|
||||
podman volume create --opt $VOLOPTS $VOLUME || true ; \
|
||||
VOLPTH=$(podman unshare podman volume mount $VOLUME)
|
||||
podman unshare chown -c -R 1000:1000 $VOLPTH && \
|
||||
podman unshare chmod -c 02770 $VOLPTH && \
|
||||
podman unshare podman volume unmount $VOLUME ; \
|
||||
done
|
||||
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
|
||||
$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.<label>)
|
||||
```
|
||||
|
||||
If you get `podman system service` startup permission-denied errors, or
|
||||
errors from gitlab-runner, unable to connect to the podman socket, this is
|
||||
likely the cause. You can fix it after-the-fact using the same commands
|
||||
above.
|
||||
#### Runner registration (step 1)
|
||||
|
||||
#### Runner registration
|
||||
All runners must be connected to a project or group runner configuration
|
||||
on your gitlab instance (or `gitlab.com`). This is done using a special
|
||||
registration *runlabel*. The command can (and probably should) be run
|
||||
more than once (using the same `config.toml`) to configure and register
|
||||
multiple runners. This is necessary for the *pipglr* container to execute
|
||||
multiple jobs in parallel. For example, if you want to support running
|
||||
four jobs at the same time, you would use the `register` *runlabel*
|
||||
four times.
|
||||
|
||||
Each time the registration command is run, a new runner is added into
|
||||
the configuration. If however, you simply need to update/modify the
|
||||
configuration, please edit the `config.toml` file directly after mounting
|
||||
(default) `pipglr-runner-config` (`/home/podman/.gitlab-runner/`) volume.
|
||||
For modern versions of podman, registration can be performed with the
|
||||
following commands:
|
||||
Before using the `register` *runlabel*, you must set your unique
|
||||
*registration* (a.k.a. *activation*) token as a podman *secret*. This
|
||||
secret may be removed once the registration step is complete. The
|
||||
**<actual registration token>** value (below) should be replaced with
|
||||
the value obtained from the "runners" settings page of a gitlab
|
||||
group or project's *CI/CD Settings*. Gitlab version 16 and later
|
||||
refers to this value as an *activation* token, but the usage is the same.
|
||||
|
||||
```bash
|
||||
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
|
||||
$ echo '<actual registration token>' | podman secret create REGISTRATION_TOKEN -
|
||||
$ touch ./config.toml # important: file must exist, even if empty.
|
||||
$ podman container runlabel register $IMAGE
|
||||
...repeat as desired...
|
||||
$ podman secret rm REGISTRATION_TOKEN # if desired
|
||||
```
|
||||
|
||||
Where `<actual registration token>` is the value obtained from the "runners"
|
||||
settings page of a gitlab group or project. When you're finished registering
|
||||
as many runners as you want, the secret is no-longer needed and may be removed:
|
||||
#### Runner Configuration (step 2)
|
||||
|
||||
During the registration process (above), a boiler-plate (default) `config.toml` file
|
||||
will be created/updated for you. At this point you may edit the configuration
|
||||
if desired before committing it as a *podman secret*. Please refer to the
|
||||
[gitlab runner documentation](https://docs.gitlab.com/runner/configuration/)
|
||||
for details.
|
||||
|
||||
```bash
|
||||
$ podman secret rm REGISTRATION_TOKEN
|
||||
$ $EDITOR ./config.toml # if desired
|
||||
$ podman secret create config.toml ./config.toml
|
||||
$ rm ./config.toml # if desired
|
||||
```
|
||||
|
||||
##### Note
|
||||
#### Volume setup (step 3)
|
||||
|
||||
Some versions of podman don't support the `container runlabel` sub-command.
|
||||
If this is the case, you may simulate it with the following command (in addition
|
||||
to the other example commands above):
|
||||
Since several users are utilized inside the container volumes must be
|
||||
specifically configured to permit access. This is done using several
|
||||
*runlabels* as follows:
|
||||
|
||||
```bash
|
||||
$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.register)
|
||||
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
|
||||
$ podman container runlabel setupstorage $IMAGE
|
||||
$ podman container runlabel setupcache $IMAGE
|
||||
```
|
||||
|
||||
#### Runner Startup
|
||||
Note: These volumes generally do not contain any critical operational data,
|
||||
they may be re-created anytime to quickly free up host disk-space if
|
||||
it's running low. Simply remove them with the command
|
||||
`podman volume rm pipglr-storage pipglr-cache`. The reuse the `setupstorage`
|
||||
and `setupcache` *runlabels* as in the above example.
|
||||
|
||||
With one or more runners successfully registered and configured, the GitLab
|
||||
runner container may be launched with the following commands:
|
||||
#### Runner Startup (step 4)
|
||||
|
||||
With the runner configuration saved as a Podman secret, and the runner volumes
|
||||
created, the GitLab runner container may be launched with the following commands:
|
||||
|
||||
```bash
|
||||
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
|
||||
$ podman container runlabel run $IMAGE
|
||||
```
|
||||
|
||||
##### Note
|
||||
### Configuration Editing
|
||||
|
||||
As above, if you're missing the `container runlabel` sub-command, the following
|
||||
may be used instead (assuming `$IMAGE` remains set):
|
||||
The gitlab-runner configuration contains some sensitive values which
|
||||
should be protected. The pipglr container assumes the entire configuration
|
||||
will be passed in as a Podman secret. This makes editing it slightly
|
||||
convoluted, so a handy *runlabel* `dumpconfig` is available.
|
||||
It's intended use is as follows:
|
||||
|
||||
```bash
|
||||
$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.run)
|
||||
$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
|
||||
$ podman container runlabel dumpconfig $IMAGE > ./config.toml
|
||||
$ $EDITOR ./config.toml
|
||||
$ podman secret rm config.toml
|
||||
$ podman secret create config.toml ./config.toml
|
||||
$ rm ./config.toml # if desired
|
||||
```
|
||||
|
||||
#### Runner configuration
|
||||
### Debugging
|
||||
|
||||
You may inspect/modify the gitlab-runner configuration as you see fit, just be
|
||||
sure to use the `podman unshare` command-wrapper to enter the usernamespace.
|
||||
For example, to display the config:
|
||||
|
||||
```bash
|
||||
$ podman unshare cat $(podman unshare podman volume mount pipglr-config)/config.toml
|
||||
```
|
||||
|
||||
Edit the config with your favorite `$EDITOR`:
|
||||
|
||||
```bash
|
||||
$ podman unshare $EDITOR $(podman unshare podman volume mount pipglr-config)/config.toml
|
||||
```
|
||||
|
||||
#### Debugging
|
||||
|
||||
The first thing to check is the container output:
|
||||
The first thing to check is the container output. This shows three things:
|
||||
Systemd, Podman, and GitLab-Runner output. For example:
|
||||
|
||||
```bash
|
||||
$ podman logs --since 0 pipglr
|
||||
```
|
||||
|
||||
Next, try running pipglr after an `export PODMAN_RUNNER_DEBUG=debug` to enable
|
||||
debugging on the inner-podman. If more runner detail is needed, you can instead/additionally
|
||||
set `export LOG_LEVEL=debug` to debug the gitlab-runner itself.
|
||||
Next, try running a pipglr image built with more verbose logging. Both
|
||||
the `runner.service` and `podman.service` files have a `log-level` option.
|
||||
Simply increase one or both to the "info", or "debug" level. Start the
|
||||
debug container, and reproduce the problem.
|
||||
|
||||
## Building
|
||||
|
||||
@@ -140,69 +150,34 @@ $ podman build -t registry.gitlab.com/qontainers/pipglr:latest .
|
||||
This will utilize the latest stable version of podman and the latest
|
||||
stable version of the gitlab runner.
|
||||
|
||||
### Notes
|
||||
|
||||
* If you wish to use the `testing` or `upstream` flavors of the podman base image,
|
||||
simply build with `--build-arg FLAVOR=testing` (or `upstream`).
|
||||
|
||||
* Additionally or alternatively, you may specify a specific podman base image tag
|
||||
with `--build-arg BASE_TAG=<value>`. Where `<value>` is either `latest`, the
|
||||
podman image version (e.g. `v4`, `v4.2`, `v4.2.0`, etc.)
|
||||
|
||||
### Build-args
|
||||
|
||||
Several build arguments are available to control the output image:
|
||||
|
||||
* `FLAVOR` - Choose from 'stable', 'testing', or 'upstream'. These
|
||||
select the podman base-image to utilize - which may affect the
|
||||
podman version, features, and stability. For more information
|
||||
see [the podmanimage README](https://github.com/containers/podman/blob/main/contrib/podmanimage/README.md).
|
||||
* `BASE_TAG` - When `FLAVOR="stable"`, allows granular choice over the
|
||||
exact podman version. Possible values include, `latest`, `vX`, `vX.Y`,
|
||||
and `vX.Y.Z` (where, `X`, `Y`, and `Z` represent the podman semantic
|
||||
version numbers). It's also possible to specify an image SHA.
|
||||
* `CLEAN_INTERVAL` - A `sleep` (command) compatible time-argument that
|
||||
determines how often to clean out podman storage of disused containers and
|
||||
images. Defaults to 24-hours, but should be adjusted based on desired caching-effect
|
||||
versus available storage space and rate of job execution.
|
||||
* `EXCLUDE_PACKAGES` - A space-separated list of RPM packages to prevent
|
||||
their existence in the final image. This is intended as a security measure
|
||||
to limit the attack-surface should a gitlab-runner process escape it's
|
||||
inner-container.
|
||||
* `PRUNE_INTERVAL` - A systemd.timer compatible `OnCalendar` value that
|
||||
determines how often to prune Podman's storage of disused containers and
|
||||
images. Defaults to "daily", but should be adjusted based on desired
|
||||
caching-effect balanced against available storage space and job
|
||||
execution rate.
|
||||
* `RUNNER_VERSION` - Allows specifying an exact gitlab runner version.
|
||||
By default the `latest` is used, assuming the user is building a tagged
|
||||
image anyway. Valid versions may be found on the [runner
|
||||
release page](https://gitlab.com/gitlab-org/gitlab-runner/-/releases).
|
||||
* `DNFCMD` - By default this is set to `dnf --setopt=tsflags=nodocs -y`.
|
||||
However, if you'd like to volume-mount in `/var/cache/dnf` then you'll
|
||||
need to use
|
||||
`--build-arg DNFCMD="dnf --setopt=tsflags=nodocs -y --setopt keepcache=true`"
|
||||
Note: Changing `DNFCMD` will cause build-time cache cleanup to be disabled.
|
||||
* `TARGETARCH` - Supports inclusion of non-x86_64 gitlab runners. This
|
||||
value is assumed to match the image's architecture. If using the
|
||||
`--platform` build argument, it will be set automatically.
|
||||
* `RUNNER_LISTEN_ADDRESS` - Disabled by default, setting this to the FQDN
|
||||
and port supports various observability and debugging features of the
|
||||
gitlab runner. For more information see the [gitlab runner advanced
|
||||
configuration documentation](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section).
|
||||
* `PRIVILEGED_RUNNER` - Defaults to 'true', may be set 'true' if you're brave.
|
||||
However this may result in the gitlab-runner failing to launch inner-containers.
|
||||
Setting it false will also prevent building container images using the runner.
|
||||
* `RUNNER_TAGS` - Defaults to `podman_in_podman`, may be set to any comma-separated
|
||||
list (with no spaces!) of tags. These show up in GitLab (not the runner
|
||||
configuration), and determines where jobs are run.
|
||||
* `RUNNER_UNTAGED` - Defaults to `true`, may be set to `false`. Allows
|
||||
the runner to service jobs without any tags on them at all.
|
||||
`--platform` build argument, it will be set automatically. Note:
|
||||
as of this writing, only `amd64` and `arm64` builds of the gitlab-runner
|
||||
are available.
|
||||
* `NESTED_PRIVILEGED` - Defaults to 'true', may be set 'false' to prevent
|
||||
nested containers running in `--privileged` mode. This will affect
|
||||
the ability to build container images in CI jobs using tools like
|
||||
podman or buildah.
|
||||
|
||||
### Environment variables
|
||||
|
||||
Nearly every option to every gitlab-runner sub-command may be specified via
|
||||
environment variable. Many important/required options are set in the
|
||||
`Containerfile`. However it's entirely possible to pass them in via
|
||||
either of the `podman container runlabel...` container commands. To
|
||||
discover them, simply append `--help` to the end of the command.
|
||||
For example:
|
||||
|
||||
```bash
|
||||
podman container runlabel $IMAGE register --help
|
||||
```
|
||||
environment variable. Some of these are set in the `Containerfile` for
|
||||
the `register` *runlabel*. If you need to set additional runtime
|
||||
env. vars., please do so via additional `Environment` optionns in the
|
||||
`runner.service` file. See the *systemd.nspawn* man page for important
|
||||
value-format details.
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
|
||||
# Ref: https://docs.gitlab.com/runner/configuration/advanced-configuration.html
|
||||
|
||||
concurrent = 8
|
||||
# N/B: This is DIVIDED among the number of registered runners
|
||||
check_interval = 10
|
||||
listen_address = "@@RUNNER_LISTEN_ADDRESS@@" # Will be removed if undefined
|
||||
13
containers.conf
Normal file
13
containers.conf
Normal file
@@ -0,0 +1,13 @@
|
||||
[containers]
|
||||
netns="host"
|
||||
userns="host"
|
||||
ipcns="host"
|
||||
utsns="private"
|
||||
cgroupns="host"
|
||||
cgroups="disabled"
|
||||
log_driver = "k8s-file"
|
||||
|
||||
[engine]
|
||||
cgroup_manager = "cgroupfs"
|
||||
events_logger="file"
|
||||
runtime="crun"
|
||||
@@ -1,28 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script is intended to be called as the entrypoint for
|
||||
# a podman-in-podman gitlab runner container. Any usage
|
||||
# outside that context is not supported and may cause harm.
|
||||
|
||||
set -e
|
||||
|
||||
unset _debug_args
|
||||
if [[ -n "$PODMAN_RUNNER_DEBUG" ]]; then
|
||||
_debug_args="--log-level=$PODMAN_RUNNER_DEBUG"
|
||||
fi
|
||||
|
||||
SOCKET="/tmp/podman-run-1000/podman/podman.sock"
|
||||
if [[ "$1" == "run" ]] && [[ ! -S "$SOCKET" ]]; then
|
||||
podman $_debug_args system service -t 0 &
|
||||
/usr/local/bin/podman-in-podman-maintenance &
|
||||
# Prevent SIGHUP propagation to podman process
|
||||
disown -ar
|
||||
sleep 1s # Give podman a chance to get going
|
||||
# Verify podman is listening on it's socket
|
||||
if [[ ! -S "$SOCKET" ]]; then
|
||||
echo "ERROR: Inner-podman system service failed to start, expecting to find socket '$SOCKET'. Are all volume's owned & writeable by $(id -u podman):$(id -g podman)?" > /dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
exec gitlab-runner "$@"
|
||||
3
kmsglog.conf
Normal file
3
kmsglog.conf
Normal file
@@ -0,0 +1,3 @@
|
||||
[Manager]
|
||||
LogTarget=kmsg
|
||||
LogColor=yes
|
||||
@@ -1,26 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script is intended to be called by the entrypoint for
|
||||
# a podman-in-podman gitlab runner container. Any usage
|
||||
# outside that context is not supported and may cause harm.
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
maintain_podman() {
|
||||
# Two days seems to be a good happy-medium beween filling up
|
||||
# about 40gig of storage space from moderate CI activity,
|
||||
# and maintaining a useful level of caching.
|
||||
while sleep "$CLEAN_INTERVAL"; do
|
||||
if [[ -n "$PODMAN_RUNNER_DEBUG" ]]; then
|
||||
echo "$(date --iso-8601=second) ${BASH_SOURCE[0] performing podman maintenance}"
|
||||
fi
|
||||
podman system prune --all --force
|
||||
done
|
||||
}
|
||||
|
||||
if [[ -z "$CLEAN_INTERVAL" ]]; then
|
||||
echo "ERROR: Empty/unset \$CLEAN_INTERVAL"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
maintain_podman
|
||||
18
podman.service
Normal file
18
podman.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Podman API Service
|
||||
Requires=podman.socket
|
||||
After=podman.socket
|
||||
Documentation=man:podman-system-service(1)
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
Delegate=true
|
||||
Type=exec
|
||||
KillMode=process
|
||||
Environment=LOGGING="--log-level=warn"
|
||||
ExecStart=/usr/bin/podman $LOGGING system service
|
||||
StandardOutput=journal+console
|
||||
StandardError=inherit
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
11
podman.socket
Normal file
11
podman.socket
Normal file
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Podman API Socket
|
||||
Documentation=man:podman-system-service(1)
|
||||
|
||||
[Socket]
|
||||
ListenStream=/home/runner/podman.sock
|
||||
SocketGroup=runner
|
||||
SocketMode=0660
|
||||
|
||||
[Install]
|
||||
WantedBy=sockets.target
|
||||
6
prune.service
Normal file
6
prune.service
Normal file
@@ -0,0 +1,6 @@
|
||||
[Unit]
|
||||
Description=Prune all disused podman volumes, images, and containers
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/podman system prune --all --force
|
||||
6
prune.timer
Normal file
6
prune.timer
Normal file
@@ -0,0 +1,6 @@
|
||||
[Unit]
|
||||
Description=Execute the prune service periodically
|
||||
|
||||
[Timer]
|
||||
OnCalendar=@@@PRUNE_INTERVAL@@@
|
||||
RemainAfterElapse=no
|
||||
7
runner.service
Normal file
7
runner.service
Normal file
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Gitlab-runner service
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/bin/gitlab-runner --log-level=warn run --user runner --working-directory=/home/runner
|
||||
StandardOutput=journal+console
|
||||
StandardError=inherit
|
||||
109
setup.sh
Normal file
109
setup.sh
Normal file
@@ -0,0 +1,109 @@
|
||||
|
||||
|
||||
# This script is intended to be run during container-image build. Any
|
||||
# other usage outside this context is likely to cause harm.
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
for varname in PRUNE_INTERVAL RUNNER_VERSION TARGETARCH; do
|
||||
if [[ -z "${!varname}" ]]; then
|
||||
echo "Error: \$$varname must be non-empty."
|
||||
fi
|
||||
done
|
||||
|
||||
# Make image smaller by not installing docs.
|
||||
DNF="dnf --setopt=tsflags=nodocs -y"
|
||||
|
||||
for rpm in $(egrep -v '^(# )+' < /root/xpackages.txt); do
|
||||
x+="--exclude=$rpm ";
|
||||
done
|
||||
|
||||
set -x # show what's happening to make debugging easier
|
||||
|
||||
# DNF itself or a dependence may need upgrading, take care of it first.
|
||||
$DNF upgrade
|
||||
|
||||
$DNF $x install \
|
||||
podman \
|
||||
systemd
|
||||
|
||||
# Gitlab-runner package contains scriptlets which do not function properly inside a
|
||||
# container-build environment where systemd is not active/running.
|
||||
$DNF $x --setopt=tsflags=noscripts install \
|
||||
https://gitlab-runner-downloads.s3.amazonaws.com/$RUNNER_VERSION/rpm/gitlab-runner_${TARGETARCH}.rpm
|
||||
|
||||
# Allow removing dnf, sudo, etc. packages. Also don't start unnecessary or broken
|
||||
# systemd services, like anything kernel related or login gettys.
|
||||
rm -rf \
|
||||
/etc/dnf/protected.d/* \
|
||||
/etc/sytemd/system/getty.target.wants/* \
|
||||
/etc/sytemd/system/multi-user.target.wants/* \
|
||||
/etc/sytemd/system/sysinit.target.wants/* \
|
||||
/etc/sytemd/system/timers.target.wants/* \
|
||||
/lib/systemd/system/graphical.target.wants/* \
|
||||
/lib/systemd/system/multi-user.target.wants/{getty.target,systemd-ask-password-wall.path} \
|
||||
/lib/systemd/system/sys-kernel*.mount
|
||||
|
||||
# Remove unnecessary packages, see xpackages.txt to learn how this list was generated.
|
||||
# This makes the image smaller and reduces the attack-surface.
|
||||
dnf remove -y $(egrep -v '^(# )+' /root/xpackages.txt)
|
||||
|
||||
# Wipe out the DNF cache, then remove it entirely, again to make the image smaller.
|
||||
$DNF clean all
|
||||
rm -rf /var/cache/dnf /var/log/dnf* /var/log/yum.*
|
||||
rpm -e dnf
|
||||
|
||||
# Workaround https://bugzilla.redhat.com/show_bug.cgi?id=1995337
|
||||
rpm --setcaps shadow-utils
|
||||
|
||||
# Prevent copying of skel since it can interfere with the gitlab-runner
|
||||
mkdir -p /home/podman /home/runner
|
||||
# Guarantee uid/gid 1000 for user 'podman' / 1001 for user 'runner'.
|
||||
groupadd -g 1000 podman
|
||||
groupadd -g 1001 runner
|
||||
# Separate users for services to increase process isolation.
|
||||
# The 'podman' user's socket service writes /home/runner/podman.socket
|
||||
useradd -M -u 1000 -g podman -G runner podman
|
||||
useradd -M -u 1001 -g runner runner
|
||||
# Allow 'podman' user to create socket file under /home/runner.
|
||||
chmod 770 /home/runner
|
||||
|
||||
# Overwrite defaults, only user 'podman' permited to have a user-namespace
|
||||
# Split the namespaced ID's around the containers root (ID 0) and the user
|
||||
# IDs 1000 and 1001 (defined above) to prevent hijacking from a nested container.
|
||||
echo -e "podman:1:999\npodman:1002:64533" | tee /etc/subuid > /etc/subgid
|
||||
# Host volume mount necessary for nested-podman to use overlayfs2 for container & volume storage.
|
||||
mkdir -p /home/podman/.local/share/containers
|
||||
# Nested-container's local container-cache volume mount, recommended by gitlab-runner docs.
|
||||
mkdir -p /cache
|
||||
# Both the gitlab-runner and podman need access to the cache directory / volume mount.
|
||||
chown podman:runner /cache
|
||||
|
||||
# Setup persistent 'podman' user services to start & run without a login.
|
||||
mkdir -p /var/lib/systemd/linger
|
||||
touch /var/lib/systemd/linger/podman
|
||||
# Setup 'podman' socket and a container-storage pruning service for 'podman' user.
|
||||
mkdir -p /home/podman/.config/systemd/user/{sockets.target.wants,default.target.wants}
|
||||
cd /home/podman/.config/systemd/user/
|
||||
ln -s $PWD/podman.socket ./sockets.target.wants/ # Added from Containerfile
|
||||
ln -s $PWD/prune.timer ./default.target.wants/ # also from Containerfile
|
||||
# Substitute value from --build-arg if specified, otherwise use default from Containerfile.
|
||||
sed -i -e "s/@@@PRUNE_INTERVAL@@@/$PRUNE_INTERVAL/" ./prune.timer
|
||||
# Containerfile ADD instruction does not properly set ownership/permissions.
|
||||
chown -R 1000:1000 /home/podman
|
||||
chmod 700 /home/podman
|
||||
|
||||
# Setup persistent 'runner' user services to start & run without a login.
|
||||
touch /var/lib/systemd/linger/runner
|
||||
mkdir -p /home/runner/.config/systemd/user/default.target.wants
|
||||
cd /home/runner/.config/systemd/user/
|
||||
# Does not depend on podman.socket file availablility, will retry if not present.
|
||||
ln -s $PWD/runner.service ./default.target.wants/
|
||||
# gitlab-runner will create side-car '.runner_system_id' file next to 'config.toml'
|
||||
# on first startup. Ensure access is allowed. Also link to future config file
|
||||
# presented as a container-secret.
|
||||
mkdir -p /home/runner/.gitlab-runner
|
||||
ln -s /var/run/secrets/config.toml /home/runner/.gitlab-runner/config.toml
|
||||
# Containerfile ADD instruction does not properly set ownership/permissions.
|
||||
chown -R runner:runner /home/runner
|
||||
chmod -R 700 /home/runner/.gitlab-runner
|
||||
21
xpackages.txt
Normal file
21
xpackages.txt
Normal file
@@ -0,0 +1,21 @@
|
||||
# This list was formed by running the following commands in the base image:
|
||||
# for package in $(rpm -qa); do if dnf erase $package; then echo "$package" >> remove; fi; done
|
||||
# cat remove
|
||||
# Including those packages in this file. Finally, repeatedly running the container build
|
||||
# untill no dependency errors were raised.
|
||||
|
||||
criu
|
||||
criu-libs
|
||||
crypto-policies-scripts
|
||||
dejavu-sans-fonts
|
||||
findutils
|
||||
fonts-filesystem
|
||||
gdb-gdbserver
|
||||
langpacks-core-en
|
||||
langpacks-core-font-en
|
||||
langpacks-en
|
||||
libnet
|
||||
protobuf-c
|
||||
rootfiles
|
||||
vim-minimal
|
||||
yum
|
||||
Reference in New Issue
Block a user