From df8f46eb2d1bf0c9e77df32f47d44c123dc74e24 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 22 Nov 2022 14:49:25 -0500 Subject: [PATCH] Update docs and Containerfile to match Fully tested README.md instructions end-to-end on F36. Signed-off-by: Chris Evich --- Containerfile | 11 +++++--- README.md | 77 ++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 62 insertions(+), 26 deletions(-) diff --git a/Containerfile b/Containerfile index d14e8c5..085646b 100644 --- a/Containerfile +++ b/Containerfile @@ -97,6 +97,7 @@ RUN sed -i -r \ chmod +x /usr/local/bin/gitlab-runner-wrapper && \ chmod +x /usr/local/bin/podman-in-podman-maintenance && \ chown -R podman:podman /home/podman && \ + chmod u+s /usr/bin/new{uid,gid}map && \ rm -f /home/podman/.bash* && \ echo DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" > /etc/profile.d/podman.sh @@ -113,7 +114,8 @@ RUN mkdir -p .local/share/containers/storage # Gitlab-runner configuration options. Default to unprivileged (nested) # runner. Privileged is required to permit nested container image building. ARG RUNNER_NAME="qontainers-pipglr" -ARG PRIVILEGED_RUNNER="false" +# Running inner-podman privileged is necessary at the time of this commit. +ARG PRIVILEGED_RUNNER="true" # Tags allow pinning jobs to specific runners, comma-separated list of # tags to add to runner (no spaces!) ARG RUNNER_TAGS="podman-in-podman" @@ -132,15 +134,16 @@ ENV REGISTER_NON_INTERACTIVE="true" \ DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" \ DOCKER_DEVICES="/dev/fuse" \ DOCKER_IMAGE="registry.fedoraproject.org/fedora-minimal:latest" \ - DOCKER_CACHE_DIR="/home/podman/.cache/gitlab-runner" \ + DOCKER_CACHE_DIR="/cache" \ + DOCKER_VOLUMES="/cache" \ DOCKER_NETWORK_MODE="host" \ DOCKER_PRIVILEGED="$PRIVILEGED_RUNNER" # Not a real build-arg. Simply here to save lots of typing. -ARG _pm="--systemd=true --device=/dev/fuse --security-opt label=disable --user podman --volume pipglr-podman-root:/home/podman/.local/share/containers/storage:Z --volume pipglr-runner-config:/home/podman/.gitlab-runner:Z -e PODMAN_RUNNER_DEBUG -e LOG_LEVEL" +ARG _pm="--systemd=true --device=/dev/fuse --security-opt label=disable --user podman --volume pipglr-podman-root:/home/podman/.local/share/containers/storage --volume pipglr-config:/home/podman/.gitlab-runner -v pipglr-podman-cache:/cache -e PODMAN_RUNNER_DEBUG -e LOG_LEVEL" # These labels simply make it easier to register and execute the runner. # Define them last so they are absent should a image-build failure occur. LABEL register="podman run -it --rm $_pm --secret REGISTRATION_TOKEN,type=env \$IMAGE register" # Note: Privileged mode is required to permit building container images with inner-podman -LABEL run="podman run -d --rm --privileged --name gitlab-runner $_pm \$IMAGE run" +LABEL run="podman run -d --privileged --name pipglr $_pm \$IMAGE run" diff --git a/README.md b/README.md index c9d1bf9..c6bac29 100644 --- a/README.md +++ b/README.md @@ -27,21 +27,28 @@ lacks this feature, Several labels are set on the image to support easy registration and execution of a runner container using a special bash command. See the examples below for more information. -#### Volume Ownership Bug +#### [Volume Ownership Bug](https://github.com/containers/podman/issues/16576) -Some versions of podman contain a bug where named volumes aren't owned -by the namespaced user within a rootless container (i.e. in conjunction -with the --user option). Since the `podman` user/group inside the `pipglr` -container is known, it's possible to manually set/reset ownership: +Some versions of podman contain a bug where named local volumes aren't owned +by the namespaced user within a rootless container (i.e. the 'podman' user). +Since the `podman` user/group inside the `pipglr` container is known, it's +possible to manually setup ownership ahead of time. This should be be done +once, prior to registering your runners: ```bash -VOLUME=pipglr-podman-root -podman volume create $VOLUME -cd $(podman unshare podman volume mount $VOLUME) -podman unshare chown 1000:1000 -podman volume unmount $VOLUME +$ for VOLUME in pipglr-podman-root pipglr-config pipglr-podman-cache; do \ + PUPVM="podman unshare podman volume mount $VOLUME" + podman volume create $VOLUME && \ + podman unshare chown 1000:1000 $($PUPVM) && \ + podman unshare chmod 02770 $($PUPVM) && \ + podman unshare ls -land $($PUPVM) ; \ + done ``` +If you get `podman system service` startup permission-denied errors, or +errors from gitlab-runner, unable to connect to the podman socket, this is +likely the cause. You can fix it after-the-fact using the same commands as above, just add a `-R` option to the `chown`/`chmod`, and additionally target `./*`. + #### Runner registration Each time the registration command is run, a new runner is added into @@ -52,20 +59,27 @@ For modern versions of podman, registration can be performed with the following commands: ```bash -IMAGE="=registry.gitlab.com/qontainers/pipglr:latest" -echo '' | podman secret create REGISTRATION_TOKEN - -podman container runlabel $IMAGE register --secret REGISTRATION_TOKEN,type=env +$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest" +$ echo '' | podman secret create REGISTRATION_TOKEN - +$ podman container runlabel register $IMAGE ``` Where `` is the value obtained from the "runners" -settings page of a gitlab group or project. +settings page of a gitlab group or project. When you're finished registering +as many runners as you want, the secret is no-longer needed and may be removed: -Note: Some versions of podman don't support the `container runlabel` sub-command. +```bash +$ podman secret rm REGISTRATION_TOKEN +``` + +##### Note + +Some versions of podman don't support the `container runlabel` sub-command. If this is the case, you may simulate it with the following command (in addition to the other example commands above): ```bash -eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.register) +$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.register) ``` #### Runner Startup @@ -74,9 +88,11 @@ With one or more runners successfully registered and configured, the GitLab runner container may be launched with the following commands: ```bash -podman container runlabel $IMAGE run +$ podman container runlabel run $IMAGE ``` +##### Note + As above, if you're missing the `container runlabel` sub-command, the following may be used instead (assuming `$IMAGE` remains set): @@ -84,8 +100,24 @@ may be used instead (assuming `$IMAGE` remains set): $ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.run) ``` +#### Runner configuration + +You may inspect/modify the gitlab-runner configuration as you see fit, just be +sure to use the `podman unshare` command-wrapper to enter the usernamespace. +For example, to display the config: + +```bash +$ podman unshare $(podman unshare podman volume mount pipglr-config)/config.toml +``` + #### Debugging +The first thing to check is the container output: + +```bash +$ podman logs --since 0 pipglr +``` + Before starting the runner, you may `export PODMAN_RUNNER_DEBUG=debug` to enable debugging on the inner-podman. Whereas `export LOG_LEVEL=debug` can be used to debug the gitlab-runner itself. @@ -94,7 +126,9 @@ debug the gitlab-runner itself. This image may be built simply with: -`podman build -t registry.gitlab.com/qontainers/pipglr:latest .` +```bash +$ podman build -t registry.gitlab.com/qontainers/pipglr:latest . +``` This will utilize the latest stable version of podman and the latest stable version of the gitlab runner. @@ -140,10 +174,9 @@ Several build arguments are available to control the output image: and port supports various observability and debugging features of the gitlab runner. For more information see the [gitlab runner advanced configuration documentation](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section). -* `PRIVILEGED_RUNNER` - Defaults to 'false', may be set 'true'. When - `true`, this causes inner-containers to be created with the `--privileged` - flag. This is a potential security weakness, but is necessary for - (among other things) allowing nested container image builds. +* `PRIVILEGED_RUNNER` - Defaults to 'true', may be set 'true' if you're brave. + However this may result in the gitlab-runner failing to launch inner-containers. + Setting it false will also prevent building container images using the runner. * `RUNNER_TAGS` - Defaults to `podman_in_podman`, may be set to any comma-separated list (with no spaces!) of tags. These show up in GitLab (not the runner configuration), and determines where jobs are run.