Attempt fix v2.0.0 newuidmap permission denied

Recently, I believe an update or packaging problem has been causing
podman to throw errors like:

```
level=error msg="running `/usr/bin/newuidmap ...`: newuidmap: open of
uid_map failed: Permission denied\n"
```

This seems to have something to do with the shadow-utils package, which
owns this binary.  I've examined the file attribuites and permissions
along with /etc/sub{uid,gid} contents.  The only thing that seems to
resolve the issue is reinstalling shadow-utils.  Attempt that fix here
and hope it clears up the problem (present in v2.0.0)

Signed-off-by: Chris Evich <chris_gitlab@icuc.me>

.gitlab-ci.yml

@@ -1,25 +1,39 @@
 ---
 
-stages:
-    - build
-
-build:
-  stage: build
+default:
+  image: quay.io/buildah/stable:v1.28.0
   tags:
     - docker
     - linux
-  image:
-    name: gcr.io/kaniko-project/executor:v1.6.0-debug
-    entrypoint: ["/busybox/sh", "-c"]
+
+envars:
+  stage: test
+  script: |
+    echo "Select CI env. vars.:";
+    printenv | egrep '^CI_' | sort
+
+commit_check:
+  stage: test
   variables:
-    BASE_TAG: latest
-    FLAVOR: stable
+    BADRX: '^(squash!)|(fixup!)'
+  script: |
+    dnf install -y git
+    shortlogtmp=$(mktemp -p '' commit_check_tmp_XXXX)
+    git log --oneline --no-show-signature "${CI_MERGE_REQUEST_DIFF_BASE_SHA}..HEAD" > "$shortlogtmp"
+    if egrep -q "$BADRX" "$shortlogtmp"; then
+        egrep "$BADRX" "$shortlogtmp"
+        die "Found the above commits matching '$BADRX'"
+    fi
+
+build:
+  stage: deploy
+  variables:
+    BUILDAH_FORMAT: docker
+    BUILDAH_ISOLATION: chroot
+    STORAGE_DRIVER: vfs
+  before_script:
+    - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
   script:
-    - 'mkdir -p /kaniko/.docker'
-    - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json'
-    - |
-        echo "Select CI env. vars.:";
-        printenv | egrep '^CI_' | sort
     # N/B: There could be more than one merge-request open with this branch's HEAD
     - |
         IMAGE_TAG="${CI_COMMIT_BRANCH}";
@@ -31,10 +45,12 @@ build:
             IMAGE_TAG="latest";
         fi
         echo "Building/Pushing to: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG}";
-    - |
-        /kaniko/executor \
-            --context $CI_PROJECT_DIR \
-            --dockerfile $CI_PROJECT_DIR/Containerfile \
-            --destination "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}" \
-            --build-arg "BASE_TAG=$BASE_TAG" \
-            --build-arg "FLAVOR=$FLAVOR"
+    - >-
+      buildah build \
+        --label "org.opencontainers.image.source=${CI_PROJECT_URL}" \
+        --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" \
+        --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" \
+        --label "org.opencontainers.image.version=${IMAGE_TAG}" \
+        -t "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}" "$CI_PROJECT_DIR"
+    - buildah images
+    - buildah push "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}"

Containerfile

@@ -6,9 +6,7 @@
 # https://docs.gitlab.com/runner/executors/docker.html#use-podman-to-run-docker-commands
 #
 
-ARG FLAVOR="stable"
-ARG BASE_TAG="latest"
-FROM quay.io/podman/$FLAVOR:$BASE_TAG
+FROM quay.io/podman/stable:v4.3.1
 
 # This is a list of packages to remove and/or exclude from the image.
 # Primarily this is done for security reasons, should a runner process
@@ -59,6 +57,7 @@ RUN for rpm in ${EXCLUDE_PACKAGES}; do x+="--exclude=$rpm "; done && \
     $DNFCMD update && \
     $DNFCMD install $x $RUNNER_RPM_URL && \
     $DNFCMD upgrade && \
+    $DNFCMD reinstall shadow-utils && \
     if [[ "${DNFCMD}" == "${_DNFCMD}" ]]; then \
       dnf clean all && \
       rm -rf /var/cache/dnf; \
@@ -97,13 +96,15 @@ RUN sed -i -r \
     chmod +x /usr/local/bin/gitlab-runner-wrapper && \
     chmod +x /usr/local/bin/podman-in-podman-maintenance && \
     chown -R podman:podman /home/podman && \
+    chmod u+s /usr/bin/new{uid,gid}map && \
     rm -f /home/podman/.bash* && \
-    echo DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" > /etc/profile.d/podman.sh
+    echo DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" > /etc/profile.d/podman.sh && \
+    echo "podman:10000:10000" | tee /etc/subuid > /etc/subgid
 
 # Runtime rootless-mode configuration
 USER podman
-VOLUME ["/home/podman/.local/share/containers/storage/",\
-        "/home/podman/.gitlab-runner/"]
+# N/B: Volumes are cumulative with the base image
+VOLUME ["/home/podman/.gitlab-runner/", "/cache"]
 WORKDIR /home/podman
 ENTRYPOINT ["/usr/local/bin/gitlab-runner-wrapper"]
 
@@ -113,13 +114,17 @@ RUN mkdir -p .local/share/containers/storage
 # Gitlab-runner configuration options.  Default to unprivileged (nested)
 # runner.  Privileged is required to permit nested container image building.
 ARG RUNNER_NAME="qontainers-pipglr"
-ARG PRIVILEGED_RUNNER="false"
+# Running inner-podman privileged is necessary at the time of this commit.
+ARG PRIVILEGED_RUNNER="true"
 # Tags allow pinning jobs to specific runners, comma-separated list of
 # tags to add to runner (no spaces!)
 ARG RUNNER_TAGS="podman-in-podman"
 # Permit running jobs without any tag at all
 ARG RUNNER_UNTAGGED="true"
-ENV REGISTER_NON_INTERACTIVE="true" \
+# Adjust based on usage and storage size to prevent ENOSPACE problems
+ARG CLEAN_INTERVAL="24h"
+ENV CLEAN_INTERVAL="$CLEAN_INTERVAL" \
+    REGISTER_NON_INTERACTIVE="true" \
     RUNNER_TAG_LIST="$RUNNER_TAGS" \
     REGISTER_RUN_UNTAGGED="$RUNNER_UNTAGGED" \
     REGISTER_ACCESS_LEVEL="ref_protected" \
@@ -132,15 +137,19 @@ ENV REGISTER_NON_INTERACTIVE="true" \
     DOCKER_HOST="unix:///tmp/podman-run-1000/podman/podman.sock" \
     DOCKER_DEVICES="/dev/fuse" \
     DOCKER_IMAGE="registry.fedoraproject.org/fedora-minimal:latest" \
-    DOCKER_CACHE_DIR="/home/podman/.cache/gitlab-runner" \
+    DOCKER_CACHE_DIR="/cache" \
+    DOCKER_VOLUMES="/cache" \
     DOCKER_NETWORK_MODE="host" \
     DOCKER_PRIVILEGED="$PRIVILEGED_RUNNER"
 
 # Not a real build-arg.  Simply here to save lots of typing.
-ARG _pm="--systemd=true --device=/dev/fuse --security-opt label=disable --user podman --volume pipglr-podman-root:/home/podman/.local/share/containers/storage:Z --volume pipglr-runner-config:/home/podman/.gitlab-runner:Z -e PODMAN_RUNNER_DEBUG -e LOG_LEVEL"
+ARG _pm="--systemd=true --device=/dev/fuse --security-opt label=disable --user podman --volume pipglr-podman-root:/home/podman/.local/share/containers --volume pipglr-config:/home/podman/.gitlab-runner -v pipglr-podman-cache:/cache --tmpfs /var/lib/containers,ro,size=1k -e PODMAN_RUNNER_DEBUG -e LOG_LEVEL"
 
 # These labels simply make it easier to register and execute the runner.
 # Define them last so they are absent should a image-build failure occur.
 LABEL register="podman run -it --rm $_pm --secret REGISTRATION_TOKEN,type=env \$IMAGE register"
 # Note: Privileged mode is required to permit building container images with inner-podman
-LABEL run="podman run -d --rm --privileged --name gitlab-runner $_pm \$IMAGE run"
+LABEL run="podman run -d --privileged --name pipglr $_pm \$IMAGE run"
+
+# In case it's helpful, include the documentation
+ADD /README.md /home/podman/

README.md

@@ -27,21 +27,29 @@ lacks this feature, Several labels are set on the image to support
 easy registration and execution of a runner container using a special
 bash command.  See the examples below for more information.
 
-#### Volume Ownership Bug
+#### [Volume setup]
 
-Some versions of podman contain a bug where named volumes aren't owned
-by the namespaced user within a rootless container (i.e. in conjunction
-with the --user option).  Since the `podman` user/group inside the `pipglr`
-container is known, it's possible to manually set/reset ownership:
+Since podman inside the container runs as user `podman`, the volumes
+used by it need to be pre-created with ownership information.  While,
+we're at it, might as well add the performance-improving `noatime`,
+option as well.
 
 ```bash
-VOLUME=pipglr-podman-root
-podman volume create $VOLUME
-cd $(podman unshare podman volume mount $VOLUME)
-podman unshare chown 1000:1000
-podman volume unmount $VOLUME
+$ VOLOPTS="o=uid=1000,gid=1000,noatime"; \
+  for VOLUME in pipglr-podman-root pipglr-config pipglr-podman-cache; do \
+    podman volume create --opt $VOLOPTS $VOLUME || true ; \
+    VOLPTH=$(podman unshare podman volume mount $VOLUME)
+    podman unshare chown -c -R 1000:1000 $VOLPTH && \
+    podman unshare chmod -c 02770 $VOLPTH && \
+    podman unshare podman volume unmount $VOLUME ; \
+  done
 ```
 
+If you get `podman system service` startup permission-denied errors, or
+errors from gitlab-runner, unable to connect to the podman socket, this is
+likely the cause.  You can fix it after-the-fact using the same commands
+above.
+
 #### Runner registration
 
 Each time the registration command is run, a new runner is added into
@@ -52,20 +60,27 @@ For modern versions of podman, registration can be performed with the
 following commands:
 
 ```bash
-IMAGE="=registry.gitlab.com/qontainers/pipglr:latest"
-echo '<actual registration token>' | podman secret create REGISTRATION_TOKEN -
-podman container runlabel $IMAGE register --secret REGISTRATION_TOKEN,type=env
+$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
+$ echo '<actual registration token>' | podman secret create REGISTRATION_TOKEN -
+$ podman container runlabel register $IMAGE
 ```
 
 Where `<actual registration token>` is the value obtained from the "runners"
-settings page of a gitlab group or project.
+settings page of a gitlab group or project.  When you're finished registering
+as many runners as you want, the secret is no-longer needed and may be removed:
 
-Note: Some versions of podman don't support the `container runlabel` sub-command.
+```bash
+$ podman secret rm REGISTRATION_TOKEN
+```
+
+##### Note
+
+Some versions of podman don't support the `container runlabel` sub-command.
 If this is the case, you may simulate it with the following command (in addition
 to the other example commands above):
 
 ```bash
-eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.register)
+$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.register)
 ```
 
 #### Runner Startup
@@ -74,9 +89,11 @@ With one or more runners successfully registered and configured, the GitLab
 runner container may be launched with the following commands:
 
 ```bash
-podman container runlabel $IMAGE run
+$ podman container runlabel run $IMAGE
 ```
 
+##### Note
+
 As above, if you're missing the `container runlabel` sub-command, the following
 may be used instead (assuming `$IMAGE` remains set):
 
@@ -84,17 +101,41 @@ may be used instead (assuming `$IMAGE` remains set):
 $ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.run)
 ```
 
+#### Runner configuration
+
+You may inspect/modify the gitlab-runner configuration as you see fit, just be
+sure to use the `podman unshare` command-wrapper to enter the usernamespace.
+For example, to display the config:
+
+```bash
+$ podman unshare cat $(podman unshare podman volume mount pipglr-config)/config.toml
+```
+
+Edit the config with your favorite `$EDITOR`:
+
+```bash
+$ podman unshare $EDITOR $(podman unshare podman volume mount pipglr-config)/config.toml
+```
+
 #### Debugging
 
-Before starting the runner, you may `export PODMAN_RUNNER_DEBUG=debug` to enable
-debugging on the inner-podman.  Whereas `export LOG_LEVEL=debug` can be used to
-debug the gitlab-runner itself.
+The first thing to check is the container output:
+
+```bash
+$ podman logs --since 0 pipglr
+```
+
+Next, try running pipglr after an `export PODMAN_RUNNER_DEBUG=debug` to enable
+debugging on the inner-podman.  If more runner detail is needed, you can instead/additionally
+set `export LOG_LEVEL=debug` to debug the gitlab-runner itself.
 
 ## Building
 
 This image may be built simply with:
 
-`podman build -t registry.gitlab.com/qontainers/pipglr:latest .`
+```bash
+$ podman build -t registry.gitlab.com/qontainers/pipglr:latest .
+```
 
 This will utilize the latest stable version of podman and the latest
 stable version of the gitlab runner.
@@ -120,6 +161,10 @@ Several build arguments are available to control the output image:
   exact podman version.  Possible values include, `latest`, `vX`, `vX.Y`,
   and `vX.Y.Z` (where, `X`, `Y`, and `Z` represent the podman semantic
   version numbers).  It's also possible to specify an image SHA.
+* `CLEAN_INTERVAL` - A `sleep` (command) compatible time-argument that
+  determines how often to clean out podman storage of disused containers and
+  images.  Defaults to 24-hours, but should be adjusted based on desired caching-effect
+  versus available storage space and rate of job execution.
 * `EXCLUDE_PACKAGES` - A space-separated list of RPM packages to prevent
   their existence in the final image.  This is intended as a security measure
   to limit the attack-surface should a gitlab-runner process escape it's
@@ -140,10 +185,9 @@ Several build arguments are available to control the output image:
   and port supports various observability and debugging features of the
   gitlab runner.  For more information see the [gitlab runner advanced
   configuration documentation](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section).
-* `PRIVILEGED_RUNNER` - Defaults to 'false', may be set 'true'.  When
-  `true`, this causes inner-containers to be created with the `--privileged`
-  flag.  This is a potential security weakness, but is necessary for
-  (among other things) allowing nested container image builds.
+* `PRIVILEGED_RUNNER` - Defaults to 'true', may be set 'true' if you're brave.
+  However this may result in the gitlab-runner failing to launch inner-containers.
+  Setting it false will also prevent building container images using the runner.
 * `RUNNER_TAGS` - Defaults to `podman_in_podman`, may be set to any comma-separated
   list (with no spaces!) of tags.  These show up in GitLab (not the runner
   configuration), and determines where jobs are run.

podman-in-podman-maintenance

@@ -4,16 +4,23 @@
 # a podman-in-podman gitlab runner container.  Any usage
 # outside that context is not supported and may cause harm.
 
-set -e
+set -eo pipefail
 
 maintain_podman() {
   # Two days seems to be a good happy-medium beween filling up
   # about 40gig of storage space from moderate CI activity,
   # and maintaining a useful level of caching.
-  while sleep 2d; do
+  while sleep "$CLEAN_INTERVAL"; do
     if [[ -n "$PODMAN_RUNNER_DEBUG" ]]; then
       echo "$(date --iso-8601=second) ${BASH_SOURCE[0] performing podman maintenance}"
     fi
     podman system prune --all --force
   done
 }
+
+if [[ -z "$CLEAN_INTERVAL" ]]; then
+  echo "ERROR: Empty/unset \$CLEAN_INTERVAL"
+  exit 1
+fi
+
+maintain_podman