Merge branch 'configuration-directory-fix' into 'main'

Update runner installation and attempt to fix storage volume errors

See merge request qontainers/pipglr!51

.gitlab-ci.yml

@@ -1,5 +1,4 @@
 ---
-
 default:
   image: quay.io/buildah/stable:v1.32
   tags:
@@ -11,6 +10,7 @@ workflow:
     - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
       when: never
     - if: $CI_COMMIT_BRANCH
+    - if: $CI_COMMIT_TAG
 
 include:
   - component: gitlab.com/blue42u/ci.pre-commit/lite@0.2.0
@@ -29,7 +29,7 @@ commit_check:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
     - when: never
   variables:
-    BADRX: '^(squash!)|(fixup!)'
+    BADRX: "^(squash!)|(fixup!)"
   script: |
     dnf install -y git
     shortlogtmp=$(mktemp -p '' commit_check_tmp_XXXX)
@@ -44,29 +44,11 @@ build:
     - saas-linux-medium-amd64
   stage: deploy
   variables:
+    FF_GITLAB_REGISTRY_HELPER_IMAGE: 0
     BUILDAH_FORMAT: docker
     BUILDAH_ISOLATION: chroot
     STORAGE_DRIVER: vfs
   before_script:
     - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
   script:
-    # N/B: There could be more than one merge-request open with this branch's HEAD
+    - scripts/build.sh
-    - |
-        IMAGE_TAG="${CI_COMMIT_REF_SLUG}";
-        if [[ -n "$CI_COMMIT_TAG" ]]; then
-            IMAGE_TAG="${CI_COMMIT_TAG}";
-        elif [[ -n "$CI_OPEN_MERGE_REQUESTS" ]]; then
-            IMAGE_TAG=mr$(echo "${CI_OPEN_MERGE_REQUESTS}" | cut -d, -f -1 | cut -d\! -f 2);
-        elif [[ "$CI_COMMIT_BRANCH" == "main" ]]; then
-            IMAGE_TAG="latest";
-        fi
-        echo "Building/Pushing to: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG}";
-    - >-
-      buildah build \
-        --label "org.opencontainers.image.source=${CI_PROJECT_URL}" \
-        --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" \
-        --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" \
-        --label "org.opencontainers.image.version=${IMAGE_TAG}" \
-        -t "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}" "$CI_PROJECT_DIR"
-    - buildah images
-    - buildah push "${CI_REGISTRY_IMAGE}:${IMAGE_TAG}"

Containerfile

@@ -20,6 +20,10 @@ ADD /root/ /root/
 ADD /etc/ /etc/
 ADD /home/ /home/
 
+# The build type: either `dev` or `prod`
+# In `dev` mode: the package manager will not be deleted.
+ARG BUILD_TYPE=prod
+
 # Allow image-builders to choose another version besides "latest" should
 # an incompatible change be introduced.
 ARG RUNNER_VERSION=latest
@@ -35,9 +39,11 @@ ARG TARGETARCH=amd64
 ARG PRUNE_INTERVAL=daily  # see systemd.timer for allowable values
 
 # All-in-one packaging/image-setup script to keep things simple.
-RUN PRUNE_INTERVAL=${PRUNE_INTERVAL} \
+RUN \
-    RUNNER_VERSION=${RUNNER_VERSION} \
+ PRUNE_INTERVAL=${PRUNE_INTERVAL} \
-    bash /root/setup.sh
+ RUNNER_VERSION=${RUNNER_VERSION} \
+ BUILD_TYPE=${BUILD_TYPE} \
+ bash /root/setup.sh
 
 VOLUME /cache /home/podman/.local/share/containers
 ENTRYPOINT /lib/systemd/systemd
@@ -64,6 +70,7 @@ ARG ENABLE_FIPS=true
 # may be changed if you know what you're doing.
 LABEL register="podman run -it --rm \
  --secret=REGISTRATION_TOKEN,type=env \
+ --user=root \
  -v ./config.toml:/etc/gitlab-runner/config.toml:Z \
  -e REGISTER_NON_INTERACTIVE=true \
  -e CI_SERVER_URL=${GITLAB_URL} \
@@ -89,6 +96,7 @@ LABEL setupcache="podman volume create --opt o=uid=1000,gid=1001 pipglr-cache"
 
 # Helper to extract the current configuration secret to allow editing.
 LABEL dumpconfig="podman run -it --rm \
+ --user=root \
  --secret config.toml --entrypoint=/bin/cat \
  \$IMAGE /var/run/secrets/config.toml"
 
@@ -97,6 +105,7 @@ LABEL dumpconfig="podman run -it --rm \
 # Note: The contents of the volumes are not critical, they may be removed
 # and re-created (see above) to quickly free-up disk space.
 LABEL run="podman run -dt --name pipglr \
+ --user=root \
  --secret config.toml,uid=1001,gid=1001 \
  -v pipglr-storage:/home/podman/.local/share/containers \
  -v pipglr-cache:/cache \

README.md

@@ -36,79 +36,55 @@ container. While it's possible to use the container with your own command-line,
 it's highly recommended to base them off of one of the labels. See the examples
 below for more information.
 
-**_Note:_** Some older versions of podman don't support the `container runlabel`
+*_Only podman versions 4.8.x and later are supported by pipglr._*
-sub-command. If this is the case, you may simulate it with the following,
-substituting `<label>` with one of the predefined values (i.e. `register`,
-`setupconfig`, etc.):
-
-```bash
-$ IMAGE="registry.gitlab.com/qontainers/pipglr:latest"
-$ eval $(podman inspect --format=json $IMAGE | jq -r .[].Labels.<label>)
-```
 
 #### Persistent Containers (step 1)
 
 By default on many distributions, regular users aren't permitted to leave
 background processes running after they log out. Since this is likely desired
 for running the pipglr container long-term, `systemd` needs to be configured to
-override this policy. For this, you (`$USER`) will need root access on the
+override this policy.  Doing so requires root or `sudo` access on the
-system.
+system.  Assuming the user `johndoe` will be executing the pipglr container,
+linger may be enabled (as the admin user or root) with a command like:
 
 ```bash
-$ sudo loginctl enable-linger $USER
+$ sudo loginctl enable-linger `johndoe`
 ```
 
-Side-effect: This will allow your user to persist other user-level systemd
+Side-effect: This will allow the user (`johndoe` for example) to persist
-services as well. For example `podman.socket` is handy to enable for
+other user-level systemd services as well. For example `podman.socket` is
-`podman remote` access. You could also
+handy to enable for `podman remote` access. You could also
 [setup quadlet](https://www.redhat.com/sysadmin/quadlet-podman) or a systemd
 unit so pipglr starts up on system boot.
 
-#### Expanded User-Namespace (step 2) **_This is probably important_**
+#### Expanded Host-system User-Namespace (step 2)
 
-As an added protection/safety measure, pipglr excludes three UID/GIDs from being
+**_This is really important and frequently the cause of pipglr issues_**
-used by job-level containers. One for `root`, another for `runner` and a third
-for `podman`. However, some container images you may want to use for jobs
-(mainly Debian/Ubuntu), assign one/more essential users a high UID/GID value
-(like `65535`).
 
-At the same time, most distributions also set `65536` as the default maximum
+On the host, as root, edit the two files `/etc/subuid` and `/etc/subgid` to
-number (including ID `0`) of IDs to allocate for user-namespaces (via
+expand the pipglr user's ID allocation by 3.  For example if the host user
-`/etc/login.defs`). This creates a problem you won't realize until the runner
+running the pipglr container is named `johndoe`, this entry in the files should
-actually picks up a job. The main symptom of this issue will be messages in
+be edited like:
-the pipglr containers log, similar to (abbreviated):
-
-```text
-...cut...
-running `/usr/bin/newuidmap ...cut...`: newuidmap: write to uid_map failed: Operation not permitted
-Error: cannot set up namespace using "/usr/bin/newuidmap": exit status 1
-...cut...
-```
-
-or
-
-```text
-E: setgroups 65534 failed - setgroups (22: Invalid argument)
-```
-
-**_The good news is, working around this is relatively simple:_**
-
-As root, edit the two files `/etc/subuid` and `/etc/subgid` to expand the by 3
-IDs. For example assuming a user running the pipglr container is called
-`johndoe`, the contents of these files should be edited to allocate `65539` IDs
-like:
 
 `johndoe:<some number>:65539`
 
-Where `<some number>` was set by your OS when the `johndoe` user was created
+Where `<some number>` was set by the OS when the `johndoe` user was created
-(you can ignore this). Only the last number needs to be increased. This change
+(you can ignore this).  Only the last number needs to be increased by three.
-will be effective on next login, or immediately by running:
+This change will be effective on the user's next login, or immediately
+by running (as the host pipglr user):
 
 `podman system migrate`
 
 _Note:_ This will stop any currently running containers.
 
-#### Runner Registration (step 3)
+_Details:_ As an added protection/safety measure, pipglr excludes three UID/GIDs
+from being used by job-level (nested) containers. One for `root`, another for
+`runner` and a third for `podman`.  This necessitates expanding the number of
+available UIDs/GIDs from the host machine, to the pipglr container.  This way,
+the full set of 65535 UIDs/GIDs may be utilized by job-level (nested)
+sub-containers.
+
+#### Runner Activation/Registration (step 3)
 
 All runners must be connected to a project or group runner configuration on your
 gitlab instance (or `gitlab.com`). This is done using a special registration
@@ -238,6 +214,9 @@ version of the gitlab runner.
 
 Several build arguments are available to control the output image:
 
+- `BUILD_TYPE`: The build type, either `prod` or `dev`. In `dev` mode, the package
+  manager is not deleted for development and debugging purposes. Please see
+  [`build.sh`](scripts/build.sh) for more details.
 - `PRUNE_INTERVAL`: A systemd.timer compatible `OnCalendar` value that
   determines how often to prune Podman's storage of disused containers and
   images. Defaults to `daily`, but should be adjusted based on desired
@@ -257,6 +236,9 @@ Several build arguments are available to control the output image:
   containers running in `--privileged` mode. This will affect the ability to
   build container images in CI jobs using tools like podman or buildah.
 
+Additional build-args are available as well.  See the `Containerfile` comments
+for more details.
+
 ### Environment Variables
 
 Nearly every option to every gitlab-runner sub-command may be specified via

root/setup.sh

@@ -33,6 +33,10 @@ function check_vars() {
       die "Env. variable '$varname' must be non-empty."
     fi
   done
+
+  if [[ ! "$BUILD_TYPE" =~ dev|prod ]]; then
+    die "Build type must be 'dev' or 'prod': '$BUILD_TYPE'."
+  fi
 }
 
 function main() {
@@ -47,10 +51,15 @@ function main() {
   setup_service_podman
   setup_service_runner
   setup_gitlab_config
+  setup_volumes
 
   finalize_ownership
 }
 
+function is_release() {
+  [ "$BUILD_TYPE" = "prod" ] || return 1
+}
+
 function install_packages() {
   readarray xpackages < <(grep -vE '^(# )+' </root/xpackages.txt)
   local exclude_args=()
@@ -67,27 +76,30 @@ function install_packages() {
   # Gitlab-runner package contains scriptlets which do not function properly inside a
   # container-build environment where systemd is not active/running.
   if [[ ${ENABLE_FIPS} == true && $(cat /proc/sys/crypto/fips_enabled) == 1 ]]; then
-    PACKAGE_FILE="gitlab-runner_${TARGETARCH}-fips.rpm"
+    PACKAGE_FILES=(
+      "https://s3.dualstack.us-east-1.amazonaws.com/gitlab-runner-downloads/$RUNNER_VERSION/rpm/gitlab-runner_${TARGETARCH}-fips.rpm"
+    )
   else
-    PACKAGE_FILE="gitlab-runner_${TARGETARCH}.rpm"
+    PACKAGE_FILES=(
+      "https://s3.dualstack.us-east-1.amazonaws.com/gitlab-runner-downloads/$RUNNER_VERSION/rpm/gitlab-runner_${TARGETARCH}.rpm"
+      "https://s3.dualstack.us-east-1.amazonaws.com/gitlab-runner-downloads/$RUNNER_VERSION/rpm/gitlab-runner-helper-images.rpm"
+    )
   fi
 
   "${dnf[@]}" "${exclude_args[@]}" \
     --setopt=tsflags=noscripts install \
-    "https://gitlab-runner-downloads.s3.amazonaws.com/$RUNNER_VERSION/rpm/${PACKAGE_FILE}"
+    ${PACKAGE_FILES[@]}
 
-
+  # Also don't start unnecessary or broken
-# Allow removing dnf, sudo, etc. packages.  Also don't start unnecessary or broken
+  # systemd services, like anything kernel related or login gettys.
-# systemd services, like anything kernel related or login gettys.
+  rm -rf \
-rm -rf \
+    /etc/systemd/system/getty.target.wants/* \
-  /etc/dnf/protected.d/* \
+    /etc/systemd/system/multi-user.target.wants/* \
-  /etc/systemd/system/getty.target.wants/* \
+    /etc/systemd/system/sysinit.target.wants/* \
-  /etc/systemd/system/multi-user.target.wants/* \
+    /etc/systemd/system/timers.target.wants/* \
-  /etc/systemd/system/sysinit.target.wants/* \
+    /lib/systemd/system/graphical.target.wants/* \
-  /etc/systemd/system/timers.target.wants/* \
+    /lib/systemd/system/multi-user.target.wants/{getty.target,systemd-ask-password-wall.path} \
-  /lib/systemd/system/graphical.target.wants/* \
+    /lib/systemd/system/sys-kernel*.mount
-  /lib/systemd/system/multi-user.target.wants/{getty.target,systemd-ask-password-wall.path} \
-  /lib/systemd/system/sys-kernel*.mount
 
   # Allow removing dnf, sudo, etc. packages.
   rm -rf \
@@ -96,13 +108,16 @@ rm -rf \
   # This makes the image smaller and reduces the attack-surface.
   dnf remove -y "${xpackages[@]}"
 
-  # Wipe out the DNF cache, then remove it entirely, again to make the image smaller.
+  if is_release; then
-  "${dnf[@]}" clean all
+    # Wipe out the DNF cache, then remove it entirely, again to make the image smaller.
-  rm -rf /var/cache/dnf /var/log/dnf* /var/log/yum.*
+    "${dnf[@]}" clean all
-  rpm -e dnf
+    rm -rf /var/cache/dnf /var/log/dnf* /var/log/yum.*
+    rpm -e dnf
+  fi
 
-  # Workaround base-image failing to confer capabilties properly on
+  # Workaround https://bugzilla.redhat.com/show_bug.cgi?id=1995337
-  # /usr/bin/new{u,g}idmap to `cap_set{u,g}id=ep` in new image layers.
+  # Base-image failing to confer capabilities properly on
+  # /usr/bin/new{u,g}idmap to `cap_set{u,g}id=ep` in new image layers
   rpm --setcaps shadow-utils
 }
 
@@ -139,9 +154,11 @@ function setup_user() {
 function setup_volumes() {
   # Host volume mount necessary for nested-podman to use overlayfs2 for container & volume storage.
   mkdir -p /home/podman/.local/share/containers
+  touch /home/podman/.local/share/containers/.placeholder
 
   # Nested-container's local container-cache volume mount, recommended by gitlab-runner docs.
   mkdir -p /cache
+  touch /cache/.placeholder
 
   # Both the gitlab-runner and podman need access to the cache directory / volume mount.
   chown podman:runner /cache

scripts/build.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Usage: build.sh [<build-type>]
+#
+# If the build type (second arg. `<build-type>`) is `prod`
+# the images are build in `release` mode. For all other build types
+# the images are build for development and testing purposes
+# By default the build type is `prod`.
+
+set -eu
+set -o pipefail
+
+ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)/.."
+cd "$ROOT_DIR"
+
+function ci_running() {
+  [ "${CI:-}" = "true" ] && return 0
+  return 1
+}
+
+# Define the image tag depending on the context.
+function get_image_tag() {
+  local build_type="$1"
+  local image_tag="${build_type}-latest"
+
+  # Define image tag.
+  if ci_running; then
+    # The image tag gets adjusted depending on
+    # if it is a merge request or build on the
+    # main branch or on a tag.
+
+    image_tag="${CI_COMMIT_REF_SLUG:-}"
+
+    if [[ -n "${CI_COMMIT_TAG:-}" ]]; then
+      image_tag="${CI_COMMIT_TAG}"
+    elif [[ -n "${CI_OPEN_MERGE_REQUESTS:-}" ]]; then
+      image_tag=mr$(echo "${CI_OPEN_MERGE_REQUESTS}" | cut -d, -f -1 | cut -d\! -f 2)
+    elif [[ "${CI_COMMIT_BRANCH:-}" == "main" ]]; then
+      image_tag="latest"
+    fi
+  fi
+
+  echo "$image_tag"
+}
+
+function main() {
+  # Define common build variables.
+  local container_mgr=${CI_CONTAINER_MGR:-buildah}
+  local project_dir=${CI_PROJECT_DIR:-.}
+  local registry_name=${CI_REGISTRY_IMAGE:-"containers-storage:pipglr"}
+  local build_type=${CI_BUILD_TYPE:-${1:-prod}}
+
+  # Define image name and tag.
+  local image_tag image_name
+  image_tag=$(get_image_tag "$build_type")
+  image_name="${registry_name}:${image_tag}"
+
+  # Define OpenContainers labels.
+  local oc_project_url=${CI_PROJECT_URL:-file://$ROOT_DIR}
+  local oc_commit_sha=${CI_COMMIT_SHA:-$(git rev-parse HEAD)}
+  local oc_job_started_at=${CI_JOB_STARTED_AT:-$(date -u --iso-8601=seconds)}
+  local oc_version="${image_tag}"
+
+  BUILD_CMD=(
+    "${container_mgr}" build
+    --label "org.opencontainers.image.source=${oc_project_url}"
+    --label "org.opencontainers.image.revision=${oc_commit_sha}"
+    --label "org.opencontainers.image.created=${oc_job_started_at}"
+    --label "org.opencontainers.image.version=${oc_version}"
+    --build-arg "BUILD_TYPE=${build_type}"
+    -t "$image_name"
+    "${project_dir}")
+
+  echo "Build image: '$image_name'"
+  echo -e "Build command:\n" "${BUILD_CMD[@]}"
+
+  "${BUILD_CMD[@]}"
+
+  echo "Images are:"
+  "${container_mgr}" images
+
+  if ci_running; then
+    echo "Pushing image: ${image_name}"
+    "${container_mgr}" push "${image_name}"
+  fi
+}
+
+main "$@"