diff --git a/.gitea/workflows/quality-gates.yml b/.gitea/workflows/quality-gates.yml index a2b9c247..5bec0833 100644 --- a/.gitea/workflows/quality-gates.yml +++ b/.gitea/workflows/quality-gates.yml @@ -106,7 +106,7 @@ jobs: run: corepack pnpm ci:gate -- FE-GATE-007 - name: "FE-GATE-013 / security" if: ${{ !cancelled() }} - run: corepack pnpm ci:gate -- FE-GATE-013 + run: scripts/run-in-system-container.sh corepack pnpm ci:gate -- FE-GATE-013 - name: "FE-GATE-020 / removability" if: ${{ !cancelled() }} run: corepack pnpm ci:gate -- FE-GATE-020 diff --git a/config/ci/gates.json b/config/ci/gates.json index 1d8fe31c..5ec9f082 100644 --- a/config/ci/gates.json +++ b/config/ci/gates.json @@ -1802,7 +1802,8 @@ "hostRequirements": [ "Linux with executable /usr/bin/bwrap (with --size), /usr/bin/prlimit, /usr/bin/systemd-run, /usr/bin/systemctl and /usr/bin/tar, an active systemd user manager 254 or newer, and unified cgroup v2 with delegated memory, pids and cpu controllers", "unprivileged user and network namespaces for bwrap (the provider sandbox runs bwrap --unshare-pid --unshare-ipc --unshare-uts --unshare-net); Ubuntu 23.10 and later refuse them while kernel.apparmor_restrict_unprivileged_userns=1 unless /usr/bin/bwrap has an AppArmor profile that allows userns; scripts/check-system-test-prerequisites.ts probes this before test:system runs" - ] + ], + "hostEnvironment": "system-container" }, { "id": "FE-GATE-014", diff --git a/config/ci/system-container.Dockerfile b/config/ci/system-container.Dockerfile new file mode 100644 index 00000000..0fc092d4 --- /dev/null +++ b/config/ci/system-container.Dockerfile @@ -0,0 +1,16 @@ +# The host FE-GATE-013 runs on (scripts/run-in-system-container.sh): systemd as +# PID 1 with a lingering root user manager, cgroup v2 delegation, bubblewrap and +# the tools the provider sandbox calls (config/ci/gates.json hostRequirements). +# Node and pnpm are not installed here: the script copies in the caller's own, +# so the gate runs with the same versions as the job that called it. +FROM docker.io/library/ubuntu:24.04@sha256:fe81560990fae86dc4743202b2cb4d3df98d0bcf949173cecef15ce1127ef852 + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + bubblewrap ca-certificates dbus dbus-user-session git systemd systemd-sysv tar util-linux \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /var/lib/systemd/linger \ + && touch /var/lib/systemd/linger/root + +STOPSIGNAL SIGRTMIN+3 +CMD ["/sbin/init"] diff --git a/scripts/contracts/ci-gates.ts b/scripts/contracts/ci-gates.ts index 9241c91a..16e67306 100644 --- a/scripts/contracts/ci-gates.ts +++ b/scripts/contracts/ci-gates.ts @@ -299,6 +299,14 @@ const gateSchema = z * contract baseline, like any other change to what a gate demands. */ hostRequirements: z.array(nonEmpty.max(512)).min(1).max(8).optional(), + /** + * Where the gate runs when the job container cannot meet its host + * requirements. "system-container": scripts/run-in-system-container.sh + * runs the gate in a disposable privileged systemd container + * (config/ci/system-container.Dockerfile), in CI through the runner's + * Docker daemon and on desktop through scripts/verify.sh. + */ + hostEnvironment: z.enum(["system-container"]).optional(), }) .strict(); diff --git a/scripts/generate-ci-workflow.ts b/scripts/generate-ci-workflow.ts index 1def38b1..85b69b21 100644 --- a/scripts/generate-ci-workflow.ts +++ b/scripts/generate-ci-workflow.ts @@ -254,6 +254,15 @@ function renderHostRequirements( ); } +/** + * A gate whose host requirements the job container cannot meet runs in the + * environment its contract names (`hostEnvironment`). FE-GATE-013's provider + * sandbox needs systemd and bubblewrap, which the runner's job image lacks. + */ +function gateHostPrefix(gate: Readonly<{ hostEnvironment?: "system-container" | undefined }> | undefined): string { + return gate?.hostEnvironment === "system-container" ? "scripts/run-in-system-container.sh " : ""; +} + function renderStep( job: CiWorkflowJob, step: CiWorkflowStep, @@ -303,7 +312,7 @@ function renderStep( return job.gateIds.flatMap((gateId) => [ ` - name: ${yamlScalar(`${gateId} / ${index.gates.get(gateId)?.name ?? ""}`)}`, " if: ${{ !cancelled() }}", - ` run: corepack pnpm ci:gate -- ${gateId}`, + ` run: ${gateHostPrefix(index.gates.get(gateId))}corepack pnpm ci:gate -- ${gateId}`, ]); } // 12-02: every gate the job owns runs, in order. Rendering only the first @@ -533,9 +542,11 @@ export function workflowGateExecutionFailures( ) { productFailures.push(`product gates do not hold the release stage: ${PRODUCT_GATES_HOLD} does not need ${PRODUCT_GATE_JOB}`); } + const hostPrefixes = new Map(contract.gates.map((gate) => [gate.id, gateHostPrefix(gate)])); for (const job of jobs) { - for (const match of job.matchAll(/^ +run: corepack pnpm ci:gate -- (FE-GATE-\d{3})$/gmu)) { - executed.add(match[1]!); + for (const match of job.matchAll(/^ +run: (\S+ )?corepack pnpm ci:gate -- (FE-GATE-\d{3})$/gmu)) { + // A gate runs through the prefix its contract names, and only that one. + if ((match[1] ?? "") === hostPrefixes.get(match[2]!)) executed.add(match[2]!); } if (/^ +run: corepack pnpm ci:gate -- \$\{\{ matrix\.gate \}\}$/mu.test(job)) { for (const match of job.matchAll(/^ +- \{ gate: (FE-GATE-\d{3}),/gmu)) { diff --git a/scripts/run-in-system-container.sh b/scripts/run-in-system-container.sh new file mode 100755 index 00000000..3fc7aa58 --- /dev/null +++ b/scripts/run-in-system-container.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Runs one command in a disposable systemd container, for a gate whose host requirements a CI job +# container or a developer shell does not meet (config/ci/gates.json `hostEnvironment: "system-container"`, +# FE-GATE-013's provider sandbox: bubblewrap, a systemd user manager, delegated cgroup v2). +# +# scripts/run-in-system-container.sh e.g. corepack pnpm ci:gate -- FE-GATE-013 +# +# The container is privileged with its own cgroup namespace and boots systemd (config/ci/system-container.Dockerfile). +# The command runs as root there, so bubblewrap needs no unprivileged user namespace from the host kernel +# (Ubuntu 23.10+ refuses those to unconfined processes). Node and pnpm come from the caller, copied in with the +# working tree and its git history; artifacts/ is copied back and the container removed, whatever the result. +# Docker: the CI job reaches the runner's own daemon through its Docker proxy +# (platform-core infrastructure/ci/gitea-runner); on desktop it is the host daemon. +# Environment: SYSTEM_CONTAINER_DOCKER (default docker), SYSTEM_CONTAINER_KEEP=1 keeps the container for debugging. +# Exit code: the command's, or 3 when the container could not be prepared. +set -euo pipefail + +[ $# -gt 0 ] || { sed -n '2,13p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; } +docker=${SYSTEM_CONTAINER_DOCKER:-docker} +root=$(git rev-parse --show-toplevel) +cd "$root" +dockerfile=config/ci/system-container.Dockerfile +tag="fe-system-container:$(sha256sum "$dockerfile" | cut -c1-16)" +node_root=$(dirname "$(dirname "$(command -v node)")") +corepack_home=${COREPACK_HOME:-$HOME/.cache/node/corepack} +# The environment the gate reads (scripts/verify.sh ci_env, the workflow's global env). +pass_env=(CI VITE_BUILD_ID VITE_COMMIT_SHA RELEASE_ID CI_RUNNER_IMAGE SOURCE_DATE_EPOCH APP_PROFILE RELEASE_TARGET) + +say() { echo "system-container: $*" >&2; } +fail() { say "$*"; exit 3; } + +if ! $docker image inspect "$tag" > /dev/null 2>&1; then + say "building $tag" + $docker build -q -t "$tag" -f "$dockerfile" config/ci > /dev/null || fail "image build failed" +fi +cid=$($docker run -d --privileged --cgroupns=private --tmpfs /run --tmpfs /run/lock "$tag") \ + || fail "container did not start" +cleanup() { + if [ "${SYSTEM_CONTAINER_KEEP:-0}" = 1 ]; then say "kept container $cid"; else $docker rm -f "$cid" > /dev/null 2>&1 || :; fi +} +trap cleanup EXIT + +# systemd reaches a final state, then root's lingering user manager opens its bus. Until PID 1 has its own +# bus up, systemctl cannot connect and answers nothing, so ask again for up to a minute. +state="" +for _ in $(seq 1 120); do + state=$($docker exec "$cid" systemctl is-system-running --wait 2>/dev/null || :) + case "$state" in running | degraded) break ;; esac + sleep 0.5 +done +case "$state" in running | degraded) ;; *) fail "systemd did not start (state: ${state:-none})" ;; esac +$docker exec "$cid" sh -c 'for i in $(seq 1 120); do [ -S /run/user/0/bus ] && exit 0; sleep 0.5; done; exit 1' \ + || fail "the root user manager did not start" + +# The working tree as it is (node_modules included) and its history. A linked worktree's .git is a file +# naming a directory outside the tree, so the history goes in as a bundle of HEAD instead. +tar -C "$root" --exclude=./.git --exclude=./.tmp -cf - . | $docker exec -i "$cid" sh -c 'mkdir -p /work && tar -C /work --no-same-owner -xf -' \ + || fail "copying the working tree failed" +if [ -d .git ]; then + tar -C "$root" -cf - .git | $docker exec -i "$cid" tar -C /work --no-same-owner -xf - || fail "copying .git failed" +else + bundle=$(mktemp) + git bundle create -q "$bundle" HEAD + $docker exec -i "$cid" sh -c 'cat > /tmp/head.bundle' < "$bundle" + rm -f "$bundle" + $docker exec -w /work "$cid" sh -c 'git init -q . && git fetch -q /tmp/head.bundle HEAD && git update-ref HEAD FETCH_HEAD && git reset -q' \ + || fail "restoring the git history failed" +fi +tar -C "$node_root" -cf - . | $docker exec -i "$cid" sh -c 'mkdir -p /opt/node && tar -C /opt/node --no-same-owner -xf -' \ + || fail "copying node failed" +if [ -d "$corepack_home" ]; then + tar -C "$corepack_home" -cf - . | $docker exec -i "$cid" sh -c 'mkdir -p /opt/corepack && tar -C /opt/corepack --no-same-owner -xf -' \ + || fail "copying the corepack cache failed" +fi + +exec_env=(-e PATH=/opt/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + -e COREPACK_HOME=/opt/corepack -e HOME=/root + -e XDG_RUNTIME_DIR=/run/user/0 -e DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus) +for name in "${pass_env[@]}"; do + [ -z "${!name+x}" ] || exec_env+=(-e "$name=${!name}") +done +rc=0 +$docker exec -w /work "${exec_env[@]}" "$cid" "$@" || rc=$? +# The gate's log and evidence come back even when it failed. +mkdir -p artifacts +$docker exec "$cid" sh -c '[ -d /work/artifacts ] && tar -C /work -cf - artifacts' | tar -C "$root" -xf - \ + || say "copying artifacts back failed" +exit "$rc" diff --git a/scripts/tests/test_verify.py b/scripts/tests/test_verify.py index 3e8b92c6..d10998fc 100644 --- a/scripts/tests/test_verify.py +++ b/scripts/tests/test_verify.py @@ -45,12 +45,29 @@ class StagePlanTest(unittest.TestCase): def test_parts_runs_each_command_of_the_gate(self) -> None: contract = gates() - gate = next(g for g in contract["gates"] if g["id"] == "FE-GATE-013") + gate = next(g for g in contract["gates"] if g["id"] == "FE-GATE-005") scripts = {c["id"]: c["script"] for c in contract["commands"]} - result = sh([str(VERIFY), "parts", "FE-GATE-013", "--dry-run"], cwd=REPO) + result = sh([str(VERIFY), "parts", "FE-GATE-005", "--dry-run"], cwd=REPO) self.assertEqual([line.split(": ", 1)[1] for line in result.stdout.splitlines()], [f"corepack pnpm -s {scripts[c]}" for c in gate["commandIds"]]) + def test_a_gate_with_a_host_environment_runs_in_it_as_in_the_workflow(self) -> None: + # FE-GATE-013 needs systemd and bubblewrap; the workflow and verify.sh both run it through + # scripts/run-in-system-container.sh (config/ci/gates.json hostEnvironment). + contract = gates() + hosted = [g for g in contract["gates"] if g.get("hostEnvironment") == "system-container"] + self.assertEqual([g["id"] for g in hosted], ["FE-GATE-013"]) + scripts = {c["id"]: c["script"] for c in contract["commands"]} + wrapper = "scripts/run-in-system-container.sh" + result = sh([str(VERIFY), "gate", "FE-GATE-013", "FE-GATE-005", "--dry-run"], cwd=REPO) + self.assertEqual(result.stdout.splitlines(), [f"FE-GATE-013: {wrapper} corepack pnpm -s ci:gate FE-GATE-013", + "FE-GATE-005: corepack pnpm -s ci:gate FE-GATE-005"]) + result = sh([str(VERIFY), "parts", "FE-GATE-013", "--dry-run"], cwd=REPO) + self.assertEqual([line.split(": ", 1)[1] for line in result.stdout.splitlines()], + [f"{wrapper} corepack pnpm -s {scripts[c]}" for c in hosted[0]["commandIds"]]) + workflow = (REPO / ".gitea/workflows/quality-gates.yml").read_text(encoding="utf-8") + self.assertIn(f"run: {wrapper} corepack pnpm ci:gate -- FE-GATE-013", workflow) + def test_parts_passes_each_command_its_arguments_as_ci_gate_does(self) -> None: # run-ci-gate.ts runs `pnpm