From f21ea48911bf7327c82aa0d9bff93d7c0b4a6ecc Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 12:51:47 +0900 Subject: [PATCH] =?UTF-8?q?ci(gates):=20=EC=9D=98=EC=A1=B4=EC=84=B1=20?= =?UTF-8?q?=EC=BA=90=EC=8B=9C=EB=A5=BC=20checkout=EA=B3=BC=20=EA=B0=99?= =?UTF-8?q?=EC=9D=80=20=ED=8C=8C=EC=9D=BC=EC=8B=9C=EC=8A=A4=ED=85=9C?= =?UTF-8?q?=EC=9D=98=20=ED=8F=B4=EB=8D=94=EC=97=90=20=EB=91=94=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 문제: #41의 merge job 캐시는 hit해도 설치가 빨라지지 않았다. PR run 230에서 세 job 모두 복원했지만(222MB, 4.8초) pnpm 설치는 16.9·26.4·22.4초로 캐시 없는 run(13.8~22.8초)과 같았다. `reused 404, downloaded 0` 뒤 node_modules에 넣는 데만 15초 넘게 걸렸다. desktop 재현: 같은 파일시스템 2.1초, tar 왕복 뒤 7.6초, store가 다른 파일시스템이면 58.8초(하드링크 대신 복사). store가 job의 `/tmp`에 있어 checkout과 다른 파일시스템일 가능성이 크다. 고친 것(권하는 안으로 정함, 총괄): merge job의 첫 캐시 단계가 `scripts/ci/dependency-cache-root.sh "$RUNNER_TEMP" "$(dirname "$GITHUB_WORKSPACE")"` 를 돈다. 후보 중 checkout과 `stat -c %d`가 같고 checkout 밖인 첫 폴더를 고르고, 없으면 첫 후보를 쓴다. 고른 폴더, 견준 장치 번호, job 시작 때의 부하를 한 줄로 로그에 남기고, `COREPACK_HOME`·`PNPM_CONFIG_STORE_DIR`· `PNPM_CONFIG_CACHE_DIR`를 `$GITHUB_ENV`로 넘긴다(job env 고정값을 뺌). 캐시 경로는 그 단계 출력을 쓴다. checkout 안은 후보가 아니다(`git status`, repo 배치 검사, 산출물에 잡히지 않게). 버린 대안: checkout 안 `.tmp` 아래(배치 검사·산출물에 걸림), workspace 부모를 고정 경로로(파일시스템이 같은지 runner마다 다를 수 있다). 위험·한계: 고른 폴더가 run마다 다르면 캐시 항목의 경로가 달라 hit하지 않는다 (같은 runner에서는 같다). 같은 파일시스템이 없으면 지금과 같다. runner에서의 효과는 main 저장 → 다음 PR 복원으로 잰다. 검증(desktop): 새 시험 `tests/unit/dependency-cache-root.test.ts`(가짜 `stat`으로 같은 파일시스템 고르기·checkout 안 제외·대체 경로) 통과, 최대 47ms. 생성 시험은 바꾸기 전 생성기에서 실패(RED), 고친 뒤 ci-workflow-generation·test-timeout-declarations·repository-layout 등 460개 통과. check:ci·check:types·eslint·verify:documentation rc=0. --- .gitea/workflows/quality-gates.yml | 32 +++--- docs/operations/ci-quality-gates.md | 5 +- scripts/ci/dependency-cache-root.sh | 43 ++++++++ scripts/contracts/ci-gates.ts | 26 +++-- scripts/generate-ci-workflow.ts | 13 +-- .../ci-workflow-generation.test.ts.snap | 32 +++--- tests/unit/ci-workflow-generation.test.ts | 18 ++-- tests/unit/dependency-cache-root.test.ts | 101 ++++++++++++++++++ 8 files changed, 214 insertions(+), 56 deletions(-) create mode 100755 scripts/ci/dependency-cache-root.sh create mode 100644 tests/unit/dependency-cache-root.test.ts diff --git a/.gitea/workflows/quality-gates.yml b/.gitea/workflows/quality-gates.yml index 4898c239..2c76e9eb 100644 --- a/.gitea/workflows/quality-gates.yml +++ b/.gitea/workflows/quality-gates.yml @@ -41,21 +41,20 @@ jobs: if: ${{ gitea.event_name != 'workflow_dispatch' || (inputs.stage != 'documentation' && inputs.stage != 'main') }} runs-on: ubuntu-latest timeout-minutes: 45 - env: - COREPACK_HOME: /tmp/fe-dependency-cache/corepack - PNPM_CONFIG_STORE_DIR: /tmp/fe-dependency-cache/pnpm-store - PNPM_CONFIG_CACHE_DIR: /tmp/fe-dependency-cache/pnpm-cache steps: - uses: https://github.com/actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 with: persist-credentials: false + - name: Choose the dependency cache folder + id: dependency-cache-root + run: bash scripts/ci/dependency-cache-root.sh "$RUNNER_TEMP" "$(dirname "$GITHUB_WORKSPACE")" - name: Restore and save the dependency cache if: ${{ gitea.ref == 'refs/heads/main' }} uses: https://github.com/actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + ${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-${{ runner.os }}-${{ runner.arch }}- @@ -65,7 +64,7 @@ jobs: with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + ${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-${{ runner.os }}-${{ runner.arch }}- @@ -109,21 +108,20 @@ jobs: # FE-GATE-013 runner requirement: unprivileged user and network namespaces for bwrap (the provider sandbox runs bwrap --unshare-pid --unshare-ipc --unshare-uts --unshare-net); Ubuntu 23.10 and later refuse them while kernel.apparmor_restrict_unprivileged_userns=1 unless /usr/bin/bwrap has an AppArmor profile that allows userns; scripts/check-system-test-prerequisites.ts probes this before test:system runs and scripts/run-and-validate-provider.ts before a provider runs runs-on: ubuntu-latest timeout-minutes: 45 - env: - COREPACK_HOME: /tmp/fe-dependency-cache/corepack - PNPM_CONFIG_STORE_DIR: /tmp/fe-dependency-cache/pnpm-store - PNPM_CONFIG_CACHE_DIR: /tmp/fe-dependency-cache/pnpm-cache steps: - uses: https://github.com/actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 with: persist-credentials: false + - name: Choose the dependency cache folder + id: dependency-cache-root + run: bash scripts/ci/dependency-cache-root.sh "$RUNNER_TEMP" "$(dirname "$GITHUB_WORKSPACE")" - name: Restore and save the dependency cache if: ${{ gitea.ref == 'refs/heads/main' }} uses: https://github.com/actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + ${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-${{ runner.os }}-${{ runner.arch }}- @@ -133,7 +131,7 @@ jobs: with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + ${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-${{ runner.os }}-${{ runner.arch }}- @@ -179,20 +177,20 @@ jobs: image: "mcr.microsoft.com/playwright:v1.62.0-noble@sha256:baed2032d533817f3dbe6425de795788430ba345e819a1201337009ba17c9d07" env: CI_RUNNER_IMAGE: "mcr.microsoft.com/playwright:v1.62.0-noble@sha256:baed2032d533817f3dbe6425de795788430ba345e819a1201337009ba17c9d07" - COREPACK_HOME: /tmp/fe-dependency-cache/corepack - PNPM_CONFIG_STORE_DIR: /tmp/fe-dependency-cache/pnpm-store - PNPM_CONFIG_CACHE_DIR: /tmp/fe-dependency-cache/pnpm-cache steps: - uses: https://github.com/actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 with: persist-credentials: false + - name: Choose the dependency cache folder + id: dependency-cache-root + run: bash scripts/ci/dependency-cache-root.sh "$RUNNER_TEMP" "$(dirname "$GITHUB_WORKSPACE")" - name: Restore and save the dependency cache if: ${{ gitea.ref == 'refs/heads/main' }} uses: https://github.com/actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + ${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-${{ runner.os }}-${{ runner.arch }}- @@ -202,7 +200,7 @@ jobs: with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + ${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-${{ runner.os }}-${{ runner.arch }}- diff --git a/docs/operations/ci-quality-gates.md b/docs/operations/ci-quality-gates.md index 7c7ab0f4..5be5aff5 100644 --- a/docs/operations/ci-quality-gates.md +++ b/docs/operations/ci-quality-gates.md @@ -16,7 +16,10 @@ Node toolcache, corepack이 받은 pnpm, pnpm store와 메타데이터 캐시를 입력이다. pnpm은 같은 lockfile의 공급망 검증 기록이 캐시에 있으면 검증을 다시 하지 않는다. 그래서 브랜치 run은 복원만 한다. frozen install은 캐시가 맞든 없든 늘 돈다. 항목이 없거나 깨지면 pnpm이 모자란 패키지만 내려받고 node_modules는 같다. FE-GATE-013의 시스템 컨테이너도 -같은 store와 메타데이터 캐시를 받는다. release·promotion·provider job은 공급망 증거를 +같은 store와 메타데이터 캐시를 받는다. 캐시 폴더는 checkout 밖, checkout과 같은 파일시스템에 +둔다(`scripts/ci/dependency-cache-root.sh`). 다른 파일시스템이면 pnpm이 node_modules를 복사해, +캐시가 맞아도 설치가 빨라지지 않는다. 고른 폴더, 견준 장치 번호, job 시작 때의 부하를 job 로그에 +한 줄 남긴다. release·promotion·provider job은 공급망 증거를 만들므로 매번 새로 설치한다. 이 규칙은 `scripts/contracts/ci-gates.ts`의 `DEPENDENCY_CACHE`와 계약 검사가 지킨다. runner의 캐시 저장소는 짝 백엔드와 함께 쓰고 크기 상한이 있다(platform-core `infrastructure/ci/gitea-runner/config.yaml`). 검사기 자신의 음성 fixture와 diff --git a/scripts/ci/dependency-cache-root.sh b/scripts/ci/dependency-cache-root.sh new file mode 100755 index 00000000..4f76bc5d --- /dev/null +++ b/scripts/ci/dependency-cache-root.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Chooses where a merge job keeps its dependency cache (scripts/contracts/ci-gates.ts DEPENDENCY_CACHE): the first +# candidate folder on the checkout's filesystem, so pnpm hardlinks node_modules from the restored store. A store on +# another filesystem is copied file by file, and a warm install took longer than a cold one (desktop: 2.1 s on the +# same filesystem, 58.8 s across filesystems, 2026-11-13). A folder inside the checkout is never chosen: a cache +# there would show in `git status`, the repository layout checks and the uploaded artifacts. +# +# scripts/ci/dependency-cache-root.sh ... +# +# Writes COREPACK_HOME, PNPM_CONFIG_STORE_DIR and PNPM_CONFIG_CACHE_DIR to $GITHUB_ENV and root= to +# $GITHUB_OUTPUT, and logs one line: the folder, whether it shares the checkout's filesystem, each candidate's +# device and the load average at the job's start (the runner shares the desktop with other work). With no +# candidate on that filesystem it takes the first usable candidate, and the install still runs. +set -euo pipefail + +workspace=$(cd "${GITHUB_WORKSPACE:?}" && pwd -P) +checkout=$(stat -c %d "$workspace") +chosen="" +fallback="" +devices="" +for candidate in "$@"; do + [ -n "$candidate" ] && [ -d "$candidate" ] && [ -w "$candidate" ] || continue + real=$(cd "$candidate" && pwd -P) + case "$real/" in "$workspace/"*) continue ;; esac + device=$(stat -c %d "$real") + devices="$devices $real=$device" + [ -n "$fallback" ] || fallback=$real + if [ -z "$chosen" ] && [ "$device" = "$checkout" ]; then chosen=$real; fi +done +same=yes +if [ -z "$chosen" ]; then + same=no + chosen=${fallback:-/tmp} +fi +root="${chosen%/}/fe-dependency-cache" + +echo "dependency cache: $root (on the checkout's filesystem: $same; devices: checkout=$checkout$devices; load average at job start: $(cut -d' ' -f1-3 /proc/loadavg))" +{ + echo "COREPACK_HOME=$root/corepack" + echo "PNPM_CONFIG_STORE_DIR=$root/pnpm-store" + echo "PNPM_CONFIG_CACHE_DIR=$root/pnpm-cache" +} >> "${GITHUB_ENV:?}" +echo "root=$root" >> "${GITHUB_OUTPUT:?}" diff --git a/scripts/contracts/ci-gates.ts b/scripts/contracts/ci-gates.ts index 71499d29..45f41d23 100644 --- a/scripts/contracts/ci-gates.ts +++ b/scripts/contracts/ci-gates.ts @@ -123,17 +123,29 @@ export function resolveCiSubActionUses(actionId: CiActionId, subpath: "restore") * (desktop, 2026-11-13: cold 13.9 s, warm 1.6 s, 40 broken store and 40 * broken metadata files 3.6 s with 27 packages fetched again; the same tree * hash in all five installs). + * + * The cache folder is outside the checkout, on its filesystem + * (scripts/ci/dependency-cache-root.sh): with the store in the job's `/tmp`, + * a warm runner install still took 17 to 26 s, as pnpm copied node_modules + * across filesystems (run 230). The script logs the devices it compared and + * the load average at the job's start, and sets the variables below through + * `$GITHUB_ENV`, so every pnpm call of the job, the system container's + * included, reads the same folder. */ export const DEPENDENCY_CACHE = Object.freeze({ keyPrefix: "fe-dependencies-${{ runner.os }}-${{ runner.arch }}-", // `.nvmrc` and `pnpm-lock.yaml` (config/repository-layout.json). keyInputs: selectRepositoryPaths("dependency-cache-key"), - paths: Object.freeze(["/opt/hostedtoolcache/node", "/tmp/fe-dependency-cache"] as const), - environment: Object.freeze([ - Object.freeze({ name: "COREPACK_HOME", value: "/tmp/fe-dependency-cache/corepack" }), - Object.freeze({ name: "PNPM_CONFIG_STORE_DIR", value: "/tmp/fe-dependency-cache/pnpm-store" }), - Object.freeze({ name: "PNPM_CONFIG_CACHE_DIR", value: "/tmp/fe-dependency-cache/pnpm-cache" }), - ]), + rootStepId: "dependency-cache-root", + rootScript: "scripts/ci/dependency-cache-root.sh", + /** Folders outside the checkout to try, in order (shell words). */ + rootCandidates: Object.freeze(['"$RUNNER_TEMP"', '"$(dirname "$GITHUB_WORKSPACE")"'] as const), + paths: Object.freeze([ + "/opt/hostedtoolcache/node", + "${{ steps.dependency-cache-root.outputs.root }}", + ] as const), + /** What the script writes to `$GITHUB_ENV`. */ + environmentNames: Object.freeze(["COREPACK_HOME", "PNPM_CONFIG_STORE_DIR", "PNPM_CONFIG_CACHE_DIR"] as const), /** The jobs that may hold the step: the three merge jobs. */ jobIds: Object.freeze(["merge_static", "merge_test", "merge_browser"] as const), }); @@ -868,7 +880,7 @@ function validateContractSemantics( if (!(DEPENDENCY_CACHE.jobIds as readonly string[]).includes(job.id)) { issue(`the dependency cache is for the merge jobs only: ${job.id}`); } - for (const { name } of DEPENDENCY_CACHE.environment) { + for (const name of DEPENDENCY_CACHE.environmentNames) { if (job.environment.some((binding) => binding.name === name)) { issue(`job ${job.id} binds ${name}, which its dependency cache sets`); } diff --git a/scripts/generate-ci-workflow.ts b/scripts/generate-ci-workflow.ts index 50a68b1e..dd1a400a 100644 --- a/scripts/generate-ci-workflow.ts +++ b/scripts/generate-ci-workflow.ts @@ -236,16 +236,9 @@ function renderJob( const jobEnvironment = job.environment.filter((binding) => !isStepScopedSecret(job, binding)); // The build records the image it ran in (scripts/lib/build-environment.ts): // a job in its own image names that one, not the runner label's. - // Every pnpm and corepack call of the job reads the cached directories, the - // install and the system container's own install alike. - const cacheEnvironment = job.steps.some(({ kind }) => kind === "dependency-cache") - ? DEPENDENCY_CACHE.environment - : []; const environmentLines = [ ...(image ? [` CI_RUNNER_IMAGE: ${yamlScalar(image)}`] : []), - ...[...cacheEnvironment, ...jobEnvironment].map( - (binding) => ` ${binding.name}: ${yamlScalar(binding.value)}`, - ), + ...jobEnvironment.map((binding) => ` ${binding.name}: ${yamlScalar(binding.value)}`), ]; if (environmentLines.length > 0) lines.push(" env:", ...environmentLines); if (job.kind === "gate-matrix") { @@ -338,6 +331,10 @@ function renderStep( ` ${DEPENDENCY_CACHE.keyPrefix}`, ]; return [ + // The folder on the checkout's filesystem, and the job's pnpm variables. + " - name: Choose the dependency cache folder", + ` id: ${DEPENDENCY_CACHE.rootStepId}`, + ` run: bash ${DEPENDENCY_CACHE.rootScript} ${DEPENDENCY_CACHE.rootCandidates.join(" ")}`, " - name: Restore and save the dependency cache", " if: ${{ gitea.ref == 'refs/heads/main' }}", ` uses: ${requiredStepActionUses(step.kind)}`, diff --git a/tests/unit/__snapshots__/ci-workflow-generation.test.ts.snap b/tests/unit/__snapshots__/ci-workflow-generation.test.ts.snap index c5c3eb7a..4f08cf2c 100644 --- a/tests/unit/__snapshots__/ci-workflow-generation.test.ts.snap +++ b/tests/unit/__snapshots__/ci-workflow-generation.test.ts.snap @@ -44,21 +44,20 @@ jobs: if: \${{ gitea.event_name != 'workflow_dispatch' || (inputs.stage != 'documentation' && inputs.stage != 'main') }} runs-on: ubuntu-latest timeout-minutes: 45 - env: - COREPACK_HOME: /tmp/fe-dependency-cache/corepack - PNPM_CONFIG_STORE_DIR: /tmp/fe-dependency-cache/pnpm-store - PNPM_CONFIG_CACHE_DIR: /tmp/fe-dependency-cache/pnpm-cache steps: - uses: https://github.com/actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 with: persist-credentials: false + - name: Choose the dependency cache folder + id: dependency-cache-root + run: bash scripts/ci/dependency-cache-root.sh "$RUNNER_TEMP" "$(dirname "$GITHUB_WORKSPACE")" - name: Restore and save the dependency cache if: \${{ gitea.ref == 'refs/heads/main' }} uses: https://github.com/actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + \${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}-\${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}- @@ -68,7 +67,7 @@ jobs: with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + \${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}-\${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}- @@ -112,21 +111,20 @@ jobs: # FE-GATE-013 runner requirement: unprivileged user and network namespaces for bwrap (the provider sandbox runs bwrap --unshare-pid --unshare-ipc --unshare-uts --unshare-net); Ubuntu 23.10 and later refuse them while kernel.apparmor_restrict_unprivileged_userns=1 unless /usr/bin/bwrap has an AppArmor profile that allows userns; scripts/check-system-test-prerequisites.ts probes this before test:system runs and scripts/run-and-validate-provider.ts before a provider runs runs-on: ubuntu-latest timeout-minutes: 45 - env: - COREPACK_HOME: /tmp/fe-dependency-cache/corepack - PNPM_CONFIG_STORE_DIR: /tmp/fe-dependency-cache/pnpm-store - PNPM_CONFIG_CACHE_DIR: /tmp/fe-dependency-cache/pnpm-cache steps: - uses: https://github.com/actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 with: persist-credentials: false + - name: Choose the dependency cache folder + id: dependency-cache-root + run: bash scripts/ci/dependency-cache-root.sh "$RUNNER_TEMP" "$(dirname "$GITHUB_WORKSPACE")" - name: Restore and save the dependency cache if: \${{ gitea.ref == 'refs/heads/main' }} uses: https://github.com/actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + \${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}-\${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}- @@ -136,7 +134,7 @@ jobs: with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + \${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}-\${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}- @@ -182,20 +180,20 @@ jobs: image: "mcr.microsoft.com/playwright:v1.62.0-noble@sha256:baed2032d533817f3dbe6425de795788430ba345e819a1201337009ba17c9d07" env: CI_RUNNER_IMAGE: "mcr.microsoft.com/playwright:v1.62.0-noble@sha256:baed2032d533817f3dbe6425de795788430ba345e819a1201337009ba17c9d07" - COREPACK_HOME: /tmp/fe-dependency-cache/corepack - PNPM_CONFIG_STORE_DIR: /tmp/fe-dependency-cache/pnpm-store - PNPM_CONFIG_CACHE_DIR: /tmp/fe-dependency-cache/pnpm-cache steps: - uses: https://github.com/actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 with: persist-credentials: false + - name: Choose the dependency cache folder + id: dependency-cache-root + run: bash scripts/ci/dependency-cache-root.sh "$RUNNER_TEMP" "$(dirname "$GITHUB_WORKSPACE")" - name: Restore and save the dependency cache if: \${{ gitea.ref == 'refs/heads/main' }} uses: https://github.com/actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + \${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}-\${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}- @@ -205,7 +203,7 @@ jobs: with: path: | /opt/hostedtoolcache/node - /tmp/fe-dependency-cache + \${{ steps.dependency-cache-root.outputs.root }} key: fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}-\${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }} restore-keys: | fe-dependencies-\${{ runner.os }}-\${{ runner.arch }}- diff --git a/tests/unit/ci-workflow-generation.test.ts b/tests/unit/ci-workflow-generation.test.ts index 0e67e3c3..749cb3d3 100644 --- a/tests/unit/ci-workflow-generation.test.ts +++ b/tests/unit/ci-workflow-generation.test.ts @@ -1824,11 +1824,17 @@ describe("CI workflow generation", () => { for (const jobId of DEPENDENCY_CACHE.jobIds) { const job = jobSection(workflow, jobId); const steps = job.split("\n - "); + const rootIndex = steps.findIndex((step) => step.startsWith("name: Choose the dependency cache folder")); const saveIndex = steps.findIndex((step) => step.includes(`uses: ${save}`)); const restoreIndex = steps.findIndex((step) => step.includes(`uses: ${restore}`)); const setupIndex = steps.findIndex((step) => step.startsWith("uses: https://github.com/actions/setup-node@")); const installIndex = steps.findIndex((step) => step.startsWith("name: Frozen install")); - expect([saveIndex, restoreIndex].every((index) => index > 0 && index < setupIndex), jobId).toBe(true); + expect(rootIndex, jobId).toBeGreaterThan(0); + expect([saveIndex, restoreIndex].every((index) => index > rootIndex && index < setupIndex), jobId).toBe(true); + // The folder is chosen on the checkout's filesystem, outside it, and logged. + expect(steps[rootIndex]).toBe( + 'name: Choose the dependency cache folder\n id: dependency-cache-root\n run: bash scripts/ci/dependency-cache-root.sh "$RUNNER_TEMP" "$(dirname "$GITHUB_WORKSPACE")"', + ); expect(setupIndex).toBeLessThan(installIndex); expect(steps[saveIndex]).toContain("if: ${{ gitea.ref == 'refs/heads/main' }}"); expect(steps[restoreIndex]).toContain("if: ${{ gitea.ref != 'refs/heads/main' }}"); @@ -1836,17 +1842,17 @@ describe("CI workflow generation", () => { expect(step).toContain( "key: fe-dependencies-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', 'pnpm-lock.yaml') }}", ); - expect(step).toContain(" /opt/hostedtoolcache/node\n /tmp/fe-dependency-cache\n"); + expect(step).toContain( + " /opt/hostedtoolcache/node\n ${{ steps.dependency-cache-root.outputs.root }}\n", + ); } // The install does not depend on a hit: a missing or damaged entry // only makes pnpm fetch what it lacks. expect(steps[installIndex]).toBe( "name: Frozen install\n run: |\n corepack enable\n corepack pnpm install --frozen-lockfile --ignore-scripts", ); - for (const { name, value } of DEPENDENCY_CACHE.environment) { - expect(job).toContain(` ${name}: ${value}\n`); - expect(value.startsWith("/tmp/fe-dependency-cache/")).toBe(true); - } + // The script sets the pnpm variables through $GITHUB_ENV, not the job. + for (const name of DEPENDENCY_CACHE.environmentNames) expect(job).not.toContain(` ${name}:`); } }); diff --git a/tests/unit/dependency-cache-root.test.ts b/tests/unit/dependency-cache-root.test.ts new file mode 100644 index 00000000..df0d1052 --- /dev/null +++ b/tests/unit/dependency-cache-root.test.ts @@ -0,0 +1,101 @@ +import { spawnSync } from "node:child_process"; +import { chmod, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { DEPENDENCY_CACHE } from "../../scripts/contracts/ci-gates.ts"; +import { childProcessTimeout } from "../helpers/child-process-timeout.ts"; + +/** + * scripts/ci/dependency-cache-root.sh picks the merge job's dependency cache + * folder: the first candidate on the checkout's filesystem and outside it + * (run 230: with the store in the job's /tmp, pnpm copied node_modules across + * filesystems and a warm install was no faster than a cold one). A fake + * `stat` stands in for the devices, so the cases do not depend on how this + * host mounts its folders: a path under FAKE_SAME is device 11, any other 22. + */ +const roots: string[] = []; + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +async function layout() { + const root = await realpath(await mkdtemp(path.join(tmpdir(), "dependency-cache-root-"))); + roots.push(root); + const bin = path.join(root, "bin"); + await mkdir(bin); + await writeFile( + path.join(bin, "stat"), + ['#!/usr/bin/env bash', 'target=${@: -1}', 'case "$target/" in "$FAKE_SAME"/*) echo 11 ;; *) echo 22 ;; esac', ""].join("\n"), + ); + await chmod(path.join(bin, "stat"), 0o755); + const folders = { + same: path.join(root, "same"), + checkout: path.join(root, "same", "checkout"), + insideCheckout: path.join(root, "same", "checkout", "tmp"), + sameTemp: path.join(root, "same", "runner-temp"), + other: path.join(root, "other"), + otherTemp: path.join(root, "other", "runner-temp"), + missing: path.join(root, "missing"), + }; + for (const folder of [folders.insideCheckout, folders.sameTemp, folders.otherTemp]) { + await mkdir(folder, { recursive: true }); + } + const environmentFile = path.join(root, "github-env"); + const outputFile = path.join(root, "github-output"); + const run = (...candidates: string[]) => + spawnSync("bash", [DEPENDENCY_CACHE.rootScript, ...candidates], { + encoding: "utf8", + env: { + PATH: `${bin}:${process.env.PATH ?? ""}`, + FAKE_SAME: folders.same, + GITHUB_WORKSPACE: folders.checkout, + GITHUB_ENV: environmentFile, + GITHUB_OUTPUT: outputFile, + }, + }); + const written = async () => ({ + environment: (await readFile(environmentFile, "utf8")).trim().split("\n"), + output: (await readFile(outputFile, "utf8")).trim(), + }); + return { folders, run, written }; +} + +const variables = (root: string) => [ + `COREPACK_HOME=${root}/corepack`, + `PNPM_CONFIG_STORE_DIR=${root}/pnpm-store`, + `PNPM_CONFIG_CACHE_DIR=${root}/pnpm-cache`, +]; + +describe("the merge job's dependency cache folder", { timeout: childProcessTimeout(47) }, () => { + it("is the first candidate on the checkout's filesystem, never one inside the checkout", async () => { + const { folders, run, written } = await layout(); + + const result = run(folders.otherTemp, folders.insideCheckout, folders.sameTemp); + + expect(result.status, result.stderr).toBe(0); + const root = `${folders.sameTemp}/fe-dependency-cache`; + expect(await written()).toEqual({ environment: variables(root), output: `root=${root}` }); + expect(result.stdout).toMatch( + new RegExp( + String.raw`^dependency cache: ${root} \(on the checkout's filesystem: yes; devices: checkout=11 ${folders.otherTemp}=22 ${folders.sameTemp}=11; load average at job start: [0-9.]+ [0-9.]+ [0-9.]+\)\n$`, + "u", + ), + ); + expect(DEPENDENCY_CACHE.environmentNames).toEqual(variables(root).map((line) => line.split("=")[0])); + }); + + it("falls back to the first usable candidate when none shares the checkout's filesystem", async () => { + const { folders, run, written } = await layout(); + + const result = run(folders.missing, "", folders.otherTemp, folders.other); + + expect(result.status, result.stderr).toBe(0); + const root = `${folders.otherTemp}/fe-dependency-cache`; + expect(await written()).toEqual({ environment: variables(root), output: `root=${root}` }); + expect(result.stdout).toContain(`dependency cache: ${root} (on the checkout's filesystem: no; devices: checkout=11 ${folders.otherTemp}=22 ${folders.other}=22;`); + }); +}); -- 2.54.0