diff --git a/docs/operations/release-cache-rollback.md b/docs/operations/release-cache-rollback.md index 8237b3d2..d0f71f34 100644 --- a/docs/operations/release-cache-rollback.md +++ b/docs/operations/release-cache-rollback.md @@ -12,6 +12,32 @@ cache action, and repeats the smoke checks. Rebuilding an old commit, replacing HTML alone, or declaring recovery from cache-purge completion is prohibited. Recovery is established by old/new reachability probes. +Before deploying a promoted release later, or rolling back to one, re-verify +its promoted bundle (the five files uploaded by the `promotion` job) with the +archived mode (review 13-04). Provider evidence expires two hours after issue, +so the normal mode refuses any release promoted before that. The archived mode +still checks signatures, digests and the exact five members, and judges +freshness at the release's recorded promotion time +(`promotion-verification.json` `verifiedAt`, which may not lie in the future). +New promotion never uses it (`scripts/lib/exact-promotion-bundle.ts`). + +```bash +VULNERABILITY_PUBLIC_KEY_PATH=... VULNERABILITY_KEY_ID=... \ +PROVENANCE_PUBLIC_KEY_PATH=... PROVENANCE_KEY_ID=... \ +EXPECTED_PROMOTION_RUN_ID= EXPECTED_PROMOTION_RUN_ATTEMPT= \ +EXPECTED_PROMOTION_SOURCE_REVISION= \ +EXPECTED_PROMOTION_ARCHIVE_SHA256= \ +corepack pnpm verify:promotion -- --root --archived + +mkdir && tar -xzf /release-candidate.tar.gz -C +corepack pnpm drill:runbook -- FE-RB-005 --release-root +``` + +The expected values come from the release record, not from the bundle. The +drill then checks that release's build/config/manifest tuple and its assets, +not the local build. It ends `UNVERIFIED` until the provider switch and smoke +are recorded, because those happen outside this repository. + The provider-independent cache defaults are: - hashed assets: `public, max-age=31536000, immutable` diff --git a/docs/runbooks/FE-RB-005.md b/docs/runbooks/FE-RB-005.md index 6a38a2dc..83bb919d 100644 --- a/docs/runbooks/FE-RB-005.md +++ b/docs/runbooks/FE-RB-005.md @@ -11,3 +11,8 @@ Close only when compatibility and release-coherence gates pass, critical smoke passes, repeated `DEPLOY_MISMATCH` is absent, and the incident timeline records the restored release ID. Pointer-switch or cache-purge completion alone is not recovery evidence. + +Verify the prior release before switching to it: `verify:promotion -- --root + --archived` and `drill:runbook -- FE-RB-005 --release-root `. The commands and inputs are in +[`docs/operations/release-cache-rollback.md`](../operations/release-cache-rollback.md). diff --git a/scripts/check-supply-chain-provider-fixtures.ts b/scripts/check-supply-chain-provider-fixtures.ts index 69cbef8f..21646494 100644 --- a/scripts/check-supply-chain-provider-fixtures.ts +++ b/scripts/check-supply-chain-provider-fixtures.ts @@ -249,6 +249,31 @@ try { }, nowEpochMs: () => now, } as const; + // 13-04: the same bundle three hours later. Current freshness refuses the + // expired provider evidence; archived freshness accepts it at the recorded + // promotion time, and still refuses a tampered archive. + const threeHoursLater = now + 3 * 60 * 60 * 1_000; + const archivedAfterWindow = await captureRejection(async () => { + await verifyExactPromotionBundle(exactFiles, { + ...bundleTrust, + nowEpochMs: () => threeHoursLater, + freshness: "archived", + }); + }); + const currentAfterWindow = await captureRejection(async () => { + await verifyExactPromotionBundle(exactFiles, { + ...bundleTrust, + nowEpochMs: () => threeHoursLater, + }); + }); + const tamperedArchive = Buffer.from(exactFiles["release-candidate.tar.gz"]); + tamperedArchive[tamperedArchive.length - 1] ^= 0xff; + const archivedTampered = await captureRejection(async () => { + await verifyExactPromotionBundle( + { ...exactFiles, "release-candidate.tar.gz": tamperedArchive }, + { ...bundleTrust, nowEpochMs: () => threeHoursLater, freshness: "archived" }, + ); + }); const absentFiles = { ...exactFiles } as Partial; delete absentFiles["provider-verification.json"]; const absent = await captureRejection(async () => { @@ -282,7 +307,8 @@ try { await captureCiCandidateArchive({ archivePath, expectedSha256: "0".repeat(64) }); }); const passed = - validImmutable && absent.rejected && invalidTar.rejected && wrongDigest.rejected; + validImmutable && absent.rejected && invalidTar.rejected && wrongDigest.rejected && + !archivedAfterWindow.rejected && currentAfterWindow.rejected && archivedTampered.rejected; await cleanupFinalizedPromotion({ runnerTempRoot, stagingRoot: finalized.stagingRoot, @@ -314,12 +340,24 @@ try { status: invalidTar.rejected ? "FAIL_UNVERIFIED" : "PASS", failures: [invalidTar.failure], }, + archivedAfterWindow: { + status: archivedAfterWindow.rejected ? "FAIL_UNVERIFIED" : "PASS", + failures: archivedAfterWindow.rejected ? [archivedAfterWindow.failure] : [], + }, + currentAfterWindow: { + status: currentAfterWindow.rejected ? "FAIL_UNVERIFIED" : "PASS", + failures: [currentAfterWindow.failure], + }, + archivedTampered: { + status: archivedTampered.rejected ? "FAIL_UNVERIFIED" : "PASS", + failures: [archivedTampered.failure], + }, }, externalTreeCanary: { status: validImmutable ? "PASS" : "FAIL_UNVERIFIED", failures: validImmutable ? [] : ["external tree canary influenced captured archive"], }, - passingFixtureCount: validImmutable ? 1 : 0, + passingFixtureCount: (validImmutable ? 1 : 0) + (archivedAfterWindow.rejected ? 0 : 1), status: passed ? "PASS" : "FAIL", }, }); diff --git a/scripts/contracts/release-artifacts.ts b/scripts/contracts/release-artifacts.ts index 2d74a89c..baa63804 100644 --- a/scripts/contracts/release-artifacts.ts +++ b/scripts/contracts/release-artifacts.ts @@ -1297,6 +1297,10 @@ export const supplyChainProviderFixturesArtifactSchema = z validImmutable: providerFixtureResultSchema, wrongDigest: providerFixtureResultSchema, invalidTar: providerFixtureResultSchema, + // 13-04: archived re-verification of the same bundle three hours on. + archivedAfterWindow: providerFixtureResultSchema, + currentAfterWindow: providerFixtureResultSchema, + archivedTampered: providerFixtureResultSchema, }) .strict(), externalTreeCanary: providerFixtureResultSchema, @@ -1308,11 +1312,19 @@ export const supplyChainProviderFixturesArtifactSchema = z const expectedPass = artifact.actualDefaultVerifier.status === "PASS" && artifact.fixtures.validImmutable.status === "PASS" && + artifact.fixtures.archivedAfterWindow.status === "PASS" && artifact.externalTreeCanary.status === "PASS" && - [artifact.fixtures.absent, artifact.fixtures.wrongDigest, artifact.fixtures.invalidTar] - .every(({ status }) => status === "FAIL_UNVERIFIED"); - if (artifact.passingFixtureCount !== (artifact.fixtures.validImmutable.status === "PASS" ? 1 : 0)) { - context.addIssue({ code: "custom", path: ["passingFixtureCount"], message: "must count the passing immutable fixture" }); + [ + artifact.fixtures.absent, + artifact.fixtures.wrongDigest, + artifact.fixtures.invalidTar, + artifact.fixtures.currentAfterWindow, + artifact.fixtures.archivedTampered, + ].every(({ status }) => status === "FAIL_UNVERIFIED"); + const passing = [artifact.fixtures.validImmutable, artifact.fixtures.archivedAfterWindow] + .filter(({ status }) => status === "PASS").length; + if (artifact.passingFixtureCount !== passing) { + context.addIssue({ code: "custom", path: ["passingFixtureCount"], message: "must count the passing fixtures" }); } if ((artifact.status === "PASS") !== expectedPass) { context.addIssue({ code: "custom", path: ["status"], message: "must agree with required fixture outcomes" }); diff --git a/scripts/drill-runbook.ts b/scripts/drill-runbook.ts index 43444f7d..fe702bec 100644 --- a/scripts/drill-runbook.ts +++ b/scripts/drill-runbook.ts @@ -72,8 +72,12 @@ export type RollbackCoherenceOptions = Readonly<{ }>; export type DrillOptions = Readonly<{ - /** Directory holding `dist/` and `config/runbooks/`. Defaults to the cwd. */ - root?: string; + /** + * Directory holding the release's `dist/`. Defaults to the cwd (the local + * build). 13-04: pass an extracted candidate or promoted archive to drill + * the release that is actually deployed or rolled back to. + */ + releaseRoot?: string; contractPackages?: readonly InstalledContractPackageIdentity[]; }>; @@ -389,11 +393,11 @@ export async function verifyRollbackReleaseCoherence( export async function runRunbookDrill(runbookId: string, options: DrillOptions = {}) { const context: DrillContext = { - root: options.root ?? process.cwd(), + root: options.releaseRoot ?? process.cwd(), contractPackages: options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES, }; const document = JSON.parse( - await readFile(path.join(context.root, "config/runbooks/runbooks.json"), "utf8"), + await readFile("config/runbooks/runbooks.json", "utf8"), ) as RunbookDocument; const specification = document.runbooks[runbookId]; const drill = drillById[runbookId]; @@ -435,12 +439,16 @@ async function main(): Promise { const runbookId = process.argv .slice(2) .find((argument) => /^FE-RB-00[1-5]$/.test(argument)); - if (!runbookId) { - process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n"); + const releaseRootFlag = process.argv.indexOf("--release-root"); + const releaseRoot = releaseRootFlag >= 0 ? process.argv[releaseRootFlag + 1] : undefined; + if (!runbookId || (releaseRootFlag >= 0 && !releaseRoot)) { + process.stderr.write( + "Usage: drill:runbook -- FE-RB-001..FE-RB-005 [--release-root ]\n", + ); process.exit(2); } - const { gateId, ...record } = await runRunbookDrill(runbookId); + const { gateId, ...record } = await runRunbookDrill(runbookId, releaseRoot ? { releaseRoot } : {}); const artifactDirectory = `artifacts/runbooks/${runbookId}`; await mkdir(artifactDirectory, { recursive: true }); await writeValidatedJsonArtifact({ diff --git a/scripts/lib/exact-promotion-bundle.ts b/scripts/lib/exact-promotion-bundle.ts index 412779e4..9757a2d3 100644 --- a/scripts/lib/exact-promotion-bundle.ts +++ b/scripts/lib/exact-promotion-bundle.ts @@ -9,6 +9,7 @@ import { import { PROMOTION_VERIFIER_ID, PROMOTION_VERIFIER_VERSION, + PROVIDER_FUTURE_SKEW_MS, assertDistinctProviderTrust, evaluatePromotionEvidence, providerVerificationArtifactSchema, @@ -35,6 +36,29 @@ export type ExactPromotionExpectedContext = Readonly<{ lockfileSha256?: string; }>; +/** + * 13-04. `current` checks provider evidence against the clock, as a new + * promotion must. `archived` re-verifies a release promoted earlier (deploying + * it later, or rolling back to it): signatures, digests and members are checked + * the same way, but freshness is judged at the promotion's recorded + * `verifiedAt`, which may not lie in the future. Promotion itself + * (promotion-stager.ts) never passes `archived`. + */ +export type PromotionFreshness = "current" | "archived"; + +export function promotionEvidenceTime( + freshness: PromotionFreshness, + verifiedAt: string, + now: number, +): number { + if (freshness === "current") return now; + const verified = Date.parse(verifiedAt); + if (!Number.isFinite(verified) || verified > now + PROVIDER_FUTURE_SKEW_MS) { + throw new Error("archived promotion verifiedAt is in the future or invalid"); + } + return verified; +} + export async function verifyExactPromotionBundle( files: ExactPromotionBundle, options: Readonly<{ @@ -42,6 +66,7 @@ export async function verifyExactPromotionBundle( provenanceTrust: ProviderTrust; expected: ExactPromotionExpectedContext; nowEpochMs?: () => number; + freshness?: PromotionFreshness; }>, ): Promise> { assertDistinctProviderTrust(options); @@ -286,7 +311,12 @@ export async function verifyExactPromotionBundle( provenanceAttestation: provenance, vulnerabilityTrust: options.vulnerabilityTrust, provenanceTrust: options.provenanceTrust, - nowEpochMs: options.nowEpochMs, + nowEpochMs: () => + promotionEvidenceTime( + options.freshness ?? "current", + promotion.verifiedAt, + (options.nowEpochMs ?? Date.now)(), + ), }); if ( reevaluated.status !== "PASS" || diff --git a/scripts/verify-exact-promotion-bundle.ts b/scripts/verify-exact-promotion-bundle.ts index 15d75343..102e6e8b 100644 --- a/scripts/verify-exact-promotion-bundle.ts +++ b/scripts/verify-exact-promotion-bundle.ts @@ -55,9 +55,13 @@ const files = Object.fromEntries( ] as const), ), ); +// 13-04: `--archived` re-verifies a release promoted earlier (deploy later or +// roll back to it); freshness is judged at its recorded promotion time. +const freshness = process.argv.includes("--archived") ? "archived" : "current"; await verifyExactPromotionBundle(files, { vulnerabilityTrust, provenanceTrust, + freshness, expected: { run: { id: required("EXPECTED_PROMOTION_RUN_ID"), @@ -71,7 +75,7 @@ await verifyExactPromotionBundle(files, { ...optionalDigest("lockfileSha256", "EXPECTED_PROMOTION_LOCKFILE_SHA256"), }, }); -process.stdout.write("Exact promotion bundle verification: PASS\n"); +process.stdout.write(`Exact promotion bundle verification (${freshness}): PASS\n`); function asciiCompare(left: string, right: string): number { return left < right ? -1 : left > right ? 1 : 0; diff --git a/tests/unit/archived-promotion.test.ts b/tests/unit/archived-promotion.test.ts new file mode 100644 index 00000000..b814c4b5 --- /dev/null +++ b/tests/unit/archived-promotion.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; + +import { promotionEvidenceTime } from "../../scripts/lib/exact-promotion-bundle.ts"; +import { evaluatePromotionEvidence } from "../../scripts/lib/provider-evidence.ts"; +import { syntheticSignedPromotionBundle } from "./security-followup-fixture.ts"; + +// 13-04. Provider evidence lives two hours at most, so a promoted release could +// not be re-verified the next day, which the rollback runbook requires. The +// archived mode checks freshness at the recorded promotion time instead. +const promotedAt = Date.parse("2026-08-02T01:00:00.000Z"); +const threeHoursLater = promotedAt + 3 * 60 * 60 * 1_000; + +function evidenceAt(freshness: "current" | "archived", now: number) { + const fixture = syntheticSignedPromotionBundle(); + const provider = JSON.parse(fixture.files["provider-verification.json"].toString("utf8")); + return evaluatePromotionEvidence({ + expected: { + run: provider.run, + source: provider.source, + candidate: provider.candidate, + vulnerabilityInvocationNonce: provider.providerEvidence.vulnerabilityInvocationNonce, + provenanceInvocationNonce: provider.providerEvidence.provenanceInvocationNonce, + secretScanAttestation: provider.providerEvidence.secretScanAttestation, + }, + localStatus: "PASS", + vulnerabilityReport: JSON.parse(fixture.files["vulnerability-report.json"].toString("utf8")), + provenanceAttestation: JSON.parse(fixture.files["provenance-attestation.json"].toString("utf8")), + vulnerabilityTrust: fixture.verification.vulnerabilityTrust, + provenanceTrust: fixture.verification.provenanceTrust, + nowEpochMs: () => promotionEvidenceTime(freshness, provider.verifiedAt, now), + }); +} + +describe("archived promotion re-verification", () => { + it("accepts evidence three hours after promotion in archived mode only", () => { + expect(evidenceAt("archived", threeHoursLater).status).toBe("PASS"); + expect(evidenceAt("current", threeHoursLater)).toMatchObject({ + status: "FAIL_UNVERIFIED", + failures: expect.arrayContaining(["vulnerability report is expired"]), + }); + }); + + it("refuses a promotion time recorded in the future", () => { + expect(() => + promotionEvidenceTime("archived", "2026-08-02T05:00:00.000Z", promotedAt), + ).toThrow(/verifiedAt is in the future/u); + }); +}); diff --git a/tests/unit/runbook-drill.test.ts b/tests/unit/runbook-drill.test.ts index 0a474d6f..38d85701 100644 --- a/tests/unit/runbook-drill.test.ts +++ b/tests/unit/runbook-drill.test.ts @@ -32,11 +32,6 @@ afterEach(async () => { async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?: boolean }> = {}) { const root = await mkdtemp(path.join(tmpdir(), "runbook-drill-")); roots.push(root); - await mkdir(path.join(root, "config/runbooks"), { recursive: true }); - await writeFile( - path.join(root, "config/runbooks/runbooks.json"), - await readFile("config/runbooks/runbooks.json"), - ); if (options.withoutDist) return root; const write = async (file: string, content: string) => { await mkdir(path.dirname(path.join(root, file)), { recursive: true }); @@ -103,7 +98,7 @@ async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?: describe("runbook drill verdicts", () => { it.each(RUNBOOK_IDS)("%s on a complete local build is UNVERIFIED, not PASS", async (runbookId) => { - const record = await runRunbookDrill(runbookId, { root: await builtDist(), contractPackages }); + const record = await runRunbookDrill(runbookId, { releaseRoot: await builtDist(), contractPackages }); expect(record.result).toBe("UNVERIFIED"); expect(record.recoveryAssertions.some(({ result }) => result === "FAIL")).toBe(false); }); @@ -112,7 +107,7 @@ describe("runbook drill verdicts", () => { "%s fails when the build lost its JavaScript assets", async (runbookId) => { const record = await runRunbookDrill(runbookId, { - root: await builtDist({ withoutJs: true }), + releaseRoot: await builtDist({ withoutJs: true }), contractPackages, }); expect(record.result).toBe("FAIL"); @@ -121,7 +116,7 @@ describe("runbook drill verdicts", () => { it("FE-RB-005 fails without a build instead of reading public/ placeholders", async () => { await expect( - runRunbookDrill("FE-RB-005", { root: await builtDist({ withoutDist: true }), contractPackages }), + runRunbookDrill("FE-RB-005", { releaseRoot: await builtDist({ withoutDist: true }), contractPackages }), ).rejects.toThrow(/dist\/release-manifest\.json/u); });