feat(release): re-verify a promoted release in archived mode (13-04)

Provider evidence expires two hours after issue, so no repo tool could
re-verify a release before deploying it later or rolling back to it.
verify:promotion --archived keeps every signature, digest and member check and
judges freshness at the recorded promotion time; the FE-RB-005 drill takes the
extracted release with --release-root.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-25 22:58:37 +09:00
co-authored by Claude Opus 5.5
parent 1a1cfe6605
commit 839205c134
9 changed files with 189 additions and 23 deletions
+26
View File
@@ -12,6 +12,32 @@ cache action, and repeats the smoke checks. Rebuilding an old commit, replacing
HTML alone, or declaring recovery from cache-purge completion is prohibited.
Recovery is established by old/new reachability probes.
Before deploying a promoted release later, or rolling back to one, re-verify
its promoted bundle (the five files uploaded by the `promotion` job) with the
archived mode (review 13-04). Provider evidence expires two hours after issue,
so the normal mode refuses any release promoted before that. The archived mode
still checks signatures, digests and the exact five members, and judges
freshness at the release's recorded promotion time
(`promotion-verification.json` `verifiedAt`, which may not lie in the future).
New promotion never uses it (`scripts/lib/exact-promotion-bundle.ts`).
```bash
VULNERABILITY_PUBLIC_KEY_PATH=... VULNERABILITY_KEY_ID=... \
PROVENANCE_PUBLIC_KEY_PATH=... PROVENANCE_KEY_ID=... \
EXPECTED_PROMOTION_RUN_ID=<run id> EXPECTED_PROMOTION_RUN_ATTEMPT=<attempt> \
EXPECTED_PROMOTION_SOURCE_REVISION=<commit> \
EXPECTED_PROMOTION_ARCHIVE_SHA256=<archive sha256> \
corepack pnpm verify:promotion -- --root <promoted bundle directory> --archived
mkdir <release dir> && tar -xzf <promoted bundle directory>/release-candidate.tar.gz -C <release dir>
corepack pnpm drill:runbook -- FE-RB-005 --release-root <release dir>
```
The expected values come from the release record, not from the bundle. The
drill then checks that release's build/config/manifest tuple and its assets,
not the local build. It ends `UNVERIFIED` until the provider switch and smoke
are recorded, because those happen outside this repository.
The provider-independent cache defaults are:
- hashed assets: `public, max-age=31536000, immutable`
+5
View File
@@ -11,3 +11,8 @@ Close only when compatibility and release-coherence gates pass, critical smoke
passes, repeated `DEPLOY_MISMATCH` is absent, and the incident timeline records
the restored release ID. Pointer-switch or cache-purge completion alone is not
recovery evidence.
Verify the prior release before switching to it: `verify:promotion -- --root
<bundle> --archived` and `drill:runbook -- FE-RB-005 --release-root <extracted
archive>`. The commands and inputs are in
[`docs/operations/release-cache-rollback.md`](../operations/release-cache-rollback.md).
@@ -249,6 +249,31 @@ try {
},
nowEpochMs: () => now,
} as const;
// 13-04: the same bundle three hours later. Current freshness refuses the
// expired provider evidence; archived freshness accepts it at the recorded
// promotion time, and still refuses a tampered archive.
const threeHoursLater = now + 3 * 60 * 60 * 1_000;
const archivedAfterWindow = await captureRejection(async () => {
await verifyExactPromotionBundle(exactFiles, {
...bundleTrust,
nowEpochMs: () => threeHoursLater,
freshness: "archived",
});
});
const currentAfterWindow = await captureRejection(async () => {
await verifyExactPromotionBundle(exactFiles, {
...bundleTrust,
nowEpochMs: () => threeHoursLater,
});
});
const tamperedArchive = Buffer.from(exactFiles["release-candidate.tar.gz"]);
tamperedArchive[tamperedArchive.length - 1] ^= 0xff;
const archivedTampered = await captureRejection(async () => {
await verifyExactPromotionBundle(
{ ...exactFiles, "release-candidate.tar.gz": tamperedArchive },
{ ...bundleTrust, nowEpochMs: () => threeHoursLater, freshness: "archived" },
);
});
const absentFiles = { ...exactFiles } as Partial<typeof exactFiles>;
delete absentFiles["provider-verification.json"];
const absent = await captureRejection(async () => {
@@ -282,7 +307,8 @@ try {
await captureCiCandidateArchive({ archivePath, expectedSha256: "0".repeat(64) });
});
const passed =
validImmutable && absent.rejected && invalidTar.rejected && wrongDigest.rejected;
validImmutable && absent.rejected && invalidTar.rejected && wrongDigest.rejected &&
!archivedAfterWindow.rejected && currentAfterWindow.rejected && archivedTampered.rejected;
await cleanupFinalizedPromotion({
runnerTempRoot,
stagingRoot: finalized.stagingRoot,
@@ -314,12 +340,24 @@ try {
status: invalidTar.rejected ? "FAIL_UNVERIFIED" : "PASS",
failures: [invalidTar.failure],
},
archivedAfterWindow: {
status: archivedAfterWindow.rejected ? "FAIL_UNVERIFIED" : "PASS",
failures: archivedAfterWindow.rejected ? [archivedAfterWindow.failure] : [],
},
currentAfterWindow: {
status: currentAfterWindow.rejected ? "FAIL_UNVERIFIED" : "PASS",
failures: [currentAfterWindow.failure],
},
archivedTampered: {
status: archivedTampered.rejected ? "FAIL_UNVERIFIED" : "PASS",
failures: [archivedTampered.failure],
},
},
externalTreeCanary: {
status: validImmutable ? "PASS" : "FAIL_UNVERIFIED",
failures: validImmutable ? [] : ["external tree canary influenced captured archive"],
},
passingFixtureCount: validImmutable ? 1 : 0,
passingFixtureCount: (validImmutable ? 1 : 0) + (archivedAfterWindow.rejected ? 0 : 1),
status: passed ? "PASS" : "FAIL",
},
});
+16 -4
View File
@@ -1297,6 +1297,10 @@ export const supplyChainProviderFixturesArtifactSchema = z
validImmutable: providerFixtureResultSchema,
wrongDigest: providerFixtureResultSchema,
invalidTar: providerFixtureResultSchema,
// 13-04: archived re-verification of the same bundle three hours on.
archivedAfterWindow: providerFixtureResultSchema,
currentAfterWindow: providerFixtureResultSchema,
archivedTampered: providerFixtureResultSchema,
})
.strict(),
externalTreeCanary: providerFixtureResultSchema,
@@ -1308,11 +1312,19 @@ export const supplyChainProviderFixturesArtifactSchema = z
const expectedPass =
artifact.actualDefaultVerifier.status === "PASS" &&
artifact.fixtures.validImmutable.status === "PASS" &&
artifact.fixtures.archivedAfterWindow.status === "PASS" &&
artifact.externalTreeCanary.status === "PASS" &&
[artifact.fixtures.absent, artifact.fixtures.wrongDigest, artifact.fixtures.invalidTar]
.every(({ status }) => status === "FAIL_UNVERIFIED");
if (artifact.passingFixtureCount !== (artifact.fixtures.validImmutable.status === "PASS" ? 1 : 0)) {
context.addIssue({ code: "custom", path: ["passingFixtureCount"], message: "must count the passing immutable fixture" });
[
artifact.fixtures.absent,
artifact.fixtures.wrongDigest,
artifact.fixtures.invalidTar,
artifact.fixtures.currentAfterWindow,
artifact.fixtures.archivedTampered,
].every(({ status }) => status === "FAIL_UNVERIFIED");
const passing = [artifact.fixtures.validImmutable, artifact.fixtures.archivedAfterWindow]
.filter(({ status }) => status === "PASS").length;
if (artifact.passingFixtureCount !== passing) {
context.addIssue({ code: "custom", path: ["passingFixtureCount"], message: "must count the passing fixtures" });
}
if ((artifact.status === "PASS") !== expectedPass) {
context.addIssue({ code: "custom", path: ["status"], message: "must agree with required fixture outcomes" });
+15 -7
View File
@@ -72,8 +72,12 @@ export type RollbackCoherenceOptions = Readonly<{
}>;
export type DrillOptions = Readonly<{
/** Directory holding `dist/` and `config/runbooks/`. Defaults to the cwd. */
root?: string;
/**
* Directory holding the release's `dist/`. Defaults to the cwd (the local
* build). 13-04: pass an extracted candidate or promoted archive to drill
* the release that is actually deployed or rolled back to.
*/
releaseRoot?: string;
contractPackages?: readonly InstalledContractPackageIdentity[];
}>;
@@ -389,11 +393,11 @@ export async function verifyRollbackReleaseCoherence(
export async function runRunbookDrill(runbookId: string, options: DrillOptions = {}) {
const context: DrillContext = {
root: options.root ?? process.cwd(),
root: options.releaseRoot ?? process.cwd(),
contractPackages: options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES,
};
const document = JSON.parse(
await readFile(path.join(context.root, "config/runbooks/runbooks.json"), "utf8"),
await readFile("config/runbooks/runbooks.json", "utf8"),
) as RunbookDocument;
const specification = document.runbooks[runbookId];
const drill = drillById[runbookId];
@@ -435,12 +439,16 @@ async function main(): Promise<void> {
const runbookId = process.argv
.slice(2)
.find((argument) => /^FE-RB-00[1-5]$/.test(argument));
if (!runbookId) {
process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n");
const releaseRootFlag = process.argv.indexOf("--release-root");
const releaseRoot = releaseRootFlag >= 0 ? process.argv[releaseRootFlag + 1] : undefined;
if (!runbookId || (releaseRootFlag >= 0 && !releaseRoot)) {
process.stderr.write(
"Usage: drill:runbook -- FE-RB-001..FE-RB-005 [--release-root <extracted release archive>]\n",
);
process.exit(2);
}
const { gateId, ...record } = await runRunbookDrill(runbookId);
const { gateId, ...record } = await runRunbookDrill(runbookId, releaseRoot ? { releaseRoot } : {});
const artifactDirectory = `artifacts/runbooks/${runbookId}`;
await mkdir(artifactDirectory, { recursive: true });
await writeValidatedJsonArtifact({
+31 -1
View File
@@ -9,6 +9,7 @@ import {
import {
PROMOTION_VERIFIER_ID,
PROMOTION_VERIFIER_VERSION,
PROVIDER_FUTURE_SKEW_MS,
assertDistinctProviderTrust,
evaluatePromotionEvidence,
providerVerificationArtifactSchema,
@@ -35,6 +36,29 @@ export type ExactPromotionExpectedContext = Readonly<{
lockfileSha256?: string;
}>;
/**
* 13-04. `current` checks provider evidence against the clock, as a new
* promotion must. `archived` re-verifies a release promoted earlier (deploying
* it later, or rolling back to it): signatures, digests and members are checked
* the same way, but freshness is judged at the promotion's recorded
* `verifiedAt`, which may not lie in the future. Promotion itself
* (promotion-stager.ts) never passes `archived`.
*/
export type PromotionFreshness = "current" | "archived";
export function promotionEvidenceTime(
freshness: PromotionFreshness,
verifiedAt: string,
now: number,
): number {
if (freshness === "current") return now;
const verified = Date.parse(verifiedAt);
if (!Number.isFinite(verified) || verified > now + PROVIDER_FUTURE_SKEW_MS) {
throw new Error("archived promotion verifiedAt is in the future or invalid");
}
return verified;
}
export async function verifyExactPromotionBundle(
files: ExactPromotionBundle,
options: Readonly<{
@@ -42,6 +66,7 @@ export async function verifyExactPromotionBundle(
provenanceTrust: ProviderTrust;
expected: ExactPromotionExpectedContext;
nowEpochMs?: () => number;
freshness?: PromotionFreshness;
}>,
): Promise<Readonly<{ status: "PASS" }>> {
assertDistinctProviderTrust(options);
@@ -286,7 +311,12 @@ export async function verifyExactPromotionBundle(
provenanceAttestation: provenance,
vulnerabilityTrust: options.vulnerabilityTrust,
provenanceTrust: options.provenanceTrust,
nowEpochMs: options.nowEpochMs,
nowEpochMs: () =>
promotionEvidenceTime(
options.freshness ?? "current",
promotion.verifiedAt,
(options.nowEpochMs ?? Date.now)(),
),
});
if (
reevaluated.status !== "PASS" ||
+5 -1
View File
@@ -55,9 +55,13 @@ const files = Object.fromEntries(
] as const),
),
);
// 13-04: `--archived` re-verifies a release promoted earlier (deploy later or
// roll back to it); freshness is judged at its recorded promotion time.
const freshness = process.argv.includes("--archived") ? "archived" : "current";
await verifyExactPromotionBundle(files, {
vulnerabilityTrust,
provenanceTrust,
freshness,
expected: {
run: {
id: required("EXPECTED_PROMOTION_RUN_ID"),
@@ -71,7 +75,7 @@ await verifyExactPromotionBundle(files, {
...optionalDigest("lockfileSha256", "EXPECTED_PROMOTION_LOCKFILE_SHA256"),
},
});
process.stdout.write("Exact promotion bundle verification: PASS\n");
process.stdout.write(`Exact promotion bundle verification (${freshness}): PASS\n`);
function asciiCompare(left: string, right: string): number {
return left < right ? -1 : left > right ? 1 : 0;
+48
View File
@@ -0,0 +1,48 @@
import { describe, expect, it } from "vitest";
import { promotionEvidenceTime } from "../../scripts/lib/exact-promotion-bundle.ts";
import { evaluatePromotionEvidence } from "../../scripts/lib/provider-evidence.ts";
import { syntheticSignedPromotionBundle } from "./security-followup-fixture.ts";
// 13-04. Provider evidence lives two hours at most, so a promoted release could
// not be re-verified the next day, which the rollback runbook requires. The
// archived mode checks freshness at the recorded promotion time instead.
const promotedAt = Date.parse("2026-08-02T01:00:00.000Z");
const threeHoursLater = promotedAt + 3 * 60 * 60 * 1_000;
function evidenceAt(freshness: "current" | "archived", now: number) {
const fixture = syntheticSignedPromotionBundle();
const provider = JSON.parse(fixture.files["provider-verification.json"].toString("utf8"));
return evaluatePromotionEvidence({
expected: {
run: provider.run,
source: provider.source,
candidate: provider.candidate,
vulnerabilityInvocationNonce: provider.providerEvidence.vulnerabilityInvocationNonce,
provenanceInvocationNonce: provider.providerEvidence.provenanceInvocationNonce,
secretScanAttestation: provider.providerEvidence.secretScanAttestation,
},
localStatus: "PASS",
vulnerabilityReport: JSON.parse(fixture.files["vulnerability-report.json"].toString("utf8")),
provenanceAttestation: JSON.parse(fixture.files["provenance-attestation.json"].toString("utf8")),
vulnerabilityTrust: fixture.verification.vulnerabilityTrust,
provenanceTrust: fixture.verification.provenanceTrust,
nowEpochMs: () => promotionEvidenceTime(freshness, provider.verifiedAt, now),
});
}
describe("archived promotion re-verification", () => {
it("accepts evidence three hours after promotion in archived mode only", () => {
expect(evidenceAt("archived", threeHoursLater).status).toBe("PASS");
expect(evidenceAt("current", threeHoursLater)).toMatchObject({
status: "FAIL_UNVERIFIED",
failures: expect.arrayContaining(["vulnerability report is expired"]),
});
});
it("refuses a promotion time recorded in the future", () => {
expect(() =>
promotionEvidenceTime("archived", "2026-08-02T05:00:00.000Z", promotedAt),
).toThrow(/verifiedAt is in the future/u);
});
});
+3 -8
View File
@@ -32,11 +32,6 @@ afterEach(async () => {
async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?: boolean }> = {}) {
const root = await mkdtemp(path.join(tmpdir(), "runbook-drill-"));
roots.push(root);
await mkdir(path.join(root, "config/runbooks"), { recursive: true });
await writeFile(
path.join(root, "config/runbooks/runbooks.json"),
await readFile("config/runbooks/runbooks.json"),
);
if (options.withoutDist) return root;
const write = async (file: string, content: string) => {
await mkdir(path.dirname(path.join(root, file)), { recursive: true });
@@ -103,7 +98,7 @@ async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?:
describe("runbook drill verdicts", () => {
it.each(RUNBOOK_IDS)("%s on a complete local build is UNVERIFIED, not PASS", async (runbookId) => {
const record = await runRunbookDrill(runbookId, { root: await builtDist(), contractPackages });
const record = await runRunbookDrill(runbookId, { releaseRoot: await builtDist(), contractPackages });
expect(record.result).toBe("UNVERIFIED");
expect(record.recoveryAssertions.some(({ result }) => result === "FAIL")).toBe(false);
});
@@ -112,7 +107,7 @@ describe("runbook drill verdicts", () => {
"%s fails when the build lost its JavaScript assets",
async (runbookId) => {
const record = await runRunbookDrill(runbookId, {
root: await builtDist({ withoutJs: true }),
releaseRoot: await builtDist({ withoutJs: true }),
contractPackages,
});
expect(record.result).toBe("FAIL");
@@ -121,7 +116,7 @@ describe("runbook drill verdicts", () => {
it("FE-RB-005 fails without a build instead of reading public/ placeholders", async () => {
await expect(
runRunbookDrill("FE-RB-005", { root: await builtDist({ withoutDist: true }), contractPackages }),
runRunbookDrill("FE-RB-005", { releaseRoot: await builtDist({ withoutDist: true }), contractPackages }),
).rejects.toThrow(/dist\/release-manifest\.json/u);
});