feat(release): re-verify a promoted release in archived mode (13-04)
Provider evidence expires two hours after issue, so no repo tool could re-verify a release before deploying it later or rolling back to it. verify:promotion --archived keeps every signature, digest and member check and judges freshness at the recorded promotion time; the FE-RB-005 drill takes the extracted release with --release-root. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
1a1cfe6605
commit
839205c134
@@ -12,6 +12,32 @@ cache action, and repeats the smoke checks. Rebuilding an old commit, replacing
|
||||
HTML alone, or declaring recovery from cache-purge completion is prohibited.
|
||||
Recovery is established by old/new reachability probes.
|
||||
|
||||
Before deploying a promoted release later, or rolling back to one, re-verify
|
||||
its promoted bundle (the five files uploaded by the `promotion` job) with the
|
||||
archived mode (review 13-04). Provider evidence expires two hours after issue,
|
||||
so the normal mode refuses any release promoted before that. The archived mode
|
||||
still checks signatures, digests and the exact five members, and judges
|
||||
freshness at the release's recorded promotion time
|
||||
(`promotion-verification.json` `verifiedAt`, which may not lie in the future).
|
||||
New promotion never uses it (`scripts/lib/exact-promotion-bundle.ts`).
|
||||
|
||||
```bash
|
||||
VULNERABILITY_PUBLIC_KEY_PATH=... VULNERABILITY_KEY_ID=... \
|
||||
PROVENANCE_PUBLIC_KEY_PATH=... PROVENANCE_KEY_ID=... \
|
||||
EXPECTED_PROMOTION_RUN_ID=<run id> EXPECTED_PROMOTION_RUN_ATTEMPT=<attempt> \
|
||||
EXPECTED_PROMOTION_SOURCE_REVISION=<commit> \
|
||||
EXPECTED_PROMOTION_ARCHIVE_SHA256=<archive sha256> \
|
||||
corepack pnpm verify:promotion -- --root <promoted bundle directory> --archived
|
||||
|
||||
mkdir <release dir> && tar -xzf <promoted bundle directory>/release-candidate.tar.gz -C <release dir>
|
||||
corepack pnpm drill:runbook -- FE-RB-005 --release-root <release dir>
|
||||
```
|
||||
|
||||
The expected values come from the release record, not from the bundle. The
|
||||
drill then checks that release's build/config/manifest tuple and its assets,
|
||||
not the local build. It ends `UNVERIFIED` until the provider switch and smoke
|
||||
are recorded, because those happen outside this repository.
|
||||
|
||||
The provider-independent cache defaults are:
|
||||
|
||||
- hashed assets: `public, max-age=31536000, immutable`
|
||||
|
||||
@@ -11,3 +11,8 @@ Close only when compatibility and release-coherence gates pass, critical smoke
|
||||
passes, repeated `DEPLOY_MISMATCH` is absent, and the incident timeline records
|
||||
the restored release ID. Pointer-switch or cache-purge completion alone is not
|
||||
recovery evidence.
|
||||
|
||||
Verify the prior release before switching to it: `verify:promotion -- --root
|
||||
<bundle> --archived` and `drill:runbook -- FE-RB-005 --release-root <extracted
|
||||
archive>`. The commands and inputs are in
|
||||
[`docs/operations/release-cache-rollback.md`](../operations/release-cache-rollback.md).
|
||||
|
||||
@@ -249,6 +249,31 @@ try {
|
||||
},
|
||||
nowEpochMs: () => now,
|
||||
} as const;
|
||||
// 13-04: the same bundle three hours later. Current freshness refuses the
|
||||
// expired provider evidence; archived freshness accepts it at the recorded
|
||||
// promotion time, and still refuses a tampered archive.
|
||||
const threeHoursLater = now + 3 * 60 * 60 * 1_000;
|
||||
const archivedAfterWindow = await captureRejection(async () => {
|
||||
await verifyExactPromotionBundle(exactFiles, {
|
||||
...bundleTrust,
|
||||
nowEpochMs: () => threeHoursLater,
|
||||
freshness: "archived",
|
||||
});
|
||||
});
|
||||
const currentAfterWindow = await captureRejection(async () => {
|
||||
await verifyExactPromotionBundle(exactFiles, {
|
||||
...bundleTrust,
|
||||
nowEpochMs: () => threeHoursLater,
|
||||
});
|
||||
});
|
||||
const tamperedArchive = Buffer.from(exactFiles["release-candidate.tar.gz"]);
|
||||
tamperedArchive[tamperedArchive.length - 1] ^= 0xff;
|
||||
const archivedTampered = await captureRejection(async () => {
|
||||
await verifyExactPromotionBundle(
|
||||
{ ...exactFiles, "release-candidate.tar.gz": tamperedArchive },
|
||||
{ ...bundleTrust, nowEpochMs: () => threeHoursLater, freshness: "archived" },
|
||||
);
|
||||
});
|
||||
const absentFiles = { ...exactFiles } as Partial<typeof exactFiles>;
|
||||
delete absentFiles["provider-verification.json"];
|
||||
const absent = await captureRejection(async () => {
|
||||
@@ -282,7 +307,8 @@ try {
|
||||
await captureCiCandidateArchive({ archivePath, expectedSha256: "0".repeat(64) });
|
||||
});
|
||||
const passed =
|
||||
validImmutable && absent.rejected && invalidTar.rejected && wrongDigest.rejected;
|
||||
validImmutable && absent.rejected && invalidTar.rejected && wrongDigest.rejected &&
|
||||
!archivedAfterWindow.rejected && currentAfterWindow.rejected && archivedTampered.rejected;
|
||||
await cleanupFinalizedPromotion({
|
||||
runnerTempRoot,
|
||||
stagingRoot: finalized.stagingRoot,
|
||||
@@ -314,12 +340,24 @@ try {
|
||||
status: invalidTar.rejected ? "FAIL_UNVERIFIED" : "PASS",
|
||||
failures: [invalidTar.failure],
|
||||
},
|
||||
archivedAfterWindow: {
|
||||
status: archivedAfterWindow.rejected ? "FAIL_UNVERIFIED" : "PASS",
|
||||
failures: archivedAfterWindow.rejected ? [archivedAfterWindow.failure] : [],
|
||||
},
|
||||
currentAfterWindow: {
|
||||
status: currentAfterWindow.rejected ? "FAIL_UNVERIFIED" : "PASS",
|
||||
failures: [currentAfterWindow.failure],
|
||||
},
|
||||
archivedTampered: {
|
||||
status: archivedTampered.rejected ? "FAIL_UNVERIFIED" : "PASS",
|
||||
failures: [archivedTampered.failure],
|
||||
},
|
||||
},
|
||||
externalTreeCanary: {
|
||||
status: validImmutable ? "PASS" : "FAIL_UNVERIFIED",
|
||||
failures: validImmutable ? [] : ["external tree canary influenced captured archive"],
|
||||
},
|
||||
passingFixtureCount: validImmutable ? 1 : 0,
|
||||
passingFixtureCount: (validImmutable ? 1 : 0) + (archivedAfterWindow.rejected ? 0 : 1),
|
||||
status: passed ? "PASS" : "FAIL",
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1297,6 +1297,10 @@ export const supplyChainProviderFixturesArtifactSchema = z
|
||||
validImmutable: providerFixtureResultSchema,
|
||||
wrongDigest: providerFixtureResultSchema,
|
||||
invalidTar: providerFixtureResultSchema,
|
||||
// 13-04: archived re-verification of the same bundle three hours on.
|
||||
archivedAfterWindow: providerFixtureResultSchema,
|
||||
currentAfterWindow: providerFixtureResultSchema,
|
||||
archivedTampered: providerFixtureResultSchema,
|
||||
})
|
||||
.strict(),
|
||||
externalTreeCanary: providerFixtureResultSchema,
|
||||
@@ -1308,11 +1312,19 @@ export const supplyChainProviderFixturesArtifactSchema = z
|
||||
const expectedPass =
|
||||
artifact.actualDefaultVerifier.status === "PASS" &&
|
||||
artifact.fixtures.validImmutable.status === "PASS" &&
|
||||
artifact.fixtures.archivedAfterWindow.status === "PASS" &&
|
||||
artifact.externalTreeCanary.status === "PASS" &&
|
||||
[artifact.fixtures.absent, artifact.fixtures.wrongDigest, artifact.fixtures.invalidTar]
|
||||
.every(({ status }) => status === "FAIL_UNVERIFIED");
|
||||
if (artifact.passingFixtureCount !== (artifact.fixtures.validImmutable.status === "PASS" ? 1 : 0)) {
|
||||
context.addIssue({ code: "custom", path: ["passingFixtureCount"], message: "must count the passing immutable fixture" });
|
||||
[
|
||||
artifact.fixtures.absent,
|
||||
artifact.fixtures.wrongDigest,
|
||||
artifact.fixtures.invalidTar,
|
||||
artifact.fixtures.currentAfterWindow,
|
||||
artifact.fixtures.archivedTampered,
|
||||
].every(({ status }) => status === "FAIL_UNVERIFIED");
|
||||
const passing = [artifact.fixtures.validImmutable, artifact.fixtures.archivedAfterWindow]
|
||||
.filter(({ status }) => status === "PASS").length;
|
||||
if (artifact.passingFixtureCount !== passing) {
|
||||
context.addIssue({ code: "custom", path: ["passingFixtureCount"], message: "must count the passing fixtures" });
|
||||
}
|
||||
if ((artifact.status === "PASS") !== expectedPass) {
|
||||
context.addIssue({ code: "custom", path: ["status"], message: "must agree with required fixture outcomes" });
|
||||
|
||||
@@ -72,8 +72,12 @@ export type RollbackCoherenceOptions = Readonly<{
|
||||
}>;
|
||||
|
||||
export type DrillOptions = Readonly<{
|
||||
/** Directory holding `dist/` and `config/runbooks/`. Defaults to the cwd. */
|
||||
root?: string;
|
||||
/**
|
||||
* Directory holding the release's `dist/`. Defaults to the cwd (the local
|
||||
* build). 13-04: pass an extracted candidate or promoted archive to drill
|
||||
* the release that is actually deployed or rolled back to.
|
||||
*/
|
||||
releaseRoot?: string;
|
||||
contractPackages?: readonly InstalledContractPackageIdentity[];
|
||||
}>;
|
||||
|
||||
@@ -389,11 +393,11 @@ export async function verifyRollbackReleaseCoherence(
|
||||
|
||||
export async function runRunbookDrill(runbookId: string, options: DrillOptions = {}) {
|
||||
const context: DrillContext = {
|
||||
root: options.root ?? process.cwd(),
|
||||
root: options.releaseRoot ?? process.cwd(),
|
||||
contractPackages: options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES,
|
||||
};
|
||||
const document = JSON.parse(
|
||||
await readFile(path.join(context.root, "config/runbooks/runbooks.json"), "utf8"),
|
||||
await readFile("config/runbooks/runbooks.json", "utf8"),
|
||||
) as RunbookDocument;
|
||||
const specification = document.runbooks[runbookId];
|
||||
const drill = drillById[runbookId];
|
||||
@@ -435,12 +439,16 @@ async function main(): Promise<void> {
|
||||
const runbookId = process.argv
|
||||
.slice(2)
|
||||
.find((argument) => /^FE-RB-00[1-5]$/.test(argument));
|
||||
if (!runbookId) {
|
||||
process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n");
|
||||
const releaseRootFlag = process.argv.indexOf("--release-root");
|
||||
const releaseRoot = releaseRootFlag >= 0 ? process.argv[releaseRootFlag + 1] : undefined;
|
||||
if (!runbookId || (releaseRootFlag >= 0 && !releaseRoot)) {
|
||||
process.stderr.write(
|
||||
"Usage: drill:runbook -- FE-RB-001..FE-RB-005 [--release-root <extracted release archive>]\n",
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const { gateId, ...record } = await runRunbookDrill(runbookId);
|
||||
const { gateId, ...record } = await runRunbookDrill(runbookId, releaseRoot ? { releaseRoot } : {});
|
||||
const artifactDirectory = `artifacts/runbooks/${runbookId}`;
|
||||
await mkdir(artifactDirectory, { recursive: true });
|
||||
await writeValidatedJsonArtifact({
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
import {
|
||||
PROMOTION_VERIFIER_ID,
|
||||
PROMOTION_VERIFIER_VERSION,
|
||||
PROVIDER_FUTURE_SKEW_MS,
|
||||
assertDistinctProviderTrust,
|
||||
evaluatePromotionEvidence,
|
||||
providerVerificationArtifactSchema,
|
||||
@@ -35,6 +36,29 @@ export type ExactPromotionExpectedContext = Readonly<{
|
||||
lockfileSha256?: string;
|
||||
}>;
|
||||
|
||||
/**
|
||||
* 13-04. `current` checks provider evidence against the clock, as a new
|
||||
* promotion must. `archived` re-verifies a release promoted earlier (deploying
|
||||
* it later, or rolling back to it): signatures, digests and members are checked
|
||||
* the same way, but freshness is judged at the promotion's recorded
|
||||
* `verifiedAt`, which may not lie in the future. Promotion itself
|
||||
* (promotion-stager.ts) never passes `archived`.
|
||||
*/
|
||||
export type PromotionFreshness = "current" | "archived";
|
||||
|
||||
export function promotionEvidenceTime(
|
||||
freshness: PromotionFreshness,
|
||||
verifiedAt: string,
|
||||
now: number,
|
||||
): number {
|
||||
if (freshness === "current") return now;
|
||||
const verified = Date.parse(verifiedAt);
|
||||
if (!Number.isFinite(verified) || verified > now + PROVIDER_FUTURE_SKEW_MS) {
|
||||
throw new Error("archived promotion verifiedAt is in the future or invalid");
|
||||
}
|
||||
return verified;
|
||||
}
|
||||
|
||||
export async function verifyExactPromotionBundle(
|
||||
files: ExactPromotionBundle,
|
||||
options: Readonly<{
|
||||
@@ -42,6 +66,7 @@ export async function verifyExactPromotionBundle(
|
||||
provenanceTrust: ProviderTrust;
|
||||
expected: ExactPromotionExpectedContext;
|
||||
nowEpochMs?: () => number;
|
||||
freshness?: PromotionFreshness;
|
||||
}>,
|
||||
): Promise<Readonly<{ status: "PASS" }>> {
|
||||
assertDistinctProviderTrust(options);
|
||||
@@ -286,7 +311,12 @@ export async function verifyExactPromotionBundle(
|
||||
provenanceAttestation: provenance,
|
||||
vulnerabilityTrust: options.vulnerabilityTrust,
|
||||
provenanceTrust: options.provenanceTrust,
|
||||
nowEpochMs: options.nowEpochMs,
|
||||
nowEpochMs: () =>
|
||||
promotionEvidenceTime(
|
||||
options.freshness ?? "current",
|
||||
promotion.verifiedAt,
|
||||
(options.nowEpochMs ?? Date.now)(),
|
||||
),
|
||||
});
|
||||
if (
|
||||
reevaluated.status !== "PASS" ||
|
||||
|
||||
@@ -55,9 +55,13 @@ const files = Object.fromEntries(
|
||||
] as const),
|
||||
),
|
||||
);
|
||||
// 13-04: `--archived` re-verifies a release promoted earlier (deploy later or
|
||||
// roll back to it); freshness is judged at its recorded promotion time.
|
||||
const freshness = process.argv.includes("--archived") ? "archived" : "current";
|
||||
await verifyExactPromotionBundle(files, {
|
||||
vulnerabilityTrust,
|
||||
provenanceTrust,
|
||||
freshness,
|
||||
expected: {
|
||||
run: {
|
||||
id: required("EXPECTED_PROMOTION_RUN_ID"),
|
||||
@@ -71,7 +75,7 @@ await verifyExactPromotionBundle(files, {
|
||||
...optionalDigest("lockfileSha256", "EXPECTED_PROMOTION_LOCKFILE_SHA256"),
|
||||
},
|
||||
});
|
||||
process.stdout.write("Exact promotion bundle verification: PASS\n");
|
||||
process.stdout.write(`Exact promotion bundle verification (${freshness}): PASS\n`);
|
||||
|
||||
function asciiCompare(left: string, right: string): number {
|
||||
return left < right ? -1 : left > right ? 1 : 0;
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { promotionEvidenceTime } from "../../scripts/lib/exact-promotion-bundle.ts";
|
||||
import { evaluatePromotionEvidence } from "../../scripts/lib/provider-evidence.ts";
|
||||
import { syntheticSignedPromotionBundle } from "./security-followup-fixture.ts";
|
||||
|
||||
// 13-04. Provider evidence lives two hours at most, so a promoted release could
|
||||
// not be re-verified the next day, which the rollback runbook requires. The
|
||||
// archived mode checks freshness at the recorded promotion time instead.
|
||||
const promotedAt = Date.parse("2026-08-02T01:00:00.000Z");
|
||||
const threeHoursLater = promotedAt + 3 * 60 * 60 * 1_000;
|
||||
|
||||
function evidenceAt(freshness: "current" | "archived", now: number) {
|
||||
const fixture = syntheticSignedPromotionBundle();
|
||||
const provider = JSON.parse(fixture.files["provider-verification.json"].toString("utf8"));
|
||||
return evaluatePromotionEvidence({
|
||||
expected: {
|
||||
run: provider.run,
|
||||
source: provider.source,
|
||||
candidate: provider.candidate,
|
||||
vulnerabilityInvocationNonce: provider.providerEvidence.vulnerabilityInvocationNonce,
|
||||
provenanceInvocationNonce: provider.providerEvidence.provenanceInvocationNonce,
|
||||
secretScanAttestation: provider.providerEvidence.secretScanAttestation,
|
||||
},
|
||||
localStatus: "PASS",
|
||||
vulnerabilityReport: JSON.parse(fixture.files["vulnerability-report.json"].toString("utf8")),
|
||||
provenanceAttestation: JSON.parse(fixture.files["provenance-attestation.json"].toString("utf8")),
|
||||
vulnerabilityTrust: fixture.verification.vulnerabilityTrust,
|
||||
provenanceTrust: fixture.verification.provenanceTrust,
|
||||
nowEpochMs: () => promotionEvidenceTime(freshness, provider.verifiedAt, now),
|
||||
});
|
||||
}
|
||||
|
||||
describe("archived promotion re-verification", () => {
|
||||
it("accepts evidence three hours after promotion in archived mode only", () => {
|
||||
expect(evidenceAt("archived", threeHoursLater).status).toBe("PASS");
|
||||
expect(evidenceAt("current", threeHoursLater)).toMatchObject({
|
||||
status: "FAIL_UNVERIFIED",
|
||||
failures: expect.arrayContaining(["vulnerability report is expired"]),
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses a promotion time recorded in the future", () => {
|
||||
expect(() =>
|
||||
promotionEvidenceTime("archived", "2026-08-02T05:00:00.000Z", promotedAt),
|
||||
).toThrow(/verifiedAt is in the future/u);
|
||||
});
|
||||
});
|
||||
@@ -32,11 +32,6 @@ afterEach(async () => {
|
||||
async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?: boolean }> = {}) {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "runbook-drill-"));
|
||||
roots.push(root);
|
||||
await mkdir(path.join(root, "config/runbooks"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(root, "config/runbooks/runbooks.json"),
|
||||
await readFile("config/runbooks/runbooks.json"),
|
||||
);
|
||||
if (options.withoutDist) return root;
|
||||
const write = async (file: string, content: string) => {
|
||||
await mkdir(path.dirname(path.join(root, file)), { recursive: true });
|
||||
@@ -103,7 +98,7 @@ async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?:
|
||||
|
||||
describe("runbook drill verdicts", () => {
|
||||
it.each(RUNBOOK_IDS)("%s on a complete local build is UNVERIFIED, not PASS", async (runbookId) => {
|
||||
const record = await runRunbookDrill(runbookId, { root: await builtDist(), contractPackages });
|
||||
const record = await runRunbookDrill(runbookId, { releaseRoot: await builtDist(), contractPackages });
|
||||
expect(record.result).toBe("UNVERIFIED");
|
||||
expect(record.recoveryAssertions.some(({ result }) => result === "FAIL")).toBe(false);
|
||||
});
|
||||
@@ -112,7 +107,7 @@ describe("runbook drill verdicts", () => {
|
||||
"%s fails when the build lost its JavaScript assets",
|
||||
async (runbookId) => {
|
||||
const record = await runRunbookDrill(runbookId, {
|
||||
root: await builtDist({ withoutJs: true }),
|
||||
releaseRoot: await builtDist({ withoutJs: true }),
|
||||
contractPackages,
|
||||
});
|
||||
expect(record.result).toBe("FAIL");
|
||||
@@ -121,7 +116,7 @@ describe("runbook drill verdicts", () => {
|
||||
|
||||
it("FE-RB-005 fails without a build instead of reading public/ placeholders", async () => {
|
||||
await expect(
|
||||
runRunbookDrill("FE-RB-005", { root: await builtDist({ withoutDist: true }), contractPackages }),
|
||||
runRunbookDrill("FE-RB-005", { releaseRoot: await builtDist({ withoutDist: true }), contractPackages }),
|
||||
).rejects.toThrow(/dist\/release-manifest\.json/u);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user