fix(release): record unobserved runbook drill verdicts as UNVERIFIED (11-01, 13-03)

Drills held constant true verdicts and read public/ placeholders, so they
passed with no build or with JS assets deleted. They now check the local dist,
mark boot, live traffic and provider steps UNVERIFIED (exit 3), and FE-GATE-016
is retired because it repeated FE-GATE-025's FE-RB-005 drill.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-25 22:15:59 +09:00
co-authored by Claude Opus 5.5
parent 7298275271
commit 7f889a6d83
17 changed files with 482 additions and 334 deletions
+16 -6
View File
@@ -438,6 +438,19 @@ export type CiGateContractIndex = Readonly<{
jobs: ReadonlyMap<string, CiWorkflowJob>;
retentionClasses: ReadonlyMap<string, CiGateContract["retention"]["classes"][number]>;
}>;
/**
* D-14. FE-GATE-016 (rollback-drill) ran the same FE-RB-005 drill as
* FE-GATE-025 and counted it twice toward production readiness. Its id stays
* retired so no later gate reuses it with a different meaning.
*/
export const RETIRED_GATE_IDS: readonly string[] = Object.freeze(["FE-GATE-016"]);
export const CANONICAL_GATE_IDS: readonly string[] = Object.freeze(
Array.from({ length: 28 }, (_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`)
.filter((gateId) => !RETIRED_GATE_IDS.includes(gateId)),
);
/**
* D-13. The contract's size and gate shape are a reviewed baseline, not code
* constants: a project that removes a sample or adds a gate records the new
@@ -789,12 +802,9 @@ function validateContractSemantics(
issue(`stage dependency cycle: ${cycle.join(" -> ")}`);
}
const expectedGateIds = Array.from(
{ length: 28 },
(_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`,
);
const expectedGateIds = CANONICAL_GATE_IDS;
if (JSON.stringify(contract.gates.map(({ id }) => id)) !== JSON.stringify(expectedGateIds)) {
issue("gate registry must contain FE-GATE-001..028 in canonical order");
issue(`gate registry must contain FE-GATE-001..028 without retired ${RETIRED_GATE_IDS.join(", ")} in canonical order`);
}
const expectedStages: ReadonlyArray<readonly [string, string, readonly string[], readonly string[]]> = [
["merge", "MERGE_READY", [], PROMOTION_FORMULA.MERGE_READY],
@@ -869,7 +879,7 @@ function validateContractSemantics(
vulnerability_provider: [],
provenance_provider: [],
promotion: [],
production_gate: ["FE-GATE-016", "FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"],
production_gate: ["FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"],
field_gate: ["FE-GATE-018"],
documentation_gate: ["FE-GATE-017"],
};
+44 -8
View File
@@ -718,31 +718,67 @@ export const releaseVerificationArtifactSchema = z
})
.strict();
// 11-01 / 13-03 (D-14): each verdict is PASS or FAIL only when observed on the
// local build; anything that needs a boot, live traffic or the provider is
// UNVERIFIED, and so is a drill with provider scope.
const drillVerdict = z.enum(["PASS", "FAIL", "UNVERIFIED"]);
export type DrillVerdict = z.infer<typeof drillVerdict>;
/** FAIL beats UNVERIFIED beats PASS; provider scope alone keeps a drill UNVERIFIED. */
export function combineDrillVerdicts(
verdicts: readonly DrillVerdict[],
providerVerificationRequired: boolean,
): DrillVerdict {
if (verdicts.includes("FAIL")) return "FAIL";
if (verdicts.includes("UNVERIFIED") || providerVerificationRequired) return "UNVERIFIED";
return "PASS";
}
export const runbookRecordArtifactSchema = z
.object({
schemaVersion: z.literal(1),
schemaVersion: z.literal(2),
runbookId: z.string().regex(/^FE-RB-00[1-5]$/u),
releaseId: nonEmptyString,
drillTimestamp: timestamp,
triggerInjected: nonEmptyString,
triggerAsserted: z.boolean(),
containmentAsserted: z.boolean(),
escalationPathAsserted: z.boolean(),
triggerAsserted: drillVerdict,
containmentAsserted: drillVerdict,
escalationPathAsserted: drillVerdict,
recoveryAssertions: z.array(
z
.object({
assertion: nonEmptyString,
evidence: nonEmptyString,
passed: z.boolean(),
result: drillVerdict,
})
.strict(),
),
negativeFixtureFailedAsExpected: z.boolean(),
negativeFixtureFailedAsExpected: drillVerdict,
windowObservedBucket: nonEmptyString,
providerVerificationRequired: z.boolean(),
passed: z.boolean(),
result: drillVerdict,
})
.strict();
.strict()
.superRefine((record, context) => {
const expected = combineDrillVerdicts(
[
record.triggerAsserted,
record.containmentAsserted,
record.escalationPathAsserted,
...record.recoveryAssertions.map(({ result }) => result),
record.negativeFixtureFailedAsExpected,
],
record.providerVerificationRequired,
);
if (record.result !== expected) {
context.addIssue({
code: "custom",
path: ["result"],
message: `result must be ${expected} for these verdicts`,
});
}
});
const failureList = z.array(nonEmptyString).max(4_096);
const sourceOrFixtureMode = z.enum(["source", "negative-fixture"]);