fix(release): record unobserved runbook drill verdicts as UNVERIFIED (11-01, 13-03)
Drills held constant true verdicts and read public/ placeholders, so they passed with no build or with JS assets deleted. They now check the local dist, mark boot, live traffic and provider steps UNVERIFIED (exit 3), and FE-GATE-016 is retired because it repeated FE-GATE-025's FE-RB-005 drill. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
7298275271
commit
7f889a6d83
@@ -438,6 +438,19 @@ export type CiGateContractIndex = Readonly<{
|
||||
jobs: ReadonlyMap<string, CiWorkflowJob>;
|
||||
retentionClasses: ReadonlyMap<string, CiGateContract["retention"]["classes"][number]>;
|
||||
}>;
|
||||
|
||||
/**
|
||||
* D-14. FE-GATE-016 (rollback-drill) ran the same FE-RB-005 drill as
|
||||
* FE-GATE-025 and counted it twice toward production readiness. Its id stays
|
||||
* retired so no later gate reuses it with a different meaning.
|
||||
*/
|
||||
export const RETIRED_GATE_IDS: readonly string[] = Object.freeze(["FE-GATE-016"]);
|
||||
|
||||
export const CANONICAL_GATE_IDS: readonly string[] = Object.freeze(
|
||||
Array.from({ length: 28 }, (_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`)
|
||||
.filter((gateId) => !RETIRED_GATE_IDS.includes(gateId)),
|
||||
);
|
||||
|
||||
/**
|
||||
* D-13. The contract's size and gate shape are a reviewed baseline, not code
|
||||
* constants: a project that removes a sample or adds a gate records the new
|
||||
@@ -789,12 +802,9 @@ function validateContractSemantics(
|
||||
issue(`stage dependency cycle: ${cycle.join(" -> ")}`);
|
||||
}
|
||||
|
||||
const expectedGateIds = Array.from(
|
||||
{ length: 28 },
|
||||
(_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`,
|
||||
);
|
||||
const expectedGateIds = CANONICAL_GATE_IDS;
|
||||
if (JSON.stringify(contract.gates.map(({ id }) => id)) !== JSON.stringify(expectedGateIds)) {
|
||||
issue("gate registry must contain FE-GATE-001..028 in canonical order");
|
||||
issue(`gate registry must contain FE-GATE-001..028 without retired ${RETIRED_GATE_IDS.join(", ")} in canonical order`);
|
||||
}
|
||||
const expectedStages: ReadonlyArray<readonly [string, string, readonly string[], readonly string[]]> = [
|
||||
["merge", "MERGE_READY", [], PROMOTION_FORMULA.MERGE_READY],
|
||||
@@ -869,7 +879,7 @@ function validateContractSemantics(
|
||||
vulnerability_provider: [],
|
||||
provenance_provider: [],
|
||||
promotion: [],
|
||||
production_gate: ["FE-GATE-016", "FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"],
|
||||
production_gate: ["FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"],
|
||||
field_gate: ["FE-GATE-018"],
|
||||
documentation_gate: ["FE-GATE-017"],
|
||||
};
|
||||
|
||||
@@ -718,31 +718,67 @@ export const releaseVerificationArtifactSchema = z
|
||||
})
|
||||
.strict();
|
||||
|
||||
// 11-01 / 13-03 (D-14): each verdict is PASS or FAIL only when observed on the
|
||||
// local build; anything that needs a boot, live traffic or the provider is
|
||||
// UNVERIFIED, and so is a drill with provider scope.
|
||||
const drillVerdict = z.enum(["PASS", "FAIL", "UNVERIFIED"]);
|
||||
|
||||
export type DrillVerdict = z.infer<typeof drillVerdict>;
|
||||
|
||||
/** FAIL beats UNVERIFIED beats PASS; provider scope alone keeps a drill UNVERIFIED. */
|
||||
export function combineDrillVerdicts(
|
||||
verdicts: readonly DrillVerdict[],
|
||||
providerVerificationRequired: boolean,
|
||||
): DrillVerdict {
|
||||
if (verdicts.includes("FAIL")) return "FAIL";
|
||||
if (verdicts.includes("UNVERIFIED") || providerVerificationRequired) return "UNVERIFIED";
|
||||
return "PASS";
|
||||
}
|
||||
|
||||
export const runbookRecordArtifactSchema = z
|
||||
.object({
|
||||
schemaVersion: z.literal(1),
|
||||
schemaVersion: z.literal(2),
|
||||
runbookId: z.string().regex(/^FE-RB-00[1-5]$/u),
|
||||
releaseId: nonEmptyString,
|
||||
drillTimestamp: timestamp,
|
||||
triggerInjected: nonEmptyString,
|
||||
triggerAsserted: z.boolean(),
|
||||
containmentAsserted: z.boolean(),
|
||||
escalationPathAsserted: z.boolean(),
|
||||
triggerAsserted: drillVerdict,
|
||||
containmentAsserted: drillVerdict,
|
||||
escalationPathAsserted: drillVerdict,
|
||||
recoveryAssertions: z.array(
|
||||
z
|
||||
.object({
|
||||
assertion: nonEmptyString,
|
||||
evidence: nonEmptyString,
|
||||
passed: z.boolean(),
|
||||
result: drillVerdict,
|
||||
})
|
||||
.strict(),
|
||||
),
|
||||
negativeFixtureFailedAsExpected: z.boolean(),
|
||||
negativeFixtureFailedAsExpected: drillVerdict,
|
||||
windowObservedBucket: nonEmptyString,
|
||||
providerVerificationRequired: z.boolean(),
|
||||
passed: z.boolean(),
|
||||
result: drillVerdict,
|
||||
})
|
||||
.strict();
|
||||
.strict()
|
||||
.superRefine((record, context) => {
|
||||
const expected = combineDrillVerdicts(
|
||||
[
|
||||
record.triggerAsserted,
|
||||
record.containmentAsserted,
|
||||
record.escalationPathAsserted,
|
||||
...record.recoveryAssertions.map(({ result }) => result),
|
||||
record.negativeFixtureFailedAsExpected,
|
||||
],
|
||||
record.providerVerificationRequired,
|
||||
);
|
||||
if (record.result !== expected) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["result"],
|
||||
message: `result must be ${expected} for these verdicts`,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const failureList = z.array(nonEmptyString).max(4_096);
|
||||
const sourceOrFixtureMode = z.enum(["source", "negative-fixture"]);
|
||||
|
||||
Reference in New Issue
Block a user