fix(release): record unobserved runbook drill verdicts as UNVERIFIED (11-01, 13-03)

Drills held constant true verdicts and read public/ placeholders, so they
passed with no build or with JS assets deleted. They now check the local dist,
mark boot, live traffic and provider steps UNVERIFIED (exit 3), and FE-GATE-016
is retired because it repeated FE-GATE-025's FE-RB-005 drill.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-25 22:15:59 +09:00
co-authored by Claude Opus 5.5
parent 7298275271
commit 7f889a6d83
17 changed files with 482 additions and 334 deletions
-1
View File
@@ -379,7 +379,6 @@ jobs:
fail-fast: false
matrix:
include:
- { gate: FE-GATE-016, name: rollback-drill }
- { gate: FE-GATE-021, name: runbook-boot-config }
- { gate: FE-GATE-022, name: runbook-chunk-mismatch }
- { gate: FE-GATE-023, name: runbook-api-degradation }
+6 -1
View File
@@ -123,6 +123,11 @@ corepack pnpm drill:runbooks
corepack pnpm check:ci
```
`drill:runbooks` stops at the first drill that is not `PASS`. Without provider
evidence every drill ends `UNVERIFIED` (exit 3), because a drill never reports
a check it could not run locally as `PASS`; see "Runbook jobs" in
[`docs/operations/ci-quality-gates.md`](docs/operations/ci-quality-gates.md).
`test:all` is the ordinary product-development loop and intentionally excludes
host-level CI-runner assurance. Run `corepack pnpm test:system` only on the
compatible Linux assurance host described in
@@ -168,7 +173,7 @@ Live release verification additionally requires `HOSTING_BASE_URL`.
## CI and evidence
The 28-gate registry is `config/ci/gates.json`; the Gitea workflow is
The 27-gate registry is `config/ci/gates.json`; the Gitea workflow is
`.gitea/workflows/quality-gates.yml`. It follows:
```text
+5 -5
View File
@@ -1,11 +1,11 @@
{
"schemaVersion": 1,
"owner": "frontend-platform",
"reason": "08-06: the signed manual accessibility review moves to release gate FE-GATE-028; merge gate FE-GATE-009 checks only the record format",
"gates": 28,
"reason": "11-01, 13-03 (D-14): FE-GATE-016 retired; it ran the same FE-RB-005 drill as FE-GATE-025",
"gates": 27,
"commands": 94,
"commandReferences": 106,
"artifacts": 110,
"commandReferences": 104,
"artifacts": 109,
"stages": 5,
"gateShapeSha256": "111e82fa1fd8341f8ee1f3d10d14365de382d83da5f8f9c5a1e3fcfecc5cafae"
"gateShapeSha256": "3ad2407c5929a7ec43dadebc68a3dc69e265793bf1c80c355b6d2fdeae019821"
}
-21
View File
@@ -1502,12 +1502,6 @@
"build-release-candidate"
]
},
{
"id": "artifact-artifacts-quality-gates-FE-GATE-016-txt",
"path": "artifacts/quality/gates/FE-GATE-016.txt",
"schemaId": "text",
"production": "runner-generated"
},
{
"id": "artifact-artifacts-runbooks-FE-RB-005-record-json",
"path": "artifacts/runbooks/FE-RB-005/record.json",
@@ -2015,19 +2009,6 @@
],
"retentionClassId": "release-coherence"
},
{
"id": "FE-GATE-016",
"name": "rollback-drill",
"commandIds": [
"build",
"drill-runbook"
],
"logArtifactId": "artifact-artifacts-quality-gates-FE-GATE-016-txt",
"evidenceArtifactIds": [
"artifact-artifacts-runbooks-FE-RB-005-record-json"
],
"retentionClassId": "prod-drill"
},
{
"id": "FE-GATE-017",
"name": "diagram-review",
@@ -2243,7 +2224,6 @@
"release"
],
"gateIds": [
"FE-GATE-016",
"FE-GATE-021",
"FE-GATE-022",
"FE-GATE-023",
@@ -2774,7 +2754,6 @@
"condition": "production",
"timeoutMinutes": 45,
"gateIds": [
"FE-GATE-016",
"FE-GATE-021",
"FE-GATE-022",
"FE-GATE-023",
@@ -1,6 +1,6 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "ART-FE-RUNBOOK-DRILL@1",
"$id": "ART-FE-RUNBOOK-DRILL@2",
"type": "object",
"required": [
"schemaVersion",
@@ -14,29 +14,39 @@
"recoveryAssertions",
"negativeFixtureFailedAsExpected",
"windowObservedBucket",
"passed"
"providerVerificationRequired",
"result"
],
"$defs": {
"verdict": { "enum": ["PASS", "FAIL", "UNVERIFIED"] }
},
"properties": {
"schemaVersion": { "const": 1 },
"schemaVersion": { "const": 2 },
"runbookId": { "pattern": "^FE-RB-00[1-5]$" },
"releaseId": { "type": "string", "minLength": 1 },
"drillTimestamp": { "type": "string", "format": "date-time" },
"triggerInjected": { "type": "string" },
"triggerAsserted": { "type": "boolean" },
"containmentAsserted": { "type": "boolean" },
"escalationPathAsserted": { "type": "boolean" },
"triggerAsserted": { "$ref": "#/$defs/verdict" },
"containmentAsserted": { "$ref": "#/$defs/verdict" },
"escalationPathAsserted": { "$ref": "#/$defs/verdict" },
"recoveryAssertions": {
"type": "array",
"minItems": 4,
"items": {
"type": "object",
"required": ["assertion", "evidence", "passed"]
"required": ["assertion", "evidence", "result"],
"properties": {
"assertion": { "type": "string", "minLength": 1 },
"evidence": { "type": "string", "minLength": 1 },
"result": { "$ref": "#/$defs/verdict" }
},
"additionalProperties": false
}
},
"negativeFixtureFailedAsExpected": { "type": "boolean" },
"negativeFixtureFailedAsExpected": { "$ref": "#/$defs/verdict" },
"windowObservedBucket": { "type": "string" },
"providerVerificationRequired": { "type": "boolean" },
"passed": { "type": "boolean" }
"result": { "$ref": "#/$defs/verdict" }
},
"additionalProperties": false
}
+15 -2
View File
@@ -1,6 +1,6 @@
# CI quality-gate orchestration
`config/ci/gates.json` is the executable registry for all 28 gates and the sole
`config/ci/gates.json` is the executable registry for all 27 gates and the sole
workflow model. `.gitea/workflows/quality-gates.yml` is generated from that
strict v2 contract; it is not an independent configuration authority. Update
the registry, run `corepack pnpm generate:ci-workflow`, and commit the generated
@@ -269,7 +269,20 @@ promotion, and field evidence through aggregation.
Runbook jobs write to the stable path
`artifacts/runbooks/<runbook-id>/record.json`; the dynamic release identity is
stored inside the record. This keeps gate evidence lookup independent from
stored inside the record.
A runbook drill (`drill:runbook`, FE-GATE-021..025) records each verdict as
`PASS`, `FAIL` or `UNVERIFIED` (`scripts/drill-runbook.ts`, record schema in
`scripts/contracts/release-artifacts.ts`). `PASS`/`FAIL` means the drill checked
it against the local `dist/` build; it reads no `public/` placeholder, so a
missing build fails. Browser boot, live API traffic, smoke after a provider
switch, and anything the runbook marks as needing provider verification are
`UNVERIFIED`. One `FAIL` makes the drill `FAIL` (exit 1). Otherwise any
`UNVERIFIED` makes it `UNVERIFIED` (exit 3), and the gate is not `PASS`. So
without provider evidence the production stage stays open. That is the intended
result: a local drill cannot show that the promoted release can be rolled back.
FE-GATE-016 (`rollback-drill`) is retired because it ran the same FE-RB-005
drill as FE-GATE-025 (`RETIRED_GATE_IDS` in `scripts/contracts/ci-gates.ts`). This keeps gate evidence lookup independent from
slashes or other provider-specific characters in `RELEASE_ID`, while each
workflow artifact remains scoped to its own run.
+3 -5
View File
@@ -8,6 +8,7 @@ import {
type GateResult,
} from "./lib/promotion-readiness.ts";
import {
CANONICAL_GATE_IDS,
loadCiGateContract,
indexCiGateContract,
} from "./contracts/ci-gates.ts";
@@ -104,10 +105,7 @@ if (!architecture?.evidenceArtifactIds.some((id) => index.artifacts.get(id)?.pat
failures.push("FE-GATE-010 must publish the typed CI contract report");
}
const expectedGateIds = Array.from(
{ length: 28 },
(_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`,
);
const expectedGateIds = CANONICAL_GATE_IDS;
const passingResults: Record<string, GateResult> = Object.fromEntries(
expectedGateIds.map((gateId) => [gateId, "PASS"]),
);
@@ -168,4 +166,4 @@ if (failures.length > 0) {
process.stderr.write(`CI contract failed:\n${failures.join("\n")}\n`);
process.exit(1);
}
process.stdout.write("CI contract: 28 gates, strict v2 graph and generated workflow model PASS\n");
process.stdout.write(`CI contract: ${CANONICAL_GATE_IDS.length} gates, strict v2 graph and generated workflow model PASS\n`);
+16 -6
View File
@@ -438,6 +438,19 @@ export type CiGateContractIndex = Readonly<{
jobs: ReadonlyMap<string, CiWorkflowJob>;
retentionClasses: ReadonlyMap<string, CiGateContract["retention"]["classes"][number]>;
}>;
/**
* D-14. FE-GATE-016 (rollback-drill) ran the same FE-RB-005 drill as
* FE-GATE-025 and counted it twice toward production readiness. Its id stays
* retired so no later gate reuses it with a different meaning.
*/
export const RETIRED_GATE_IDS: readonly string[] = Object.freeze(["FE-GATE-016"]);
export const CANONICAL_GATE_IDS: readonly string[] = Object.freeze(
Array.from({ length: 28 }, (_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`)
.filter((gateId) => !RETIRED_GATE_IDS.includes(gateId)),
);
/**
* D-13. The contract's size and gate shape are a reviewed baseline, not code
* constants: a project that removes a sample or adds a gate records the new
@@ -789,12 +802,9 @@ function validateContractSemantics(
issue(`stage dependency cycle: ${cycle.join(" -> ")}`);
}
const expectedGateIds = Array.from(
{ length: 28 },
(_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`,
);
const expectedGateIds = CANONICAL_GATE_IDS;
if (JSON.stringify(contract.gates.map(({ id }) => id)) !== JSON.stringify(expectedGateIds)) {
issue("gate registry must contain FE-GATE-001..028 in canonical order");
issue(`gate registry must contain FE-GATE-001..028 without retired ${RETIRED_GATE_IDS.join(", ")} in canonical order`);
}
const expectedStages: ReadonlyArray<readonly [string, string, readonly string[], readonly string[]]> = [
["merge", "MERGE_READY", [], PROMOTION_FORMULA.MERGE_READY],
@@ -869,7 +879,7 @@ function validateContractSemantics(
vulnerability_provider: [],
provenance_provider: [],
promotion: [],
production_gate: ["FE-GATE-016", "FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"],
production_gate: ["FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"],
field_gate: ["FE-GATE-018"],
documentation_gate: ["FE-GATE-017"],
};
+44 -8
View File
@@ -718,31 +718,67 @@ export const releaseVerificationArtifactSchema = z
})
.strict();
// 11-01 / 13-03 (D-14): each verdict is PASS or FAIL only when observed on the
// local build; anything that needs a boot, live traffic or the provider is
// UNVERIFIED, and so is a drill with provider scope.
const drillVerdict = z.enum(["PASS", "FAIL", "UNVERIFIED"]);
export type DrillVerdict = z.infer<typeof drillVerdict>;
/** FAIL beats UNVERIFIED beats PASS; provider scope alone keeps a drill UNVERIFIED. */
export function combineDrillVerdicts(
verdicts: readonly DrillVerdict[],
providerVerificationRequired: boolean,
): DrillVerdict {
if (verdicts.includes("FAIL")) return "FAIL";
if (verdicts.includes("UNVERIFIED") || providerVerificationRequired) return "UNVERIFIED";
return "PASS";
}
export const runbookRecordArtifactSchema = z
.object({
schemaVersion: z.literal(1),
schemaVersion: z.literal(2),
runbookId: z.string().regex(/^FE-RB-00[1-5]$/u),
releaseId: nonEmptyString,
drillTimestamp: timestamp,
triggerInjected: nonEmptyString,
triggerAsserted: z.boolean(),
containmentAsserted: z.boolean(),
escalationPathAsserted: z.boolean(),
triggerAsserted: drillVerdict,
containmentAsserted: drillVerdict,
escalationPathAsserted: drillVerdict,
recoveryAssertions: z.array(
z
.object({
assertion: nonEmptyString,
evidence: nonEmptyString,
passed: z.boolean(),
result: drillVerdict,
})
.strict(),
),
negativeFixtureFailedAsExpected: z.boolean(),
negativeFixtureFailedAsExpected: drillVerdict,
windowObservedBucket: nonEmptyString,
providerVerificationRequired: z.boolean(),
passed: z.boolean(),
result: drillVerdict,
})
.strict();
.strict()
.superRefine((record, context) => {
const expected = combineDrillVerdicts(
[
record.triggerAsserted,
record.containmentAsserted,
record.escalationPathAsserted,
...record.recoveryAssertions.map(({ result }) => result),
record.negativeFixtureFailedAsExpected,
],
record.providerVerificationRequired,
);
if (record.result !== expected) {
context.addIssue({
code: "custom",
path: ["result"],
message: `result must be ${expected} for these verdicts`,
});
}
});
const failureList = z.array(nonEmptyString).max(4_096);
const sourceOrFixtureMode = z.enum(["source", "negative-fixture"]);
+219 -204
View File
@@ -1,4 +1,5 @@
import { access, mkdir, readFile } from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { shouldRetry } from "../src/adapters/http/retry-policy.ts";
@@ -15,21 +16,30 @@ import {
} from "../src/contracts/release-artifacts.ts";
import { projectTelemetryEvent } from "../src/contracts/telemetry.ts";
import { EXPECTED_CONTRACT_SET_PACKAGES } from "../src/features/installed-contract-contributions.ts";
import { runbookRecordArtifactSchema } from "./contracts/release-artifacts.ts";
import {
combineDrillVerdicts,
runbookRecordArtifactSchema,
type DrillVerdict,
} from "./contracts/release-artifacts.ts";
import { verifyReleaseRuntimeCoherence } from "./lib/release-runtime-coherence.ts";
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
// 11-01 / 13-03 (D-14). A verdict is PASS or FAIL only when the drill observed
// it on the local build. What needs a browser boot, live traffic or the hosting
// provider is UNVERIFIED, and so is every drill whose runbook needs provider
// verification: a local drill never claims the release can be recovered.
type RecoveryAssertion = Readonly<{
assertion: string;
evidence: string;
passed: boolean;
result: DrillVerdict;
}>;
type DrillResult = Readonly<{
triggerAsserted: boolean;
containmentAsserted: boolean;
triggerAsserted: DrillVerdict;
containmentAsserted: DrillVerdict;
recoveryAssertions: RecoveryAssertion[];
negativeFixtureFailedAsExpected: boolean;
negativeFixtureFailedAsExpected: DrillVerdict;
providerVerificationRequired: boolean;
}>;
@@ -51,10 +61,8 @@ type RunbookDocument = Readonly<{
export type JsonArtifactReader = (path: string) => Promise<unknown>;
export type RollbackArtifactPaths = Readonly<{
primaryRelease: string;
fallbackRelease: string;
primaryRuntime: string;
fallbackRuntime: string;
release: string;
runtime: string;
}>;
export type RollbackCoherenceOptions = Readonly<{
@@ -63,27 +71,37 @@ export type RollbackCoherenceOptions = Readonly<{
paths?: RollbackArtifactPaths;
}>;
export type DrillOptions = Readonly<{
/** Directory holding `dist/` and `config/runbooks/`. Defaults to the cwd. */
root?: string;
contractPackages?: readonly InstalledContractPackageIdentity[];
}>;
type DrillContext = Readonly<{
root: string;
contractPackages: readonly InstalledContractPackageIdentity[];
}>;
// 13-03: the drill reads the built release only. It used to fall back to the
// placeholders in public/, so it passed without any build at all.
const DEFAULT_ROLLBACK_PATHS = Object.freeze({
primaryRelease: "dist/release-manifest.json",
fallbackRelease: "public/release-manifest.json",
primaryRuntime: "dist/config.json",
fallbackRuntime: "public/config.json",
release: "dist/release-manifest.json",
runtime: "dist/config.json",
});
async function readJsonArtifact(path: string): Promise<unknown> {
return JSON.parse(await readFile(path, "utf8"));
try {
return JSON.parse(await readFile(path, "utf8"));
} catch (error) {
if (hasErrorCode(error, "ENOENT")) {
throw new Error(`${path} is missing; build the release before running the drill`, { cause: error });
}
throw error;
}
}
async function releaseManifest(
readArtifact: JsonArtifactReader = readJsonArtifact,
paths: RollbackArtifactPaths = DEFAULT_ROLLBACK_PATHS,
): Promise<ReleaseArtifact> {
const value = await readPrimaryOrFallback(
readArtifact,
paths.primaryRelease,
paths.fallbackRelease,
);
return parseReleaseArtifact(value);
function rootedReader(root: string): JsonArtifactReader {
return (file) => readJsonArtifact(path.join(root, file));
}
const validConfig = {
@@ -100,12 +118,48 @@ const validConfig = {
RELEASE_ID: "local-release",
};
function verdict(passed: boolean): DrillVerdict {
return passed ? "PASS" : "FAIL";
}
function assertion(
assertion: string,
evidence: string,
passed: boolean,
result: DrillVerdict,
): RecoveryAssertion {
return { assertion, evidence, passed };
return { assertion, evidence, result };
}
/** An assertion the local drill cannot observe; `needs` names what can. */
function notObserved(name: string, needs: string): RecoveryAssertion {
return assertion(name, `not observed locally; needs ${needs}`, "UNVERIFIED");
}
/**
* Every file the Vite manifest names (entries, lazy chunks, CSS, assets) exists
* in the local dist. Hosting reachability still needs the provider smoke.
*/
async function builtAssetsPresent(root: string): Promise<boolean> {
const manifest = await readJsonArtifact(path.join(root, "dist/.vite/manifest.json"));
if (!manifest || typeof manifest !== "object") return false;
const files = new Set<string>();
for (const chunk of Object.values(manifest as Record<string, Record<string, unknown>>)) {
for (const key of ["file", "css", "assets"] as const) {
const value = chunk[key];
for (const file of Array.isArray(value) ? value : [value]) {
if (typeof file === "string") files.add(file);
}
}
}
if (![...files].some((file) => file.endsWith(".js"))) return false;
try {
await Promise.all(
["index.html", ...files].map((file) => access(path.join(root, "dist", file))),
);
return true;
} catch {
return false;
}
}
async function drillBoot(): Promise<DrillResult> {
@@ -115,19 +169,22 @@ async function drillBoot(): Promise<DrillResult> {
API_BASE_URL: "http://insecure.invalid",
});
const recovered = validateRuntimeConfig(validConfig);
const injectedMountFailure = true;
const injectedMountFailureRecovery =
recovered.success && !injectedMountFailure;
return {
triggerAsserted: !invalid.success,
containmentAsserted: !invalid.success,
triggerAsserted: verdict(!invalid.success),
containmentAsserted: verdict(!invalid.success),
recoveryAssertions: [
assertion("clean-session boot", "valid runtime schema parse", recovered.success),
assertion("product root mount", "boot precondition satisfied", recovered.success),
assertion("config validation", "invalid fixture rejected", !invalid.success),
assertion("no repeated boot error", "valid fixture remains valid", recovered.success),
notObserved("clean-session boot", "a browser boot of the promoted release"),
notObserved("product root mount", "a browser boot of the promoted release"),
assertion(
"config validation",
"invalid fixture rejected, valid fixture accepted",
verdict(!invalid.success && recovered.success),
),
notObserved("no repeated boot error", "boot telemetry from the promoted release"),
],
negativeFixtureFailedAsExpected: !injectedMountFailureRecovery,
// The runbook's negative fixture injects a mount failure after a valid
// config; that needs a mounted app, which a Node drill does not have.
negativeFixtureFailedAsExpected: "UNVERIFIED",
providerVerificationRequired: false,
};
}
@@ -144,7 +201,7 @@ function memoryStorage(): StoragePort {
};
}
async function drillChunkMismatch(): Promise<DrillResult> {
async function drillChunkMismatch(context: DrillContext): Promise<DrillResult> {
const storage = memoryStorage();
const input: Parameters<typeof decideChunkRecovery>[0] = {
failureKind: "DEPLOY_MISMATCH",
@@ -157,29 +214,26 @@ async function drillChunkMismatch(): Promise<DrillResult> {
};
const first = decideChunkRecovery(input);
const second = decideChunkRecovery(input);
const manifest = await releaseManifest();
let assetsReachable = true;
try {
await access("dist/index.html");
await access("dist/.vite/manifest.json");
} catch {
assetsReachable = false;
}
const { coherence } = await verifyRollbackReleaseCoherence({
readArtifact: rootedReader(context.root),
contractPackages: context.contractPackages,
});
return {
triggerAsserted: first.action === "reload-once",
containmentAsserted:
triggerAsserted: verdict(first.action === "reload-once"),
containmentAsserted: verdict(
first.action === "reload-once" && second.action === "support",
),
recoveryAssertions: [
assertion("entry and lazy assets reachable", "local dist access", assetsReachable),
assertion(
"release tuple coherent",
"release manifest has generated asset hash",
manifest.assetManifestHash !== "generated-during-build",
"entry and lazy assets reachable",
"every Vite manifest file exists in the local dist",
verdict(await builtAssetsPresent(context.root)),
),
assertion("second reload blocked", "reload guard decision", second.action === "support"),
assertion("critical route smoke", "built index available", assetsReachable),
assertion("release tuple coherent", "build/config/manifest tuple", verdict(coherence.compatible)),
assertion("second reload blocked", "reload guard decision", verdict(second.action === "support")),
notObserved("critical route smoke", "a browser smoke against the hosted release"),
],
negativeFixtureFailedAsExpected: second.action !== "reload-once",
negativeFixtureFailedAsExpected: verdict(second.action !== "reload-once"),
providerVerificationRequired: true,
};
}
@@ -196,15 +250,20 @@ async function drillApiDegradation(): Promise<DrillResult> {
0,
);
return {
triggerAsserted: true,
containmentAsserted: !unkeyedRetry,
// The trigger is a failure rate over live traffic.
triggerAsserted: "UNVERIFIED",
containmentAsserted: verdict(!unkeyedRetry && safeRetry),
recoveryAssertions: [
assertion("failure rate at baseline", "deterministic recovery window", true),
assertion("no retry amplification", "unkeyed retry policy", !unkeyedRetry),
assertion("critical read/write smoke", "safe read and protected mutation", safeRetry && !unkeyedRetry),
assertion("schema fixtures", "schema mismatch is not retryable", !shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)),
notObserved("terminal failure rate at baseline", "API failure telemetry from the promoted release"),
assertion("no retry amplification", "unkeyed retry policy", verdict(!unkeyedRetry)),
notObserved("critical read/write smoke", "a read and a write against the live API"),
assertion(
"schema fixtures",
"schema mismatch is not retryable",
verdict(!shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)),
),
],
negativeFixtureFailedAsExpected: !unkeyedRetry,
negativeFixtureFailedAsExpected: verdict(!unkeyedRetry),
providerVerificationRequired: true,
};
}
@@ -224,7 +283,10 @@ async function drillTelemetry(): Promise<DrillResult> {
attempt_count_bucket: "1",
route_id: "APP_HOME",
});
await adapter.flush();
const flushResolved = await adapter.flush().then(
() => true,
() => false,
);
const projected = projectTelemetryEvent("api.request.failed", {
error_kind: "SERVER_FAILURE",
http_status_group: "5xx",
@@ -235,22 +297,25 @@ async function drillTelemetry(): Promise<DrillResult> {
const redacted =
projected.success && !JSON.stringify(projected).includes("raw_url");
return {
triggerAsserted: adapter.droppedCount() === 1,
containmentAsserted: adapter.pendingCount() === 0,
triggerAsserted: verdict(adapter.droppedCount() === 1),
containmentAsserted: verdict(adapter.pendingCount() === 0),
recoveryAssertions: [
assertion("product flow unaffected", "adapter flush resolves", true),
assertion("delivery self-check", "sink failure counted", adapter.droppedCount() === 1),
assertion("queue drained within bound", "pending queue count", adapter.pendingCount() === 0),
assertion("forbidden attributes absent", "default-deny projection", redacted),
assertion("product flow unaffected", "adapter flush resolves despite the sink failure", verdict(flushResolved)),
assertion("delivery self-check", "sink failure counted", verdict(adapter.droppedCount() === 1)),
assertion("queue drained within bound", "pending queue count", verdict(adapter.pendingCount() === 0)),
assertion("forbidden attributes absent", "default-deny projection", verdict(redacted)),
],
negativeFixtureFailedAsExpected: redacted,
negativeFixtureFailedAsExpected: verdict(redacted),
providerVerificationRequired: true,
};
}
async function drillRollback(): Promise<DrillResult> {
const verified = await verifyRollbackReleaseCoherence();
const { release, coherence: coherent } = verified;
async function drillRollback(context: DrillContext): Promise<DrillResult> {
const { release, coherence: coherent } = await verifyRollbackReleaseCoherence({
readArtifact: rootedReader(context.root),
contractPackages: context.contractPackages,
});
const assetsPresent = await builtAssetsPresent(context.root);
const mixed = verifyCompatibilityTuple({
frontend: {
buildId: "build-a",
@@ -268,20 +333,25 @@ async function drillRollback(): Promise<DrillResult> {
},
});
return {
triggerAsserted: true,
containmentAsserted: coherent.compatible,
// A release-blocking defect is declared by a person during an incident.
triggerAsserted: "UNVERIFIED",
containmentAsserted: verdict(coherent.compatible && assetsPresent),
recoveryAssertions: [
assertion("compatibility gate", "typed version comparison", coherent.compatible),
assertion("release coherence gate", "build/config/manifest tuple", coherent.compatible),
assertion("critical smoke", "built or public runtime set parsed", true),
assertion("release ID in timeline", "drill record releaseId", Boolean(release.releaseId)),
assertion("compatibility gate", "typed version comparison", verdict(coherent.compatible)),
assertion(
"release coherence gate",
"build/config/manifest tuple and every Vite manifest file in the local dist",
verdict(coherent.compatible && assetsPresent),
),
notObserved("critical smoke", "a browser smoke after the provider switches to the prior release"),
assertion("release ID in timeline", "drill record releaseId", verdict(Boolean(release.releaseId))),
],
negativeFixtureFailedAsExpected: !mixed.compatible,
negativeFixtureFailedAsExpected: verdict(!mixed.compatible),
providerVerificationRequired: true,
};
}
const drillById: Record<string, () => Promise<DrillResult>> = {
const drillById: Record<string, (context: DrillContext) => Promise<DrillResult>> = {
"FE-RB-001": drillBoot,
"FE-RB-002": drillChunkMismatch,
"FE-RB-003": drillApiDegradation,
@@ -294,12 +364,12 @@ export async function verifyRollbackReleaseCoherence(
) {
const readArtifact = options.readArtifact ?? readJsonArtifact;
const paths = options.paths ?? DEFAULT_ROLLBACK_PATHS;
const artifactPair = await selectRollbackArtifactPair(
readArtifact,
paths,
);
const release = parseReleaseArtifact(artifactPair.release);
const runtimeArtifact = parseRuntimeConfigArtifact(artifactPair.runtime);
const [releaseValue, runtimeValue] = await Promise.all([
readArtifact(paths.release),
readArtifact(paths.runtime),
]);
const release = parseReleaseArtifact(releaseValue);
const runtimeArtifact = parseRuntimeConfigArtifact(runtimeValue);
const runtime = {
...runtimeArtifact,
BUILD_ID: requireIdentity(runtimeArtifact.BUILD_ID, "runtime BUILD_ID"),
@@ -317,43 +387,60 @@ export async function verifyRollbackReleaseCoherence(
return Object.freeze({ release, runtime, coherence });
}
export async function runRunbookDrill(runbookId: string, options: DrillOptions = {}) {
const context: DrillContext = {
root: options.root ?? process.cwd(),
contractPackages: options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES,
};
const document = JSON.parse(
await readFile(path.join(context.root, "config/runbooks/runbooks.json"), "utf8"),
) as RunbookDocument;
const specification = document.runbooks[runbookId];
const drill = drillById[runbookId];
if (!specification || !drill) throw new RangeError(`unknown runbook: ${runbookId}`);
const release = parseReleaseArtifact(
await rootedReader(context.root)(DEFAULT_ROLLBACK_PATHS.release),
);
const result = await drill(context);
const escalationPathAsserted = verdict(specification.escalation.length >= 2);
return {
schemaVersion: 2 as const,
runbookId,
gateId: specification.gateId,
releaseId: release.releaseId,
drillTimestamp: new Date().toISOString(),
triggerInjected: specification.triggerKinds[0] ?? "",
triggerAsserted: result.triggerAsserted,
containmentAsserted: result.containmentAsserted,
escalationPathAsserted,
recoveryAssertions: result.recoveryAssertions,
negativeFixtureFailedAsExpected: result.negativeFixtureFailedAsExpected,
windowObservedBucket: specification.window,
providerVerificationRequired: result.providerVerificationRequired,
result: combineDrillVerdicts(
[
result.triggerAsserted,
result.containmentAsserted,
escalationPathAsserted,
...result.recoveryAssertions.map((item) => item.result),
result.negativeFixtureFailedAsExpected,
],
result.providerVerificationRequired,
),
};
}
async function main(): Promise<void> {
const runbookId = process.argv
.slice(2)
.find((argument) => /^FE-RB-00[1-5]$/.test(argument));
const document = JSON.parse(
await readFile("config/runbooks/runbooks.json", "utf8"),
) as RunbookDocument;
const specification = runbookId ? document.runbooks[runbookId] : undefined;
if (!runbookId || !specification) {
if (!runbookId) {
process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n");
process.exit(2);
}
const drill = await drillById[runbookId]();
const escalationPathAsserted = specification.escalation.length >= 2;
const passed =
drill.triggerAsserted &&
drill.containmentAsserted &&
escalationPathAsserted &&
drill.recoveryAssertions.every((item) => item.passed) &&
drill.negativeFixtureFailedAsExpected;
const release = await releaseManifest();
const record = {
schemaVersion: 1,
runbookId,
releaseId: release.releaseId,
drillTimestamp: new Date().toISOString(),
triggerInjected: specification.triggerKinds[0],
triggerAsserted: drill.triggerAsserted,
containmentAsserted: drill.containmentAsserted,
escalationPathAsserted,
recoveryAssertions: drill.recoveryAssertions,
negativeFixtureFailedAsExpected: drill.negativeFixtureFailedAsExpected,
windowObservedBucket: specification.window,
providerVerificationRequired: drill.providerVerificationRequired,
passed,
};
const { gateId, ...record } = await runRunbookDrill(runbookId);
const artifactDirectory = `artifacts/runbooks/${runbookId}`;
await mkdir(artifactDirectory, { recursive: true });
await writeValidatedJsonArtifact({
@@ -361,15 +448,25 @@ async function main(): Promise<void> {
schema: runbookRecordArtifactSchema,
value: record,
});
if (!passed) {
process.stderr.write(`${runbookId} drill failed.\n`);
if (record.result === "FAIL") {
process.stderr.write(`${runbookId} drill: FAIL (${gateId})\n`);
process.exit(1);
}
process.stdout.write(
`${runbookId} drill: PASS (${specification.gateId}; provider verification ${
drill.providerVerificationRequired ? "still required" : "not required"
})\n`,
);
if (record.result === "UNVERIFIED") {
// Not a pass: the gate stays red until provider evidence exists (D-14).
const open = [
["trigger", record.triggerAsserted],
["containment", record.containmentAsserted],
...record.recoveryAssertions.map((item) => [item.assertion, item.result] as const),
["negative fixture", record.negativeFixtureFailedAsExpected],
["provider verification", record.providerVerificationRequired ? "UNVERIFIED" : "PASS"],
].filter(([, result]) => result === "UNVERIFIED").map(([name]) => name);
process.stderr.write(
`${runbookId} drill: UNVERIFIED (${gateId}; no local check failed; not observed locally: ${open.join(", ")})\n`,
);
process.exit(3);
}
process.stdout.write(`${runbookId} drill: PASS (${gateId})\n`);
}
function requireIdentity(value: string | undefined, label: string): string {
@@ -379,88 +476,6 @@ function requireIdentity(value: string | undefined, label: string): string {
return value;
}
async function selectRollbackArtifactPair(
readArtifact: JsonArtifactReader,
paths: RollbackArtifactPaths,
): Promise<Readonly<{ release: unknown; runtime: unknown }>> {
const primary = await readArtifactPair(
readArtifact,
paths.primaryRelease,
paths.primaryRuntime,
);
if (
primary.release.status === "fulfilled" &&
primary.runtime.status === "fulfilled"
) {
return Object.freeze({
release: primary.release.value,
runtime: primary.runtime.value,
});
}
const releaseMissing =
primary.release.status === "rejected" &&
hasErrorCode(primary.release.reason, "ENOENT");
const runtimeMissing =
primary.runtime.status === "rejected" &&
hasErrorCode(primary.runtime.reason, "ENOENT");
if (releaseMissing && runtimeMissing) {
const fallback = await readArtifactPair(
readArtifact,
paths.fallbackRelease,
paths.fallbackRuntime,
);
if (fallback.release.status === "rejected") {
throw fallback.release.reason;
}
if (fallback.runtime.status === "rejected") {
throw fallback.runtime.reason;
}
return Object.freeze({
release: fallback.release.value,
runtime: fallback.runtime.value,
});
}
if (primary.release.status === "rejected" && !releaseMissing) {
throw primary.release.reason;
}
if (primary.runtime.status === "rejected" && !runtimeMissing) {
throw primary.runtime.reason;
}
throw new Error("primary rollback artifact pair is incomplete");
}
async function readArtifactPair(
readArtifact: JsonArtifactReader,
releasePath: string,
runtimePath: string,
): Promise<
Readonly<{
release: PromiseSettledResult<unknown>;
runtime: PromiseSettledResult<unknown>;
}>
> {
const [release, runtime] = await Promise.allSettled([
Promise.resolve().then(() => readArtifact(releasePath)),
Promise.resolve().then(() => readArtifact(runtimePath)),
]);
return Object.freeze({ release, runtime });
}
async function readPrimaryOrFallback(
readArtifact: JsonArtifactReader,
primary: string,
fallback: string,
): Promise<unknown> {
try {
return await readArtifact(primary);
} catch (error) {
if (!hasErrorCode(error, "ENOENT")) throw error;
return readArtifact(fallback);
}
}
function hasErrorCode(error: unknown, code: string): boolean {
return Boolean(
error &&
+1 -1
View File
@@ -4,7 +4,7 @@ export const ciContractReportSchema = z
.object({
schemaVersion: z.literal(2),
nodeVersion: z.string().regex(/^\d+\.\d+\.\d+$/u),
gateCount: z.literal(28),
gateCount: z.literal(27),
commandDefinitionCount: z.number().int().positive(),
commandReferenceCount: z.number().int().positive(),
artifactCount: z.number().int().positive(),
+1 -1
View File
@@ -28,7 +28,7 @@ export const PROMOTION_FORMULA = Object.freeze({
"FE-GATE-028",
]),
PROD_PROMOTION_READY: Object.freeze([
"FE-GATE-016",
// FE-GATE-016 is retired (D-14): it repeated FE-GATE-025's FE-RB-005 drill.
"FE-GATE-021",
"FE-GATE-022",
"FE-GATE-023",
@@ -382,7 +382,6 @@ jobs:
fail-fast: false
matrix:
include:
- { gate: FE-GATE-016, name: rollback-drill }
- { gate: FE-GATE-021, name: runbook-boot-config }
- { gate: FE-GATE-022, name: runbook-chunk-mismatch }
- { gate: FE-GATE-023, name: runbook-api-degradation }
+4 -4
View File
@@ -191,14 +191,14 @@ describe("CI gate contract", () => {
expect(contract.gates.map(({ id }) => id)).toEqual(
Array.from({ length: 28 }, (_, index) =>
`FE-GATE-${String(index + 1).padStart(3, "0")}`,
),
).filter((gateId) => gateId !== "FE-GATE-016"),
);
expect(contract.jobs).toHaveLength(9);
expect(contract.commands).toHaveLength(94);
expect(contract.gates.reduce((total, gate) => total + gate.commandIds.length, 0)).toBe(106);
expect(contract.gates.reduce((total, gate) => total + gate.commandIds.length, 0)).toBe(104);
expect(contract.commands.filter(({ expect }) => expect === "fail")).toHaveLength(31);
expect(contract.gates.reduce((total, gate) => total + gate.evidenceArtifactIds.length, 0)).toBe(88);
expect(contract.artifacts).toHaveLength(110);
expect(contract.gates.reduce((total, gate) => total + gate.evidenceArtifactIds.length, 0)).toBe(87);
expect(contract.artifacts).toHaveLength(109);
expect(contract.stages).toHaveLength(5);
expect(contract.retention.classes).toHaveLength(5);
expect(index.gates.get("FE-GATE-015")?.commandIds).toHaveLength(2);
+1 -1
View File
@@ -25,7 +25,7 @@ describe("promotion readiness formula", () => {
it.each([
["FE-GATE-001", "MERGE_READY"],
["FE-GATE-012", "RELEASE_READY"],
["FE-GATE-016", "PROD_PROMOTION_READY"],
["FE-GATE-021", "PROD_PROMOTION_READY"],
["FE-GATE-018", "FIELD_SLO_READY"],
["FE-GATE-017", "DOCUMENTATION_READY"],
] as const)("fails closed when %s fails", (failedGate, readiness) => {
+14 -64
View File
@@ -281,32 +281,32 @@ describe("release coherence", () => {
}
});
it("falls back to public rollback artifacts only when primary dist is absent", async () => {
// 13-03: the rollback drill checks the built release only. Reading the
// public/ placeholders when dist was absent let it pass without any build.
it("reads only the built dist pair and never falls back to public placeholders", async () => {
const exactV2 = await releaseV2With(contractPackages);
const tamperedPrimary = await releaseV2With(contractPackages.slice(0, 1));
const validPublic = {
"public/release-manifest.json": exactV2,
"public/config.json": runtimeV2,
};
const unavailablePrimary = [
new SyntaxError("invalid primary JSON"),
Object.assign(new Error("unreadable primary"), { code: "EACCES" }),
{ ...exactV2, unexpected: "tampered" },
];
for (const primaryFailure of unavailablePrimary) {
for (const primary of [
{},
{ "dist/config.json": runtimeV2 },
{ "dist/release-manifest.json": exactV2 },
{ "dist/release-manifest.json": new SyntaxError("invalid primary JSON"), "dist/config.json": runtimeV2 },
{ "dist/release-manifest.json": { ...exactV2, unexpected: "tampered" }, "dist/config.json": runtimeV2 },
]) {
const reads: string[] = [];
await expect(
verifyRollbackReleaseCoherence({
readArtifact: artifactReader({
"dist/release-manifest.json": primaryFailure,
"dist/config.json": runtimeV2,
...validPublic,
}),
readArtifact: artifactReader({ ...primary, ...validPublic }, reads),
contractPackages,
}),
).rejects.toBeDefined();
expect(reads.some((read) => read.startsWith("public/"))).toBe(false);
}
const tamperedPrimary = await releaseV2With(contractPackages.slice(0, 1));
await expect(
verifyRollbackReleaseCoherence({
readArtifact: artifactReader({
@@ -317,55 +317,5 @@ describe("release coherence", () => {
contractPackages,
}),
).resolves.toMatchObject({ coherence: { compatible: false } });
const releaseOnlyMissingReads: string[] = [];
await expect(
verifyRollbackReleaseCoherence({
readArtifact: artifactReader(
{
"dist/config.json": runtimeV2,
...validPublic,
},
releaseOnlyMissingReads,
),
contractPackages,
}),
).rejects.toThrow("primary rollback artifact pair");
expect(releaseOnlyMissingReads).toEqual([
"dist/release-manifest.json",
"dist/config.json",
]);
const runtimeOnlyMissingReads: string[] = [];
await expect(
verifyRollbackReleaseCoherence({
readArtifact: artifactReader(
{
"dist/release-manifest.json": exactV2,
...validPublic,
},
runtimeOnlyMissingReads,
),
contractPackages,
}),
).rejects.toThrow("primary rollback artifact pair");
expect(runtimeOnlyMissingReads).toEqual([
"dist/release-manifest.json",
"dist/config.json",
]);
const missingPairReads: string[] = [];
await expect(
verifyRollbackReleaseCoherence({
readArtifact: artifactReader(validPublic, missingPairReads),
contractPackages,
}),
).resolves.toMatchObject({ coherence: { compatible: true } });
expect(missingPairReads).toEqual([
"dist/release-manifest.json",
"dist/config.json",
"public/release-manifest.json",
"public/config.json",
]);
});
});
+134
View File
@@ -0,0 +1,134 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { runRunbookDrill } from "../../scripts/drill-runbook.ts";
import { computeContractSetDigest } from "../../src/contracts/contract-set-canonical.ts";
import type { InstalledContractPackageIdentity } from "../../src/contracts/external-contract-runtime.ts";
// 11-01 / 13-03 (D-14). A drill reports PASS only for what it observed on the
// local build. Boot, live traffic, smoke and the provider switch are not
// observable here, so a drill with provider scope ends UNVERIFIED, never PASS.
const contractPackages = [
{
packageId: "@example/accounts",
version: "1.2.3",
digest: `sha256:${"1".repeat(64)}`,
runtimeProtocolVersion: 1,
sourceRevision: "a".repeat(40),
},
] as const satisfies readonly InstalledContractPackageIdentity[];
const RUNBOOK_IDS = ["FE-RB-001", "FE-RB-002", "FE-RB-003", "FE-RB-004", "FE-RB-005"] as const;
const roots: string[] = [];
afterEach(async () => {
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
});
async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?: boolean }> = {}) {
const root = await mkdtemp(path.join(tmpdir(), "runbook-drill-"));
roots.push(root);
await mkdir(path.join(root, "config/runbooks"), { recursive: true });
await writeFile(
path.join(root, "config/runbooks/runbooks.json"),
await readFile("config/runbooks/runbooks.json"),
);
if (options.withoutDist) return root;
const write = async (file: string, content: string) => {
await mkdir(path.dirname(path.join(root, file)), { recursive: true });
await writeFile(path.join(root, file), content);
};
await write("dist/index.html", "<!doctype html><script type=module src=/assets/index-a.js></script>");
await write(
"dist/.vite/manifest.json",
JSON.stringify({
"index.html": { file: "assets/index-a.js", isEntry: true, css: ["assets/index-a.css"], imports: ["_shared-b.js"] },
"_shared-b.js": { file: "assets/shared-b.js" },
"src/pages/lazy.tsx": { file: "assets/lazy-c.js", isDynamicEntry: true },
}),
);
await write("dist/assets/index-a.css", "body{}");
if (!options.withoutJs) {
for (const file of ["assets/index-a.js", "assets/shared-b.js", "assets/lazy-c.js"]) {
await write(`dist/${file}`, "export {};");
}
}
await write(
"dist/release-manifest.json",
JSON.stringify({
schemaVersion: 2,
appVersion: "1.0.0",
buildId: "build-a",
commitSha: "abc1234",
configSchemaVersion: "2.0",
assetManifestHash: "assets-a",
releaseId: "release-a",
builtAt: "2026-08-01T00:00:00.000Z",
routeChunks: {},
contractSet: {
setAlgorithm: "CA_CONTRACT_SET_V1",
setDigest: await computeContractSetDigest(contractPackages),
packages: [...contractPackages],
},
}),
);
await write(
"dist/config.json",
JSON.stringify({
APP_ENV: "local",
API_BASE_URL: "http://localhost:8080/",
REQUEST_TIMEOUT_MS: 10_000,
MAX_RETRY_ATTEMPTS: 2,
TELEMETRY_ENABLED: false,
AUTH_MODE: "external",
CONFIG_SCHEMA_VERSION: "2.0",
RELEASE_MANIFEST_URL: "/release-manifest.json",
BUILD_ID: "build-a",
RELEASE_ID: "release-a",
CAPABILITY_OVERRIDES: {
REALTIME: "DEFAULT",
WEB_WORKER: "DEFAULT",
SERVICE_WORKER: "DEFAULT",
OFFLINE_COMMANDS: "DEFAULT",
},
FEATURE_OVERRIDES: {},
}),
);
return root;
}
describe("runbook drill verdicts", () => {
it.each(RUNBOOK_IDS)("%s on a complete local build is UNVERIFIED, not PASS", async (runbookId) => {
const record = await runRunbookDrill(runbookId, { root: await builtDist(), contractPackages });
expect(record.result).toBe("UNVERIFIED");
expect(record.recoveryAssertions.some(({ result }) => result === "FAIL")).toBe(false);
});
it.each(["FE-RB-002", "FE-RB-005"] as const)(
"%s fails when the build lost its JavaScript assets",
async (runbookId) => {
const record = await runRunbookDrill(runbookId, {
root: await builtDist({ withoutJs: true }),
contractPackages,
});
expect(record.result).toBe("FAIL");
},
);
it("FE-RB-005 fails without a build instead of reading public/ placeholders", async () => {
await expect(
runRunbookDrill("FE-RB-005", { root: await builtDist({ withoutDist: true }), contractPackages }),
).rejects.toThrow(/dist\/release-manifest\.json/u);
});
it("keeps no constant recovery verdict in the drill source", async () => {
const source = await readFile("scripts/drill-runbook.ts", "utf8");
expect(source).not.toMatch(/assertion\([^;]*,\s*true\s*\)/u);
expect(source).not.toMatch(/Asserted:\s*true\b/u);
expect(source).not.toMatch(/injectedMountFailure\s*=\s*true/u);
});
});