fix(release): record unobserved runbook drill verdicts as UNVERIFIED (11-01, 13-03)

Drills held constant true verdicts and read public/ placeholders, so they
passed with no build or with JS assets deleted. They now check the local dist,
mark boot, live traffic and provider steps UNVERIFIED (exit 3), and FE-GATE-016
is retired because it repeated FE-GATE-025's FE-RB-005 drill.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-25 22:15:59 +09:00
co-authored by Claude Opus 5.5
parent 7298275271
commit 7f889a6d83
17 changed files with 482 additions and 334 deletions
+3 -5
View File
@@ -8,6 +8,7 @@ import {
type GateResult,
} from "./lib/promotion-readiness.ts";
import {
CANONICAL_GATE_IDS,
loadCiGateContract,
indexCiGateContract,
} from "./contracts/ci-gates.ts";
@@ -104,10 +105,7 @@ if (!architecture?.evidenceArtifactIds.some((id) => index.artifacts.get(id)?.pat
failures.push("FE-GATE-010 must publish the typed CI contract report");
}
const expectedGateIds = Array.from(
{ length: 28 },
(_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`,
);
const expectedGateIds = CANONICAL_GATE_IDS;
const passingResults: Record<string, GateResult> = Object.fromEntries(
expectedGateIds.map((gateId) => [gateId, "PASS"]),
);
@@ -168,4 +166,4 @@ if (failures.length > 0) {
process.stderr.write(`CI contract failed:\n${failures.join("\n")}\n`);
process.exit(1);
}
process.stdout.write("CI contract: 28 gates, strict v2 graph and generated workflow model PASS\n");
process.stdout.write(`CI contract: ${CANONICAL_GATE_IDS.length} gates, strict v2 graph and generated workflow model PASS\n`);
+16 -6
View File
@@ -438,6 +438,19 @@ export type CiGateContractIndex = Readonly<{
jobs: ReadonlyMap<string, CiWorkflowJob>;
retentionClasses: ReadonlyMap<string, CiGateContract["retention"]["classes"][number]>;
}>;
/**
* D-14. FE-GATE-016 (rollback-drill) ran the same FE-RB-005 drill as
* FE-GATE-025 and counted it twice toward production readiness. Its id stays
* retired so no later gate reuses it with a different meaning.
*/
export const RETIRED_GATE_IDS: readonly string[] = Object.freeze(["FE-GATE-016"]);
export const CANONICAL_GATE_IDS: readonly string[] = Object.freeze(
Array.from({ length: 28 }, (_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`)
.filter((gateId) => !RETIRED_GATE_IDS.includes(gateId)),
);
/**
* D-13. The contract's size and gate shape are a reviewed baseline, not code
* constants: a project that removes a sample or adds a gate records the new
@@ -789,12 +802,9 @@ function validateContractSemantics(
issue(`stage dependency cycle: ${cycle.join(" -> ")}`);
}
const expectedGateIds = Array.from(
{ length: 28 },
(_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`,
);
const expectedGateIds = CANONICAL_GATE_IDS;
if (JSON.stringify(contract.gates.map(({ id }) => id)) !== JSON.stringify(expectedGateIds)) {
issue("gate registry must contain FE-GATE-001..028 in canonical order");
issue(`gate registry must contain FE-GATE-001..028 without retired ${RETIRED_GATE_IDS.join(", ")} in canonical order`);
}
const expectedStages: ReadonlyArray<readonly [string, string, readonly string[], readonly string[]]> = [
["merge", "MERGE_READY", [], PROMOTION_FORMULA.MERGE_READY],
@@ -869,7 +879,7 @@ function validateContractSemantics(
vulnerability_provider: [],
provenance_provider: [],
promotion: [],
production_gate: ["FE-GATE-016", "FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"],
production_gate: ["FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"],
field_gate: ["FE-GATE-018"],
documentation_gate: ["FE-GATE-017"],
};
+44 -8
View File
@@ -718,31 +718,67 @@ export const releaseVerificationArtifactSchema = z
})
.strict();
// 11-01 / 13-03 (D-14): each verdict is PASS or FAIL only when observed on the
// local build; anything that needs a boot, live traffic or the provider is
// UNVERIFIED, and so is a drill with provider scope.
const drillVerdict = z.enum(["PASS", "FAIL", "UNVERIFIED"]);
export type DrillVerdict = z.infer<typeof drillVerdict>;
/** FAIL beats UNVERIFIED beats PASS; provider scope alone keeps a drill UNVERIFIED. */
export function combineDrillVerdicts(
verdicts: readonly DrillVerdict[],
providerVerificationRequired: boolean,
): DrillVerdict {
if (verdicts.includes("FAIL")) return "FAIL";
if (verdicts.includes("UNVERIFIED") || providerVerificationRequired) return "UNVERIFIED";
return "PASS";
}
export const runbookRecordArtifactSchema = z
.object({
schemaVersion: z.literal(1),
schemaVersion: z.literal(2),
runbookId: z.string().regex(/^FE-RB-00[1-5]$/u),
releaseId: nonEmptyString,
drillTimestamp: timestamp,
triggerInjected: nonEmptyString,
triggerAsserted: z.boolean(),
containmentAsserted: z.boolean(),
escalationPathAsserted: z.boolean(),
triggerAsserted: drillVerdict,
containmentAsserted: drillVerdict,
escalationPathAsserted: drillVerdict,
recoveryAssertions: z.array(
z
.object({
assertion: nonEmptyString,
evidence: nonEmptyString,
passed: z.boolean(),
result: drillVerdict,
})
.strict(),
),
negativeFixtureFailedAsExpected: z.boolean(),
negativeFixtureFailedAsExpected: drillVerdict,
windowObservedBucket: nonEmptyString,
providerVerificationRequired: z.boolean(),
passed: z.boolean(),
result: drillVerdict,
})
.strict();
.strict()
.superRefine((record, context) => {
const expected = combineDrillVerdicts(
[
record.triggerAsserted,
record.containmentAsserted,
record.escalationPathAsserted,
...record.recoveryAssertions.map(({ result }) => result),
record.negativeFixtureFailedAsExpected,
],
record.providerVerificationRequired,
);
if (record.result !== expected) {
context.addIssue({
code: "custom",
path: ["result"],
message: `result must be ${expected} for these verdicts`,
});
}
});
const failureList = z.array(nonEmptyString).max(4_096);
const sourceOrFixtureMode = z.enum(["source", "negative-fixture"]);
+219 -204
View File
@@ -1,4 +1,5 @@
import { access, mkdir, readFile } from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { shouldRetry } from "../src/adapters/http/retry-policy.ts";
@@ -15,21 +16,30 @@ import {
} from "../src/contracts/release-artifacts.ts";
import { projectTelemetryEvent } from "../src/contracts/telemetry.ts";
import { EXPECTED_CONTRACT_SET_PACKAGES } from "../src/features/installed-contract-contributions.ts";
import { runbookRecordArtifactSchema } from "./contracts/release-artifacts.ts";
import {
combineDrillVerdicts,
runbookRecordArtifactSchema,
type DrillVerdict,
} from "./contracts/release-artifacts.ts";
import { verifyReleaseRuntimeCoherence } from "./lib/release-runtime-coherence.ts";
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
// 11-01 / 13-03 (D-14). A verdict is PASS or FAIL only when the drill observed
// it on the local build. What needs a browser boot, live traffic or the hosting
// provider is UNVERIFIED, and so is every drill whose runbook needs provider
// verification: a local drill never claims the release can be recovered.
type RecoveryAssertion = Readonly<{
assertion: string;
evidence: string;
passed: boolean;
result: DrillVerdict;
}>;
type DrillResult = Readonly<{
triggerAsserted: boolean;
containmentAsserted: boolean;
triggerAsserted: DrillVerdict;
containmentAsserted: DrillVerdict;
recoveryAssertions: RecoveryAssertion[];
negativeFixtureFailedAsExpected: boolean;
negativeFixtureFailedAsExpected: DrillVerdict;
providerVerificationRequired: boolean;
}>;
@@ -51,10 +61,8 @@ type RunbookDocument = Readonly<{
export type JsonArtifactReader = (path: string) => Promise<unknown>;
export type RollbackArtifactPaths = Readonly<{
primaryRelease: string;
fallbackRelease: string;
primaryRuntime: string;
fallbackRuntime: string;
release: string;
runtime: string;
}>;
export type RollbackCoherenceOptions = Readonly<{
@@ -63,27 +71,37 @@ export type RollbackCoherenceOptions = Readonly<{
paths?: RollbackArtifactPaths;
}>;
export type DrillOptions = Readonly<{
/** Directory holding `dist/` and `config/runbooks/`. Defaults to the cwd. */
root?: string;
contractPackages?: readonly InstalledContractPackageIdentity[];
}>;
type DrillContext = Readonly<{
root: string;
contractPackages: readonly InstalledContractPackageIdentity[];
}>;
// 13-03: the drill reads the built release only. It used to fall back to the
// placeholders in public/, so it passed without any build at all.
const DEFAULT_ROLLBACK_PATHS = Object.freeze({
primaryRelease: "dist/release-manifest.json",
fallbackRelease: "public/release-manifest.json",
primaryRuntime: "dist/config.json",
fallbackRuntime: "public/config.json",
release: "dist/release-manifest.json",
runtime: "dist/config.json",
});
async function readJsonArtifact(path: string): Promise<unknown> {
return JSON.parse(await readFile(path, "utf8"));
try {
return JSON.parse(await readFile(path, "utf8"));
} catch (error) {
if (hasErrorCode(error, "ENOENT")) {
throw new Error(`${path} is missing; build the release before running the drill`, { cause: error });
}
throw error;
}
}
async function releaseManifest(
readArtifact: JsonArtifactReader = readJsonArtifact,
paths: RollbackArtifactPaths = DEFAULT_ROLLBACK_PATHS,
): Promise<ReleaseArtifact> {
const value = await readPrimaryOrFallback(
readArtifact,
paths.primaryRelease,
paths.fallbackRelease,
);
return parseReleaseArtifact(value);
function rootedReader(root: string): JsonArtifactReader {
return (file) => readJsonArtifact(path.join(root, file));
}
const validConfig = {
@@ -100,12 +118,48 @@ const validConfig = {
RELEASE_ID: "local-release",
};
function verdict(passed: boolean): DrillVerdict {
return passed ? "PASS" : "FAIL";
}
function assertion(
assertion: string,
evidence: string,
passed: boolean,
result: DrillVerdict,
): RecoveryAssertion {
return { assertion, evidence, passed };
return { assertion, evidence, result };
}
/** An assertion the local drill cannot observe; `needs` names what can. */
function notObserved(name: string, needs: string): RecoveryAssertion {
return assertion(name, `not observed locally; needs ${needs}`, "UNVERIFIED");
}
/**
* Every file the Vite manifest names (entries, lazy chunks, CSS, assets) exists
* in the local dist. Hosting reachability still needs the provider smoke.
*/
async function builtAssetsPresent(root: string): Promise<boolean> {
const manifest = await readJsonArtifact(path.join(root, "dist/.vite/manifest.json"));
if (!manifest || typeof manifest !== "object") return false;
const files = new Set<string>();
for (const chunk of Object.values(manifest as Record<string, Record<string, unknown>>)) {
for (const key of ["file", "css", "assets"] as const) {
const value = chunk[key];
for (const file of Array.isArray(value) ? value : [value]) {
if (typeof file === "string") files.add(file);
}
}
}
if (![...files].some((file) => file.endsWith(".js"))) return false;
try {
await Promise.all(
["index.html", ...files].map((file) => access(path.join(root, "dist", file))),
);
return true;
} catch {
return false;
}
}
async function drillBoot(): Promise<DrillResult> {
@@ -115,19 +169,22 @@ async function drillBoot(): Promise<DrillResult> {
API_BASE_URL: "http://insecure.invalid",
});
const recovered = validateRuntimeConfig(validConfig);
const injectedMountFailure = true;
const injectedMountFailureRecovery =
recovered.success && !injectedMountFailure;
return {
triggerAsserted: !invalid.success,
containmentAsserted: !invalid.success,
triggerAsserted: verdict(!invalid.success),
containmentAsserted: verdict(!invalid.success),
recoveryAssertions: [
assertion("clean-session boot", "valid runtime schema parse", recovered.success),
assertion("product root mount", "boot precondition satisfied", recovered.success),
assertion("config validation", "invalid fixture rejected", !invalid.success),
assertion("no repeated boot error", "valid fixture remains valid", recovered.success),
notObserved("clean-session boot", "a browser boot of the promoted release"),
notObserved("product root mount", "a browser boot of the promoted release"),
assertion(
"config validation",
"invalid fixture rejected, valid fixture accepted",
verdict(!invalid.success && recovered.success),
),
notObserved("no repeated boot error", "boot telemetry from the promoted release"),
],
negativeFixtureFailedAsExpected: !injectedMountFailureRecovery,
// The runbook's negative fixture injects a mount failure after a valid
// config; that needs a mounted app, which a Node drill does not have.
negativeFixtureFailedAsExpected: "UNVERIFIED",
providerVerificationRequired: false,
};
}
@@ -144,7 +201,7 @@ function memoryStorage(): StoragePort {
};
}
async function drillChunkMismatch(): Promise<DrillResult> {
async function drillChunkMismatch(context: DrillContext): Promise<DrillResult> {
const storage = memoryStorage();
const input: Parameters<typeof decideChunkRecovery>[0] = {
failureKind: "DEPLOY_MISMATCH",
@@ -157,29 +214,26 @@ async function drillChunkMismatch(): Promise<DrillResult> {
};
const first = decideChunkRecovery(input);
const second = decideChunkRecovery(input);
const manifest = await releaseManifest();
let assetsReachable = true;
try {
await access("dist/index.html");
await access("dist/.vite/manifest.json");
} catch {
assetsReachable = false;
}
const { coherence } = await verifyRollbackReleaseCoherence({
readArtifact: rootedReader(context.root),
contractPackages: context.contractPackages,
});
return {
triggerAsserted: first.action === "reload-once",
containmentAsserted:
triggerAsserted: verdict(first.action === "reload-once"),
containmentAsserted: verdict(
first.action === "reload-once" && second.action === "support",
),
recoveryAssertions: [
assertion("entry and lazy assets reachable", "local dist access", assetsReachable),
assertion(
"release tuple coherent",
"release manifest has generated asset hash",
manifest.assetManifestHash !== "generated-during-build",
"entry and lazy assets reachable",
"every Vite manifest file exists in the local dist",
verdict(await builtAssetsPresent(context.root)),
),
assertion("second reload blocked", "reload guard decision", second.action === "support"),
assertion("critical route smoke", "built index available", assetsReachable),
assertion("release tuple coherent", "build/config/manifest tuple", verdict(coherence.compatible)),
assertion("second reload blocked", "reload guard decision", verdict(second.action === "support")),
notObserved("critical route smoke", "a browser smoke against the hosted release"),
],
negativeFixtureFailedAsExpected: second.action !== "reload-once",
negativeFixtureFailedAsExpected: verdict(second.action !== "reload-once"),
providerVerificationRequired: true,
};
}
@@ -196,15 +250,20 @@ async function drillApiDegradation(): Promise<DrillResult> {
0,
);
return {
triggerAsserted: true,
containmentAsserted: !unkeyedRetry,
// The trigger is a failure rate over live traffic.
triggerAsserted: "UNVERIFIED",
containmentAsserted: verdict(!unkeyedRetry && safeRetry),
recoveryAssertions: [
assertion("failure rate at baseline", "deterministic recovery window", true),
assertion("no retry amplification", "unkeyed retry policy", !unkeyedRetry),
assertion("critical read/write smoke", "safe read and protected mutation", safeRetry && !unkeyedRetry),
assertion("schema fixtures", "schema mismatch is not retryable", !shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)),
notObserved("terminal failure rate at baseline", "API failure telemetry from the promoted release"),
assertion("no retry amplification", "unkeyed retry policy", verdict(!unkeyedRetry)),
notObserved("critical read/write smoke", "a read and a write against the live API"),
assertion(
"schema fixtures",
"schema mismatch is not retryable",
verdict(!shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)),
),
],
negativeFixtureFailedAsExpected: !unkeyedRetry,
negativeFixtureFailedAsExpected: verdict(!unkeyedRetry),
providerVerificationRequired: true,
};
}
@@ -224,7 +283,10 @@ async function drillTelemetry(): Promise<DrillResult> {
attempt_count_bucket: "1",
route_id: "APP_HOME",
});
await adapter.flush();
const flushResolved = await adapter.flush().then(
() => true,
() => false,
);
const projected = projectTelemetryEvent("api.request.failed", {
error_kind: "SERVER_FAILURE",
http_status_group: "5xx",
@@ -235,22 +297,25 @@ async function drillTelemetry(): Promise<DrillResult> {
const redacted =
projected.success && !JSON.stringify(projected).includes("raw_url");
return {
triggerAsserted: adapter.droppedCount() === 1,
containmentAsserted: adapter.pendingCount() === 0,
triggerAsserted: verdict(adapter.droppedCount() === 1),
containmentAsserted: verdict(adapter.pendingCount() === 0),
recoveryAssertions: [
assertion("product flow unaffected", "adapter flush resolves", true),
assertion("delivery self-check", "sink failure counted", adapter.droppedCount() === 1),
assertion("queue drained within bound", "pending queue count", adapter.pendingCount() === 0),
assertion("forbidden attributes absent", "default-deny projection", redacted),
assertion("product flow unaffected", "adapter flush resolves despite the sink failure", verdict(flushResolved)),
assertion("delivery self-check", "sink failure counted", verdict(adapter.droppedCount() === 1)),
assertion("queue drained within bound", "pending queue count", verdict(adapter.pendingCount() === 0)),
assertion("forbidden attributes absent", "default-deny projection", verdict(redacted)),
],
negativeFixtureFailedAsExpected: redacted,
negativeFixtureFailedAsExpected: verdict(redacted),
providerVerificationRequired: true,
};
}
async function drillRollback(): Promise<DrillResult> {
const verified = await verifyRollbackReleaseCoherence();
const { release, coherence: coherent } = verified;
async function drillRollback(context: DrillContext): Promise<DrillResult> {
const { release, coherence: coherent } = await verifyRollbackReleaseCoherence({
readArtifact: rootedReader(context.root),
contractPackages: context.contractPackages,
});
const assetsPresent = await builtAssetsPresent(context.root);
const mixed = verifyCompatibilityTuple({
frontend: {
buildId: "build-a",
@@ -268,20 +333,25 @@ async function drillRollback(): Promise<DrillResult> {
},
});
return {
triggerAsserted: true,
containmentAsserted: coherent.compatible,
// A release-blocking defect is declared by a person during an incident.
triggerAsserted: "UNVERIFIED",
containmentAsserted: verdict(coherent.compatible && assetsPresent),
recoveryAssertions: [
assertion("compatibility gate", "typed version comparison", coherent.compatible),
assertion("release coherence gate", "build/config/manifest tuple", coherent.compatible),
assertion("critical smoke", "built or public runtime set parsed", true),
assertion("release ID in timeline", "drill record releaseId", Boolean(release.releaseId)),
assertion("compatibility gate", "typed version comparison", verdict(coherent.compatible)),
assertion(
"release coherence gate",
"build/config/manifest tuple and every Vite manifest file in the local dist",
verdict(coherent.compatible && assetsPresent),
),
notObserved("critical smoke", "a browser smoke after the provider switches to the prior release"),
assertion("release ID in timeline", "drill record releaseId", verdict(Boolean(release.releaseId))),
],
negativeFixtureFailedAsExpected: !mixed.compatible,
negativeFixtureFailedAsExpected: verdict(!mixed.compatible),
providerVerificationRequired: true,
};
}
const drillById: Record<string, () => Promise<DrillResult>> = {
const drillById: Record<string, (context: DrillContext) => Promise<DrillResult>> = {
"FE-RB-001": drillBoot,
"FE-RB-002": drillChunkMismatch,
"FE-RB-003": drillApiDegradation,
@@ -294,12 +364,12 @@ export async function verifyRollbackReleaseCoherence(
) {
const readArtifact = options.readArtifact ?? readJsonArtifact;
const paths = options.paths ?? DEFAULT_ROLLBACK_PATHS;
const artifactPair = await selectRollbackArtifactPair(
readArtifact,
paths,
);
const release = parseReleaseArtifact(artifactPair.release);
const runtimeArtifact = parseRuntimeConfigArtifact(artifactPair.runtime);
const [releaseValue, runtimeValue] = await Promise.all([
readArtifact(paths.release),
readArtifact(paths.runtime),
]);
const release = parseReleaseArtifact(releaseValue);
const runtimeArtifact = parseRuntimeConfigArtifact(runtimeValue);
const runtime = {
...runtimeArtifact,
BUILD_ID: requireIdentity(runtimeArtifact.BUILD_ID, "runtime BUILD_ID"),
@@ -317,43 +387,60 @@ export async function verifyRollbackReleaseCoherence(
return Object.freeze({ release, runtime, coherence });
}
export async function runRunbookDrill(runbookId: string, options: DrillOptions = {}) {
const context: DrillContext = {
root: options.root ?? process.cwd(),
contractPackages: options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES,
};
const document = JSON.parse(
await readFile(path.join(context.root, "config/runbooks/runbooks.json"), "utf8"),
) as RunbookDocument;
const specification = document.runbooks[runbookId];
const drill = drillById[runbookId];
if (!specification || !drill) throw new RangeError(`unknown runbook: ${runbookId}`);
const release = parseReleaseArtifact(
await rootedReader(context.root)(DEFAULT_ROLLBACK_PATHS.release),
);
const result = await drill(context);
const escalationPathAsserted = verdict(specification.escalation.length >= 2);
return {
schemaVersion: 2 as const,
runbookId,
gateId: specification.gateId,
releaseId: release.releaseId,
drillTimestamp: new Date().toISOString(),
triggerInjected: specification.triggerKinds[0] ?? "",
triggerAsserted: result.triggerAsserted,
containmentAsserted: result.containmentAsserted,
escalationPathAsserted,
recoveryAssertions: result.recoveryAssertions,
negativeFixtureFailedAsExpected: result.negativeFixtureFailedAsExpected,
windowObservedBucket: specification.window,
providerVerificationRequired: result.providerVerificationRequired,
result: combineDrillVerdicts(
[
result.triggerAsserted,
result.containmentAsserted,
escalationPathAsserted,
...result.recoveryAssertions.map((item) => item.result),
result.negativeFixtureFailedAsExpected,
],
result.providerVerificationRequired,
),
};
}
async function main(): Promise<void> {
const runbookId = process.argv
.slice(2)
.find((argument) => /^FE-RB-00[1-5]$/.test(argument));
const document = JSON.parse(
await readFile("config/runbooks/runbooks.json", "utf8"),
) as RunbookDocument;
const specification = runbookId ? document.runbooks[runbookId] : undefined;
if (!runbookId || !specification) {
if (!runbookId) {
process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n");
process.exit(2);
}
const drill = await drillById[runbookId]();
const escalationPathAsserted = specification.escalation.length >= 2;
const passed =
drill.triggerAsserted &&
drill.containmentAsserted &&
escalationPathAsserted &&
drill.recoveryAssertions.every((item) => item.passed) &&
drill.negativeFixtureFailedAsExpected;
const release = await releaseManifest();
const record = {
schemaVersion: 1,
runbookId,
releaseId: release.releaseId,
drillTimestamp: new Date().toISOString(),
triggerInjected: specification.triggerKinds[0],
triggerAsserted: drill.triggerAsserted,
containmentAsserted: drill.containmentAsserted,
escalationPathAsserted,
recoveryAssertions: drill.recoveryAssertions,
negativeFixtureFailedAsExpected: drill.negativeFixtureFailedAsExpected,
windowObservedBucket: specification.window,
providerVerificationRequired: drill.providerVerificationRequired,
passed,
};
const { gateId, ...record } = await runRunbookDrill(runbookId);
const artifactDirectory = `artifacts/runbooks/${runbookId}`;
await mkdir(artifactDirectory, { recursive: true });
await writeValidatedJsonArtifact({
@@ -361,15 +448,25 @@ async function main(): Promise<void> {
schema: runbookRecordArtifactSchema,
value: record,
});
if (!passed) {
process.stderr.write(`${runbookId} drill failed.\n`);
if (record.result === "FAIL") {
process.stderr.write(`${runbookId} drill: FAIL (${gateId})\n`);
process.exit(1);
}
process.stdout.write(
`${runbookId} drill: PASS (${specification.gateId}; provider verification ${
drill.providerVerificationRequired ? "still required" : "not required"
})\n`,
);
if (record.result === "UNVERIFIED") {
// Not a pass: the gate stays red until provider evidence exists (D-14).
const open = [
["trigger", record.triggerAsserted],
["containment", record.containmentAsserted],
...record.recoveryAssertions.map((item) => [item.assertion, item.result] as const),
["negative fixture", record.negativeFixtureFailedAsExpected],
["provider verification", record.providerVerificationRequired ? "UNVERIFIED" : "PASS"],
].filter(([, result]) => result === "UNVERIFIED").map(([name]) => name);
process.stderr.write(
`${runbookId} drill: UNVERIFIED (${gateId}; no local check failed; not observed locally: ${open.join(", ")})\n`,
);
process.exit(3);
}
process.stdout.write(`${runbookId} drill: PASS (${gateId})\n`);
}
function requireIdentity(value: string | undefined, label: string): string {
@@ -379,88 +476,6 @@ function requireIdentity(value: string | undefined, label: string): string {
return value;
}
async function selectRollbackArtifactPair(
readArtifact: JsonArtifactReader,
paths: RollbackArtifactPaths,
): Promise<Readonly<{ release: unknown; runtime: unknown }>> {
const primary = await readArtifactPair(
readArtifact,
paths.primaryRelease,
paths.primaryRuntime,
);
if (
primary.release.status === "fulfilled" &&
primary.runtime.status === "fulfilled"
) {
return Object.freeze({
release: primary.release.value,
runtime: primary.runtime.value,
});
}
const releaseMissing =
primary.release.status === "rejected" &&
hasErrorCode(primary.release.reason, "ENOENT");
const runtimeMissing =
primary.runtime.status === "rejected" &&
hasErrorCode(primary.runtime.reason, "ENOENT");
if (releaseMissing && runtimeMissing) {
const fallback = await readArtifactPair(
readArtifact,
paths.fallbackRelease,
paths.fallbackRuntime,
);
if (fallback.release.status === "rejected") {
throw fallback.release.reason;
}
if (fallback.runtime.status === "rejected") {
throw fallback.runtime.reason;
}
return Object.freeze({
release: fallback.release.value,
runtime: fallback.runtime.value,
});
}
if (primary.release.status === "rejected" && !releaseMissing) {
throw primary.release.reason;
}
if (primary.runtime.status === "rejected" && !runtimeMissing) {
throw primary.runtime.reason;
}
throw new Error("primary rollback artifact pair is incomplete");
}
async function readArtifactPair(
readArtifact: JsonArtifactReader,
releasePath: string,
runtimePath: string,
): Promise<
Readonly<{
release: PromiseSettledResult<unknown>;
runtime: PromiseSettledResult<unknown>;
}>
> {
const [release, runtime] = await Promise.allSettled([
Promise.resolve().then(() => readArtifact(releasePath)),
Promise.resolve().then(() => readArtifact(runtimePath)),
]);
return Object.freeze({ release, runtime });
}
async function readPrimaryOrFallback(
readArtifact: JsonArtifactReader,
primary: string,
fallback: string,
): Promise<unknown> {
try {
return await readArtifact(primary);
} catch (error) {
if (!hasErrorCode(error, "ENOENT")) throw error;
return readArtifact(fallback);
}
}
function hasErrorCode(error: unknown, code: string): boolean {
return Boolean(
error &&
+1 -1
View File
@@ -4,7 +4,7 @@ export const ciContractReportSchema = z
.object({
schemaVersion: z.literal(2),
nodeVersion: z.string().regex(/^\d+\.\d+\.\d+$/u),
gateCount: z.literal(28),
gateCount: z.literal(27),
commandDefinitionCount: z.number().int().positive(),
commandReferenceCount: z.number().int().positive(),
artifactCount: z.number().int().positive(),
+1 -1
View File
@@ -28,7 +28,7 @@ export const PROMOTION_FORMULA = Object.freeze({
"FE-GATE-028",
]),
PROD_PROMOTION_READY: Object.freeze([
"FE-GATE-016",
// FE-GATE-016 is retired (D-14): it repeated FE-GATE-025's FE-RB-005 drill.
"FE-GATE-021",
"FE-GATE-022",
"FE-GATE-023",