fix(release): record unobserved runbook drill verdicts as UNVERIFIED (11-01, 13-03)
Drills held constant true verdicts and read public/ placeholders, so they passed with no build or with JS assets deleted. They now check the local dist, mark boot, live traffic and provider steps UNVERIFIED (exit 3), and FE-GATE-016 is retired because it repeated FE-GATE-025's FE-RB-005 drill. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
7298275271
commit
7f889a6d83
@@ -379,7 +379,6 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { gate: FE-GATE-016, name: rollback-drill }
|
||||
- { gate: FE-GATE-021, name: runbook-boot-config }
|
||||
- { gate: FE-GATE-022, name: runbook-chunk-mismatch }
|
||||
- { gate: FE-GATE-023, name: runbook-api-degradation }
|
||||
|
||||
@@ -123,6 +123,11 @@ corepack pnpm drill:runbooks
|
||||
corepack pnpm check:ci
|
||||
```
|
||||
|
||||
`drill:runbooks` stops at the first drill that is not `PASS`. Without provider
|
||||
evidence every drill ends `UNVERIFIED` (exit 3), because a drill never reports
|
||||
a check it could not run locally as `PASS`; see "Runbook jobs" in
|
||||
[`docs/operations/ci-quality-gates.md`](docs/operations/ci-quality-gates.md).
|
||||
|
||||
`test:all` is the ordinary product-development loop and intentionally excludes
|
||||
host-level CI-runner assurance. Run `corepack pnpm test:system` only on the
|
||||
compatible Linux assurance host described in
|
||||
@@ -168,7 +173,7 @@ Live release verification additionally requires `HOSTING_BASE_URL`.
|
||||
|
||||
## CI and evidence
|
||||
|
||||
The 28-gate registry is `config/ci/gates.json`; the Gitea workflow is
|
||||
The 27-gate registry is `config/ci/gates.json`; the Gitea workflow is
|
||||
`.gitea/workflows/quality-gates.yml`. It follows:
|
||||
|
||||
```text
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"owner": "frontend-platform",
|
||||
"reason": "08-06: the signed manual accessibility review moves to release gate FE-GATE-028; merge gate FE-GATE-009 checks only the record format",
|
||||
"gates": 28,
|
||||
"reason": "11-01, 13-03 (D-14): FE-GATE-016 retired; it ran the same FE-RB-005 drill as FE-GATE-025",
|
||||
"gates": 27,
|
||||
"commands": 94,
|
||||
"commandReferences": 106,
|
||||
"artifacts": 110,
|
||||
"commandReferences": 104,
|
||||
"artifacts": 109,
|
||||
"stages": 5,
|
||||
"gateShapeSha256": "111e82fa1fd8341f8ee1f3d10d14365de382d83da5f8f9c5a1e3fcfecc5cafae"
|
||||
"gateShapeSha256": "3ad2407c5929a7ec43dadebc68a3dc69e265793bf1c80c355b6d2fdeae019821"
|
||||
}
|
||||
|
||||
@@ -1502,12 +1502,6 @@
|
||||
"build-release-candidate"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "artifact-artifacts-quality-gates-FE-GATE-016-txt",
|
||||
"path": "artifacts/quality/gates/FE-GATE-016.txt",
|
||||
"schemaId": "text",
|
||||
"production": "runner-generated"
|
||||
},
|
||||
{
|
||||
"id": "artifact-artifacts-runbooks-FE-RB-005-record-json",
|
||||
"path": "artifacts/runbooks/FE-RB-005/record.json",
|
||||
@@ -2015,19 +2009,6 @@
|
||||
],
|
||||
"retentionClassId": "release-coherence"
|
||||
},
|
||||
{
|
||||
"id": "FE-GATE-016",
|
||||
"name": "rollback-drill",
|
||||
"commandIds": [
|
||||
"build",
|
||||
"drill-runbook"
|
||||
],
|
||||
"logArtifactId": "artifact-artifacts-quality-gates-FE-GATE-016-txt",
|
||||
"evidenceArtifactIds": [
|
||||
"artifact-artifacts-runbooks-FE-RB-005-record-json"
|
||||
],
|
||||
"retentionClassId": "prod-drill"
|
||||
},
|
||||
{
|
||||
"id": "FE-GATE-017",
|
||||
"name": "diagram-review",
|
||||
@@ -2243,7 +2224,6 @@
|
||||
"release"
|
||||
],
|
||||
"gateIds": [
|
||||
"FE-GATE-016",
|
||||
"FE-GATE-021",
|
||||
"FE-GATE-022",
|
||||
"FE-GATE-023",
|
||||
@@ -2774,7 +2754,6 @@
|
||||
"condition": "production",
|
||||
"timeoutMinutes": 45,
|
||||
"gateIds": [
|
||||
"FE-GATE-016",
|
||||
"FE-GATE-021",
|
||||
"FE-GATE-022",
|
||||
"FE-GATE-023",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "ART-FE-RUNBOOK-DRILL@1",
|
||||
"$id": "ART-FE-RUNBOOK-DRILL@2",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"schemaVersion",
|
||||
@@ -14,29 +14,39 @@
|
||||
"recoveryAssertions",
|
||||
"negativeFixtureFailedAsExpected",
|
||||
"windowObservedBucket",
|
||||
"passed"
|
||||
"providerVerificationRequired",
|
||||
"result"
|
||||
],
|
||||
"$defs": {
|
||||
"verdict": { "enum": ["PASS", "FAIL", "UNVERIFIED"] }
|
||||
},
|
||||
"properties": {
|
||||
"schemaVersion": { "const": 1 },
|
||||
"schemaVersion": { "const": 2 },
|
||||
"runbookId": { "pattern": "^FE-RB-00[1-5]$" },
|
||||
"releaseId": { "type": "string", "minLength": 1 },
|
||||
"drillTimestamp": { "type": "string", "format": "date-time" },
|
||||
"triggerInjected": { "type": "string" },
|
||||
"triggerAsserted": { "type": "boolean" },
|
||||
"containmentAsserted": { "type": "boolean" },
|
||||
"escalationPathAsserted": { "type": "boolean" },
|
||||
"triggerAsserted": { "$ref": "#/$defs/verdict" },
|
||||
"containmentAsserted": { "$ref": "#/$defs/verdict" },
|
||||
"escalationPathAsserted": { "$ref": "#/$defs/verdict" },
|
||||
"recoveryAssertions": {
|
||||
"type": "array",
|
||||
"minItems": 4,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["assertion", "evidence", "passed"]
|
||||
"required": ["assertion", "evidence", "result"],
|
||||
"properties": {
|
||||
"assertion": { "type": "string", "minLength": 1 },
|
||||
"evidence": { "type": "string", "minLength": 1 },
|
||||
"result": { "$ref": "#/$defs/verdict" }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"negativeFixtureFailedAsExpected": { "type": "boolean" },
|
||||
"negativeFixtureFailedAsExpected": { "$ref": "#/$defs/verdict" },
|
||||
"windowObservedBucket": { "type": "string" },
|
||||
"providerVerificationRequired": { "type": "boolean" },
|
||||
"passed": { "type": "boolean" }
|
||||
"result": { "$ref": "#/$defs/verdict" }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# CI quality-gate orchestration
|
||||
|
||||
`config/ci/gates.json` is the executable registry for all 28 gates and the sole
|
||||
`config/ci/gates.json` is the executable registry for all 27 gates and the sole
|
||||
workflow model. `.gitea/workflows/quality-gates.yml` is generated from that
|
||||
strict v2 contract; it is not an independent configuration authority. Update
|
||||
the registry, run `corepack pnpm generate:ci-workflow`, and commit the generated
|
||||
@@ -269,7 +269,20 @@ promotion, and field evidence through aggregation.
|
||||
|
||||
Runbook jobs write to the stable path
|
||||
`artifacts/runbooks/<runbook-id>/record.json`; the dynamic release identity is
|
||||
stored inside the record. This keeps gate evidence lookup independent from
|
||||
stored inside the record.
|
||||
|
||||
A runbook drill (`drill:runbook`, FE-GATE-021..025) records each verdict as
|
||||
`PASS`, `FAIL` or `UNVERIFIED` (`scripts/drill-runbook.ts`, record schema in
|
||||
`scripts/contracts/release-artifacts.ts`). `PASS`/`FAIL` means the drill checked
|
||||
it against the local `dist/` build; it reads no `public/` placeholder, so a
|
||||
missing build fails. Browser boot, live API traffic, smoke after a provider
|
||||
switch, and anything the runbook marks as needing provider verification are
|
||||
`UNVERIFIED`. One `FAIL` makes the drill `FAIL` (exit 1). Otherwise any
|
||||
`UNVERIFIED` makes it `UNVERIFIED` (exit 3), and the gate is not `PASS`. So
|
||||
without provider evidence the production stage stays open. That is the intended
|
||||
result: a local drill cannot show that the promoted release can be rolled back.
|
||||
FE-GATE-016 (`rollback-drill`) is retired because it ran the same FE-RB-005
|
||||
drill as FE-GATE-025 (`RETIRED_GATE_IDS` in `scripts/contracts/ci-gates.ts`). This keeps gate evidence lookup independent from
|
||||
slashes or other provider-specific characters in `RELEASE_ID`, while each
|
||||
workflow artifact remains scoped to its own run.
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
type GateResult,
|
||||
} from "./lib/promotion-readiness.ts";
|
||||
import {
|
||||
CANONICAL_GATE_IDS,
|
||||
loadCiGateContract,
|
||||
indexCiGateContract,
|
||||
} from "./contracts/ci-gates.ts";
|
||||
@@ -104,10 +105,7 @@ if (!architecture?.evidenceArtifactIds.some((id) => index.artifacts.get(id)?.pat
|
||||
failures.push("FE-GATE-010 must publish the typed CI contract report");
|
||||
}
|
||||
|
||||
const expectedGateIds = Array.from(
|
||||
{ length: 28 },
|
||||
(_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`,
|
||||
);
|
||||
const expectedGateIds = CANONICAL_GATE_IDS;
|
||||
const passingResults: Record<string, GateResult> = Object.fromEntries(
|
||||
expectedGateIds.map((gateId) => [gateId, "PASS"]),
|
||||
);
|
||||
@@ -168,4 +166,4 @@ if (failures.length > 0) {
|
||||
process.stderr.write(`CI contract failed:\n${failures.join("\n")}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
process.stdout.write("CI contract: 28 gates, strict v2 graph and generated workflow model PASS\n");
|
||||
process.stdout.write(`CI contract: ${CANONICAL_GATE_IDS.length} gates, strict v2 graph and generated workflow model PASS\n`);
|
||||
|
||||
@@ -438,6 +438,19 @@ export type CiGateContractIndex = Readonly<{
|
||||
jobs: ReadonlyMap<string, CiWorkflowJob>;
|
||||
retentionClasses: ReadonlyMap<string, CiGateContract["retention"]["classes"][number]>;
|
||||
}>;
|
||||
|
||||
/**
|
||||
* D-14. FE-GATE-016 (rollback-drill) ran the same FE-RB-005 drill as
|
||||
* FE-GATE-025 and counted it twice toward production readiness. Its id stays
|
||||
* retired so no later gate reuses it with a different meaning.
|
||||
*/
|
||||
export const RETIRED_GATE_IDS: readonly string[] = Object.freeze(["FE-GATE-016"]);
|
||||
|
||||
export const CANONICAL_GATE_IDS: readonly string[] = Object.freeze(
|
||||
Array.from({ length: 28 }, (_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`)
|
||||
.filter((gateId) => !RETIRED_GATE_IDS.includes(gateId)),
|
||||
);
|
||||
|
||||
/**
|
||||
* D-13. The contract's size and gate shape are a reviewed baseline, not code
|
||||
* constants: a project that removes a sample or adds a gate records the new
|
||||
@@ -789,12 +802,9 @@ function validateContractSemantics(
|
||||
issue(`stage dependency cycle: ${cycle.join(" -> ")}`);
|
||||
}
|
||||
|
||||
const expectedGateIds = Array.from(
|
||||
{ length: 28 },
|
||||
(_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`,
|
||||
);
|
||||
const expectedGateIds = CANONICAL_GATE_IDS;
|
||||
if (JSON.stringify(contract.gates.map(({ id }) => id)) !== JSON.stringify(expectedGateIds)) {
|
||||
issue("gate registry must contain FE-GATE-001..028 in canonical order");
|
||||
issue(`gate registry must contain FE-GATE-001..028 without retired ${RETIRED_GATE_IDS.join(", ")} in canonical order`);
|
||||
}
|
||||
const expectedStages: ReadonlyArray<readonly [string, string, readonly string[], readonly string[]]> = [
|
||||
["merge", "MERGE_READY", [], PROMOTION_FORMULA.MERGE_READY],
|
||||
@@ -869,7 +879,7 @@ function validateContractSemantics(
|
||||
vulnerability_provider: [],
|
||||
provenance_provider: [],
|
||||
promotion: [],
|
||||
production_gate: ["FE-GATE-016", "FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"],
|
||||
production_gate: ["FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"],
|
||||
field_gate: ["FE-GATE-018"],
|
||||
documentation_gate: ["FE-GATE-017"],
|
||||
};
|
||||
|
||||
@@ -718,31 +718,67 @@ export const releaseVerificationArtifactSchema = z
|
||||
})
|
||||
.strict();
|
||||
|
||||
// 11-01 / 13-03 (D-14): each verdict is PASS or FAIL only when observed on the
|
||||
// local build; anything that needs a boot, live traffic or the provider is
|
||||
// UNVERIFIED, and so is a drill with provider scope.
|
||||
const drillVerdict = z.enum(["PASS", "FAIL", "UNVERIFIED"]);
|
||||
|
||||
export type DrillVerdict = z.infer<typeof drillVerdict>;
|
||||
|
||||
/** FAIL beats UNVERIFIED beats PASS; provider scope alone keeps a drill UNVERIFIED. */
|
||||
export function combineDrillVerdicts(
|
||||
verdicts: readonly DrillVerdict[],
|
||||
providerVerificationRequired: boolean,
|
||||
): DrillVerdict {
|
||||
if (verdicts.includes("FAIL")) return "FAIL";
|
||||
if (verdicts.includes("UNVERIFIED") || providerVerificationRequired) return "UNVERIFIED";
|
||||
return "PASS";
|
||||
}
|
||||
|
||||
export const runbookRecordArtifactSchema = z
|
||||
.object({
|
||||
schemaVersion: z.literal(1),
|
||||
schemaVersion: z.literal(2),
|
||||
runbookId: z.string().regex(/^FE-RB-00[1-5]$/u),
|
||||
releaseId: nonEmptyString,
|
||||
drillTimestamp: timestamp,
|
||||
triggerInjected: nonEmptyString,
|
||||
triggerAsserted: z.boolean(),
|
||||
containmentAsserted: z.boolean(),
|
||||
escalationPathAsserted: z.boolean(),
|
||||
triggerAsserted: drillVerdict,
|
||||
containmentAsserted: drillVerdict,
|
||||
escalationPathAsserted: drillVerdict,
|
||||
recoveryAssertions: z.array(
|
||||
z
|
||||
.object({
|
||||
assertion: nonEmptyString,
|
||||
evidence: nonEmptyString,
|
||||
passed: z.boolean(),
|
||||
result: drillVerdict,
|
||||
})
|
||||
.strict(),
|
||||
),
|
||||
negativeFixtureFailedAsExpected: z.boolean(),
|
||||
negativeFixtureFailedAsExpected: drillVerdict,
|
||||
windowObservedBucket: nonEmptyString,
|
||||
providerVerificationRequired: z.boolean(),
|
||||
passed: z.boolean(),
|
||||
result: drillVerdict,
|
||||
})
|
||||
.strict();
|
||||
.strict()
|
||||
.superRefine((record, context) => {
|
||||
const expected = combineDrillVerdicts(
|
||||
[
|
||||
record.triggerAsserted,
|
||||
record.containmentAsserted,
|
||||
record.escalationPathAsserted,
|
||||
...record.recoveryAssertions.map(({ result }) => result),
|
||||
record.negativeFixtureFailedAsExpected,
|
||||
],
|
||||
record.providerVerificationRequired,
|
||||
);
|
||||
if (record.result !== expected) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["result"],
|
||||
message: `result must be ${expected} for these verdicts`,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const failureList = z.array(nonEmptyString).max(4_096);
|
||||
const sourceOrFixtureMode = z.enum(["source", "negative-fixture"]);
|
||||
|
||||
+219
-204
@@ -1,4 +1,5 @@
|
||||
import { access, mkdir, readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
import { shouldRetry } from "../src/adapters/http/retry-policy.ts";
|
||||
@@ -15,21 +16,30 @@ import {
|
||||
} from "../src/contracts/release-artifacts.ts";
|
||||
import { projectTelemetryEvent } from "../src/contracts/telemetry.ts";
|
||||
import { EXPECTED_CONTRACT_SET_PACKAGES } from "../src/features/installed-contract-contributions.ts";
|
||||
import { runbookRecordArtifactSchema } from "./contracts/release-artifacts.ts";
|
||||
import {
|
||||
combineDrillVerdicts,
|
||||
runbookRecordArtifactSchema,
|
||||
type DrillVerdict,
|
||||
} from "./contracts/release-artifacts.ts";
|
||||
import { verifyReleaseRuntimeCoherence } from "./lib/release-runtime-coherence.ts";
|
||||
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
|
||||
|
||||
// 11-01 / 13-03 (D-14). A verdict is PASS or FAIL only when the drill observed
|
||||
// it on the local build. What needs a browser boot, live traffic or the hosting
|
||||
// provider is UNVERIFIED, and so is every drill whose runbook needs provider
|
||||
// verification: a local drill never claims the release can be recovered.
|
||||
|
||||
type RecoveryAssertion = Readonly<{
|
||||
assertion: string;
|
||||
evidence: string;
|
||||
passed: boolean;
|
||||
result: DrillVerdict;
|
||||
}>;
|
||||
|
||||
type DrillResult = Readonly<{
|
||||
triggerAsserted: boolean;
|
||||
containmentAsserted: boolean;
|
||||
triggerAsserted: DrillVerdict;
|
||||
containmentAsserted: DrillVerdict;
|
||||
recoveryAssertions: RecoveryAssertion[];
|
||||
negativeFixtureFailedAsExpected: boolean;
|
||||
negativeFixtureFailedAsExpected: DrillVerdict;
|
||||
providerVerificationRequired: boolean;
|
||||
}>;
|
||||
|
||||
@@ -51,10 +61,8 @@ type RunbookDocument = Readonly<{
|
||||
export type JsonArtifactReader = (path: string) => Promise<unknown>;
|
||||
|
||||
export type RollbackArtifactPaths = Readonly<{
|
||||
primaryRelease: string;
|
||||
fallbackRelease: string;
|
||||
primaryRuntime: string;
|
||||
fallbackRuntime: string;
|
||||
release: string;
|
||||
runtime: string;
|
||||
}>;
|
||||
|
||||
export type RollbackCoherenceOptions = Readonly<{
|
||||
@@ -63,27 +71,37 @@ export type RollbackCoherenceOptions = Readonly<{
|
||||
paths?: RollbackArtifactPaths;
|
||||
}>;
|
||||
|
||||
export type DrillOptions = Readonly<{
|
||||
/** Directory holding `dist/` and `config/runbooks/`. Defaults to the cwd. */
|
||||
root?: string;
|
||||
contractPackages?: readonly InstalledContractPackageIdentity[];
|
||||
}>;
|
||||
|
||||
type DrillContext = Readonly<{
|
||||
root: string;
|
||||
contractPackages: readonly InstalledContractPackageIdentity[];
|
||||
}>;
|
||||
|
||||
// 13-03: the drill reads the built release only. It used to fall back to the
|
||||
// placeholders in public/, so it passed without any build at all.
|
||||
const DEFAULT_ROLLBACK_PATHS = Object.freeze({
|
||||
primaryRelease: "dist/release-manifest.json",
|
||||
fallbackRelease: "public/release-manifest.json",
|
||||
primaryRuntime: "dist/config.json",
|
||||
fallbackRuntime: "public/config.json",
|
||||
release: "dist/release-manifest.json",
|
||||
runtime: "dist/config.json",
|
||||
});
|
||||
|
||||
async function readJsonArtifact(path: string): Promise<unknown> {
|
||||
return JSON.parse(await readFile(path, "utf8"));
|
||||
try {
|
||||
return JSON.parse(await readFile(path, "utf8"));
|
||||
} catch (error) {
|
||||
if (hasErrorCode(error, "ENOENT")) {
|
||||
throw new Error(`${path} is missing; build the release before running the drill`, { cause: error });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function releaseManifest(
|
||||
readArtifact: JsonArtifactReader = readJsonArtifact,
|
||||
paths: RollbackArtifactPaths = DEFAULT_ROLLBACK_PATHS,
|
||||
): Promise<ReleaseArtifact> {
|
||||
const value = await readPrimaryOrFallback(
|
||||
readArtifact,
|
||||
paths.primaryRelease,
|
||||
paths.fallbackRelease,
|
||||
);
|
||||
return parseReleaseArtifact(value);
|
||||
function rootedReader(root: string): JsonArtifactReader {
|
||||
return (file) => readJsonArtifact(path.join(root, file));
|
||||
}
|
||||
|
||||
const validConfig = {
|
||||
@@ -100,12 +118,48 @@ const validConfig = {
|
||||
RELEASE_ID: "local-release",
|
||||
};
|
||||
|
||||
function verdict(passed: boolean): DrillVerdict {
|
||||
return passed ? "PASS" : "FAIL";
|
||||
}
|
||||
|
||||
function assertion(
|
||||
assertion: string,
|
||||
evidence: string,
|
||||
passed: boolean,
|
||||
result: DrillVerdict,
|
||||
): RecoveryAssertion {
|
||||
return { assertion, evidence, passed };
|
||||
return { assertion, evidence, result };
|
||||
}
|
||||
|
||||
/** An assertion the local drill cannot observe; `needs` names what can. */
|
||||
function notObserved(name: string, needs: string): RecoveryAssertion {
|
||||
return assertion(name, `not observed locally; needs ${needs}`, "UNVERIFIED");
|
||||
}
|
||||
|
||||
/**
|
||||
* Every file the Vite manifest names (entries, lazy chunks, CSS, assets) exists
|
||||
* in the local dist. Hosting reachability still needs the provider smoke.
|
||||
*/
|
||||
async function builtAssetsPresent(root: string): Promise<boolean> {
|
||||
const manifest = await readJsonArtifact(path.join(root, "dist/.vite/manifest.json"));
|
||||
if (!manifest || typeof manifest !== "object") return false;
|
||||
const files = new Set<string>();
|
||||
for (const chunk of Object.values(manifest as Record<string, Record<string, unknown>>)) {
|
||||
for (const key of ["file", "css", "assets"] as const) {
|
||||
const value = chunk[key];
|
||||
for (const file of Array.isArray(value) ? value : [value]) {
|
||||
if (typeof file === "string") files.add(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (![...files].some((file) => file.endsWith(".js"))) return false;
|
||||
try {
|
||||
await Promise.all(
|
||||
["index.html", ...files].map((file) => access(path.join(root, "dist", file))),
|
||||
);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function drillBoot(): Promise<DrillResult> {
|
||||
@@ -115,19 +169,22 @@ async function drillBoot(): Promise<DrillResult> {
|
||||
API_BASE_URL: "http://insecure.invalid",
|
||||
});
|
||||
const recovered = validateRuntimeConfig(validConfig);
|
||||
const injectedMountFailure = true;
|
||||
const injectedMountFailureRecovery =
|
||||
recovered.success && !injectedMountFailure;
|
||||
return {
|
||||
triggerAsserted: !invalid.success,
|
||||
containmentAsserted: !invalid.success,
|
||||
triggerAsserted: verdict(!invalid.success),
|
||||
containmentAsserted: verdict(!invalid.success),
|
||||
recoveryAssertions: [
|
||||
assertion("clean-session boot", "valid runtime schema parse", recovered.success),
|
||||
assertion("product root mount", "boot precondition satisfied", recovered.success),
|
||||
assertion("config validation", "invalid fixture rejected", !invalid.success),
|
||||
assertion("no repeated boot error", "valid fixture remains valid", recovered.success),
|
||||
notObserved("clean-session boot", "a browser boot of the promoted release"),
|
||||
notObserved("product root mount", "a browser boot of the promoted release"),
|
||||
assertion(
|
||||
"config validation",
|
||||
"invalid fixture rejected, valid fixture accepted",
|
||||
verdict(!invalid.success && recovered.success),
|
||||
),
|
||||
notObserved("no repeated boot error", "boot telemetry from the promoted release"),
|
||||
],
|
||||
negativeFixtureFailedAsExpected: !injectedMountFailureRecovery,
|
||||
// The runbook's negative fixture injects a mount failure after a valid
|
||||
// config; that needs a mounted app, which a Node drill does not have.
|
||||
negativeFixtureFailedAsExpected: "UNVERIFIED",
|
||||
providerVerificationRequired: false,
|
||||
};
|
||||
}
|
||||
@@ -144,7 +201,7 @@ function memoryStorage(): StoragePort {
|
||||
};
|
||||
}
|
||||
|
||||
async function drillChunkMismatch(): Promise<DrillResult> {
|
||||
async function drillChunkMismatch(context: DrillContext): Promise<DrillResult> {
|
||||
const storage = memoryStorage();
|
||||
const input: Parameters<typeof decideChunkRecovery>[0] = {
|
||||
failureKind: "DEPLOY_MISMATCH",
|
||||
@@ -157,29 +214,26 @@ async function drillChunkMismatch(): Promise<DrillResult> {
|
||||
};
|
||||
const first = decideChunkRecovery(input);
|
||||
const second = decideChunkRecovery(input);
|
||||
const manifest = await releaseManifest();
|
||||
let assetsReachable = true;
|
||||
try {
|
||||
await access("dist/index.html");
|
||||
await access("dist/.vite/manifest.json");
|
||||
} catch {
|
||||
assetsReachable = false;
|
||||
}
|
||||
const { coherence } = await verifyRollbackReleaseCoherence({
|
||||
readArtifact: rootedReader(context.root),
|
||||
contractPackages: context.contractPackages,
|
||||
});
|
||||
return {
|
||||
triggerAsserted: first.action === "reload-once",
|
||||
containmentAsserted:
|
||||
triggerAsserted: verdict(first.action === "reload-once"),
|
||||
containmentAsserted: verdict(
|
||||
first.action === "reload-once" && second.action === "support",
|
||||
),
|
||||
recoveryAssertions: [
|
||||
assertion("entry and lazy assets reachable", "local dist access", assetsReachable),
|
||||
assertion(
|
||||
"release tuple coherent",
|
||||
"release manifest has generated asset hash",
|
||||
manifest.assetManifestHash !== "generated-during-build",
|
||||
"entry and lazy assets reachable",
|
||||
"every Vite manifest file exists in the local dist",
|
||||
verdict(await builtAssetsPresent(context.root)),
|
||||
),
|
||||
assertion("second reload blocked", "reload guard decision", second.action === "support"),
|
||||
assertion("critical route smoke", "built index available", assetsReachable),
|
||||
assertion("release tuple coherent", "build/config/manifest tuple", verdict(coherence.compatible)),
|
||||
assertion("second reload blocked", "reload guard decision", verdict(second.action === "support")),
|
||||
notObserved("critical route smoke", "a browser smoke against the hosted release"),
|
||||
],
|
||||
negativeFixtureFailedAsExpected: second.action !== "reload-once",
|
||||
negativeFixtureFailedAsExpected: verdict(second.action !== "reload-once"),
|
||||
providerVerificationRequired: true,
|
||||
};
|
||||
}
|
||||
@@ -196,15 +250,20 @@ async function drillApiDegradation(): Promise<DrillResult> {
|
||||
0,
|
||||
);
|
||||
return {
|
||||
triggerAsserted: true,
|
||||
containmentAsserted: !unkeyedRetry,
|
||||
// The trigger is a failure rate over live traffic.
|
||||
triggerAsserted: "UNVERIFIED",
|
||||
containmentAsserted: verdict(!unkeyedRetry && safeRetry),
|
||||
recoveryAssertions: [
|
||||
assertion("failure rate at baseline", "deterministic recovery window", true),
|
||||
assertion("no retry amplification", "unkeyed retry policy", !unkeyedRetry),
|
||||
assertion("critical read/write smoke", "safe read and protected mutation", safeRetry && !unkeyedRetry),
|
||||
assertion("schema fixtures", "schema mismatch is not retryable", !shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)),
|
||||
notObserved("terminal failure rate at baseline", "API failure telemetry from the promoted release"),
|
||||
assertion("no retry amplification", "unkeyed retry policy", verdict(!unkeyedRetry)),
|
||||
notObserved("critical read/write smoke", "a read and a write against the live API"),
|
||||
assertion(
|
||||
"schema fixtures",
|
||||
"schema mismatch is not retryable",
|
||||
verdict(!shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)),
|
||||
),
|
||||
],
|
||||
negativeFixtureFailedAsExpected: !unkeyedRetry,
|
||||
negativeFixtureFailedAsExpected: verdict(!unkeyedRetry),
|
||||
providerVerificationRequired: true,
|
||||
};
|
||||
}
|
||||
@@ -224,7 +283,10 @@ async function drillTelemetry(): Promise<DrillResult> {
|
||||
attempt_count_bucket: "1",
|
||||
route_id: "APP_HOME",
|
||||
});
|
||||
await adapter.flush();
|
||||
const flushResolved = await adapter.flush().then(
|
||||
() => true,
|
||||
() => false,
|
||||
);
|
||||
const projected = projectTelemetryEvent("api.request.failed", {
|
||||
error_kind: "SERVER_FAILURE",
|
||||
http_status_group: "5xx",
|
||||
@@ -235,22 +297,25 @@ async function drillTelemetry(): Promise<DrillResult> {
|
||||
const redacted =
|
||||
projected.success && !JSON.stringify(projected).includes("raw_url");
|
||||
return {
|
||||
triggerAsserted: adapter.droppedCount() === 1,
|
||||
containmentAsserted: adapter.pendingCount() === 0,
|
||||
triggerAsserted: verdict(adapter.droppedCount() === 1),
|
||||
containmentAsserted: verdict(adapter.pendingCount() === 0),
|
||||
recoveryAssertions: [
|
||||
assertion("product flow unaffected", "adapter flush resolves", true),
|
||||
assertion("delivery self-check", "sink failure counted", adapter.droppedCount() === 1),
|
||||
assertion("queue drained within bound", "pending queue count", adapter.pendingCount() === 0),
|
||||
assertion("forbidden attributes absent", "default-deny projection", redacted),
|
||||
assertion("product flow unaffected", "adapter flush resolves despite the sink failure", verdict(flushResolved)),
|
||||
assertion("delivery self-check", "sink failure counted", verdict(adapter.droppedCount() === 1)),
|
||||
assertion("queue drained within bound", "pending queue count", verdict(adapter.pendingCount() === 0)),
|
||||
assertion("forbidden attributes absent", "default-deny projection", verdict(redacted)),
|
||||
],
|
||||
negativeFixtureFailedAsExpected: redacted,
|
||||
negativeFixtureFailedAsExpected: verdict(redacted),
|
||||
providerVerificationRequired: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function drillRollback(): Promise<DrillResult> {
|
||||
const verified = await verifyRollbackReleaseCoherence();
|
||||
const { release, coherence: coherent } = verified;
|
||||
async function drillRollback(context: DrillContext): Promise<DrillResult> {
|
||||
const { release, coherence: coherent } = await verifyRollbackReleaseCoherence({
|
||||
readArtifact: rootedReader(context.root),
|
||||
contractPackages: context.contractPackages,
|
||||
});
|
||||
const assetsPresent = await builtAssetsPresent(context.root);
|
||||
const mixed = verifyCompatibilityTuple({
|
||||
frontend: {
|
||||
buildId: "build-a",
|
||||
@@ -268,20 +333,25 @@ async function drillRollback(): Promise<DrillResult> {
|
||||
},
|
||||
});
|
||||
return {
|
||||
triggerAsserted: true,
|
||||
containmentAsserted: coherent.compatible,
|
||||
// A release-blocking defect is declared by a person during an incident.
|
||||
triggerAsserted: "UNVERIFIED",
|
||||
containmentAsserted: verdict(coherent.compatible && assetsPresent),
|
||||
recoveryAssertions: [
|
||||
assertion("compatibility gate", "typed version comparison", coherent.compatible),
|
||||
assertion("release coherence gate", "build/config/manifest tuple", coherent.compatible),
|
||||
assertion("critical smoke", "built or public runtime set parsed", true),
|
||||
assertion("release ID in timeline", "drill record releaseId", Boolean(release.releaseId)),
|
||||
assertion("compatibility gate", "typed version comparison", verdict(coherent.compatible)),
|
||||
assertion(
|
||||
"release coherence gate",
|
||||
"build/config/manifest tuple and every Vite manifest file in the local dist",
|
||||
verdict(coherent.compatible && assetsPresent),
|
||||
),
|
||||
notObserved("critical smoke", "a browser smoke after the provider switches to the prior release"),
|
||||
assertion("release ID in timeline", "drill record releaseId", verdict(Boolean(release.releaseId))),
|
||||
],
|
||||
negativeFixtureFailedAsExpected: !mixed.compatible,
|
||||
negativeFixtureFailedAsExpected: verdict(!mixed.compatible),
|
||||
providerVerificationRequired: true,
|
||||
};
|
||||
}
|
||||
|
||||
const drillById: Record<string, () => Promise<DrillResult>> = {
|
||||
const drillById: Record<string, (context: DrillContext) => Promise<DrillResult>> = {
|
||||
"FE-RB-001": drillBoot,
|
||||
"FE-RB-002": drillChunkMismatch,
|
||||
"FE-RB-003": drillApiDegradation,
|
||||
@@ -294,12 +364,12 @@ export async function verifyRollbackReleaseCoherence(
|
||||
) {
|
||||
const readArtifact = options.readArtifact ?? readJsonArtifact;
|
||||
const paths = options.paths ?? DEFAULT_ROLLBACK_PATHS;
|
||||
const artifactPair = await selectRollbackArtifactPair(
|
||||
readArtifact,
|
||||
paths,
|
||||
);
|
||||
const release = parseReleaseArtifact(artifactPair.release);
|
||||
const runtimeArtifact = parseRuntimeConfigArtifact(artifactPair.runtime);
|
||||
const [releaseValue, runtimeValue] = await Promise.all([
|
||||
readArtifact(paths.release),
|
||||
readArtifact(paths.runtime),
|
||||
]);
|
||||
const release = parseReleaseArtifact(releaseValue);
|
||||
const runtimeArtifact = parseRuntimeConfigArtifact(runtimeValue);
|
||||
const runtime = {
|
||||
...runtimeArtifact,
|
||||
BUILD_ID: requireIdentity(runtimeArtifact.BUILD_ID, "runtime BUILD_ID"),
|
||||
@@ -317,43 +387,60 @@ export async function verifyRollbackReleaseCoherence(
|
||||
return Object.freeze({ release, runtime, coherence });
|
||||
}
|
||||
|
||||
export async function runRunbookDrill(runbookId: string, options: DrillOptions = {}) {
|
||||
const context: DrillContext = {
|
||||
root: options.root ?? process.cwd(),
|
||||
contractPackages: options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES,
|
||||
};
|
||||
const document = JSON.parse(
|
||||
await readFile(path.join(context.root, "config/runbooks/runbooks.json"), "utf8"),
|
||||
) as RunbookDocument;
|
||||
const specification = document.runbooks[runbookId];
|
||||
const drill = drillById[runbookId];
|
||||
if (!specification || !drill) throw new RangeError(`unknown runbook: ${runbookId}`);
|
||||
|
||||
const release = parseReleaseArtifact(
|
||||
await rootedReader(context.root)(DEFAULT_ROLLBACK_PATHS.release),
|
||||
);
|
||||
const result = await drill(context);
|
||||
const escalationPathAsserted = verdict(specification.escalation.length >= 2);
|
||||
return {
|
||||
schemaVersion: 2 as const,
|
||||
runbookId,
|
||||
gateId: specification.gateId,
|
||||
releaseId: release.releaseId,
|
||||
drillTimestamp: new Date().toISOString(),
|
||||
triggerInjected: specification.triggerKinds[0] ?? "",
|
||||
triggerAsserted: result.triggerAsserted,
|
||||
containmentAsserted: result.containmentAsserted,
|
||||
escalationPathAsserted,
|
||||
recoveryAssertions: result.recoveryAssertions,
|
||||
negativeFixtureFailedAsExpected: result.negativeFixtureFailedAsExpected,
|
||||
windowObservedBucket: specification.window,
|
||||
providerVerificationRequired: result.providerVerificationRequired,
|
||||
result: combineDrillVerdicts(
|
||||
[
|
||||
result.triggerAsserted,
|
||||
result.containmentAsserted,
|
||||
escalationPathAsserted,
|
||||
...result.recoveryAssertions.map((item) => item.result),
|
||||
result.negativeFixtureFailedAsExpected,
|
||||
],
|
||||
result.providerVerificationRequired,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const runbookId = process.argv
|
||||
.slice(2)
|
||||
.find((argument) => /^FE-RB-00[1-5]$/.test(argument));
|
||||
const document = JSON.parse(
|
||||
await readFile("config/runbooks/runbooks.json", "utf8"),
|
||||
) as RunbookDocument;
|
||||
const specification = runbookId ? document.runbooks[runbookId] : undefined;
|
||||
if (!runbookId || !specification) {
|
||||
if (!runbookId) {
|
||||
process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const drill = await drillById[runbookId]();
|
||||
const escalationPathAsserted = specification.escalation.length >= 2;
|
||||
const passed =
|
||||
drill.triggerAsserted &&
|
||||
drill.containmentAsserted &&
|
||||
escalationPathAsserted &&
|
||||
drill.recoveryAssertions.every((item) => item.passed) &&
|
||||
drill.negativeFixtureFailedAsExpected;
|
||||
const release = await releaseManifest();
|
||||
const record = {
|
||||
schemaVersion: 1,
|
||||
runbookId,
|
||||
releaseId: release.releaseId,
|
||||
drillTimestamp: new Date().toISOString(),
|
||||
triggerInjected: specification.triggerKinds[0],
|
||||
triggerAsserted: drill.triggerAsserted,
|
||||
containmentAsserted: drill.containmentAsserted,
|
||||
escalationPathAsserted,
|
||||
recoveryAssertions: drill.recoveryAssertions,
|
||||
negativeFixtureFailedAsExpected: drill.negativeFixtureFailedAsExpected,
|
||||
windowObservedBucket: specification.window,
|
||||
providerVerificationRequired: drill.providerVerificationRequired,
|
||||
passed,
|
||||
};
|
||||
const { gateId, ...record } = await runRunbookDrill(runbookId);
|
||||
const artifactDirectory = `artifacts/runbooks/${runbookId}`;
|
||||
await mkdir(artifactDirectory, { recursive: true });
|
||||
await writeValidatedJsonArtifact({
|
||||
@@ -361,15 +448,25 @@ async function main(): Promise<void> {
|
||||
schema: runbookRecordArtifactSchema,
|
||||
value: record,
|
||||
});
|
||||
if (!passed) {
|
||||
process.stderr.write(`${runbookId} drill failed.\n`);
|
||||
if (record.result === "FAIL") {
|
||||
process.stderr.write(`${runbookId} drill: FAIL (${gateId})\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
process.stdout.write(
|
||||
`${runbookId} drill: PASS (${specification.gateId}; provider verification ${
|
||||
drill.providerVerificationRequired ? "still required" : "not required"
|
||||
})\n`,
|
||||
);
|
||||
if (record.result === "UNVERIFIED") {
|
||||
// Not a pass: the gate stays red until provider evidence exists (D-14).
|
||||
const open = [
|
||||
["trigger", record.triggerAsserted],
|
||||
["containment", record.containmentAsserted],
|
||||
...record.recoveryAssertions.map((item) => [item.assertion, item.result] as const),
|
||||
["negative fixture", record.negativeFixtureFailedAsExpected],
|
||||
["provider verification", record.providerVerificationRequired ? "UNVERIFIED" : "PASS"],
|
||||
].filter(([, result]) => result === "UNVERIFIED").map(([name]) => name);
|
||||
process.stderr.write(
|
||||
`${runbookId} drill: UNVERIFIED (${gateId}; no local check failed; not observed locally: ${open.join(", ")})\n`,
|
||||
);
|
||||
process.exit(3);
|
||||
}
|
||||
process.stdout.write(`${runbookId} drill: PASS (${gateId})\n`);
|
||||
}
|
||||
|
||||
function requireIdentity(value: string | undefined, label: string): string {
|
||||
@@ -379,88 +476,6 @@ function requireIdentity(value: string | undefined, label: string): string {
|
||||
return value;
|
||||
}
|
||||
|
||||
async function selectRollbackArtifactPair(
|
||||
readArtifact: JsonArtifactReader,
|
||||
paths: RollbackArtifactPaths,
|
||||
): Promise<Readonly<{ release: unknown; runtime: unknown }>> {
|
||||
const primary = await readArtifactPair(
|
||||
readArtifact,
|
||||
paths.primaryRelease,
|
||||
paths.primaryRuntime,
|
||||
);
|
||||
if (
|
||||
primary.release.status === "fulfilled" &&
|
||||
primary.runtime.status === "fulfilled"
|
||||
) {
|
||||
return Object.freeze({
|
||||
release: primary.release.value,
|
||||
runtime: primary.runtime.value,
|
||||
});
|
||||
}
|
||||
|
||||
const releaseMissing =
|
||||
primary.release.status === "rejected" &&
|
||||
hasErrorCode(primary.release.reason, "ENOENT");
|
||||
const runtimeMissing =
|
||||
primary.runtime.status === "rejected" &&
|
||||
hasErrorCode(primary.runtime.reason, "ENOENT");
|
||||
if (releaseMissing && runtimeMissing) {
|
||||
const fallback = await readArtifactPair(
|
||||
readArtifact,
|
||||
paths.fallbackRelease,
|
||||
paths.fallbackRuntime,
|
||||
);
|
||||
if (fallback.release.status === "rejected") {
|
||||
throw fallback.release.reason;
|
||||
}
|
||||
if (fallback.runtime.status === "rejected") {
|
||||
throw fallback.runtime.reason;
|
||||
}
|
||||
return Object.freeze({
|
||||
release: fallback.release.value,
|
||||
runtime: fallback.runtime.value,
|
||||
});
|
||||
}
|
||||
|
||||
if (primary.release.status === "rejected" && !releaseMissing) {
|
||||
throw primary.release.reason;
|
||||
}
|
||||
if (primary.runtime.status === "rejected" && !runtimeMissing) {
|
||||
throw primary.runtime.reason;
|
||||
}
|
||||
throw new Error("primary rollback artifact pair is incomplete");
|
||||
}
|
||||
|
||||
async function readArtifactPair(
|
||||
readArtifact: JsonArtifactReader,
|
||||
releasePath: string,
|
||||
runtimePath: string,
|
||||
): Promise<
|
||||
Readonly<{
|
||||
release: PromiseSettledResult<unknown>;
|
||||
runtime: PromiseSettledResult<unknown>;
|
||||
}>
|
||||
> {
|
||||
const [release, runtime] = await Promise.allSettled([
|
||||
Promise.resolve().then(() => readArtifact(releasePath)),
|
||||
Promise.resolve().then(() => readArtifact(runtimePath)),
|
||||
]);
|
||||
return Object.freeze({ release, runtime });
|
||||
}
|
||||
|
||||
async function readPrimaryOrFallback(
|
||||
readArtifact: JsonArtifactReader,
|
||||
primary: string,
|
||||
fallback: string,
|
||||
): Promise<unknown> {
|
||||
try {
|
||||
return await readArtifact(primary);
|
||||
} catch (error) {
|
||||
if (!hasErrorCode(error, "ENOENT")) throw error;
|
||||
return readArtifact(fallback);
|
||||
}
|
||||
}
|
||||
|
||||
function hasErrorCode(error: unknown, code: string): boolean {
|
||||
return Boolean(
|
||||
error &&
|
||||
|
||||
@@ -4,7 +4,7 @@ export const ciContractReportSchema = z
|
||||
.object({
|
||||
schemaVersion: z.literal(2),
|
||||
nodeVersion: z.string().regex(/^\d+\.\d+\.\d+$/u),
|
||||
gateCount: z.literal(28),
|
||||
gateCount: z.literal(27),
|
||||
commandDefinitionCount: z.number().int().positive(),
|
||||
commandReferenceCount: z.number().int().positive(),
|
||||
artifactCount: z.number().int().positive(),
|
||||
|
||||
@@ -28,7 +28,7 @@ export const PROMOTION_FORMULA = Object.freeze({
|
||||
"FE-GATE-028",
|
||||
]),
|
||||
PROD_PROMOTION_READY: Object.freeze([
|
||||
"FE-GATE-016",
|
||||
// FE-GATE-016 is retired (D-14): it repeated FE-GATE-025's FE-RB-005 drill.
|
||||
"FE-GATE-021",
|
||||
"FE-GATE-022",
|
||||
"FE-GATE-023",
|
||||
|
||||
@@ -382,7 +382,6 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { gate: FE-GATE-016, name: rollback-drill }
|
||||
- { gate: FE-GATE-021, name: runbook-boot-config }
|
||||
- { gate: FE-GATE-022, name: runbook-chunk-mismatch }
|
||||
- { gate: FE-GATE-023, name: runbook-api-degradation }
|
||||
|
||||
@@ -191,14 +191,14 @@ describe("CI gate contract", () => {
|
||||
expect(contract.gates.map(({ id }) => id)).toEqual(
|
||||
Array.from({ length: 28 }, (_, index) =>
|
||||
`FE-GATE-${String(index + 1).padStart(3, "0")}`,
|
||||
),
|
||||
).filter((gateId) => gateId !== "FE-GATE-016"),
|
||||
);
|
||||
expect(contract.jobs).toHaveLength(9);
|
||||
expect(contract.commands).toHaveLength(94);
|
||||
expect(contract.gates.reduce((total, gate) => total + gate.commandIds.length, 0)).toBe(106);
|
||||
expect(contract.gates.reduce((total, gate) => total + gate.commandIds.length, 0)).toBe(104);
|
||||
expect(contract.commands.filter(({ expect }) => expect === "fail")).toHaveLength(31);
|
||||
expect(contract.gates.reduce((total, gate) => total + gate.evidenceArtifactIds.length, 0)).toBe(88);
|
||||
expect(contract.artifacts).toHaveLength(110);
|
||||
expect(contract.gates.reduce((total, gate) => total + gate.evidenceArtifactIds.length, 0)).toBe(87);
|
||||
expect(contract.artifacts).toHaveLength(109);
|
||||
expect(contract.stages).toHaveLength(5);
|
||||
expect(contract.retention.classes).toHaveLength(5);
|
||||
expect(index.gates.get("FE-GATE-015")?.commandIds).toHaveLength(2);
|
||||
|
||||
@@ -25,7 +25,7 @@ describe("promotion readiness formula", () => {
|
||||
it.each([
|
||||
["FE-GATE-001", "MERGE_READY"],
|
||||
["FE-GATE-012", "RELEASE_READY"],
|
||||
["FE-GATE-016", "PROD_PROMOTION_READY"],
|
||||
["FE-GATE-021", "PROD_PROMOTION_READY"],
|
||||
["FE-GATE-018", "FIELD_SLO_READY"],
|
||||
["FE-GATE-017", "DOCUMENTATION_READY"],
|
||||
] as const)("fails closed when %s fails", (failedGate, readiness) => {
|
||||
|
||||
@@ -281,32 +281,32 @@ describe("release coherence", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to public rollback artifacts only when primary dist is absent", async () => {
|
||||
// 13-03: the rollback drill checks the built release only. Reading the
|
||||
// public/ placeholders when dist was absent let it pass without any build.
|
||||
it("reads only the built dist pair and never falls back to public placeholders", async () => {
|
||||
const exactV2 = await releaseV2With(contractPackages);
|
||||
const tamperedPrimary = await releaseV2With(contractPackages.slice(0, 1));
|
||||
const validPublic = {
|
||||
"public/release-manifest.json": exactV2,
|
||||
"public/config.json": runtimeV2,
|
||||
};
|
||||
const unavailablePrimary = [
|
||||
new SyntaxError("invalid primary JSON"),
|
||||
Object.assign(new Error("unreadable primary"), { code: "EACCES" }),
|
||||
{ ...exactV2, unexpected: "tampered" },
|
||||
];
|
||||
|
||||
for (const primaryFailure of unavailablePrimary) {
|
||||
for (const primary of [
|
||||
{},
|
||||
{ "dist/config.json": runtimeV2 },
|
||||
{ "dist/release-manifest.json": exactV2 },
|
||||
{ "dist/release-manifest.json": new SyntaxError("invalid primary JSON"), "dist/config.json": runtimeV2 },
|
||||
{ "dist/release-manifest.json": { ...exactV2, unexpected: "tampered" }, "dist/config.json": runtimeV2 },
|
||||
]) {
|
||||
const reads: string[] = [];
|
||||
await expect(
|
||||
verifyRollbackReleaseCoherence({
|
||||
readArtifact: artifactReader({
|
||||
"dist/release-manifest.json": primaryFailure,
|
||||
"dist/config.json": runtimeV2,
|
||||
...validPublic,
|
||||
}),
|
||||
readArtifact: artifactReader({ ...primary, ...validPublic }, reads),
|
||||
contractPackages,
|
||||
}),
|
||||
).rejects.toBeDefined();
|
||||
expect(reads.some((read) => read.startsWith("public/"))).toBe(false);
|
||||
}
|
||||
|
||||
const tamperedPrimary = await releaseV2With(contractPackages.slice(0, 1));
|
||||
await expect(
|
||||
verifyRollbackReleaseCoherence({
|
||||
readArtifact: artifactReader({
|
||||
@@ -317,55 +317,5 @@ describe("release coherence", () => {
|
||||
contractPackages,
|
||||
}),
|
||||
).resolves.toMatchObject({ coherence: { compatible: false } });
|
||||
|
||||
const releaseOnlyMissingReads: string[] = [];
|
||||
await expect(
|
||||
verifyRollbackReleaseCoherence({
|
||||
readArtifact: artifactReader(
|
||||
{
|
||||
"dist/config.json": runtimeV2,
|
||||
...validPublic,
|
||||
},
|
||||
releaseOnlyMissingReads,
|
||||
),
|
||||
contractPackages,
|
||||
}),
|
||||
).rejects.toThrow("primary rollback artifact pair");
|
||||
expect(releaseOnlyMissingReads).toEqual([
|
||||
"dist/release-manifest.json",
|
||||
"dist/config.json",
|
||||
]);
|
||||
|
||||
const runtimeOnlyMissingReads: string[] = [];
|
||||
await expect(
|
||||
verifyRollbackReleaseCoherence({
|
||||
readArtifact: artifactReader(
|
||||
{
|
||||
"dist/release-manifest.json": exactV2,
|
||||
...validPublic,
|
||||
},
|
||||
runtimeOnlyMissingReads,
|
||||
),
|
||||
contractPackages,
|
||||
}),
|
||||
).rejects.toThrow("primary rollback artifact pair");
|
||||
expect(runtimeOnlyMissingReads).toEqual([
|
||||
"dist/release-manifest.json",
|
||||
"dist/config.json",
|
||||
]);
|
||||
|
||||
const missingPairReads: string[] = [];
|
||||
await expect(
|
||||
verifyRollbackReleaseCoherence({
|
||||
readArtifact: artifactReader(validPublic, missingPairReads),
|
||||
contractPackages,
|
||||
}),
|
||||
).resolves.toMatchObject({ coherence: { compatible: true } });
|
||||
expect(missingPairReads).toEqual([
|
||||
"dist/release-manifest.json",
|
||||
"dist/config.json",
|
||||
"public/release-manifest.json",
|
||||
"public/config.json",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { runRunbookDrill } from "../../scripts/drill-runbook.ts";
|
||||
import { computeContractSetDigest } from "../../src/contracts/contract-set-canonical.ts";
|
||||
import type { InstalledContractPackageIdentity } from "../../src/contracts/external-contract-runtime.ts";
|
||||
|
||||
// 11-01 / 13-03 (D-14). A drill reports PASS only for what it observed on the
|
||||
// local build. Boot, live traffic, smoke and the provider switch are not
|
||||
// observable here, so a drill with provider scope ends UNVERIFIED, never PASS.
|
||||
|
||||
const contractPackages = [
|
||||
{
|
||||
packageId: "@example/accounts",
|
||||
version: "1.2.3",
|
||||
digest: `sha256:${"1".repeat(64)}`,
|
||||
runtimeProtocolVersion: 1,
|
||||
sourceRevision: "a".repeat(40),
|
||||
},
|
||||
] as const satisfies readonly InstalledContractPackageIdentity[];
|
||||
|
||||
const RUNBOOK_IDS = ["FE-RB-001", "FE-RB-002", "FE-RB-003", "FE-RB-004", "FE-RB-005"] as const;
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?: boolean }> = {}) {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "runbook-drill-"));
|
||||
roots.push(root);
|
||||
await mkdir(path.join(root, "config/runbooks"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(root, "config/runbooks/runbooks.json"),
|
||||
await readFile("config/runbooks/runbooks.json"),
|
||||
);
|
||||
if (options.withoutDist) return root;
|
||||
const write = async (file: string, content: string) => {
|
||||
await mkdir(path.dirname(path.join(root, file)), { recursive: true });
|
||||
await writeFile(path.join(root, file), content);
|
||||
};
|
||||
await write("dist/index.html", "<!doctype html><script type=module src=/assets/index-a.js></script>");
|
||||
await write(
|
||||
"dist/.vite/manifest.json",
|
||||
JSON.stringify({
|
||||
"index.html": { file: "assets/index-a.js", isEntry: true, css: ["assets/index-a.css"], imports: ["_shared-b.js"] },
|
||||
"_shared-b.js": { file: "assets/shared-b.js" },
|
||||
"src/pages/lazy.tsx": { file: "assets/lazy-c.js", isDynamicEntry: true },
|
||||
}),
|
||||
);
|
||||
await write("dist/assets/index-a.css", "body{}");
|
||||
if (!options.withoutJs) {
|
||||
for (const file of ["assets/index-a.js", "assets/shared-b.js", "assets/lazy-c.js"]) {
|
||||
await write(`dist/${file}`, "export {};");
|
||||
}
|
||||
}
|
||||
await write(
|
||||
"dist/release-manifest.json",
|
||||
JSON.stringify({
|
||||
schemaVersion: 2,
|
||||
appVersion: "1.0.0",
|
||||
buildId: "build-a",
|
||||
commitSha: "abc1234",
|
||||
configSchemaVersion: "2.0",
|
||||
assetManifestHash: "assets-a",
|
||||
releaseId: "release-a",
|
||||
builtAt: "2026-08-01T00:00:00.000Z",
|
||||
routeChunks: {},
|
||||
contractSet: {
|
||||
setAlgorithm: "CA_CONTRACT_SET_V1",
|
||||
setDigest: await computeContractSetDigest(contractPackages),
|
||||
packages: [...contractPackages],
|
||||
},
|
||||
}),
|
||||
);
|
||||
await write(
|
||||
"dist/config.json",
|
||||
JSON.stringify({
|
||||
APP_ENV: "local",
|
||||
API_BASE_URL: "http://localhost:8080/",
|
||||
REQUEST_TIMEOUT_MS: 10_000,
|
||||
MAX_RETRY_ATTEMPTS: 2,
|
||||
TELEMETRY_ENABLED: false,
|
||||
AUTH_MODE: "external",
|
||||
CONFIG_SCHEMA_VERSION: "2.0",
|
||||
RELEASE_MANIFEST_URL: "/release-manifest.json",
|
||||
BUILD_ID: "build-a",
|
||||
RELEASE_ID: "release-a",
|
||||
CAPABILITY_OVERRIDES: {
|
||||
REALTIME: "DEFAULT",
|
||||
WEB_WORKER: "DEFAULT",
|
||||
SERVICE_WORKER: "DEFAULT",
|
||||
OFFLINE_COMMANDS: "DEFAULT",
|
||||
},
|
||||
FEATURE_OVERRIDES: {},
|
||||
}),
|
||||
);
|
||||
return root;
|
||||
}
|
||||
|
||||
describe("runbook drill verdicts", () => {
|
||||
it.each(RUNBOOK_IDS)("%s on a complete local build is UNVERIFIED, not PASS", async (runbookId) => {
|
||||
const record = await runRunbookDrill(runbookId, { root: await builtDist(), contractPackages });
|
||||
expect(record.result).toBe("UNVERIFIED");
|
||||
expect(record.recoveryAssertions.some(({ result }) => result === "FAIL")).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["FE-RB-002", "FE-RB-005"] as const)(
|
||||
"%s fails when the build lost its JavaScript assets",
|
||||
async (runbookId) => {
|
||||
const record = await runRunbookDrill(runbookId, {
|
||||
root: await builtDist({ withoutJs: true }),
|
||||
contractPackages,
|
||||
});
|
||||
expect(record.result).toBe("FAIL");
|
||||
},
|
||||
);
|
||||
|
||||
it("FE-RB-005 fails without a build instead of reading public/ placeholders", async () => {
|
||||
await expect(
|
||||
runRunbookDrill("FE-RB-005", { root: await builtDist({ withoutDist: true }), contractPackages }),
|
||||
).rejects.toThrow(/dist\/release-manifest\.json/u);
|
||||
});
|
||||
|
||||
it("keeps no constant recovery verdict in the drill source", async () => {
|
||||
const source = await readFile("scripts/drill-runbook.ts", "utf8");
|
||||
expect(source).not.toMatch(/assertion\([^;]*,\s*true\s*\)/u);
|
||||
expect(source).not.toMatch(/Asserted:\s*true\b/u);
|
||||
expect(source).not.toMatch(/injectedMountFailure\s*=\s*true/u);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user