문제: provider 확인이 필요한 drill(FE-RB-002~005)은 입력이 runbook ID와 `--release-root`뿐이라 어떤 입력으로도 PASS가 될 수 없었다. 문서는 "provider 증거가 없으면 production 단계는 닫히지 않는다"고 했지만 증거를 넘길 길이 없었다. 고친 것: `--provider-evidence <파일>`을 받는다. 파일 꼴은 `providerDrillEvidenceSchema`(runbook ID, release ID, 관찰 시각, provider run ID, 항목마다 PASS/FAIL과 근거)다. 같은 runbook·같은 release이고, 관찰이 7일 안이며 5분 넘게 앞서지 않고, 로컬 drill이 UNVERIFIED로 남긴 항목만 답한 증거만 받는다. 받은 증거는 그 항목의 UNVERIFIED만 바꾸고 로컬의 PASS·FAIL은 그대로 둔다. `provider verification` 항목도 증거가 답해야 한다. 받지 않은 증거는 이유와 함께 record의 `providerEvidence`에 남고, record schema는 그 경우 PASS를 거절한다. 버린 대안: production·field 단계를 "provider 연결 전 기본 꺼짐"으로 문서화하는 (b)는 job 그래프(`config/ci/**`)를 바꾸는 일이라 하지 않았다. 위험·한계: 템플릿의 production 게이트 명령은 아직 이 인자를 넘기지 않는다. 증거 파일을 만들고 넘기는 자리는 provider를 연결한 제품 몫이다. 유효 기간 7일은 정한 기본값이다. 검증: desktop `node scripts/run-vitest.ts run tests/unit/runbook-drill.test.ts` 통과(유효 증거 PASS, FAIL 증거, 빠진 항목, 다른 release·runbook, 만료, 미래 시각, 로컬이 본 항목, 다른 꼴 거절, PASS를 주장하는 거절 record). Refs: 0929/11-02
719 lines
27 KiB
TypeScript
719 lines
27 KiB
TypeScript
import { access, mkdir, readFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
|
|
import { createContractHttpExecutor } from "../src/adapters/http/index.ts";
|
|
import { createTelemetryAdapter } from "../src/adapters/telemetry/best-effort-telemetry.ts";
|
|
import { verifyCompatibilityTuple } from "../src/contracts/compatibility.ts";
|
|
import type { StoragePort } from "../src/application/ports/storage-port.ts";
|
|
import { decideChunkRecovery } from "../src/application/use-cases/decide-chunk-recovery.ts";
|
|
import { validateRuntimeConfig } from "../src/bootstrap/runtime-config-schema.ts";
|
|
import type {
|
|
InstalledContractPackageIdentity,
|
|
RetrySemantics,
|
|
} from "../src/contracts/external-contract-runtime.ts";
|
|
import { z } from "../src/contracts/zod.ts";
|
|
import { zodValidator } from "../src/contracts/zod-validator.ts";
|
|
import {
|
|
parseReleaseArtifact,
|
|
parseRuntimeConfigArtifact,
|
|
} from "../src/contracts/release-artifacts.ts";
|
|
import { projectTelemetryEvent } from "../src/contracts/telemetry.ts";
|
|
import { EXPECTED_CONTRACT_SET_PACKAGES } from "../src/features/installed-contract-contributions.ts";
|
|
import {
|
|
combineDrillVerdicts,
|
|
providerDrillEvidenceSchema,
|
|
runbookRecordArtifactSchema,
|
|
type DrillVerdict,
|
|
type ProviderDrillEvidence,
|
|
} from "./contracts/release-artifacts.ts";
|
|
import { verifyReleaseRuntimeCoherence } from "./lib/release-runtime-coherence.ts";
|
|
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
|
|
|
|
// 11-01 / 13-03 (D-14). A verdict is PASS or FAIL only when the drill observed
|
|
// it on the local build. What needs a browser boot, live traffic or the hosting
|
|
// provider is UNVERIFIED, and so is every drill whose runbook needs provider
|
|
// verification: a local drill never claims the release can be recovered.
|
|
//
|
|
// Each recovery assertion is named after one of the runbook's recoveryEvidence
|
|
// items in config/runbooks/runbooks.json, word for word and in its order
|
|
// (tests/unit/runbook-drill.test.ts), so the record names what the runbook lists.
|
|
|
|
type RecoveryAssertion = Readonly<{
|
|
assertion: string;
|
|
evidence: string;
|
|
result: DrillVerdict;
|
|
}>;
|
|
|
|
type DrillResult = Readonly<{
|
|
triggerAsserted: DrillVerdict;
|
|
containmentAsserted: DrillVerdict;
|
|
recoveryAssertions: RecoveryAssertion[];
|
|
negativeFixtureFailedAsExpected: DrillVerdict;
|
|
providerVerificationRequired: boolean;
|
|
}>;
|
|
|
|
type RunbookSpecification = Readonly<{
|
|
title: string;
|
|
gateId: string;
|
|
triggerKinds: string[];
|
|
containment: string;
|
|
window: string;
|
|
escalation: string[];
|
|
recoveryEvidence: string[];
|
|
negativeFixture: string;
|
|
}>;
|
|
|
|
type RunbookDocument = Readonly<{
|
|
runbooks: Record<string, RunbookSpecification>;
|
|
}>;
|
|
|
|
export type JsonArtifactReader = (path: string) => Promise<unknown>;
|
|
|
|
export type RollbackArtifactPaths = Readonly<{
|
|
release: string;
|
|
runtime: string;
|
|
}>;
|
|
|
|
export type RollbackCoherenceOptions = Readonly<{
|
|
readArtifact?: JsonArtifactReader;
|
|
contractPackages?: readonly InstalledContractPackageIdentity[];
|
|
paths?: RollbackArtifactPaths;
|
|
}>;
|
|
|
|
export type DrillOptions = Readonly<{
|
|
/**
|
|
* Directory holding the release's `dist/`. Defaults to the cwd (the local
|
|
* build). 13-04: pass an extracted candidate or promoted archive to drill
|
|
* the release that is actually deployed or rolled back to.
|
|
*/
|
|
releaseRoot?: string;
|
|
contractPackages?: readonly InstalledContractPackageIdentity[];
|
|
/**
|
|
* 0929/11-02. The provider's record for this drill (`--provider-evidence`),
|
|
* parsed JSON. Without it a drill with provider scope cannot pass.
|
|
*/
|
|
providerEvidence?: unknown;
|
|
/** The clock evidence age is judged against; a test passes a fixed one. */
|
|
now?: Date;
|
|
}>;
|
|
|
|
/** Provider evidence older than this no longer describes the provider as it is. */
|
|
export const PROVIDER_EVIDENCE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1_000;
|
|
/** How far ahead of this clock a provider's timestamp may be. */
|
|
const PROVIDER_EVIDENCE_CLOCK_SKEW_MS = 5 * 60 * 1_000;
|
|
/** The item names a drill's UNVERIFIED line uses, and provider evidence answers. */
|
|
const ITEM_TRIGGER = "trigger";
|
|
const ITEM_CONTAINMENT = "containment";
|
|
const ITEM_NEGATIVE_FIXTURE = "negative fixture";
|
|
export const ITEM_PROVIDER_VERIFICATION = "provider verification";
|
|
|
|
type DrillContext = Readonly<{
|
|
root: string;
|
|
contractPackages: readonly InstalledContractPackageIdentity[];
|
|
}>;
|
|
|
|
// 13-03: the drill reads the built release only. It used to fall back to the
|
|
// placeholders in public/, so it passed without any build at all.
|
|
const DEFAULT_ROLLBACK_PATHS = Object.freeze({
|
|
release: "dist/release-manifest.json",
|
|
runtime: "dist/config.json",
|
|
});
|
|
|
|
async function readJsonArtifact(path: string): Promise<unknown> {
|
|
try {
|
|
return JSON.parse(await readFile(path, "utf8"));
|
|
} catch (error) {
|
|
if (hasErrorCode(error, "ENOENT")) {
|
|
throw new Error(`${path} is missing; build the release before running the drill`, { cause: error });
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function rootedReader(root: string): JsonArtifactReader {
|
|
return (file) => readJsonArtifact(path.join(root, file));
|
|
}
|
|
|
|
const validConfig = {
|
|
APP_ENV: "local",
|
|
API_BASE_URL: "http://localhost:8080",
|
|
REQUEST_TIMEOUT_MS: 10_000,
|
|
MAX_RETRY_ATTEMPTS: 2,
|
|
TELEMETRY_ENABLED: false,
|
|
AUTH_MODE: "external",
|
|
CONFIG_SCHEMA_VERSION: "1",
|
|
API_CONTRACT_VERSION: "1",
|
|
RELEASE_MANIFEST_URL: "/release-manifest.json",
|
|
BUILD_ID: "local-build",
|
|
RELEASE_ID: "local-release",
|
|
};
|
|
|
|
function verdict(passed: boolean): DrillVerdict {
|
|
return passed ? "PASS" : "FAIL";
|
|
}
|
|
|
|
function assertion(
|
|
assertion: string,
|
|
evidence: string,
|
|
result: DrillVerdict,
|
|
): RecoveryAssertion {
|
|
return { assertion, evidence, result };
|
|
}
|
|
|
|
/** An assertion the local drill cannot observe; `needs` names what can. */
|
|
function notObserved(name: string, needs: string): RecoveryAssertion {
|
|
return assertion(name, `not observed locally; needs ${needs}`, "UNVERIFIED");
|
|
}
|
|
|
|
/**
|
|
* Every file the Vite manifest names (entries, lazy chunks, CSS, assets) exists
|
|
* in the local dist. Hosting reachability still needs the provider smoke.
|
|
*/
|
|
async function builtAssetsPresent(root: string): Promise<boolean> {
|
|
const manifest = await readJsonArtifact(path.join(root, "dist/.vite/manifest.json"));
|
|
if (!manifest || typeof manifest !== "object") return false;
|
|
const files = new Set<string>();
|
|
for (const chunk of Object.values(manifest as Record<string, Record<string, unknown>>)) {
|
|
for (const key of ["file", "css", "assets"] as const) {
|
|
const value = chunk[key];
|
|
for (const file of Array.isArray(value) ? value : [value]) {
|
|
if (typeof file === "string") files.add(file);
|
|
}
|
|
}
|
|
}
|
|
if (![...files].some((file) => file.endsWith(".js"))) return false;
|
|
try {
|
|
await Promise.all(
|
|
["index.html", ...files].map((file) => access(path.join(root, "dist", file))),
|
|
);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function drillBoot(): Promise<DrillResult> {
|
|
const invalid = validateRuntimeConfig({
|
|
...validConfig,
|
|
APP_ENV: "production",
|
|
API_BASE_URL: "http://insecure.invalid",
|
|
});
|
|
const recovered = validateRuntimeConfig(validConfig);
|
|
return {
|
|
triggerAsserted: verdict(!invalid.success),
|
|
containmentAsserted: verdict(!invalid.success),
|
|
recoveryAssertions: [
|
|
notObserved("깨끗한 세션에서 boot", "a browser boot of the promoted release"),
|
|
notObserved("제품 root mount", "a browser boot of the promoted release"),
|
|
assertion(
|
|
"config 검증",
|
|
"invalid fixture rejected, valid fixture accepted",
|
|
verdict(!invalid.success && recovered.success),
|
|
),
|
|
notObserved("boot 오류 반복 없음", "boot telemetry from the promoted release"),
|
|
],
|
|
// The runbook's negative fixture injects a mount failure after a valid
|
|
// config; that needs a mounted app, which a Node drill does not have.
|
|
negativeFixtureFailedAsExpected: "UNVERIFIED",
|
|
providerVerificationRequired: false,
|
|
};
|
|
}
|
|
|
|
function memoryStorage(): StoragePort {
|
|
let value: unknown;
|
|
return {
|
|
read: () => ({ ok: true, value }),
|
|
write: (_key: string, next: unknown) => {
|
|
value = next;
|
|
return { ok: true };
|
|
},
|
|
remove: () => ({ ok: true }),
|
|
};
|
|
}
|
|
|
|
async function drillChunkMismatch(context: DrillContext): Promise<DrillResult> {
|
|
const storage = memoryStorage();
|
|
const input: Parameters<typeof decideChunkRecovery>[0] = {
|
|
failureKind: "DEPLOY_MISMATCH",
|
|
manifestLoaded: true,
|
|
currentBuildId: "build-a",
|
|
currentReleaseId: "release-a",
|
|
activeBuildId: "build-b",
|
|
activeReleaseId: "release-b",
|
|
storage,
|
|
};
|
|
const first = decideChunkRecovery(input);
|
|
const second = decideChunkRecovery(input);
|
|
const { coherence } = await verifyRollbackReleaseCoherence({
|
|
readArtifact: rootedReader(context.root),
|
|
contractPackages: context.contractPackages,
|
|
});
|
|
return {
|
|
triggerAsserted: verdict(first.action === "reload-once"),
|
|
containmentAsserted: verdict(
|
|
first.action === "reload-once" && second.action === "support",
|
|
),
|
|
recoveryAssertions: [
|
|
assertion(
|
|
"entry asset과 lazy asset에 접근 가능",
|
|
"every Vite manifest file exists in the local dist",
|
|
verdict(await builtAssetsPresent(context.root)),
|
|
),
|
|
assertion("release tuple 일치", "build/config/manifest tuple", verdict(coherence.compatible)),
|
|
assertion("두 번째 reload 차단", "reload guard decision", verdict(second.action === "support")),
|
|
notObserved("핵심 route smoke 시험", "a browser smoke against the hosted release"),
|
|
],
|
|
negativeFixtureFailedAsExpected: verdict(second.action !== "reload-once"),
|
|
providerVerificationRequired: true,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* One operation through the V3 executor against a fixed answer. Returns how
|
|
* many requests it sent, which is what retry amplification is measured in.
|
|
*/
|
|
async function physicalAttempts(
|
|
retrySemantics: Extract<RetrySemantics, "SAFE" | "NEVER">,
|
|
answer: () => Response,
|
|
): Promise<number> {
|
|
const read = retrySemantics === "SAFE";
|
|
let sent = 0;
|
|
const executor = createContractHttpExecutor({
|
|
baseUrl: "https://api.invalid",
|
|
maxRetryAttempts: 2,
|
|
attachCredentials: () => ({ kind: "READY" as const, headers: {} }),
|
|
fetcher: (async () => {
|
|
sent += 1;
|
|
return answer();
|
|
}) as typeof fetch,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
});
|
|
await executor.execute(
|
|
Object.freeze({
|
|
contract: Object.freeze({
|
|
operationId: "DRILL_API_DEGRADATION",
|
|
method: read ? ("GET" as const) : ("POST" as const),
|
|
pathTemplate: "/api/drill",
|
|
inputValidator: zodValidator("DrillInput", z.object({}).strict()),
|
|
outputValidator: zodValidator(
|
|
"DrillOutput",
|
|
z.object({ id: z.string() }).strict(),
|
|
),
|
|
problemValidator: zodValidator(
|
|
"DrillProblem",
|
|
z.object({ code: z.string() }).strip(),
|
|
),
|
|
acceptedStatuses: Object.freeze([200]),
|
|
emptyBodyStatuses: Object.freeze([]),
|
|
retrySemantics,
|
|
requestBody: read ? ("NONE" as const) : ("JSON" as const),
|
|
responseBody: "REQUIRED_JSON" as const,
|
|
responseEnvelope: "NONE" as const,
|
|
commandRecovery: null,
|
|
commandEffect: read
|
|
? null
|
|
: Object.freeze({
|
|
successEffect: "APPLIED_CONFIRMED" as const,
|
|
classifyProblem: () => "MAYBE_APPLIED" as const,
|
|
}),
|
|
projectRequest: () =>
|
|
Object.freeze({
|
|
pathValues: Object.freeze({}),
|
|
queryEntries: Object.freeze([]),
|
|
body: read ? null : Object.freeze({}),
|
|
}),
|
|
}),
|
|
frontend: Object.freeze({
|
|
policyId: "DRILL_API_V1",
|
|
requestByteLimit: read ? 0 : 1_024,
|
|
responseByteLimit: 4_096,
|
|
totalDeadlineMs: 5_000,
|
|
retryBudget: read ? (2 as const) : (0 as const),
|
|
authProfileId: "ANONYMOUS",
|
|
diagnosticsOperation: "drill.api",
|
|
}),
|
|
}),
|
|
Object.freeze({}),
|
|
{
|
|
routeId: "DRILL_ROUTE",
|
|
scope: Object.freeze({
|
|
generation: 1,
|
|
fingerprint: "drill-scope",
|
|
identities: Object.freeze({}) as never,
|
|
signal: new AbortController().signal,
|
|
isCurrent: () => true,
|
|
}),
|
|
},
|
|
);
|
|
return sent;
|
|
}
|
|
|
|
async function drillApiDegradation(): Promise<DrillResult> {
|
|
const unavailable = () => Response.json({ code: "TEMPORARY" }, { status: 503 });
|
|
const unkeyedRetry = (await physicalAttempts("NEVER", unavailable)) > 1;
|
|
const safeRetry = (await physicalAttempts("SAFE", unavailable)) > 1;
|
|
const schemaRetry =
|
|
(await physicalAttempts("SAFE", () => Response.json({ unexpected: true }))) > 1;
|
|
return {
|
|
// The trigger is a failure rate over live traffic.
|
|
triggerAsserted: "UNVERIFIED",
|
|
containmentAsserted: verdict(!unkeyedRetry && safeRetry),
|
|
recoveryAssertions: [
|
|
notObserved("최종(terminal) 실패율이 기준선 수준", "API failure telemetry from the promoted release"),
|
|
assertion("재시도 증폭 없음", "unkeyed retry policy", verdict(!unkeyedRetry)),
|
|
notObserved("핵심 읽기/쓰기 smoke 시험", "a read and a write against the live API"),
|
|
assertion(
|
|
"스키마 fixture",
|
|
"schema mismatch is not retryable",
|
|
verdict(!schemaRetry),
|
|
),
|
|
],
|
|
negativeFixtureFailedAsExpected: verdict(!unkeyedRetry),
|
|
providerVerificationRequired: true,
|
|
};
|
|
}
|
|
|
|
async function drillTelemetry(): Promise<DrillResult> {
|
|
const adapter = createTelemetryAdapter({
|
|
enabled: true,
|
|
endpoint: "https://telemetry.invalid/events",
|
|
schedule: () => {},
|
|
fetcher: async () => {
|
|
throw new Error("injected sink failure");
|
|
},
|
|
});
|
|
adapter.emit("api.request.failed", {
|
|
error_kind: "SERVER_FAILURE",
|
|
http_status_group: "5xx",
|
|
attempt_count_bucket: "1",
|
|
route_id: "APP_HOME",
|
|
});
|
|
const flushResolved = await adapter.flush().then(
|
|
() => true,
|
|
() => false,
|
|
);
|
|
const projected = projectTelemetryEvent("api.request.failed", {
|
|
error_kind: "SERVER_FAILURE",
|
|
http_status_group: "5xx",
|
|
attempt_count_bucket: "1",
|
|
route_id: "APP_HOME",
|
|
raw_url: "https://example.invalid/path?token=secret",
|
|
});
|
|
const redacted =
|
|
projected.success && !JSON.stringify(projected).includes("raw_url");
|
|
return {
|
|
triggerAsserted: verdict(adapter.droppedCount() === 1),
|
|
containmentAsserted: verdict(adapter.pendingCount() === 0),
|
|
recoveryAssertions: [
|
|
assertion("제품 흐름 영향 없음", "adapter flush resolves despite the sink failure", verdict(flushResolved)),
|
|
assertion("전송 자체 점검", "sink failure counted", verdict(adapter.droppedCount() === 1)),
|
|
assertion("queue가 상한 안에서 비워짐", "pending queue count", verdict(adapter.pendingCount() === 0)),
|
|
assertion("금지 속성 없음", "default-deny projection", verdict(redacted)),
|
|
],
|
|
negativeFixtureFailedAsExpected: verdict(redacted),
|
|
providerVerificationRequired: true,
|
|
};
|
|
}
|
|
|
|
async function drillRollback(context: DrillContext): Promise<DrillResult> {
|
|
const { release, coherence: coherent } = await verifyRollbackReleaseCoherence({
|
|
readArtifact: rootedReader(context.root),
|
|
contractPackages: context.contractPackages,
|
|
});
|
|
const assetsPresent = await builtAssetsPresent(context.root);
|
|
const mixed = verifyCompatibilityTuple({
|
|
frontend: {
|
|
buildId: "build-a",
|
|
configSchemaVersion: "1",
|
|
apiContractVersion: "1",
|
|
assetManifestHash: "assets-a",
|
|
releaseId: "release-a",
|
|
},
|
|
runtime: {
|
|
buildId: "build-b",
|
|
configSchemaVersion: "2",
|
|
apiContractVersion: "2",
|
|
assetManifestHash: "assets-b",
|
|
releaseId: "release-b",
|
|
},
|
|
});
|
|
return {
|
|
// A release-blocking defect is declared by a person during an incident.
|
|
triggerAsserted: "UNVERIFIED",
|
|
containmentAsserted: verdict(coherent.compatible && assetsPresent),
|
|
recoveryAssertions: [
|
|
assertion("호환성 gate", "typed version comparison", verdict(coherent.compatible)),
|
|
assertion(
|
|
"release 일관성 gate",
|
|
"build/config/manifest tuple and every Vite manifest file in the local dist",
|
|
verdict(coherent.compatible && assetsPresent),
|
|
),
|
|
notObserved("핵심 smoke 시험", "a browser smoke after the provider switches to the prior release"),
|
|
assertion("장애 타임라인에 release ID 기록", "drill record releaseId", verdict(Boolean(release.releaseId))),
|
|
],
|
|
negativeFixtureFailedAsExpected: verdict(!mixed.compatible),
|
|
providerVerificationRequired: true,
|
|
};
|
|
}
|
|
|
|
const drillById: Record<string, (context: DrillContext) => Promise<DrillResult>> = {
|
|
"FE-RB-001": drillBoot,
|
|
"FE-RB-002": drillChunkMismatch,
|
|
"FE-RB-003": drillApiDegradation,
|
|
"FE-RB-004": drillTelemetry,
|
|
"FE-RB-005": drillRollback,
|
|
};
|
|
|
|
export async function verifyRollbackReleaseCoherence(
|
|
options: RollbackCoherenceOptions = {},
|
|
) {
|
|
const readArtifact = options.readArtifact ?? readJsonArtifact;
|
|
const paths = options.paths ?? DEFAULT_ROLLBACK_PATHS;
|
|
const [releaseValue, runtimeValue] = await Promise.all([
|
|
readArtifact(paths.release),
|
|
readArtifact(paths.runtime),
|
|
]);
|
|
const release = parseReleaseArtifact(releaseValue);
|
|
const runtimeArtifact = parseRuntimeConfigArtifact(runtimeValue);
|
|
const runtime = {
|
|
...runtimeArtifact,
|
|
BUILD_ID: requireIdentity(runtimeArtifact.BUILD_ID, "runtime BUILD_ID"),
|
|
RELEASE_ID: requireIdentity(
|
|
runtimeArtifact.RELEASE_ID,
|
|
"runtime RELEASE_ID",
|
|
),
|
|
};
|
|
const coherence = await verifyReleaseRuntimeCoherence({
|
|
release,
|
|
runtime,
|
|
contractPackages:
|
|
options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES,
|
|
});
|
|
return Object.freeze({ release, runtime, coherence });
|
|
}
|
|
|
|
export async function runRunbookDrill(runbookId: string, options: DrillOptions = {}) {
|
|
const context: DrillContext = {
|
|
root: options.releaseRoot ?? process.cwd(),
|
|
contractPackages: options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES,
|
|
};
|
|
const document = JSON.parse(
|
|
await readFile("config/runbooks/runbooks.json", "utf8"),
|
|
) as RunbookDocument;
|
|
const specification = document.runbooks[runbookId];
|
|
const drill = drillById[runbookId];
|
|
if (!specification || !drill) throw new RangeError(`unknown runbook: ${runbookId}`);
|
|
|
|
const release = parseReleaseArtifact(
|
|
await rootedReader(context.root)(DEFAULT_ROLLBACK_PATHS.release),
|
|
);
|
|
const local = await drill(context);
|
|
const now = options.now ?? new Date();
|
|
const judged = options.providerEvidence === undefined
|
|
? undefined
|
|
: judgeProviderEvidence(options.providerEvidence, runbookId, release.releaseId, openItems(local), now);
|
|
const result = judged?.accepted ? applyProviderEvidence(local, judged.evidence) : local;
|
|
const providerEvidence = judged === undefined
|
|
? undefined
|
|
: judged.accepted
|
|
? {
|
|
accepted: true as const,
|
|
runId: judged.evidence.runId,
|
|
observedAt: judged.evidence.observedAt,
|
|
providerVerification: observed(judged.evidence, ITEM_PROVIDER_VERIFICATION) ?? "UNVERIFIED",
|
|
}
|
|
: { accepted: false as const, reason: judged.reason, providerVerification: "UNVERIFIED" as const };
|
|
const escalationPathAsserted = verdict(specification.escalation.length >= 2);
|
|
return {
|
|
schemaVersion: 2 as const,
|
|
runbookId,
|
|
gateId: specification.gateId,
|
|
releaseId: release.releaseId,
|
|
drillTimestamp: now.toISOString(),
|
|
triggerInjected: specification.triggerKinds[0] ?? "",
|
|
triggerAsserted: result.triggerAsserted,
|
|
containmentAsserted: result.containmentAsserted,
|
|
escalationPathAsserted,
|
|
recoveryAssertions: result.recoveryAssertions,
|
|
negativeFixtureFailedAsExpected: result.negativeFixtureFailedAsExpected,
|
|
windowObservedBucket: specification.window,
|
|
providerVerificationRequired: result.providerVerificationRequired,
|
|
...(providerEvidence === undefined ? {} : { providerEvidence }),
|
|
result: combineDrillVerdicts(
|
|
[
|
|
result.triggerAsserted,
|
|
result.containmentAsserted,
|
|
escalationPathAsserted,
|
|
...result.recoveryAssertions.map((item) => item.result),
|
|
result.negativeFixtureFailedAsExpected,
|
|
],
|
|
result.providerVerificationRequired,
|
|
providerEvidence?.providerVerification,
|
|
),
|
|
};
|
|
}
|
|
|
|
/** The items the local drill left UNVERIFIED, by the names provider evidence uses. */
|
|
function openItems(result: DrillResult): string[] {
|
|
return [
|
|
...(result.triggerAsserted === "UNVERIFIED" ? [ITEM_TRIGGER] : []),
|
|
...(result.containmentAsserted === "UNVERIFIED" ? [ITEM_CONTAINMENT] : []),
|
|
...result.recoveryAssertions.filter((item) => item.result === "UNVERIFIED").map((item) => item.assertion),
|
|
...(result.negativeFixtureFailedAsExpected === "UNVERIFIED" ? [ITEM_NEGATIVE_FIXTURE] : []),
|
|
...(result.providerVerificationRequired ? [ITEM_PROVIDER_VERIFICATION] : []),
|
|
];
|
|
}
|
|
|
|
type JudgedProviderEvidence =
|
|
| Readonly<{ accepted: true; evidence: ProviderDrillEvidence }>
|
|
| Readonly<{ accepted: false; reason: string }>;
|
|
|
|
/**
|
|
* 0929/11-02. Provider evidence counts only for this runbook, this release,
|
|
* a recent observation, and the items the local drill could not observe.
|
|
*/
|
|
export function judgeProviderEvidence(
|
|
value: unknown,
|
|
runbookId: string,
|
|
releaseId: string,
|
|
open: readonly string[],
|
|
now: Date,
|
|
): JudgedProviderEvidence {
|
|
const parsed = providerDrillEvidenceSchema.safeParse(value);
|
|
if (!parsed.success) {
|
|
const issue = parsed.error.issues[0];
|
|
return { accepted: false, reason: `not provider drill evidence (${issue?.path.join(".") || "root"}: ${issue?.message ?? "invalid"})` };
|
|
}
|
|
const evidence = parsed.data;
|
|
if (evidence.runbookId !== runbookId) {
|
|
return { accepted: false, reason: `evidence is for ${evidence.runbookId}, not ${runbookId}` };
|
|
}
|
|
if (evidence.releaseId !== releaseId) {
|
|
return { accepted: false, reason: `evidence is for release ${evidence.releaseId}, not ${releaseId}` };
|
|
}
|
|
const age = now.getTime() - Date.parse(evidence.observedAt);
|
|
if (age > PROVIDER_EVIDENCE_MAX_AGE_MS) {
|
|
return { accepted: false, reason: `evidence observed at ${evidence.observedAt} is older than ${PROVIDER_EVIDENCE_MAX_AGE_MS / 86_400_000} days` };
|
|
}
|
|
if (age < -PROVIDER_EVIDENCE_CLOCK_SKEW_MS) {
|
|
return { accepted: false, reason: `evidence observed at ${evidence.observedAt} is in the future` };
|
|
}
|
|
const extra = evidence.observations.map(({ item }) => item).filter((item) => !open.includes(item));
|
|
if (extra.length > 0) {
|
|
return { accepted: false, reason: `evidence names items the local drill observed or does not have: ${extra.join(", ")}` };
|
|
}
|
|
return { accepted: true, evidence };
|
|
}
|
|
|
|
function observed(evidence: ProviderDrillEvidence, item: string): DrillVerdict | undefined {
|
|
return evidence.observations.find((observation) => observation.item === item)?.result;
|
|
}
|
|
|
|
/** Replaces each UNVERIFIED item the evidence answers; local PASS and FAIL stay. */
|
|
function applyProviderEvidence(result: DrillResult, evidence: ProviderDrillEvidence): DrillResult {
|
|
const answer = (item: string, local: DrillVerdict): DrillVerdict =>
|
|
local === "UNVERIFIED" ? observed(evidence, item) ?? local : local;
|
|
const source = (item: string) =>
|
|
`provider evidence ${evidence.runId} at ${evidence.observedAt}: ${
|
|
evidence.observations.find((observation) => observation.item === item)?.evidence ?? ""
|
|
}`;
|
|
return {
|
|
...result,
|
|
triggerAsserted: answer(ITEM_TRIGGER, result.triggerAsserted),
|
|
containmentAsserted: answer(ITEM_CONTAINMENT, result.containmentAsserted),
|
|
recoveryAssertions: result.recoveryAssertions.map((item) =>
|
|
item.result === "UNVERIFIED" && observed(evidence, item.assertion) !== undefined
|
|
? { ...item, evidence: source(item.assertion), result: observed(evidence, item.assertion)! }
|
|
: item,
|
|
),
|
|
negativeFixtureFailedAsExpected: answer(ITEM_NEGATIVE_FIXTURE, result.negativeFixtureFailedAsExpected),
|
|
};
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
const runbookId = process.argv
|
|
.slice(2)
|
|
.find((argument) => /^FE-RB-00[1-5]$/.test(argument));
|
|
const releaseRootFlag = process.argv.indexOf("--release-root");
|
|
const releaseRoot = releaseRootFlag >= 0 ? process.argv[releaseRootFlag + 1] : undefined;
|
|
const evidenceFlag = process.argv.indexOf("--provider-evidence");
|
|
const evidencePath = evidenceFlag >= 0 ? process.argv[evidenceFlag + 1] : undefined;
|
|
if (!runbookId || (releaseRootFlag >= 0 && !releaseRoot) || (evidenceFlag >= 0 && !evidencePath)) {
|
|
process.stderr.write(
|
|
"Usage: drill:runbook -- FE-RB-001..FE-RB-005 [--release-root <extracted release archive>]" +
|
|
" [--provider-evidence <provider drill evidence JSON>]\n",
|
|
);
|
|
process.exit(2);
|
|
}
|
|
let providerEvidence: unknown;
|
|
if (evidencePath !== undefined) {
|
|
try {
|
|
providerEvidence = JSON.parse(await readFile(evidencePath, "utf8"));
|
|
} catch (error) {
|
|
process.stderr.write(`${runbookId} drill: cannot read provider evidence ${evidencePath}: ${String(error)}\n`);
|
|
process.exit(2);
|
|
}
|
|
}
|
|
|
|
const { gateId, ...record } = await runRunbookDrill(runbookId, {
|
|
...(releaseRoot ? { releaseRoot } : {}),
|
|
...(evidencePath !== undefined ? { providerEvidence } : {}),
|
|
});
|
|
const artifactDirectory = `artifacts/runbooks/${runbookId}`;
|
|
await mkdir(artifactDirectory, { recursive: true });
|
|
await writeValidatedJsonArtifact({
|
|
path: `${artifactDirectory}/record.json`,
|
|
schema: runbookRecordArtifactSchema,
|
|
value: record,
|
|
});
|
|
if (record.result === "FAIL") {
|
|
process.stderr.write(`${runbookId} drill: FAIL (${gateId})\n`);
|
|
process.exit(1);
|
|
}
|
|
if (record.result === "UNVERIFIED") {
|
|
// Not a pass: the gate stays red until provider evidence exists (D-14).
|
|
const open = [
|
|
[ITEM_TRIGGER, record.triggerAsserted],
|
|
[ITEM_CONTAINMENT, record.containmentAsserted],
|
|
...record.recoveryAssertions.map((item) => [item.assertion, item.result] as const),
|
|
[ITEM_NEGATIVE_FIXTURE, record.negativeFixtureFailedAsExpected],
|
|
[
|
|
ITEM_PROVIDER_VERIFICATION,
|
|
record.providerVerificationRequired ? record.providerEvidence?.providerVerification ?? "UNVERIFIED" : "PASS",
|
|
],
|
|
].filter(([, result]) => result === "UNVERIFIED").map(([name]) => name);
|
|
const rejected = record.providerEvidence?.accepted === false ? `; provider evidence rejected: ${record.providerEvidence.reason}` : "";
|
|
process.stderr.write(
|
|
`${runbookId} drill: UNVERIFIED (${gateId}; no local check failed; not observed locally: ${open.join(", ")}${rejected})\n`,
|
|
);
|
|
process.exit(3);
|
|
}
|
|
process.stdout.write(`${runbookId} drill: PASS (${gateId})\n`);
|
|
}
|
|
|
|
function requireIdentity(value: string | undefined, label: string): string {
|
|
if (value === undefined || value.length === 0) {
|
|
throw new TypeError(`${label} must be a non-empty string`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function hasErrorCode(error: unknown, code: string): boolean {
|
|
return Boolean(
|
|
error &&
|
|
typeof error === "object" &&
|
|
"code" in error &&
|
|
error.code === code,
|
|
);
|
|
}
|
|
|
|
const invokedPath = process.argv[1];
|
|
if (
|
|
invokedPath !== undefined &&
|
|
import.meta.url === pathToFileURL(invokedPath).href
|
|
) {
|
|
await main();
|
|
}
|