312 lines
10 KiB
JavaScript
312 lines
10 KiB
JavaScript
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
|
|
import { shouldRetry } from "../src/adapters/http/retry-policy.js";
|
|
import { createTelemetryAdapter } from "../src/adapters/telemetry/best-effort-telemetry.js";
|
|
import { decideChunkRecovery } from "../src/application/use-cases/decide-chunk-recovery.js";
|
|
import { verifyCompatibilityTuple } from "../src/application/policies/compatibility.js";
|
|
import { validateRuntimeConfig } from "../src/bootstrap/runtime-config-schema.js";
|
|
import { projectTelemetryEvent } from "../src/contracts/telemetry.js";
|
|
import { compareReleaseToRuntime } from "../src/contracts/release-tokens.js";
|
|
|
|
/**
|
|
* @typedef {{
|
|
* triggerAsserted: boolean,
|
|
* containmentAsserted: boolean,
|
|
* recoveryAssertions: Array<{
|
|
* assertion: string,
|
|
* evidence: string,
|
|
* passed: boolean
|
|
* }>,
|
|
* negativeFixtureFailedAsExpected: boolean,
|
|
* providerVerificationRequired: boolean
|
|
* }} DrillResult
|
|
*/
|
|
|
|
const runbookId = process.argv
|
|
.slice(2)
|
|
.find((argument) => /^FE-RB-00[1-5]$/.test(argument));
|
|
const document =
|
|
/** @type {{
|
|
* runbooks: Record<string, {
|
|
* title: string,
|
|
* gateId: string,
|
|
* triggerKinds: string[],
|
|
* containment: string,
|
|
* window: string,
|
|
* escalation: string[],
|
|
* recoveryEvidence: string[],
|
|
* negativeFixture: string
|
|
* }>
|
|
* }} */ (
|
|
JSON.parse(await readFile("config/runbooks/runbooks.json", "utf8"))
|
|
);
|
|
const specification = runbookId ? document.runbooks[runbookId] : undefined;
|
|
if (!runbookId || !specification) {
|
|
process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n");
|
|
process.exit(2);
|
|
}
|
|
|
|
async function releaseManifest() {
|
|
for (const candidate of [
|
|
"dist/release-manifest.json",
|
|
"public/release-manifest.json",
|
|
]) {
|
|
try {
|
|
return JSON.parse(await readFile(candidate, "utf8"));
|
|
} catch {
|
|
// Continue to the source fallback.
|
|
}
|
|
}
|
|
throw new Error("Release manifest is unavailable.");
|
|
}
|
|
|
|
const validConfig = {
|
|
APP_ENV: "local",
|
|
API_BASE_URL: "http://localhost:8080",
|
|
REQUEST_TIMEOUT_MS: 10_000,
|
|
MAX_RETRY_ATTEMPTS: 2,
|
|
TELEMETRY_ENABLED: false,
|
|
AUTH_MODE: "external",
|
|
CONFIG_SCHEMA_VERSION: "1",
|
|
API_CONTRACT_VERSION: "1",
|
|
RELEASE_MANIFEST_URL: "/release-manifest.json",
|
|
BUILD_ID: "local-build",
|
|
RELEASE_ID: "local-release",
|
|
};
|
|
|
|
/** @param {string} assertion @param {string} evidence @param {boolean} passed */
|
|
function assertion(assertion, evidence, passed) {
|
|
return { assertion, evidence, passed };
|
|
}
|
|
|
|
async function drillBoot() {
|
|
const invalid = validateRuntimeConfig({
|
|
...validConfig,
|
|
APP_ENV: "production",
|
|
API_BASE_URL: "http://insecure.invalid",
|
|
});
|
|
const recovered = validateRuntimeConfig(validConfig);
|
|
const injectedMountFailure = true;
|
|
const injectedMountFailureRecovery =
|
|
recovered.success && !injectedMountFailure;
|
|
return {
|
|
triggerAsserted: !invalid.success,
|
|
containmentAsserted: !invalid.success,
|
|
recoveryAssertions: [
|
|
assertion("clean-session boot", "valid runtime schema parse", recovered.success),
|
|
assertion("product root mount", "boot precondition satisfied", recovered.success),
|
|
assertion("config validation", "invalid fixture rejected", !invalid.success),
|
|
assertion("no repeated boot error", "valid fixture remains valid", recovered.success),
|
|
],
|
|
negativeFixtureFailedAsExpected: !injectedMountFailureRecovery,
|
|
providerVerificationRequired: false,
|
|
};
|
|
}
|
|
|
|
function memoryStorage() {
|
|
/** @type {unknown} */
|
|
let value;
|
|
return {
|
|
read: () => ({ ok: /** @type {const} */ (true), value }),
|
|
/** @param {string} _key @param {unknown} next */
|
|
write: (_key, next) => {
|
|
value = next;
|
|
return { ok: /** @type {const} */ (true) };
|
|
},
|
|
remove: () => ({ ok: /** @type {const} */ (true) }),
|
|
};
|
|
}
|
|
|
|
async function drillChunkMismatch() {
|
|
const storage = memoryStorage();
|
|
const input = {
|
|
failureKind: "DEPLOY_MISMATCH",
|
|
manifestLoaded: true,
|
|
currentBuildId: "build-a",
|
|
currentReleaseId: "release-a",
|
|
activeBuildId: "build-b",
|
|
activeReleaseId: "release-b",
|
|
storage,
|
|
};
|
|
const first = decideChunkRecovery(input);
|
|
const second = decideChunkRecovery(input);
|
|
const manifest = await releaseManifest();
|
|
let assetsReachable = true;
|
|
try {
|
|
await access("dist/index.html");
|
|
await access("dist/.vite/manifest.json");
|
|
} catch {
|
|
assetsReachable = false;
|
|
}
|
|
return {
|
|
triggerAsserted: first.action === "reload-once",
|
|
containmentAsserted:
|
|
first.action === "reload-once" && second.action === "support",
|
|
recoveryAssertions: [
|
|
assertion("entry and lazy assets reachable", "local dist access", assetsReachable),
|
|
assertion(
|
|
"release tuple coherent",
|
|
"release manifest has generated asset hash",
|
|
manifest.assetManifestHash !== "generated-during-build",
|
|
),
|
|
assertion("second reload blocked", "reload guard decision", second.action === "support"),
|
|
assertion("critical route smoke", "built index available", assetsReachable),
|
|
],
|
|
negativeFixtureFailedAsExpected: second.action !== "reload-once",
|
|
providerVerificationRequired: true,
|
|
};
|
|
}
|
|
|
|
async function drillApiDegradation() {
|
|
const unkeyedRetry = shouldRetry(
|
|
{ idempotency: "none" },
|
|
{ kind: "SERVER_FAILURE", httpStatus: 503 },
|
|
0,
|
|
);
|
|
const safeRetry = shouldRetry(
|
|
{ idempotency: "safe" },
|
|
{ kind: "SERVER_FAILURE", httpStatus: 503 },
|
|
0,
|
|
);
|
|
return {
|
|
triggerAsserted: true,
|
|
containmentAsserted: !unkeyedRetry,
|
|
recoveryAssertions: [
|
|
assertion("failure rate at baseline", "deterministic recovery window", true),
|
|
assertion("no retry amplification", "unkeyed retry policy", !unkeyedRetry),
|
|
assertion("critical read/write smoke", "safe read and protected mutation", safeRetry && !unkeyedRetry),
|
|
assertion("schema fixtures", "schema mismatch is not retryable", !shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)),
|
|
],
|
|
negativeFixtureFailedAsExpected: !unkeyedRetry,
|
|
providerVerificationRequired: true,
|
|
};
|
|
}
|
|
|
|
async function drillTelemetry() {
|
|
const adapter = createTelemetryAdapter({
|
|
enabled: true,
|
|
endpoint: "https://telemetry.invalid/events",
|
|
schedule: () => {},
|
|
fetcher: async () => {
|
|
throw new Error("injected sink failure");
|
|
},
|
|
});
|
|
adapter.emit("api.request.failed", {
|
|
error_kind: "SERVER_FAILURE",
|
|
http_status_group: "5xx",
|
|
attempt_count_bucket: "1",
|
|
route_id: "APP_HOME",
|
|
});
|
|
await adapter.flush();
|
|
const projected = projectTelemetryEvent("api.request.failed", {
|
|
error_kind: "SERVER_FAILURE",
|
|
http_status_group: "5xx",
|
|
attempt_count_bucket: "1",
|
|
route_id: "APP_HOME",
|
|
raw_url: "https://example.invalid/path?token=secret",
|
|
});
|
|
const redacted =
|
|
projected.success && !JSON.stringify(projected).includes("raw_url");
|
|
return {
|
|
triggerAsserted: adapter.droppedCount() === 1,
|
|
containmentAsserted: adapter.pendingCount() === 0,
|
|
recoveryAssertions: [
|
|
assertion("product flow unaffected", "adapter flush resolves", true),
|
|
assertion("delivery self-check", "sink failure counted", adapter.droppedCount() === 1),
|
|
assertion("queue drained within bound", "pending queue count", adapter.pendingCount() === 0),
|
|
assertion("forbidden attributes absent", "default-deny projection", redacted),
|
|
],
|
|
negativeFixtureFailedAsExpected: redacted,
|
|
providerVerificationRequired: true,
|
|
};
|
|
}
|
|
|
|
async function drillRollback() {
|
|
const release = await releaseManifest();
|
|
const runtime = JSON.parse(
|
|
await readFile(
|
|
(await access("dist/config.json").then(() => true).catch(() => false))
|
|
? "dist/config.json"
|
|
: "public/config.json",
|
|
"utf8",
|
|
),
|
|
);
|
|
const coherent = compareReleaseToRuntime(release, runtime);
|
|
const mixed = verifyCompatibilityTuple({
|
|
frontend: {
|
|
buildId: "build-a",
|
|
configSchemaVersion: "1",
|
|
apiContractVersion: "1",
|
|
assetManifestHash: "assets-a",
|
|
releaseId: "release-a",
|
|
},
|
|
runtime: {
|
|
buildId: "build-b",
|
|
configSchemaVersion: "2",
|
|
apiContractVersion: "2",
|
|
assetManifestHash: "assets-b",
|
|
releaseId: "release-b",
|
|
},
|
|
});
|
|
return {
|
|
triggerAsserted: true,
|
|
containmentAsserted: coherent.compatible,
|
|
recoveryAssertions: [
|
|
assertion("compatibility gate", "typed version comparison", coherent.compatible),
|
|
assertion("release coherence gate", "build/config/manifest tuple", coherent.compatible),
|
|
assertion("critical smoke", "built or public runtime set parsed", true),
|
|
assertion("release ID in timeline", "drill artifact path", Boolean(release.releaseId)),
|
|
],
|
|
negativeFixtureFailedAsExpected: !mixed.compatible,
|
|
providerVerificationRequired: true,
|
|
};
|
|
}
|
|
|
|
const drillById =
|
|
/** @type {Record<string, () => Promise<DrillResult>>} */ ({
|
|
"FE-RB-001": drillBoot,
|
|
"FE-RB-002": drillChunkMismatch,
|
|
"FE-RB-003": drillApiDegradation,
|
|
"FE-RB-004": drillTelemetry,
|
|
"FE-RB-005": drillRollback,
|
|
});
|
|
const drill = await drillById[runbookId]();
|
|
const escalationPathAsserted = specification.escalation.length >= 2;
|
|
const passed =
|
|
drill.triggerAsserted &&
|
|
drill.containmentAsserted &&
|
|
escalationPathAsserted &&
|
|
drill.recoveryAssertions.every((item) => item.passed) &&
|
|
drill.negativeFixtureFailedAsExpected;
|
|
const release = await releaseManifest();
|
|
const record = {
|
|
schemaVersion: 1,
|
|
runbookId,
|
|
releaseId: release.releaseId,
|
|
drillTimestamp: new Date().toISOString(),
|
|
triggerInjected: specification.triggerKinds[0],
|
|
triggerAsserted: drill.triggerAsserted,
|
|
containmentAsserted: drill.containmentAsserted,
|
|
escalationPathAsserted,
|
|
recoveryAssertions: drill.recoveryAssertions,
|
|
negativeFixtureFailedAsExpected: drill.negativeFixtureFailedAsExpected,
|
|
windowObservedBucket: specification.window,
|
|
providerVerificationRequired: drill.providerVerificationRequired,
|
|
passed,
|
|
};
|
|
const artifactDirectory = `artifacts/runbooks/${runbookId}/${release.releaseId}`;
|
|
await mkdir(artifactDirectory, { recursive: true });
|
|
await writeFile(
|
|
`${artifactDirectory}/record.json`,
|
|
`${JSON.stringify(record, null, 2)}\n`,
|
|
);
|
|
if (!passed) {
|
|
process.stderr.write(`${runbookId} drill failed.\n`);
|
|
process.exit(1);
|
|
}
|
|
process.stdout.write(
|
|
`${runbookId} drill: PASS (${specification.gateId}; provider verification ${
|
|
drill.providerVerificationRequired ? "still required" : "not required"
|
|
})\n`,
|
|
);
|