The reload-once marker only blocked a repeat of the same mismatch key. The key is buildId:code:releaseId, so a load balancer alternating R10/R11 replicas produced a new key on every boot and each one reloaded again (A->B->A->... looped). decide() now returns show-failure whenever a marker younger than the TTL exists, whatever its key. markCoherentBoot() still clears the marker and an expired marker still allows recovery. The platform test "keeps a single bounded marker and scopes it to the mismatch key" encoded the bug (R10->R11 then R11->R12 both reload-once); it is rewritten to check only the bounded marker, and the key-independent rule gets its own tests. RED before fix: - "allows one automatic reload per session within the TTL whatever the key" (A->B returned reload-once) - "cannot loop when a load balancer alternates the mismatch key" (20 of 20 alternating boots reloaded) - mixed-replica "cannot reload-loop when replicas alternate between two different mismatches" (runBootSession threw "reload loop") Gates: check:architecture, check:types:app, check:types:test, lint, test:component, check:release-coherence, test:platform pass. test:unit passes except tests/unit/removal-fixture.test.ts, which needs generated release evidence under artifacts/ that a fresh worktree does not have (unrelated to this change). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
180 lines
6.8 KiB
TypeScript
180 lines
6.8 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import {
|
|
createReleaseMismatchRecovery,
|
|
type RecoveryMarkerStore,
|
|
} from "../../../src/platform/release/release-mismatch-recovery.ts";
|
|
import {
|
|
createBootReleaseRecovery,
|
|
decideBootFailure,
|
|
releaseMismatchKey,
|
|
} from "../../../src/bootstrap/boot-release-recovery.ts";
|
|
import { ReleaseManifestError } from "../../../src/bootstrap/load-release-manifest.ts";
|
|
import { BootConfigError } from "../../../src/bootstrap/load-runtime-config.ts";
|
|
import { STORAGE_REGISTRY } from "../../../src/contracts/storage-keys.ts";
|
|
|
|
/** A Web Storage stand-in for the host's `sessionStorage`. */
|
|
function sessionStorageLike() {
|
|
const values = new Map<string, string>();
|
|
return {
|
|
values,
|
|
getItem: (key: string) => values.get(key) ?? null,
|
|
setItem: (key: string, value: string) => void values.set(key, value),
|
|
removeItem: (key: string) => void values.delete(key),
|
|
};
|
|
}
|
|
|
|
function markerStore(): RecoveryMarkerStore & { values: Map<string, string> } {
|
|
const values = new Map<string, string>();
|
|
return {
|
|
values,
|
|
read: () => values.get("marker") ?? null,
|
|
write: (value) => void values.set("marker", value),
|
|
clear: () => void values.delete("marker"),
|
|
};
|
|
}
|
|
|
|
describe("release mismatch recovery", () => {
|
|
it("allows at most one automatic recovery attempt per release mismatch key", () => {
|
|
const recovery = createReleaseMismatchRecovery(markerStore());
|
|
|
|
expect(recovery.decide("R10->R11")).toBe("reload-once");
|
|
expect(recovery.decide("R10->R11")).toBe("show-failure");
|
|
});
|
|
|
|
it("never reloads more than once however often the same mismatch repeats", () => {
|
|
const recovery = createReleaseMismatchRecovery(markerStore());
|
|
const decisions = Array.from({ length: 50 }, () => recovery.decide("R10->R11"));
|
|
expect(decisions.filter((decision) => decision === "reload-once")).toHaveLength(1);
|
|
});
|
|
|
|
it("clears the marker after a coherent boot", () => {
|
|
const storage = markerStore();
|
|
const recovery = createReleaseMismatchRecovery(storage);
|
|
recovery.decide("R10->R11");
|
|
recovery.markCoherentBoot();
|
|
expect(storage.values.size).toBe(0);
|
|
expect(recovery.decide("R10->R11")).toBe("reload-once");
|
|
});
|
|
|
|
it("keeps a single bounded marker", () => {
|
|
const storage = markerStore();
|
|
const recovery = createReleaseMismatchRecovery(storage);
|
|
expect(recovery.decide("x".repeat(10_000))).toBe("reload-once");
|
|
expect(storage.values.size).toBe(1);
|
|
expect([...storage.values.values()][0]?.length).toBeLessThan(300);
|
|
});
|
|
|
|
it("allows one automatic reload per session within the TTL whatever the key", () => {
|
|
const recovery = createReleaseMismatchRecovery(markerStore());
|
|
expect(recovery.decide("A")).toBe("reload-once");
|
|
expect(recovery.decide("B")).toBe("show-failure");
|
|
expect(recovery.decide("A")).toBe("show-failure");
|
|
});
|
|
|
|
it("cannot loop when a load balancer alternates the mismatch key", () => {
|
|
const recovery = createReleaseMismatchRecovery(markerStore());
|
|
const decisions = Array.from({ length: 20 }, (_, boot) =>
|
|
recovery.decide(boot % 2 === 0 ? "build-R10:X:R11" : "build-R11:X:R10"),
|
|
);
|
|
expect(decisions.filter((decision) => decision === "reload-once")).toHaveLength(1);
|
|
});
|
|
|
|
it("lets an expired marker recover again instead of blocking forever", () => {
|
|
let now = 0;
|
|
const recovery = createReleaseMismatchRecovery(markerStore(), {
|
|
now: () => now,
|
|
ttlMs: 1_000,
|
|
});
|
|
expect(recovery.decide("k")).toBe("reload-once");
|
|
now = 500;
|
|
expect(recovery.decide("k")).toBe("show-failure");
|
|
now = 5_000;
|
|
expect(recovery.decide("k")).toBe("reload-once");
|
|
});
|
|
|
|
it("fails closed without working storage", () => {
|
|
expect(createReleaseMismatchRecovery(null).decide("k")).toBe("show-failure");
|
|
const throwing: RecoveryMarkerStore = {
|
|
read: () => {
|
|
throw new Error("denied");
|
|
},
|
|
write: () => {
|
|
throw new Error("denied");
|
|
},
|
|
clear: () => {
|
|
throw new Error("denied");
|
|
},
|
|
};
|
|
const recovery = createReleaseMismatchRecovery(throwing);
|
|
expect(recovery.decide("k")).toBe("show-failure");
|
|
expect(() => recovery.markCoherentBoot()).not.toThrow();
|
|
const dropping: RecoveryMarkerStore = {
|
|
read: () => null,
|
|
write: () => {},
|
|
clear: () => {},
|
|
};
|
|
expect(createReleaseMismatchRecovery(dropping).decide("k")).toBe("show-failure");
|
|
});
|
|
});
|
|
|
|
describe("boot release recovery wiring", () => {
|
|
it("keys only release mismatches, never other boot failures", () => {
|
|
const mismatch = new ReleaseManifestError("MANIFEST_RELEASE_MISMATCH", {
|
|
buildId: "build-R10",
|
|
releaseId: "release-R11",
|
|
});
|
|
expect(releaseMismatchKey(mismatch)).toBe(
|
|
"build-R10:MANIFEST_RELEASE_MISMATCH:release-R11",
|
|
);
|
|
expect(
|
|
releaseMismatchKey(new BootConfigError("CONFIG_BUILD_MISMATCH", { buildId: "build-R10" })),
|
|
).toBe("build-R10:CONFIG_BUILD_MISMATCH:-");
|
|
expect(
|
|
releaseMismatchKey(new ReleaseManifestError("MANIFEST_TIMEOUT", { buildId: "b" })),
|
|
).toBeNull();
|
|
expect(
|
|
releaseMismatchKey(new BootConfigError("CONFIG_FETCH_FAILED", { buildId: "b" })),
|
|
).toBeNull();
|
|
expect(releaseMismatchKey(new Error("boom"))).toBeNull();
|
|
});
|
|
|
|
it("uses the registered RELEASE_MISMATCH_RECOVERY session key", () => {
|
|
const storage = sessionStorageLike();
|
|
const recovery = createBootReleaseRecovery({ sessionStorage: storage });
|
|
const mismatch = new ReleaseManifestError("MANIFEST_BUILD_MISMATCH", { buildId: "build-R10" });
|
|
|
|
expect(decideBootFailure(mismatch, recovery)).toBe("reload-once");
|
|
expect([...storage.values.keys()]).toEqual([
|
|
STORAGE_REGISTRY.RELEASE_MISMATCH_RECOVERY.physicalKey,
|
|
]);
|
|
expect(decideBootFailure(mismatch, recovery)).toBe("show-failure");
|
|
expect(
|
|
decideBootFailure(new ReleaseManifestError("MANIFEST_TIMEOUT", { buildId: "b" }), recovery),
|
|
).toBe("show-failure");
|
|
});
|
|
|
|
it("remembers the attempt across the reload itself (a fresh boot, same session)", () => {
|
|
const storage = sessionStorageLike();
|
|
const mismatch = new ReleaseManifestError("MANIFEST_BUILD_MISMATCH", { buildId: "build-R10" });
|
|
|
|
expect(decideBootFailure(mismatch, createBootReleaseRecovery({ sessionStorage: storage }))).toBe(
|
|
"reload-once",
|
|
);
|
|
const afterReload = createBootReleaseRecovery({ sessionStorage: storage });
|
|
expect(decideBootFailure(mismatch, afterReload)).toBe("show-failure");
|
|
afterReload.markCoherentBoot();
|
|
expect(storage.values.size).toBe(0);
|
|
});
|
|
|
|
it("degrades to show-failure when the host has no session storage", () => {
|
|
const recovery = createBootReleaseRecovery({});
|
|
expect(
|
|
decideBootFailure(
|
|
new ReleaseManifestError("MANIFEST_BUILD_MISMATCH", { buildId: "b" }),
|
|
recovery,
|
|
),
|
|
).toBe("show-failure");
|
|
});
|
|
});
|