The reload-once marker only blocked a repeat of the same mismatch key. The key is buildId:code:releaseId, so a load balancer alternating R10/R11 replicas produced a new key on every boot and each one reloaded again (A->B->A->... looped). decide() now returns show-failure whenever a marker younger than the TTL exists, whatever its key. markCoherentBoot() still clears the marker and an expired marker still allows recovery. The platform test "keeps a single bounded marker and scopes it to the mismatch key" encoded the bug (R10->R11 then R11->R12 both reload-once); it is rewritten to check only the bounded marker, and the key-independent rule gets its own tests. RED before fix: - "allows one automatic reload per session within the TTL whatever the key" (A->B returned reload-once) - "cannot loop when a load balancer alternates the mismatch key" (20 of 20 alternating boots reloaded) - mixed-replica "cannot reload-loop when replicas alternate between two different mismatches" (runBootSession threw "reload loop") Gates: check:architecture, check:types:app, check:types:test, lint, test:component, check:release-coherence, test:platform pass. test:unit passes except tests/unit/removal-fixture.test.ts, which needs generated release evidence under artifacts/ that a fresh worktree does not have (unrelated to this change). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
119 lines
4.5 KiB
TypeScript
119 lines
4.5 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import { createReleaseMismatchRecovery } from "../../src/platform/release/release-mismatch-recovery.ts";
|
|
import {
|
|
bootAgainstReplicaSequence,
|
|
createReleaseFixture,
|
|
memoryStorage,
|
|
response,
|
|
runBootSession,
|
|
type ReleaseName,
|
|
type Surface,
|
|
} from "./release-fixture.ts";
|
|
|
|
/** A deterministic load balancer that alternates replicas request by request. */
|
|
function alternatingLoadBalancer(replicas: readonly ReleaseName[], offset = 0) {
|
|
let request = offset;
|
|
return (): readonly Readonly<{ kind: Surface; release: ReleaseName }>[] =>
|
|
(["html", "config", "manifest"] as const).map((kind) =>
|
|
response(kind, replicas[request++ % replicas.length] as ReleaseName),
|
|
);
|
|
}
|
|
|
|
describe("mixed-replica routing during a rolling release", () => {
|
|
it("boots when every response comes from R10", async () => {
|
|
const result = await bootAgainstReplicaSequence([
|
|
response("html", "R10"),
|
|
response("config", "R10"),
|
|
response("manifest", "R10"),
|
|
]);
|
|
expect(result).toMatchObject({ state: "booted", applicationMounted: true, release: "release-R10" });
|
|
});
|
|
|
|
it("boots when every response comes from R11", async () => {
|
|
const result = await bootAgainstReplicaSequence([
|
|
response("html", "R11"),
|
|
response("config", "R11"),
|
|
response("manifest", "R11"),
|
|
]);
|
|
expect(result).toMatchObject({ state: "booted", applicationMounted: true, release: "release-R11" });
|
|
});
|
|
|
|
it("fails closed when HTML and runtime config come from different releases", async () => {
|
|
const result = await bootAgainstReplicaSequence([
|
|
response("html", "R10"),
|
|
response("config", "R11"),
|
|
response("manifest", "R10"),
|
|
]);
|
|
|
|
expect(result.state).toBe("release-mismatch");
|
|
expect(result.applicationMounted).toBe(false);
|
|
});
|
|
|
|
it("fails closed when the manifest comes from a different release", async () => {
|
|
const result = await bootAgainstReplicaSequence([
|
|
response("html", "R10"),
|
|
response("config", "R10"),
|
|
response("manifest", "R11"),
|
|
]);
|
|
expect(result).toMatchObject({ state: "release-mismatch", applicationMounted: false });
|
|
});
|
|
|
|
it("never produces a silently mixed boot under an alternating load balancer", async () => {
|
|
for (let offset = 0; offset < 6; offset += 1) {
|
|
const next = alternatingLoadBalancer(["R10", "R11"], offset);
|
|
for (let boot = 0; boot < 10; boot += 1) {
|
|
const sequence = next();
|
|
const releases = new Set(sequence.map((entry) => entry.release));
|
|
const result = await bootAgainstReplicaSequence(sequence);
|
|
if (releases.size > 1) {
|
|
expect(result.applicationMounted).toBe(false);
|
|
} else {
|
|
expect(result.state).toBe("booted");
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
it("recovers a stale HTML document with exactly one reload once the rollout converges", async () => {
|
|
const deployment = createReleaseFixture(["R10", "R11"]);
|
|
const sequences = [
|
|
[response("html", "R10"), response("config", "R11"), response("manifest", "R11")],
|
|
[response("html", "R11"), response("config", "R11"), response("manifest", "R11")],
|
|
];
|
|
let boot = 0;
|
|
const session = await runBootSession(
|
|
() => sequences[Math.min(boot++, sequences.length - 1)] ?? [],
|
|
createReleaseMismatchRecovery(memoryStorage()),
|
|
deployment,
|
|
);
|
|
expect(session.reloads).toBe(1);
|
|
expect(session.final).toMatchObject({ state: "booted", release: "release-R11" });
|
|
});
|
|
|
|
it("cannot reload-loop when replicas stay permanently mixed", async () => {
|
|
const storage = memoryStorage();
|
|
const session = await runBootSession(
|
|
() => [response("html", "R10"), response("config", "R11"), response("manifest", "R10")],
|
|
createReleaseMismatchRecovery(storage),
|
|
);
|
|
expect(session.reloads).toBe(1);
|
|
expect(session.final).toMatchObject({ state: "release-mismatch", applicationMounted: false });
|
|
expect(storage.size()).toBe(1);
|
|
});
|
|
|
|
it("cannot reload-loop when replicas alternate between two different mismatches", async () => {
|
|
const sequences = [
|
|
[response("html", "R10"), response("config", "R11"), response("manifest", "R10")],
|
|
[response("html", "R11"), response("config", "R10"), response("manifest", "R11")],
|
|
];
|
|
let boot = 0;
|
|
const session = await runBootSession(
|
|
() => sequences[boot++ % sequences.length] ?? [],
|
|
createReleaseMismatchRecovery(memoryStorage()),
|
|
);
|
|
expect(session.reloads).toBe(1);
|
|
expect(session.final).toMatchObject({ state: "release-mismatch", applicationMounted: false });
|
|
});
|
|
});
|