문제: control cache의 active pointer entry는 있는데 저장된 본문이 다시 쓰기 전까지 계속 읽히지 않으면(blob 손실 등) Cache Storage 관리가 모두 멈춘다. readActivePointer는 이 실패를 UNAVAILABLE/RETRY(또는 NOT_READABLE/REOPEN)로 돌려주고 CORRUPT_DATA로 보지 않는다. activation은 CORRUPT_DATA 밖의 pointer 실패에서 멈추고, cleanup·inspect·조회도 같은 실패에서 멈춘다. pointer를 쓰는 곳은 activation 하나라서 RETRY를 지켜도 돌아오지 않는다. 돌아오는 길은 adapter 밖에서 control cache를 지우는 것뿐이다. 리뷰 실험 02-R1 X3이 이것을 보였다. 고른 방법: activateRelease에 선택 옵션 replaceUnreadablePointer를 둔다. true면 이전 pointer를 읽지 못한 이유와 상관없이, 손상 pointer와 같은 0929/02-01 경로로 검증된 후보를 rollback 목록 없이 덮어쓴다. 기본값은 지금처럼 멈춤이라 "일시 오류 하나로 rollback 목록을 잃지 않는다"는 보존 조건이 지켜진다. 포트 주석, 설계 문서 8.3절, 운영 문서에 언제 쓰는지와 rollback 목록을 잃는다는 것을 적었다. 버린 대안: (가) 문서에 "control cache를 지우고 다시 activate"만 적기. 제품이 adapter 밖 코드로 소유 cache 이름을 알고 지워야 하고, 그동안 조회가 pointer 없음으로 바뀐다. (다) 연속 N회 실패를 손상으로 보기. 실패 횟수를 저장해야 하고, 긴 일시 오류에서도 rollback 목록을 잃는다. 위험·한계: 옵션을 켜면 일시 오류에서도 rollback 목록을 잃고, 다음 cleanup이 새 활성 release 밖의 owned release cache를 모두 지운다. 그래서 기본값으로 두지 않았다. 일시와 지속을 가르는 판단은 운영자 몫이다. 포트에 선택 칸이 하나 늘었다. 호출부·구현부 모두 호환이라 breaking이 아니다. CHANGELOG [계약] 행은 CHANGELOG 담당에게 남긴다. 1105/02-02의 남은 부분(release marker 본문이 계속 읽히지 않으면 stage가 marker를 다시 쓰지 않는다)은 이 변경으로 풀리지 않는다. marker는 release cache 안의 entry라 stage 경로를 바꿔야 한다. 범위 밖으로 둔다. service-worker-pwa 레시피 번들(여유 306 B)에 수십 B가 더해진다. 실제 브라우저에서 pointer 본문 읽기 실패가 지속되는지는 확인하지 못했다(리뷰 5절 미검증과 같다). 검증: desktop batch에서 확인. node scripts/run-vitest.ts run tests/unit/public-response-cache-repair.test.ts RED(이 커밋의 테스트 + 부모 src): "replaces a pointer whose body stays unreadable when the activation asks to (1106/02-01)"가 옵션을 준 activation에서 실패한다. 부모는 옵션을 모르므로 UNAVAILABLE/RETRY를 돌려주고 pointer를 다시 쓰지 않는다. 그 앞의 멈춤 단언(X3와 같은 결과)은 부모에서도 통과한다. 일시 오류 테스트 "still stops activation when the pointer read fails transiently"는 바뀌지 않았고 통과해야 한다. Refs: 1106/02-01
1061 lines
39 KiB
TypeScript
1061 lines
39 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import type {
|
|
PublicCacheAsset,
|
|
PublicCacheReleaseManifest,
|
|
} from "../../src/application/ports/browser-file-storage/cache-storage-ports.ts";
|
|
import {
|
|
activePointerRequest,
|
|
controlCacheName,
|
|
markerRequest,
|
|
} from "../../src/adapters/cache-storage/public-cache-codec.ts";
|
|
import {
|
|
createDefaultPublicCachePolicy,
|
|
resolvePublicCachePolicy,
|
|
type PublicCacheRuntimePolicy,
|
|
} from "../../src/adapters/cache-storage/public-cache-policy.ts";
|
|
import { createPublicResponseCacheAdapter } from "../../src/adapters/cache-storage/public-response-cache-adapter.ts";
|
|
import {
|
|
MemoryCacheStorage,
|
|
digestHex,
|
|
immediateLock,
|
|
manifestFor,
|
|
} from "./public-response-cache-fixture.ts";
|
|
|
|
/**
|
|
* STO-RR-04 / STO-RR-05. A release cache that is currently serving traffic is
|
|
* the last thing a repair may destroy. A transient marker read failure is not
|
|
* evidence of damage, and a repair that has not yet fetched anything has not
|
|
* yet earned the right to delete what still works.
|
|
*/
|
|
describe("public response cache repair is failure-atomic", () => {
|
|
async function stagedRelease(
|
|
releaseRegistryId: string,
|
|
observer?: (event: unknown) => void,
|
|
) {
|
|
const policy = createDefaultPublicCachePolicy(
|
|
"https://assets.example.test",
|
|
);
|
|
const firstBytes = new Uint8Array([1, 1, 1, 1]);
|
|
const secondBytes = new Uint8Array([2, 2, 2, 2]);
|
|
const assets: readonly PublicCacheAsset[] = [
|
|
{
|
|
absoluteUrl: "https://assets.example.test/first.js",
|
|
expectedByteLength: firstBytes.byteLength,
|
|
expectedContentType: "application/javascript",
|
|
integrity: {
|
|
algorithm: "SHA-256",
|
|
digestHex: await digestHex(firstBytes),
|
|
},
|
|
},
|
|
{
|
|
absoluteUrl: "https://assets.example.test/second.js",
|
|
expectedByteLength: secondBytes.byteLength,
|
|
expectedContentType: "application/javascript",
|
|
integrity: {
|
|
algorithm: "SHA-256",
|
|
digestHex: await digestHex(secondBytes),
|
|
},
|
|
},
|
|
];
|
|
const bodies = new Map<string, Uint8Array>([
|
|
[assets[0]!.absoluteUrl, firstBytes],
|
|
[assets[1]!.absoluteUrl, secondBytes],
|
|
]);
|
|
const cacheStorage = new MemoryCacheStorage();
|
|
const fetchLog: string[] = [];
|
|
let failFrom: string | null = null;
|
|
const adapter = createPublicResponseCacheAdapter({
|
|
cacheStorage: cacheStorage as unknown as CacheStorage,
|
|
crypto: globalThis.crypto,
|
|
mutationLock: immediateLock,
|
|
policy,
|
|
...(observer ? { observer } : {}),
|
|
fetcher: async (request: Request) => {
|
|
fetchLog.push(request.url);
|
|
if (failFrom !== null && request.url === failFrom) {
|
|
throw new TypeError("network is down");
|
|
}
|
|
const body = bodies.get(request.url);
|
|
if (!body) throw new TypeError(`unknown asset ${request.url}`);
|
|
return new Response(Uint8Array.from(body), {
|
|
headers: {
|
|
"cache-control": "public",
|
|
"content-type": "application/javascript",
|
|
},
|
|
});
|
|
},
|
|
});
|
|
const manifest = await manifestFor(releaseRegistryId, assets, policy);
|
|
expect(await adapter.admin.stageRelease(manifest)).toMatchObject({
|
|
ok: true,
|
|
});
|
|
expect(
|
|
await adapter.admin.activateRelease(
|
|
manifest.releaseRegistryId,
|
|
manifest.manifestDigestHex,
|
|
),
|
|
).toMatchObject({ ok: true });
|
|
const cacheName = [...cacheStorage.caches.keys()].find((name) =>
|
|
name.includes(releaseRegistryId),
|
|
);
|
|
if (!cacheName) throw new Error("staged cache missing");
|
|
|
|
return {
|
|
adapter,
|
|
assets,
|
|
cacheName,
|
|
cacheStorage,
|
|
fetchLog,
|
|
manifest,
|
|
setFailure(url: string | null) {
|
|
failFrom = url;
|
|
},
|
|
};
|
|
}
|
|
|
|
it("does not delete an active candidate when the marker read fails transiently", async () => {
|
|
const release = await stagedRelease("transient-marker");
|
|
const cache = release.cacheStorage.caches.get(release.cacheName)!;
|
|
const realMatch = cache.match.bind(cache);
|
|
let markerReads = 0;
|
|
const assetUrls = new Set(release.assets.map((asset) => asset.absoluteUrl));
|
|
cache.match = async (request: RequestInfo | URL) => {
|
|
const url =
|
|
request instanceof Request ? request.url : String(request);
|
|
if (!assetUrls.has(url)) {
|
|
markerReads += 1;
|
|
throw new DOMException("Storage is busy", "InvalidStateError");
|
|
}
|
|
return await realMatch(request);
|
|
};
|
|
|
|
const restaged = await release.adapter.admin.stageRelease(release.manifest);
|
|
|
|
expect(markerReads).toBeGreaterThan(0);
|
|
expect(restaged.ok).toBe(false);
|
|
expect(release.cacheStorage.caches.has(release.cacheName)).toBe(true);
|
|
cache.match = realMatch;
|
|
expect(
|
|
await release.adapter.responses.matchActiveExact({
|
|
absoluteUrl: release.assets[0]!.absoluteUrl,
|
|
}),
|
|
).toMatchObject({ ok: true });
|
|
});
|
|
|
|
it("reports a failed re-verification of a staged candidate as CACHE_STAGE (2026-11-01 02-03)", async () => {
|
|
const observations: Readonly<{ outcome?: string }>[] = [];
|
|
const release = await stagedRelease("reverify-unreadable", (event) =>
|
|
observations.push(event as Readonly<{ outcome?: string }>),
|
|
);
|
|
const cache = release.cacheStorage.caches.get(release.cacheName)!;
|
|
const realMatch = cache.match.bind(cache);
|
|
const assetUrls = new Set(release.assets.map((asset) => asset.absoluteUrl));
|
|
cache.match = async (request: RequestInfo | URL) => {
|
|
const url =
|
|
request instanceof Request ? request.url : String(request);
|
|
if (assetUrls.has(url)) {
|
|
throw new DOMException("Storage is busy", "InvalidStateError");
|
|
}
|
|
return await realMatch(request);
|
|
};
|
|
observations.length = 0;
|
|
|
|
const restaged = await release.adapter.admin.stageRelease(release.manifest);
|
|
|
|
expect(restaged).toMatchObject({
|
|
ok: false,
|
|
error: { code: "UNAVAILABLE", operation: "CACHE_STAGE" },
|
|
});
|
|
expect(
|
|
observations.filter(({ outcome }) => outcome === "FAILED"),
|
|
).toEqual([
|
|
expect.objectContaining({
|
|
operation: "CACHE_STAGE",
|
|
failureCode: "UNAVAILABLE",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("reports a cached body that cannot be read during activation as unreadable, not as damage (2026-11-03 02-02)", async () => {
|
|
const release = await stagedRelease("unreadable-body");
|
|
const cache = release.cacheStorage.caches.get(release.cacheName)!;
|
|
const realMatch = cache.match.bind(cache);
|
|
const assetUrls = new Set(release.assets.map((asset) => asset.absoluteUrl));
|
|
cache.match = async (request: RequestInfo | URL) => {
|
|
const url =
|
|
request instanceof Request ? request.url : String(request);
|
|
const stored = await realMatch(request);
|
|
if (!stored || !assetUrls.has(url)) return stored;
|
|
// The entry is there and its headers are intact; only reading the
|
|
// stored bytes fails, as a disk or browser I/O error does.
|
|
return new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
pull(controller) {
|
|
controller.error(new TypeError("disk read failed"));
|
|
},
|
|
}),
|
|
{ headers: stored.headers },
|
|
);
|
|
};
|
|
const unreadable = (operation: string, recovery: string) => ({
|
|
ok: false,
|
|
error: {
|
|
code: "UNAVAILABLE",
|
|
operation,
|
|
retryable: true,
|
|
recovery,
|
|
},
|
|
});
|
|
|
|
expect(
|
|
await release.adapter.responses.matchActiveExact({
|
|
absoluteUrl: release.assets[0]!.absoluteUrl,
|
|
}),
|
|
).toMatchObject(unreadable("CACHE_LOOKUP", "RETRY"));
|
|
// 1105/02-02. Activation keeps the code but points at staging the same
|
|
// manifest again, which is what fixes an entry that stays unreadable.
|
|
expect(
|
|
await release.adapter.admin.activateRelease(
|
|
release.manifest.releaseRegistryId,
|
|
release.manifest.manifestDigestHex,
|
|
),
|
|
).toMatchObject(unreadable("CACHE_ACTIVATE", "REHYDRATE"));
|
|
// 1105/02-02. The stage fast path no longer stops at an unreadable
|
|
// candidate: it fetches every asset again and writes it over the entry.
|
|
// It used to fail with no fetch, so nothing could bring the entry back.
|
|
const fetchesBefore = release.fetchLog.length;
|
|
expect(
|
|
await release.adapter.admin.stageRelease(release.manifest),
|
|
).toMatchObject({ ok: true });
|
|
expect(release.fetchLog).toHaveLength(
|
|
fetchesBefore + release.assets.length,
|
|
);
|
|
|
|
cache.match = realMatch;
|
|
expect(
|
|
await release.adapter.admin.activateRelease(
|
|
release.manifest.releaseRegistryId,
|
|
release.manifest.manifestDigestHex,
|
|
),
|
|
).toMatchObject({ ok: true });
|
|
});
|
|
|
|
it("still reports stored bytes that no longer match as an integrity failure", async () => {
|
|
const release = await stagedRelease("mismatched-body");
|
|
const cache = release.cacheStorage.caches.get(release.cacheName)!;
|
|
const index = cache.responses.findIndex(
|
|
(entry) => entry.request.url === release.assets[0]!.absoluteUrl,
|
|
);
|
|
const entry = cache.responses[index]!;
|
|
cache.responses.splice(index, 1, {
|
|
request: entry.request,
|
|
response: new Response(new Uint8Array([9, 9, 9, 9]), {
|
|
headers: entry.response.headers,
|
|
}),
|
|
});
|
|
|
|
expect(
|
|
await release.adapter.admin.activateRelease(
|
|
release.manifest.releaseRegistryId,
|
|
release.manifest.manifestDigestHex,
|
|
),
|
|
).toMatchObject({
|
|
ok: false,
|
|
error: {
|
|
code: "INTEGRITY_FAILED",
|
|
operation: "CACHE_ACTIVATE",
|
|
recovery: "REHYDRATE",
|
|
},
|
|
});
|
|
});
|
|
|
|
it("keeps every healthy asset when one repair fetch fails", async () => {
|
|
const release = await stagedRelease("partial-repair");
|
|
const cache = release.cacheStorage.caches.get(release.cacheName)!;
|
|
// Corrupt only the first asset's stored bytes.
|
|
const corrupted = cache.responses.findIndex(
|
|
(entry) => entry.request.url === release.assets[0]!.absoluteUrl,
|
|
);
|
|
expect(corrupted).toBeGreaterThanOrEqual(0);
|
|
cache.responses.splice(corrupted, 1);
|
|
|
|
release.setFailure(release.assets[0]!.absoluteUrl);
|
|
const restaged = await release.adapter.admin.stageRelease(release.manifest);
|
|
expect(restaged.ok).toBe(false);
|
|
|
|
// The cache still exists and the healthy asset is still served.
|
|
expect(release.cacheStorage.caches.has(release.cacheName)).toBe(true);
|
|
expect(
|
|
await release.adapter.responses.matchActiveExact({
|
|
absoluteUrl: release.assets[1]!.absoluteUrl,
|
|
}),
|
|
).toMatchObject({ ok: true });
|
|
});
|
|
|
|
it("still removes a candidate this call created when staging fails", async () => {
|
|
const policy = createDefaultPublicCachePolicy(
|
|
"https://assets.example.test",
|
|
);
|
|
const bytes = new Uint8Array([7, 7, 7, 7]);
|
|
const asset: PublicCacheAsset = {
|
|
absoluteUrl: "https://assets.example.test/fresh.js",
|
|
expectedByteLength: bytes.byteLength,
|
|
expectedContentType: "application/javascript",
|
|
integrity: {
|
|
algorithm: "SHA-256",
|
|
digestHex: await digestHex(bytes),
|
|
},
|
|
};
|
|
const cacheStorage = new MemoryCacheStorage();
|
|
const adapter = createPublicResponseCacheAdapter({
|
|
cacheStorage: cacheStorage as unknown as CacheStorage,
|
|
crypto: globalThis.crypto,
|
|
mutationLock: immediateLock,
|
|
policy,
|
|
fetcher: async () => {
|
|
throw new TypeError("network is down");
|
|
},
|
|
});
|
|
const manifest = await manifestFor("fresh-release", [asset], policy);
|
|
|
|
expect(await adapter.admin.stageRelease(manifest)).toMatchObject({
|
|
ok: false,
|
|
});
|
|
expect(await cacheStorage.keys()).toEqual([]);
|
|
});
|
|
|
|
it("repairs an evicted asset in place and keeps the release usable", async () => {
|
|
const release = await stagedRelease("in-place-repair");
|
|
const cache = release.cacheStorage.caches.get(release.cacheName)!;
|
|
const evicted = cache.responses.findIndex(
|
|
(entry) => entry.request.url === release.assets[1]!.absoluteUrl,
|
|
);
|
|
cache.responses.splice(evicted, 1);
|
|
|
|
expect(
|
|
await release.adapter.admin.stageRelease(release.manifest),
|
|
).toMatchObject({ ok: true });
|
|
expect(
|
|
await release.adapter.responses.matchActiveExact({
|
|
absoluteUrl: release.assets[1]!.absoluteUrl,
|
|
}),
|
|
).toMatchObject({ ok: true });
|
|
expect(
|
|
await release.adapter.responses.matchActiveExact({
|
|
absoluteUrl: release.assets[0]!.absoluteUrl,
|
|
}),
|
|
).toMatchObject({ ok: true });
|
|
});
|
|
});
|
|
|
|
/**
|
|
* 1105/02-02. After 2026-11-03 02-02 a body read failure is no damage, and the
|
|
* stage fast path stopped there with no fetch. An entry that stays unreadable
|
|
* until it is written again (a lost blob) was then never written again: the
|
|
* candidate kept failing activation, a rollback target that cleanup keeps
|
|
* could not be activated again, and the active release kept missing the
|
|
* asset. Staging the same manifest now writes such an entry again.
|
|
*/
|
|
describe("a cached body that stays unreadable until it is written again (1105/02-02)", () => {
|
|
async function deployments() {
|
|
const policy = createDefaultPublicCachePolicy(
|
|
"https://assets.example.test",
|
|
);
|
|
const firstBytes = new Uint8Array([1, 1, 1, 1]);
|
|
const secondBytes = new Uint8Array([2, 2, 2, 2]);
|
|
const assets: readonly PublicCacheAsset[] = [
|
|
{
|
|
absoluteUrl: "https://assets.example.test/first.js",
|
|
expectedByteLength: firstBytes.byteLength,
|
|
expectedContentType: "application/javascript",
|
|
integrity: {
|
|
algorithm: "SHA-256",
|
|
digestHex: await digestHex(firstBytes),
|
|
},
|
|
},
|
|
{
|
|
absoluteUrl: "https://assets.example.test/second.js",
|
|
expectedByteLength: secondBytes.byteLength,
|
|
expectedContentType: "application/javascript",
|
|
integrity: {
|
|
algorithm: "SHA-256",
|
|
digestHex: await digestHex(secondBytes),
|
|
},
|
|
},
|
|
];
|
|
const bodies = new Map<string, Uint8Array>([
|
|
[assets[0]!.absoluteUrl, firstBytes],
|
|
[assets[1]!.absoluteUrl, secondBytes],
|
|
]);
|
|
const cacheStorage = new MemoryCacheStorage();
|
|
const fetchLog: string[] = [];
|
|
const adapter = createPublicResponseCacheAdapter({
|
|
cacheStorage: cacheStorage as unknown as CacheStorage,
|
|
crypto: globalThis.crypto,
|
|
mutationLock: immediateLock,
|
|
policy,
|
|
fetcher: async (request: Request) => {
|
|
fetchLog.push(request.url);
|
|
const body = bodies.get(request.url);
|
|
if (!body) throw new TypeError(`unknown asset ${request.url}`);
|
|
return new Response(Uint8Array.from(body), {
|
|
headers: {
|
|
"cache-control": "public",
|
|
"content-type": "application/javascript",
|
|
},
|
|
});
|
|
},
|
|
});
|
|
const manifests = new Map<string, PublicCacheReleaseManifest>();
|
|
const manifest = async (releaseRegistryId: string) => {
|
|
let found = manifests.get(releaseRegistryId);
|
|
if (!found) {
|
|
found = await manifestFor(releaseRegistryId, assets, policy);
|
|
manifests.set(releaseRegistryId, found);
|
|
}
|
|
return found;
|
|
};
|
|
const cacheOf = (releaseRegistryId: string) => {
|
|
const name = [...cacheStorage.caches.keys()].find((candidate) =>
|
|
candidate.includes(`release:${releaseRegistryId}:`),
|
|
);
|
|
const cache = name ? cacheStorage.caches.get(name) : undefined;
|
|
if (!cache) throw new Error(`no cache for ${releaseRegistryId}`);
|
|
return cache;
|
|
};
|
|
return {
|
|
assets,
|
|
async stage(releaseRegistryId: string) {
|
|
const before = fetchLog.length;
|
|
const result = await adapter.admin.stageRelease(
|
|
await manifest(releaseRegistryId),
|
|
);
|
|
return { ok: result.ok, fetched: fetchLog.length - before };
|
|
},
|
|
async activate(releaseRegistryId: string) {
|
|
const { manifestDigestHex } = await manifest(releaseRegistryId);
|
|
return await adapter.admin.activateRelease(
|
|
releaseRegistryId,
|
|
manifestDigestHex,
|
|
);
|
|
},
|
|
lookup: (absoluteUrl: string) =>
|
|
adapter.responses.matchActiveExact({ absoluteUrl }),
|
|
inspect: () => adapter.admin.inspect(),
|
|
cacheOf,
|
|
/** The entry's body fails to read until something writes it again. */
|
|
unreadableUntilRewritten(
|
|
releaseRegistryId: string,
|
|
url: string,
|
|
error: () => unknown,
|
|
) {
|
|
const cache = cacheOf(releaseRegistryId);
|
|
const state = { unreadable: true };
|
|
const realMatch = cache.match.bind(cache);
|
|
const realPut = cache.put.bind(cache);
|
|
const urlOf = (request: RequestInfo | URL) =>
|
|
request instanceof Request ? request.url : String(request);
|
|
cache.put = async (request, response) => {
|
|
if (urlOf(request) === url) state.unreadable = false;
|
|
await realPut(request, response);
|
|
};
|
|
cache.match = async (request) => {
|
|
const stored = await realMatch(request);
|
|
if (!stored || urlOf(request) !== url || !state.unreadable) {
|
|
return stored;
|
|
}
|
|
return new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
pull(controller) {
|
|
controller.error(error());
|
|
},
|
|
}),
|
|
{ headers: stored.headers },
|
|
);
|
|
};
|
|
return state;
|
|
},
|
|
};
|
|
}
|
|
|
|
it.each([
|
|
["a TypeError", () => new TypeError("blob missing"), "UNAVAILABLE"],
|
|
[
|
|
"a NotReadableError",
|
|
() => new DOMException("unreadable", "NotReadableError"),
|
|
"NOT_READABLE",
|
|
],
|
|
] as const)(
|
|
"repairs a staged candidate whose entry fails with %s by staging it again (E-1a, E-1b)",
|
|
async (_label, error, code) => {
|
|
const world = await deployments();
|
|
expect(await world.stage("r1")).toEqual({ ok: true, fetched: 2 });
|
|
expect(await world.activate("r1")).toMatchObject({ ok: true });
|
|
expect(await world.stage("r2")).toEqual({ ok: true, fetched: 2 });
|
|
const entry = world.unreadableUntilRewritten(
|
|
"r2",
|
|
world.assets[0]!.absoluteUrl,
|
|
error,
|
|
);
|
|
|
|
expect(await world.activate("r2")).toMatchObject({
|
|
ok: false,
|
|
error: { code, operation: "CACHE_ACTIVATE", recovery: "REHYDRATE" },
|
|
});
|
|
expect(await world.stage("r2")).toEqual({ ok: true, fetched: 2 });
|
|
expect(entry.unreadable).toBe(false);
|
|
expect(await world.activate("r2")).toMatchObject({ ok: true });
|
|
},
|
|
);
|
|
|
|
// 1105/02-04. Which stored-entry failures count as damage decides what an
|
|
// activation reports; the stage fast path writes every one of them again.
|
|
// Each row is one code of the damage list or one body read failure outside
|
|
// it, so taking a code out of the list or adding one fails its row.
|
|
const replaced =
|
|
(make: (stored: Response) => Response) =>
|
|
(world: Awaited<ReturnType<typeof deployments>>, url: string) => {
|
|
const cache = world.cacheOf("r2");
|
|
const index = cache.responses.findIndex(
|
|
(entry) => entry.request.url === url,
|
|
);
|
|
const entry = cache.responses[index]!;
|
|
cache.responses.splice(index, 1, {
|
|
request: entry.request,
|
|
response: make(entry.response),
|
|
});
|
|
};
|
|
const withHeader = (name: string, value: string) =>
|
|
replaced((stored) => {
|
|
const headers = new Headers(stored.headers);
|
|
headers.set(name, value);
|
|
return new Response(new Uint8Array([1, 1, 1, 1]), { headers });
|
|
});
|
|
const unreadableWith =
|
|
(error: () => unknown) =>
|
|
(world: Awaited<ReturnType<typeof deployments>>, url: string) => {
|
|
world.unreadableUntilRewritten("r2", url, error);
|
|
};
|
|
|
|
it.each([
|
|
[
|
|
"bytes that no longer match the digest",
|
|
replaced(
|
|
(stored) =>
|
|
new Response(new Uint8Array([9, 9, 9, 9]), {
|
|
headers: stored.headers,
|
|
}),
|
|
),
|
|
"INTEGRITY_FAILED",
|
|
],
|
|
[
|
|
"a header outside the policy",
|
|
withHeader("cache-control", "no-store"),
|
|
"INTEGRITY_FAILED",
|
|
],
|
|
[
|
|
"a declared length over the entry limit",
|
|
withHeader("content-length", "99999999999"),
|
|
"INTEGRITY_FAILED",
|
|
],
|
|
[
|
|
"a body that fails as corrupt data",
|
|
unreadableWith(() => new DOMException("bad record", "DataError")),
|
|
"INTEGRITY_FAILED",
|
|
],
|
|
[
|
|
"a body read that fails with a TypeError",
|
|
unreadableWith(() => new TypeError("blob missing")),
|
|
"UNAVAILABLE",
|
|
],
|
|
[
|
|
"a body read that fails with a NotReadableError",
|
|
unreadableWith(() => new DOMException("unreadable", "NotReadableError")),
|
|
"NOT_READABLE",
|
|
],
|
|
] as const)(
|
|
"reports a staged entry with %s on activation and rewrites it on the next stage",
|
|
async (_label, damage, code) => {
|
|
const world = await deployments();
|
|
expect(await world.stage("r1")).toMatchObject({ ok: true });
|
|
expect(await world.activate("r1")).toMatchObject({ ok: true });
|
|
expect(await world.stage("r2")).toEqual({ ok: true, fetched: 2 });
|
|
damage(world, world.assets[0]!.absoluteUrl);
|
|
|
|
expect(await world.activate("r2")).toMatchObject({
|
|
ok: false,
|
|
error: { code, operation: "CACHE_ACTIVATE", recovery: "REHYDRATE" },
|
|
});
|
|
expect(await world.stage("r2")).toEqual({ ok: true, fetched: 2 });
|
|
expect(await world.activate("r2")).toMatchObject({ ok: true });
|
|
},
|
|
);
|
|
|
|
// 0929/02-02 (M05). `verified` is true only when every entry was read back
|
|
// and matched. A candidate inspect could not read is not verified, whether
|
|
// its body read or the lookup itself failed; reporting it verified passed
|
|
// every test before.
|
|
it("lists a candidate it could not read as unverified", async () => {
|
|
const world = await deployments();
|
|
for (const releaseRegistryId of ["r1", "r2", "r3"]) {
|
|
expect(await world.stage(releaseRegistryId)).toMatchObject({ ok: true });
|
|
}
|
|
world.unreadableUntilRewritten(
|
|
"r2",
|
|
world.assets[0]!.absoluteUrl,
|
|
() => new TypeError("blob missing"),
|
|
);
|
|
const lookupFails = world.cacheOf("r3");
|
|
const realMatch = lookupFails.match.bind(lookupFails);
|
|
const assetUrls = new Set(world.assets.map((asset) => asset.absoluteUrl));
|
|
lookupFails.match = async (request) => {
|
|
const url = request instanceof Request ? request.url : String(request);
|
|
if (assetUrls.has(url)) {
|
|
throw new DOMException("Storage is busy", "InvalidStateError");
|
|
}
|
|
return await realMatch(request);
|
|
};
|
|
|
|
const inspected = await world.inspect();
|
|
|
|
expect(inspected.ok).toBe(true);
|
|
if (!inspected.ok) return;
|
|
expect(
|
|
Object.fromEntries(
|
|
inspected.value.releaseCandidates.map((candidate) => [
|
|
candidate.releaseRegistryId,
|
|
candidate.verified,
|
|
]),
|
|
),
|
|
).toEqual({ r1: true, r2: false, r3: false });
|
|
});
|
|
|
|
it("lets a retained rollback target be activated again after staging it again (E-1c)", async () => {
|
|
const world = await deployments();
|
|
for (const releaseRegistryId of ["r1", "r2"]) {
|
|
expect(await world.stage(releaseRegistryId)).toMatchObject({ ok: true });
|
|
expect(await world.activate(releaseRegistryId)).toMatchObject({
|
|
ok: true,
|
|
});
|
|
}
|
|
world.unreadableUntilRewritten(
|
|
"r1",
|
|
world.assets[0]!.absoluteUrl,
|
|
() => new TypeError("blob missing"),
|
|
);
|
|
|
|
expect(await world.activate("r1")).toMatchObject({
|
|
ok: false,
|
|
error: { code: "UNAVAILABLE", recovery: "REHYDRATE" },
|
|
});
|
|
expect(await world.stage("r1")).toEqual({ ok: true, fetched: 2 });
|
|
expect(await world.activate("r1")).toMatchObject({ ok: true });
|
|
});
|
|
|
|
it("serves the active release's asset again once it is staged again (E-1d)", async () => {
|
|
const world = await deployments();
|
|
expect(await world.stage("r1")).toMatchObject({ ok: true });
|
|
expect(await world.activate("r1")).toMatchObject({ ok: true });
|
|
const [first, second] = world.assets;
|
|
world.unreadableUntilRewritten(
|
|
"r1",
|
|
first!.absoluteUrl,
|
|
() => new TypeError("blob missing"),
|
|
);
|
|
|
|
expect(await world.lookup(first!.absoluteUrl)).toMatchObject({
|
|
ok: false,
|
|
error: { code: "UNAVAILABLE", recovery: "RETRY" },
|
|
});
|
|
expect(await world.lookup(second!.absoluteUrl)).toMatchObject({
|
|
ok: true,
|
|
value: expect.anything(),
|
|
});
|
|
expect(await world.stage("r1")).toEqual({ ok: true, fetched: 2 });
|
|
expect(await world.lookup(first!.absoluteUrl)).toMatchObject({
|
|
ok: true,
|
|
value: expect.anything(),
|
|
});
|
|
});
|
|
});
|
|
|
|
/**
|
|
* 0929/02-01. The active pointer was checked against the policy of the build
|
|
* reading it, so lowering `retainedPreviousReleaseCount` or a damaged pointer
|
|
* stopped lookup, activation and cleanup for good.
|
|
*/
|
|
describe("public cache pointer after a policy change or damage (0929/02-01)", () => {
|
|
const origin = "https://assets.example.test";
|
|
const policyRetaining = (retainedPreviousReleaseCount: number) =>
|
|
resolvePublicCachePolicy({
|
|
...createDefaultPublicCachePolicy(origin),
|
|
retainedPreviousReleaseCount,
|
|
});
|
|
const assetUrl = (releaseRegistryId: string) =>
|
|
`${origin}/${releaseRegistryId}.js`;
|
|
const body = (releaseRegistryId: string) =>
|
|
new TextEncoder().encode(`release ${releaseRegistryId}`);
|
|
|
|
function adapterFor(
|
|
cacheStorage: MemoryCacheStorage,
|
|
policy: PublicCacheRuntimePolicy,
|
|
) {
|
|
return createPublicResponseCacheAdapter({
|
|
cacheStorage: cacheStorage as unknown as CacheStorage,
|
|
crypto: globalThis.crypto,
|
|
mutationLock: immediateLock,
|
|
policy,
|
|
fetcher: async (request: Request) =>
|
|
new Response(
|
|
Uint8Array.from(
|
|
body(new URL(request.url).pathname.slice(1, -".js".length)),
|
|
),
|
|
{
|
|
headers: {
|
|
"cache-control": "public",
|
|
"content-type": "application/javascript",
|
|
},
|
|
},
|
|
),
|
|
});
|
|
}
|
|
|
|
async function manifestOf(
|
|
releaseRegistryId: string,
|
|
policy: PublicCacheRuntimePolicy,
|
|
) {
|
|
const bytes = body(releaseRegistryId);
|
|
return await manifestFor(
|
|
releaseRegistryId,
|
|
[
|
|
{
|
|
absoluteUrl: assetUrl(releaseRegistryId),
|
|
expectedByteLength: bytes.byteLength,
|
|
expectedContentType: "application/javascript",
|
|
integrity: { algorithm: "SHA-256", digestHex: await digestHex(bytes) },
|
|
},
|
|
],
|
|
policy,
|
|
);
|
|
}
|
|
|
|
async function deploy(
|
|
adapter: ReturnType<typeof adapterFor>,
|
|
releaseRegistryId: string,
|
|
policy: PublicCacheRuntimePolicy,
|
|
) {
|
|
const manifest = await manifestOf(releaseRegistryId, policy);
|
|
expect(await adapter.admin.stageRelease(manifest)).toMatchObject({
|
|
ok: true,
|
|
});
|
|
return await adapter.admin.activateRelease(
|
|
manifest.releaseRegistryId,
|
|
manifest.manifestDigestHex,
|
|
);
|
|
}
|
|
|
|
it("keeps working after retainedPreviousReleaseCount is lowered", async () => {
|
|
const cacheStorage = new MemoryCacheStorage();
|
|
const before = policyRetaining(3);
|
|
const wide = adapterFor(cacheStorage, before);
|
|
for (const id of ["r1", "r2", "r3", "r4"]) {
|
|
expect(await deploy(wide, id, before)).toMatchObject({ ok: true });
|
|
}
|
|
|
|
const after = policyRetaining(1);
|
|
const narrow = adapterFor(cacheStorage, after);
|
|
|
|
expect(
|
|
await narrow.responses.matchActiveExact({ absoluteUrl: assetUrl("r4") }),
|
|
).toMatchObject({ ok: true, value: expect.anything() });
|
|
expect(await deploy(narrow, "r5", after)).toMatchObject({ ok: true });
|
|
expect(await narrow.admin.cleanupOwned()).toMatchObject({ ok: true });
|
|
expect(
|
|
await narrow.responses.matchActiveExact({ absoluteUrl: assetUrl("r5") }),
|
|
).toMatchObject({ ok: true, value: expect.anything() });
|
|
});
|
|
|
|
it("replaces a damaged pointer on the next verified activation", async () => {
|
|
const cacheStorage = new MemoryCacheStorage();
|
|
const policy = policyRetaining(1);
|
|
const adapter = adapterFor(cacheStorage, policy);
|
|
expect(await deploy(adapter, "r1", policy)).toMatchObject({ ok: true });
|
|
const control = await cacheStorage.open(controlCacheName(policy));
|
|
await control.put(
|
|
activePointerRequest(policy),
|
|
new Response('{"schemaVersion":1,"releaseRegis', {
|
|
headers: { "content-type": "application/json" },
|
|
}),
|
|
);
|
|
|
|
// A release that was never staged is still refused.
|
|
const unstaged = await manifestOf("r9", policy);
|
|
expect(
|
|
await adapter.admin.activateRelease(
|
|
unstaged.releaseRegistryId,
|
|
unstaged.manifestDigestHex,
|
|
),
|
|
).toMatchObject({ ok: false });
|
|
|
|
expect(await deploy(adapter, "r2", policy)).toMatchObject({ ok: true });
|
|
expect(await adapter.admin.cleanupOwned()).toMatchObject({ ok: true });
|
|
expect(
|
|
await adapter.responses.matchActiveExact({ absoluteUrl: assetUrl("r2") }),
|
|
).toMatchObject({ ok: true, value: expect.anything() });
|
|
});
|
|
|
|
it("shows a damaged pointer and the candidates it leaves behind in inspect (2026-11-03 02-05)", async () => {
|
|
const cacheStorage = new MemoryCacheStorage();
|
|
const policy = policyRetaining(1);
|
|
const adapter = adapterFor(cacheStorage, policy);
|
|
expect(await adapter.admin.inspect()).toMatchObject({
|
|
ok: true,
|
|
value: { activePointer: "NONE", activeReleaseRegistryId: null },
|
|
});
|
|
expect(await deploy(adapter, "r1", policy)).toMatchObject({ ok: true });
|
|
expect(await deploy(adapter, "r2", policy)).toMatchObject({ ok: true });
|
|
expect(await adapter.admin.inspect()).toMatchObject({
|
|
ok: true,
|
|
value: { activePointer: "READABLE", activeReleaseRegistryId: "r2" },
|
|
});
|
|
const control = await cacheStorage.open(controlCacheName(policy));
|
|
await control.put(
|
|
activePointerRequest(policy),
|
|
new Response('{"schemaVersion":1,"releaseRegis', {
|
|
headers: { "content-type": "application/json" },
|
|
}),
|
|
);
|
|
|
|
const inspected = await adapter.admin.inspect();
|
|
|
|
expect(inspected).toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
activePointer: "UNREADABLE",
|
|
activeReleaseRegistryId: null,
|
|
ownedCacheCount: 3,
|
|
unreadableOwnedCacheCount: 0,
|
|
},
|
|
});
|
|
expect(
|
|
inspected.ok
|
|
? inspected.value.releaseCandidates
|
|
.map(({ releaseRegistryId, verified }) => ({
|
|
releaseRegistryId,
|
|
verified,
|
|
}))
|
|
.sort((left, right) =>
|
|
left.releaseRegistryId.localeCompare(right.releaseRegistryId),
|
|
)
|
|
: null,
|
|
).toEqual([
|
|
{ releaseRegistryId: "r1", verified: true },
|
|
{ releaseRegistryId: "r2", verified: true },
|
|
]);
|
|
|
|
// A pointer that could not be read right now is not reported as damage.
|
|
const realMatch = control.match.bind(control);
|
|
control.match = async () => {
|
|
throw new DOMException("Storage is busy", "InvalidStateError");
|
|
};
|
|
expect(await adapter.admin.inspect()).toMatchObject({
|
|
ok: false,
|
|
error: { code: "UNAVAILABLE", retryable: true },
|
|
});
|
|
control.match = realMatch;
|
|
});
|
|
|
|
it("does not clean up on a damaged pointer, which may name the release being served (2026-11-03 02-03)", async () => {
|
|
const cacheStorage = new MemoryCacheStorage();
|
|
const policy = policyRetaining(1);
|
|
const adapter = adapterFor(cacheStorage, policy);
|
|
expect(await deploy(adapter, "r1", policy)).toMatchObject({ ok: true });
|
|
expect(await deploy(adapter, "r2", policy)).toMatchObject({ ok: true });
|
|
const control = await cacheStorage.open(controlCacheName(policy));
|
|
await control.put(
|
|
activePointerRequest(policy),
|
|
new Response('{"schemaVersion":1,"releaseRegis', {
|
|
headers: { "content-type": "application/json" },
|
|
}),
|
|
);
|
|
const before = (await cacheStorage.keys()).sort();
|
|
|
|
expect(await adapter.admin.cleanupOwned()).toMatchObject({
|
|
ok: false,
|
|
error: { code: "CORRUPT_DATA", operation: "CACHE_DELETE" },
|
|
});
|
|
expect((await cacheStorage.keys()).sort()).toEqual(before);
|
|
});
|
|
|
|
it("repairs a damaged release marker by staging the same manifest again (2026-11-03 02-03)", async () => {
|
|
const cacheStorage = new MemoryCacheStorage();
|
|
const policy = policyRetaining(1);
|
|
const adapter = adapterFor(cacheStorage, policy);
|
|
expect(await deploy(adapter, "r1", policy)).toMatchObject({ ok: true });
|
|
const releaseName = (await cacheStorage.keys()).find((name) =>
|
|
name.includes("release:r1:"),
|
|
);
|
|
if (!releaseName) throw new Error("release cache missing");
|
|
const release = await cacheStorage.open(releaseName);
|
|
await release.put(
|
|
markerRequest(policy),
|
|
new Response('{"schemaVersion":1,"verif', {
|
|
headers: { "content-type": "application/json" },
|
|
}),
|
|
);
|
|
// REHYDRATE: the damaged candidate cannot be activated as it is ...
|
|
const manifest = await manifestOf("r1", policy);
|
|
expect(
|
|
await adapter.admin.activateRelease(
|
|
manifest.releaseRegistryId,
|
|
manifest.manifestDigestHex,
|
|
),
|
|
).toMatchObject({
|
|
ok: false,
|
|
error: { code: "CORRUPT_DATA", recovery: "REHYDRATE" },
|
|
});
|
|
|
|
// ... and staging it again is the recovery that hint promises.
|
|
expect(await deploy(adapter, "r1", policy)).toMatchObject({ ok: true });
|
|
expect(
|
|
await adapter.responses.matchActiveExact({ absoluteUrl: assetUrl("r1") }),
|
|
).toMatchObject({ ok: true, value: expect.anything() });
|
|
});
|
|
|
|
// 2026-11-01 02-04 (N39). The cache name list is read before the pointer.
|
|
// An activation in between moves the pointer to a cache that list does not
|
|
// have yet; the lookup lists again rather than report it evicted.
|
|
it("lists caches again before it reports the active release evicted", async () => {
|
|
// The adapter captures `keys` when it is built, so the stale answer is
|
|
// armed through the storage itself.
|
|
class StaleOnceCacheStorage extends MemoryCacheStorage {
|
|
staleNames: string[] | null = null;
|
|
override async keys(): Promise<string[]> {
|
|
const stale = this.staleNames;
|
|
this.staleNames = null;
|
|
return stale ?? (await super.keys());
|
|
}
|
|
}
|
|
const cacheStorage = new StaleOnceCacheStorage();
|
|
const policy = policyRetaining(1);
|
|
const adapter = adapterFor(cacheStorage, policy);
|
|
expect(await deploy(adapter, "r1", policy)).toMatchObject({ ok: true });
|
|
const namesBeforeR2 = await cacheStorage.keys();
|
|
expect(await deploy(adapter, "r2", policy)).toMatchObject({ ok: true });
|
|
cacheStorage.staleNames = namesBeforeR2;
|
|
|
|
expect(
|
|
await adapter.responses.matchActiveExact({ absoluteUrl: assetUrl("r2") }),
|
|
).toMatchObject({ ok: true, value: expect.anything() });
|
|
});
|
|
|
|
it("still stops activation when the pointer read fails transiently", async () => {
|
|
const cacheStorage = new MemoryCacheStorage();
|
|
const policy = policyRetaining(1);
|
|
const adapter = adapterFor(cacheStorage, policy);
|
|
expect(await deploy(adapter, "r1", policy)).toMatchObject({ ok: true });
|
|
const control = cacheStorage.caches.get(controlCacheName(policy))!;
|
|
const realMatch = control.match.bind(control);
|
|
control.match = async () => {
|
|
throw new DOMException("Storage is busy", "InvalidStateError");
|
|
};
|
|
|
|
expect(await deploy(adapter, "r2", policy)).toMatchObject({
|
|
ok: false,
|
|
error: { code: "UNAVAILABLE", operation: "CACHE_ACTIVATE" },
|
|
});
|
|
control.match = realMatch;
|
|
expect(
|
|
await adapter.responses.matchActiveExact({ absoluteUrl: assetUrl("r1") }),
|
|
).toMatchObject({ ok: true, value: expect.anything() });
|
|
});
|
|
|
|
// 1106/02-01 (review experiment 02-R1 X3). The pointer entry is there, but
|
|
// its stored body fails to read until something writes it again (a lost
|
|
// blob). Without the option that looks like the transient failure above and
|
|
// everything stops; only an activation writes the pointer.
|
|
it("replaces a pointer whose body stays unreadable when the activation asks to (1106/02-01)", async () => {
|
|
const cacheStorage = new MemoryCacheStorage();
|
|
const policy = policyRetaining(1);
|
|
const adapter = adapterFor(cacheStorage, policy);
|
|
expect(await deploy(adapter, "r1", policy)).toMatchObject({ ok: true });
|
|
const control = cacheStorage.caches.get(controlCacheName(policy))!;
|
|
const pointerUrl = activePointerRequest(policy).url;
|
|
const pointer = { unreadable: true, rewrites: 0 };
|
|
const realMatch = control.match.bind(control);
|
|
const realPut = control.put.bind(control);
|
|
const urlOf = (request: RequestInfo | URL) =>
|
|
request instanceof Request ? request.url : new URL(String(request)).href;
|
|
control.put = async (request, response) => {
|
|
if (urlOf(request) === pointerUrl) {
|
|
pointer.unreadable = false;
|
|
pointer.rewrites += 1;
|
|
}
|
|
await realPut(request, response);
|
|
};
|
|
control.match = async (request) => {
|
|
const stored = await realMatch(request);
|
|
if (!stored || urlOf(request) !== pointerUrl || !pointer.unreadable) {
|
|
return stored;
|
|
}
|
|
return new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
pull(controller) {
|
|
controller.error(new TypeError("blob missing"));
|
|
},
|
|
}),
|
|
{ headers: stored.headers },
|
|
);
|
|
};
|
|
const next = await manifestOf("r2", policy);
|
|
const activateNext = (
|
|
options?: Readonly<{ replaceUnreadablePointer?: boolean }>,
|
|
) =>
|
|
adapter.admin.activateRelease(
|
|
next.releaseRegistryId,
|
|
next.manifestDigestHex,
|
|
options,
|
|
);
|
|
const unavailable = (operation: string) => ({
|
|
ok: false,
|
|
error: { code: "UNAVAILABLE", operation, recovery: "RETRY" },
|
|
});
|
|
|
|
// Staging does not read the pointer; everything else stops on it.
|
|
expect(await adapter.admin.stageRelease(next)).toMatchObject({ ok: true });
|
|
expect(
|
|
await adapter.responses.matchActiveExact({ absoluteUrl: assetUrl("r1") }),
|
|
).toMatchObject(unavailable("CACHE_LOOKUP"));
|
|
expect(await activateNext()).toMatchObject(unavailable("CACHE_ACTIVATE"));
|
|
expect(await adapter.admin.cleanupOwned()).toMatchObject(
|
|
unavailable("CACHE_DELETE"),
|
|
);
|
|
expect(await adapter.admin.inspect()).toMatchObject(
|
|
unavailable("CACHE_LOOKUP"),
|
|
);
|
|
expect(await activateNext()).toMatchObject(unavailable("CACHE_ACTIVATE"));
|
|
expect(pointer).toEqual({ unreadable: true, rewrites: 0 });
|
|
|
|
// The way back: this activation replaces the pointer it cannot read.
|
|
expect(
|
|
await activateNext({ replaceUnreadablePointer: true }),
|
|
).toMatchObject({ ok: true, value: { releaseRegistryId: "r2" } });
|
|
expect(pointer).toEqual({ unreadable: false, rewrites: 1 });
|
|
expect(
|
|
await adapter.responses.matchActiveExact({ absoluteUrl: assetUrl("r2") }),
|
|
).toMatchObject({ ok: true, value: expect.anything() });
|
|
expect(await adapter.admin.inspect()).toMatchObject({
|
|
ok: true,
|
|
value: { activePointer: "READABLE", activeReleaseRegistryId: "r2" },
|
|
});
|
|
// The rollback list went with the old pointer, so cleanup drops r1.
|
|
expect(await adapter.admin.cleanupOwned()).toMatchObject({
|
|
ok: true,
|
|
value: { deletedOwnedCaches: 1, retainedOwnedCaches: 2 },
|
|
});
|
|
// A later activation without the option works as before.
|
|
expect(await deploy(adapter, "r3", policy)).toMatchObject({ ok: true });
|
|
});
|
|
});
|