diff --git a/docs/architecture/browser-file-and-origin-storage.md b/docs/architecture/browser-file-and-origin-storage.md index 4d188f57..3ff7c1f3 100644 --- a/docs/architecture/browser-file-and-origin-storage.md +++ b/docs/architecture/browser-file-and-origin-storage.md @@ -969,6 +969,20 @@ build-produced manifest ``` 한 entry라도 실패하면 candidate는 incomplete이며 active로 전환하지 않는다. + +이미 있는 candidate를 같은 manifest로 다시 stage하면 entry를 하나씩 다시 +검증한다. 모두 맞으면 그대로 쓴다. entry가 없거나 바이트·헤더가 manifest와 +다르면(`INTEGRITY_FAILED`·`CORRUPT_DATA`·`POLICY_REJECTED`·`LIMIT_EXCEEDED`) +모든 asset을 다시 받아 같은 cache에 덮어쓴다. entry는 있는데 저장된 본문을 +읽지 못해도(`UNAVAILABLE`·`NOT_READABLE` 등) 같은 길로 다시 쓴다(1105/02-02). +본문 읽기 실패는 손상의 증거가 아니라서 activation은 그 code를 그대로 +돌려주지만, recovery는 `REHYDRATE`(같은 manifest로 다시 stage)다. 다시 쓰기 +전까지 계속 읽히지 않는 entry(blob 손실 등)는 이 길로만 돌아온다. rollback +대상 release와 서비스 중인 release도 같다. 덮어쓰는 바이트는 digest로 검증한 +같은 바이트이고 cache를 지우지 않으므로, 서비스 중인 release를 해치지 않는다. +일시 오류에서는 다시 받는 비용이 든다. 중단(`ABORTED`)과 `cache.match` 자체의 +실패는 다시 쓰지 않고 실패를 돌려준다. 조회(`matchActiveExact`)의 본문 읽기 +실패는 그대로 `UNAVAILABLE`/`RETRY`다. static Cache-only 조합은 waiting worker, `skipWaiting()`, `clients.claim()` 또는 controlled-client drain을 activation 전제나 성공 증거로 사용하지 않는다. composition owner가 exact release ID와 manifest digest로 diff --git a/src/adapters/cache-storage/public-response-cache-adapter.ts b/src/adapters/cache-storage/public-response-cache-adapter.ts index efd75876..41e2ce25 100644 --- a/src/adapters/cache-storage/public-response-cache-adapter.ts +++ b/src/adapters/cache-storage/public-response-cache-adapter.ts @@ -490,12 +490,20 @@ export function createPublicResponseCacheAdapter( return browserDataSuccess(summaryFromMarker(marker.value)); } if (verified.kind === "UNKNOWN") { - // Abort or an unreadable candidate is never stage success and - // never silently deletes an owned candidate. 2026-11-01 - // 02-03: the verdict is shared with activation, so the - // failure is re-labelled as this stage call's own. + // Abort or a Cache Storage that cannot be asked is never + // stage success and never silently deletes an owned + // candidate. 2026-11-01 02-03: the verdict is shared with + // activation, so the failure is re-labelled as this stage + // call's own. return rebaseFailure(verified.failure.error, "CACHE_STAGE"); } + // 1105/02-02. A damaged entry, and also one whose stored body + // cannot be read, is written again below. An entry that stays + // unreadable until it is rewritten had no other way back: the + // retry it was told to make met the same entry, and a rollback + // target that cleanup keeps could never be activated again. + // Writing digest-verified bytes over it deletes nothing, so a + // release that is serving traffic keeps serving. } } @@ -684,6 +692,17 @@ export function createPublicResponseCacheAdapter( "CACHE_ACTIVATE", ); } + if (candidate.kind === "UNREADABLE") { + // 1105/02-02. Still not reported as damage, but the way out is + // to stage the same manifest again, which rewrites the entry. + return rebaseFailure( + Object.freeze({ + ...candidate.failure.error, + recovery: "REHYDRATE" as const, + }), + "CACHE_ACTIVATE", + ); + } if (candidate.kind === "REPAIRABLE") { return browserDataFailure( "INTEGRITY_FAILED", @@ -1077,7 +1096,13 @@ type ReleaseCandidateVerdict = | Readonly<{ kind: "VERIFIED" }> /** An exact, owned entry is missing or no longer matches the manifest. */ | Readonly<{ kind: "REPAIRABLE" }> - /** Abort or an unreadable candidate: never success, never a silent delete. */ + /** + * 1105/02-02. An entry is there but reading its stored body failed + * (UNAVAILABLE, NOT_READABLE, ...). That says nothing about the bytes, so + * it is no damage; staging the same manifest writes the entry again. + */ + | Readonly<{ kind: "UNREADABLE"; failure: BrowserFailureResult }> + /** Abort or a failed lookup: never success, never a silent delete. */ | Readonly<{ kind: "UNKNOWN"; failure: BrowserFailureResult }>; const REPAIRABLE_READ_CODES: ReadonlySet = new Set([ @@ -1134,14 +1159,15 @@ async function verifyReleaseCandidate( ); if (!verified.ok) { // 2026-11-03 02-02. Only a verdict about the stored bytes or headers is - // damage. A body read that failed (UNAVAILABLE, NOT_READABLE, ABORTED) - // says nothing about them, so it stays unknown and is retried. - return REPAIRABLE_READ_CODES.has(verified.error.code) - ? Object.freeze({ kind: "REPAIRABLE" as const }) - : Object.freeze({ - kind: "UNKNOWN" as const, - failure: asFailure(verified), - }); + // damage. A body read that failed (UNAVAILABLE, NOT_READABLE) says + // nothing about them (1105/02-02: UNREADABLE); an abort is unknown. + if (REPAIRABLE_READ_CODES.has(verified.error.code)) { + return Object.freeze({ kind: "REPAIRABLE" as const }); + } + const failure = asFailure(verified); + return verified.error.code === "ABORTED" || signal?.aborted + ? Object.freeze({ kind: "UNKNOWN" as const, failure }) + : Object.freeze({ kind: "UNREADABLE" as const, failure }); } } return Object.freeze({ kind: "VERIFIED" as const }); diff --git a/tests/unit/public-response-cache-repair.test.ts b/tests/unit/public-response-cache-repair.test.ts index 554ee1b6..cd1319f4 100644 --- a/tests/unit/public-response-cache-repair.test.ts +++ b/tests/unit/public-response-cache-repair.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "vitest"; -import type { PublicCacheAsset } from "../../src/application/ports/browser-file-storage/cache-storage-ports.ts"; +import type { + PublicCacheAsset, + PublicCacheReleaseManifest, +} from "../../src/application/ports/browser-file-storage/cache-storage-ports.ts"; import { activePointerRequest, controlCacheName, @@ -174,7 +177,7 @@ describe("public response cache repair is failure-atomic", () => { ]); }); - it("reports a cached body that cannot be read during activation as retryable, not as damage (2026-11-03 02-02)", async () => { + it("reports a cached body that cannot be read during activation as unreadable, not as damage (2026-11-03 02-02)", async () => { const release = await stagedRelease("unreadable-body"); const cache = release.cacheStorage.caches.get(release.cacheName)!; const realMatch = cache.match.bind(cache); @@ -195,13 +198,13 @@ describe("public response cache repair is failure-atomic", () => { { headers: stored.headers }, ); }; - const transient = (operation: string) => ({ + const unreadable = (operation: string, recovery: string) => ({ ok: false, error: { code: "UNAVAILABLE", operation, retryable: true, - recovery: "RETRY", + recovery, }, }); @@ -209,20 +212,25 @@ describe("public response cache repair is failure-atomic", () => { await release.adapter.responses.matchActiveExact({ absoluteUrl: release.assets[0]!.absoluteUrl, }), - ).toMatchObject(transient("CACHE_LOOKUP")); + ).toMatchObject(unreadable("CACHE_LOOKUP", "RETRY")); + // 1105/02-02. Activation keeps the code but points at staging the same + // manifest again, which is what fixes an entry that stays unreadable. expect( await release.adapter.admin.activateRelease( release.manifest.releaseRegistryId, release.manifest.manifestDigestHex, ), - ).toMatchObject(transient("CACHE_ACTIVATE")); - // The stage fast path shares the verdict: an unreadable candidate is - // neither reused nor fetched again over the network. + ).toMatchObject(unreadable("CACHE_ACTIVATE", "REHYDRATE")); + // 1105/02-02. The stage fast path no longer stops at an unreadable + // candidate: it fetches every asset again and writes it over the entry. + // It used to fail with no fetch, so nothing could bring the entry back. const fetchesBefore = release.fetchLog.length; expect( await release.adapter.admin.stageRelease(release.manifest), - ).toMatchObject(transient("CACHE_STAGE")); - expect(release.fetchLog).toHaveLength(fetchesBefore); + ).toMatchObject({ ok: true }); + expect(release.fetchLog).toHaveLength( + fetchesBefore + release.assets.length, + ); cache.match = realMatch; expect( @@ -341,6 +349,214 @@ describe("public response cache repair is failure-atomic", () => { }); }); +/** + * 1105/02-02. After 2026-11-03 02-02 a body read failure is no damage, and the + * stage fast path stopped there with no fetch. An entry that stays unreadable + * until it is written again (a lost blob) was then never written again: the + * candidate kept failing activation, a rollback target that cleanup keeps + * could not be activated again, and the active release kept missing the + * asset. Staging the same manifest now writes such an entry again. + */ +describe("a cached body that stays unreadable until it is written again (1105/02-02)", () => { + async function deployments() { + const policy = createDefaultPublicCachePolicy( + "https://assets.example.test", + ); + const firstBytes = new Uint8Array([1, 1, 1, 1]); + const secondBytes = new Uint8Array([2, 2, 2, 2]); + const assets: readonly PublicCacheAsset[] = [ + { + absoluteUrl: "https://assets.example.test/first.js", + expectedByteLength: firstBytes.byteLength, + expectedContentType: "application/javascript", + integrity: { + algorithm: "SHA-256", + digestHex: await digestHex(firstBytes), + }, + }, + { + absoluteUrl: "https://assets.example.test/second.js", + expectedByteLength: secondBytes.byteLength, + expectedContentType: "application/javascript", + integrity: { + algorithm: "SHA-256", + digestHex: await digestHex(secondBytes), + }, + }, + ]; + const bodies = new Map([ + [assets[0]!.absoluteUrl, firstBytes], + [assets[1]!.absoluteUrl, secondBytes], + ]); + const cacheStorage = new MemoryCacheStorage(); + const fetchLog: string[] = []; + const adapter = createPublicResponseCacheAdapter({ + cacheStorage: cacheStorage as unknown as CacheStorage, + crypto: globalThis.crypto, + mutationLock: immediateLock, + policy, + fetcher: async (request: Request) => { + fetchLog.push(request.url); + const body = bodies.get(request.url); + if (!body) throw new TypeError(`unknown asset ${request.url}`); + return new Response(Uint8Array.from(body), { + headers: { + "cache-control": "public", + "content-type": "application/javascript", + }, + }); + }, + }); + const manifests = new Map(); + const manifest = async (releaseRegistryId: string) => { + let found = manifests.get(releaseRegistryId); + if (!found) { + found = await manifestFor(releaseRegistryId, assets, policy); + manifests.set(releaseRegistryId, found); + } + return found; + }; + const cacheOf = (releaseRegistryId: string) => { + const name = [...cacheStorage.caches.keys()].find((candidate) => + candidate.includes(`release:${releaseRegistryId}:`), + ); + const cache = name ? cacheStorage.caches.get(name) : undefined; + if (!cache) throw new Error(`no cache for ${releaseRegistryId}`); + return cache; + }; + return { + assets, + async stage(releaseRegistryId: string) { + const before = fetchLog.length; + const result = await adapter.admin.stageRelease( + await manifest(releaseRegistryId), + ); + return { ok: result.ok, fetched: fetchLog.length - before }; + }, + async activate(releaseRegistryId: string) { + const { manifestDigestHex } = await manifest(releaseRegistryId); + return await adapter.admin.activateRelease( + releaseRegistryId, + manifestDigestHex, + ); + }, + lookup: (absoluteUrl: string) => + adapter.responses.matchActiveExact({ absoluteUrl }), + cacheOf, + /** The entry's body fails to read until something writes it again. */ + unreadableUntilRewritten( + releaseRegistryId: string, + url: string, + error: () => unknown, + ) { + const cache = cacheOf(releaseRegistryId); + const state = { unreadable: true }; + const realMatch = cache.match.bind(cache); + const realPut = cache.put.bind(cache); + const urlOf = (request: RequestInfo | URL) => + request instanceof Request ? request.url : String(request); + cache.put = async (request, response) => { + if (urlOf(request) === url) state.unreadable = false; + await realPut(request, response); + }; + cache.match = async (request) => { + const stored = await realMatch(request); + if (!stored || urlOf(request) !== url || !state.unreadable) { + return stored; + } + return new Response( + new ReadableStream({ + pull(controller) { + controller.error(error()); + }, + }), + { headers: stored.headers }, + ); + }; + return state; + }, + }; + } + + it.each([ + ["a TypeError", () => new TypeError("blob missing"), "UNAVAILABLE"], + [ + "a NotReadableError", + () => new DOMException("unreadable", "NotReadableError"), + "NOT_READABLE", + ], + ] as const)( + "repairs a staged candidate whose entry fails with %s by staging it again (E-1a, E-1b)", + async (_label, error, code) => { + const world = await deployments(); + expect(await world.stage("r1")).toEqual({ ok: true, fetched: 2 }); + expect(await world.activate("r1")).toMatchObject({ ok: true }); + expect(await world.stage("r2")).toEqual({ ok: true, fetched: 2 }); + const entry = world.unreadableUntilRewritten( + "r2", + world.assets[0]!.absoluteUrl, + error, + ); + + expect(await world.activate("r2")).toMatchObject({ + ok: false, + error: { code, operation: "CACHE_ACTIVATE", recovery: "REHYDRATE" }, + }); + expect(await world.stage("r2")).toEqual({ ok: true, fetched: 2 }); + expect(entry.unreadable).toBe(false); + expect(await world.activate("r2")).toMatchObject({ ok: true }); + }, + ); + + it("lets a retained rollback target be activated again after staging it again (E-1c)", async () => { + const world = await deployments(); + for (const releaseRegistryId of ["r1", "r2"]) { + expect(await world.stage(releaseRegistryId)).toMatchObject({ ok: true }); + expect(await world.activate(releaseRegistryId)).toMatchObject({ + ok: true, + }); + } + world.unreadableUntilRewritten( + "r1", + world.assets[0]!.absoluteUrl, + () => new TypeError("blob missing"), + ); + + expect(await world.activate("r1")).toMatchObject({ + ok: false, + error: { code: "UNAVAILABLE", recovery: "REHYDRATE" }, + }); + expect(await world.stage("r1")).toEqual({ ok: true, fetched: 2 }); + expect(await world.activate("r1")).toMatchObject({ ok: true }); + }); + + it("serves the active release's asset again once it is staged again (E-1d)", async () => { + const world = await deployments(); + expect(await world.stage("r1")).toMatchObject({ ok: true }); + expect(await world.activate("r1")).toMatchObject({ ok: true }); + const [first, second] = world.assets; + world.unreadableUntilRewritten( + "r1", + first!.absoluteUrl, + () => new TypeError("blob missing"), + ); + + expect(await world.lookup(first!.absoluteUrl)).toMatchObject({ + ok: false, + error: { code: "UNAVAILABLE", recovery: "RETRY" }, + }); + expect(await world.lookup(second!.absoluteUrl)).toMatchObject({ + ok: true, + value: expect.anything(), + }); + expect(await world.stage("r1")).toEqual({ ok: true, fetched: 2 }); + expect(await world.lookup(first!.absoluteUrl)).toMatchObject({ + ok: true, + value: expect.anything(), + }); + }); +}); + /** * 0929/02-01. The active pointer was checked against the policy of the build * reading it, so lowering `retainedPreviousReleaseCount` or a damaged pointer