From aac7a3286292c27f573e58df39e49ac69cfaf969 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 24 Sep 2026 15:31:07 +0900 Subject: [PATCH] perf(realtime): split SSE lines on bytes and decode each line once (R3-5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parser now finds CR/LF with Uint8Array.indexOf before decoding, so a line's UTF-8 byte count comes from byte offsets instead of re-encoding the text. Each complete line is decoded once by a streaming fatal TextDecoder; a line that continues into the next read is stream-decoded, and the bytes of a character split across reads count once it completes, as before. Behavior kept (the push() input was already Uint8Array; no seam change in fetch-sse-connection.ts, public API unchanged): - CR / LF / CRLF, CR and LF split across reads, multi-line data - one leading BOM stripped (now done by the parser, decoder uses ignoreBOM: true, so a BOM at a later line start stays text) - line / event / incomplete-buffer byte limits, same failure point - drain() and the per-call item cap; kept bytes are copied - a pending CR or the item cap waits while only part of a character is left, as the text-based parser did - error codes (MALFORMED_EVENT / EVENT_TOO_LARGE / QUEUE_OVERFLOW / CLOSED) Differences only on invalid UTF-8: the old parser decoded a whole chunk first, so malformed bytes won over another error later in that chunk and malformed bytes past the item cap failed in push(). Now the first problem in stream order wins and those bytes fail in drain(). Both still fail closed. Fixed: Node's streaming TextDecoder (ignoreBOM: false) drops a U+FEFF right after the stream's first character when that character is split across reads, e.g. [EF] [BB BF EF BB BF] decoded to "" instead of "". The old parser inherited that; the new one keeps the text. Tests: existing parser and connection tests unchanged and passing. Added a split-BOM case and a differential test against the round-2 parser (tests/unit/realtime/sse-parser-reference.ts) over 2x1,500 seeded random ASCII / multibyte streams with random read splits, random small limits, truncated streams and skipped drains. 300,000 seeds also matched in a manual run. Benchmark (manual, not committed; Node v24.14.0, one push of one chunk, best of 150 runs, host load average ~11-14 so absolute ms are noisy): | input (one chunk) | original e59994a | round 2 | new | new vs original | new vs round 2 | |--------------------------------|------------------|---------|---------|-----------------|----------------| | ASCII, 249 KB, 209 events | 103.72 ms | 0.48 ms | 0.30 ms | 348x | 1.6x | | Korean/emoji, 249 KB, 209 ev. | 23.30 ms | 0.85 ms | 0.44 ms | 53x | 1.9x | | ASCII, 234 KB, 4 x 60 KB lines | 176.45 ms | 0.61 ms | 0.34 ms | 513x | 1.8x | | Korean/emoji, 234 KB, 4 x 60KB | 25.28 ms | 1.10 ms | 0.65 ms | 39x | 1.7x | Across five runs, multibyte new vs original stayed between 19x and 70x (target: 10x or more). Realtime bundle: 39903 / 40000 gzip bytes (baseline 39685, +218). Co-Authored-By: Claude Opus 5.5 --- src/adapters/realtime/sse/sse-parser.ts | 179 +++++++--- tests/unit/realtime/sse-parser-reference.ts | 364 ++++++++++++++++++++ tests/unit/realtime/sse-parser.test.ts | 99 ++++++ 3 files changed, 585 insertions(+), 57 deletions(-) create mode 100644 tests/unit/realtime/sse-parser-reference.ts diff --git a/src/adapters/realtime/sse/sse-parser.ts b/src/adapters/realtime/sse/sse-parser.ts index 103fef71..8e72902f 100644 --- a/src/adapters/realtime/sse/sse-parser.ts +++ b/src/adapters/realtime/sse/sse-parser.ts @@ -13,7 +13,6 @@ export const SSE_PARSER_CEILINGS = Object.freeze({ maxRetryMs: 60_000, }); -const LINE_BREAK = /[\r\n]/gu; const NO_ITEMS: readonly SseParserItem[] = Object.freeze([]); const COMMENT_ITEMS: readonly SseParserItem[] = Object.freeze([ Object.freeze({ kind: "COMMENT" as const }), @@ -52,8 +51,8 @@ export type SseParserFinish = Readonly<{ export type IncrementalSseParser = Readonly<{ /** - * Returns at most `maxItemsPerChunk` items. Text past that cap is kept and - * must be taken with `drain()` until it returns no items; pushing or + * Returns at most `maxItemsPerChunk` items. Bytes past that cap are kept + * and must be taken with `drain()` until it returns no items; pushing or * finishing before then fails with QUEUE_OVERFLOW. */ push(chunk: Uint8Array): RealtimeResult; @@ -65,19 +64,23 @@ export function createIncrementalSseParser( limits: Partial = {}, ): IncrementalSseParser { const resolved = resolveLimits(limits); - // ignoreBOM: false makes the decoder strip the one leading BOM the - // stream grammar allows; a second one is ordinary text. + // The decoder keeps U+FEFF; `append` drops the one leading BOM the stream + // grammar allows, so a BOM at the start of a later line stays text. const decoder = new TextDecoder("utf-8", { fatal: true, - ignoreBOM: false, + ignoreBOM: true, }); - const encoder = new TextEncoder(); let state: "OPEN" | "FAILED" | "FINISHED" = "OPEN"; let pendingCarriageReturn = false; + let atStreamStart = true; let line = ""; + // UTF-8 bytes of the decoded `line`. The bytes of a character the decoder + // still holds across reads (`held` of `heldNeed`) count once it completes. let lineBytes = 0; - let undrained = ""; + let held = 0; + let heldNeed = 0; + let undrained: Uint8Array | null = null; let blockBytes = 0; let dataLines: string[] = []; let eventType = ""; @@ -96,50 +99,36 @@ export function createIncrementalSseParser( if (chunk.byteLength > resolved.maxChunkBytes) { return fail("EVENT_TOO_LARGE"); } - if (undrained.length > 0) return fail("QUEUE_OVERFLOW"); - let text: string; - try { - text = decoder.decode(chunk, { stream: true }); - } catch { - return fail("MALFORMED_EVENT"); - } - return consumeText(text); + if (undrained !== null) return fail("QUEUE_OVERFLOW"); + return consumeBytes(chunk); } function drain(): RealtimeResult { if (state !== "OPEN") { return realtimeFailure("CLOSED", "DECODE"); } - const text = undrained; - undrained = ""; - return consumeText(text); + const bytes = undrained ?? new Uint8Array(); + undrained = null; + return consumeBytes(bytes); } function finish(): RealtimeResult { if (state !== "OPEN") { return realtimeFailure("CLOSED", "DECODE"); } - if (undrained.length > 0) return fail("QUEUE_OVERFLOW"); - let tail: string; + if (undrained !== null) return fail("QUEUE_OVERFLOW"); + // Throws when the stream ends inside a character. try { - tail = decoder.decode(); + decoder.decode(); } catch { return fail("MALFORMED_EVENT"); } - const consumed = consumeText(tail); - if (!consumed.ok) return consumed; - const items = [...consumed.value]; + let items = NO_ITEMS; if (pendingCarriageReturn) { pendingCarriageReturn = false; const processed = processLine(1); if (!processed.ok) return processed; - items.push(...processed.value); - } - if ( - undrained.length > 0 || - items.length > resolved.maxItemsPerChunk - ) { - return fail("QUEUE_OVERFLOW"); + items = processed.value; } const incompleteEventDiscarded = lineBytes > 0 || @@ -153,32 +142,45 @@ export function createIncrementalSseParser( state = "FINISHED"; return success( Object.freeze({ - items: Object.freeze(items), + items, incompleteEventDiscarded, }), ); } /** - * Works a line at a time: find the next CR/LF, append the text before it - * and count its UTF-8 bytes once. Byte counts only grow within a line, so - * checking the limits per segment fails exactly where a per-character check - * would. Stops at `maxItemsPerChunk` items and keeps the rest for `drain()`. + * Finds CR/LF on the raw bytes, so a line's byte length is known without + * re-encoding, and decodes each line once. CR and LF bytes never occur + * inside a multibyte UTF-8 character. Byte counts only grow within a line, + * so checking the limits per segment fails exactly where a per-character + * check would. Stops at `maxItemsPerChunk` items and keeps the rest for + * `drain()`. */ - function consumeText( - text: string, + function consumeBytes( + bytes: Uint8Array, ): RealtimeResult { const items: SseParserItem[] = []; let index = 0; + let nextLineFeed = -1; + let nextCarriageReturn = -1; - while (index < text.length) { + while (index < bytes.length) { + // Only part of a character is left: the decoder holds it, and a + // pending CR or the item cap waits for the character to complete. + const rest = bytes.length - index; + if (rest < 4 && heldAfter(bytes.subarray(index)) === held + rest) { + const appended = append(bytes.subarray(index), true); + if (!appended.ok) return appended; + break; + } if (items.length >= resolved.maxItemsPerChunk) { - undrained = text.slice(index); + // A copy, so a caller reusing its buffer cannot change kept bytes. + undrained = bytes.slice(index); break; } if (pendingCarriageReturn) { pendingCarriageReturn = false; - const lineFeed = text.charCodeAt(index) === 0x0a; + const lineFeed = bytes[index] === 0x0a; const processed = processLine(lineFeed ? 2 : 1); if (!processed.ok) return processed; items.push(...processed.value); @@ -186,23 +188,22 @@ export function createIncrementalSseParser( continue; } - LINE_BREAK.lastIndex = index; - const lineBreak = LINE_BREAK.exec(text)?.index ?? text.length; - if (lineBreak > index) { - const segment = text.slice(index, lineBreak); - line += segment; - lineBytes += encoder.encode(segment).byteLength; - if ( - lineBytes > resolved.maxLineBytes || - lineBytes + blockBytes > resolved.maxIncompleteBufferBytes - ) { - return fail("EVENT_TOO_LARGE"); - } + if (nextLineFeed < index) { + nextLineFeed = indexOrEnd(bytes, 0x0a, index); } - if (lineBreak === text.length) break; + if (nextCarriageReturn < index) { + nextCarriageReturn = indexOrEnd(bytes, 0x0d, index); + } + const lineBreak = Math.min(nextLineFeed, nextCarriageReturn); + const appended = append( + bytes.subarray(index, lineBreak), + lineBreak === bytes.length, + ); + if (!appended.ok) return appended; + if (lineBreak === bytes.length) break; index = lineBreak + 1; - if (text.charCodeAt(lineBreak) === 0x0d) { + if (bytes[lineBreak] === 0x0d) { pendingCarriageReturn = true; continue; } @@ -213,6 +214,60 @@ export function createIncrementalSseParser( return success(Object.freeze(items)); } + /** + * Decodes one line segment. `open` means the line continues in a later + * read, so the decoder may hold the bytes of an unfinished character. + */ + function append( + segment: Uint8Array, + open: boolean, + ): RealtimeResult { + let text = ""; + if (segment.length > 0 || held > 0) { + try { + text = decoder.decode(segment, { stream: open }); + } catch { + return fail("MALFORMED_EVENT"); + } + } + const heldBefore = held; + held = open ? heldAfter(segment) : 0; + lineBytes += heldBefore + segment.length - held; + if (atStreamStart && (text.length > 0 || !open)) { + atStreamStart = false; + if (text.charCodeAt(0) === 0xfeff) { + text = text.slice(1); + lineBytes -= 3; + } + } + line += text; + if ( + lineBytes > resolved.maxLineBytes || + lineBytes + blockBytes > resolved.maxIncompleteBufferBytes + ) { + return fail("EVENT_TOO_LARGE"); + } + return success(null); + } + + /** + * Bytes of the unfinished character at the end of a segment the decoder + * accepted in stream mode, so they form a valid UTF-8 prefix. + */ + function heldAfter(segment: Uint8Array): number { + const end = segment.length; + for (let index = end - 1; index >= 0 && index >= end - 4; index -= 1) { + const byte = segment[index]!; + if (byte < 0x80) return 0; + if (byte >= 0xc0) { + heldNeed = byte >= 0xf0 ? 4 : byte >= 0xe0 ? 3 : 2; + return end - index < heldNeed ? end - index : 0; + } + } + // Only continuation bytes: they extend the character already held. + return held + end < heldNeed ? held + end : 0; + } + function processLine( terminatorBytes: number, ): RealtimeResult { @@ -304,7 +359,8 @@ export function createIncrementalSseParser( state = "FAILED"; line = ""; lineBytes = 0; - undrained = ""; + held = 0; + undrained = null; clearBlock(); return realtimeFailure(kind, "DECODE"); } @@ -357,6 +413,15 @@ function success(value: Value): RealtimeResult { return realtimeSuccess(value); } +function indexOrEnd( + bytes: Uint8Array, + byte: number, + from: number, +): number { + const index = bytes.indexOf(byte, from); + return index === -1 ? bytes.length : index; +} + function positiveInteger(value: number): boolean { return Number.isSafeInteger(value) && value > 0; } diff --git a/tests/unit/realtime/sse-parser-reference.ts b/tests/unit/realtime/sse-parser-reference.ts new file mode 100644 index 00000000..8ded67a2 --- /dev/null +++ b/tests/unit/realtime/sse-parser-reference.ts @@ -0,0 +1,364 @@ +// The round-2 SSE parser (decode each chunk, then re-encode every segment to +// count its bytes), kept only as the oracle for the differential test. +import type { RealtimeResult } from "../../../src/application/ports/realtime/shared.ts"; +import { + realtimeFailure, + realtimeSuccess, +} from "../../../src/adapters/realtime/result.ts"; + +export const SSE_PARSER_CEILINGS = Object.freeze({ + maxLineBytes: 64 * 1_024, + maxEventBytes: 64 * 1_024, + maxIncompleteBufferBytes: 128 * 1_024, + maxChunkBytes: 256 * 1_024, + maxItemsPerChunk: 256, + maxRetryMs: 60_000, +}); + +const LINE_BREAK = /[\r\n]/gu; +const NO_ITEMS: readonly SseParserItem[] = Object.freeze([]); +const COMMENT_ITEMS: readonly SseParserItem[] = Object.freeze([ + Object.freeze({ kind: "COMMENT" as const }), +]); + +export type SseParserLimits = Readonly<{ + maxLineBytes: number; + maxEventBytes: number; + maxIncompleteBufferBytes: number; + maxChunkBytes: number; + maxItemsPerChunk: number; + maxRetryMs: number; +}>; + +export type ParsedSseEvent = Readonly<{ + kind: "EVENT"; + eventType: string; + data: string; + /** + * Standard SSE last-event-ID state. Consumers that require cursor-after- + * effect must additionally require `hasExplicitId` and commit independently. + */ + id: string | null; + hasExplicitId: boolean; +}>; + +export type SseParserItem = + | ParsedSseEvent + | Readonly<{ kind: "COMMENT" }> + | Readonly<{ kind: "RETRY"; retryMs: number }>; + +export type SseParserFinish = Readonly<{ + items: readonly SseParserItem[]; + incompleteEventDiscarded: boolean; +}>; + +export type IncrementalSseParser = Readonly<{ + /** + * Returns at most `maxItemsPerChunk` items. Text past that cap is kept and + * must be taken with `drain()` until it returns no items; pushing or + * finishing before then fails with QUEUE_OVERFLOW. + */ + push(chunk: Uint8Array): RealtimeResult; + drain(): RealtimeResult; + finish(): RealtimeResult; +}>; + +export function createIncrementalSseParser( + limits: Partial = {}, +): IncrementalSseParser { + const resolved = resolveLimits(limits); + // ignoreBOM: false makes the decoder strip the one leading BOM the + // stream grammar allows; a second one is ordinary text. + const decoder = new TextDecoder("utf-8", { + fatal: true, + ignoreBOM: false, + }); + const encoder = new TextEncoder(); + + let state: "OPEN" | "FAILED" | "FINISHED" = "OPEN"; + let pendingCarriageReturn = false; + let line = ""; + let lineBytes = 0; + let undrained = ""; + let blockBytes = 0; + let dataLines: string[] = []; + let eventType = ""; + let lastEventId: string | null = null; + let hasExplicitId = false; + + function push( + chunk: Uint8Array, + ): RealtimeResult { + if (state !== "OPEN") { + return realtimeFailure("CLOSED", "DECODE"); + } + if (!(chunk instanceof Uint8Array)) { + return fail("MALFORMED_EVENT"); + } + if (chunk.byteLength > resolved.maxChunkBytes) { + return fail("EVENT_TOO_LARGE"); + } + if (undrained.length > 0) return fail("QUEUE_OVERFLOW"); + let text: string; + try { + text = decoder.decode(chunk, { stream: true }); + } catch { + return fail("MALFORMED_EVENT"); + } + return consumeText(text); + } + + function drain(): RealtimeResult { + if (state !== "OPEN") { + return realtimeFailure("CLOSED", "DECODE"); + } + const text = undrained; + undrained = ""; + return consumeText(text); + } + + function finish(): RealtimeResult { + if (state !== "OPEN") { + return realtimeFailure("CLOSED", "DECODE"); + } + if (undrained.length > 0) return fail("QUEUE_OVERFLOW"); + let tail: string; + try { + tail = decoder.decode(); + } catch { + return fail("MALFORMED_EVENT"); + } + const consumed = consumeText(tail); + if (!consumed.ok) return consumed; + const items = [...consumed.value]; + if (pendingCarriageReturn) { + pendingCarriageReturn = false; + const processed = processLine(1); + if (!processed.ok) return processed; + items.push(...processed.value); + } + if ( + undrained.length > 0 || + items.length > resolved.maxItemsPerChunk + ) { + return fail("QUEUE_OVERFLOW"); + } + const incompleteEventDiscarded = + lineBytes > 0 || + blockBytes > 0 || + dataLines.length > 0 || + eventType.length > 0 || + hasExplicitId; + clearBlock(); + line = ""; + lineBytes = 0; + state = "FINISHED"; + return success( + Object.freeze({ + items: Object.freeze(items), + incompleteEventDiscarded, + }), + ); + } + + /** + * Works a line at a time: find the next CR/LF, append the text before it + * and count its UTF-8 bytes once. Byte counts only grow within a line, so + * checking the limits per segment fails exactly where a per-character check + * would. Stops at `maxItemsPerChunk` items and keeps the rest for `drain()`. + */ + function consumeText( + text: string, + ): RealtimeResult { + const items: SseParserItem[] = []; + let index = 0; + + while (index < text.length) { + if (items.length >= resolved.maxItemsPerChunk) { + undrained = text.slice(index); + break; + } + if (pendingCarriageReturn) { + pendingCarriageReturn = false; + const lineFeed = text.charCodeAt(index) === 0x0a; + const processed = processLine(lineFeed ? 2 : 1); + if (!processed.ok) return processed; + items.push(...processed.value); + if (lineFeed) index += 1; + continue; + } + + LINE_BREAK.lastIndex = index; + const lineBreak = LINE_BREAK.exec(text)?.index ?? text.length; + if (lineBreak > index) { + const segment = text.slice(index, lineBreak); + line += segment; + lineBytes += encoder.encode(segment).byteLength; + if ( + lineBytes > resolved.maxLineBytes || + lineBytes + blockBytes > resolved.maxIncompleteBufferBytes + ) { + return fail("EVENT_TOO_LARGE"); + } + } + if (lineBreak === text.length) break; + + index = lineBreak + 1; + if (text.charCodeAt(lineBreak) === 0x0d) { + pendingCarriageReturn = true; + continue; + } + const processed = processLine(1); + if (!processed.ok) return processed; + items.push(...processed.value); + } + return success(Object.freeze(items)); + } + + function processLine( + terminatorBytes: number, + ): RealtimeResult { + const currentLine = line; + const currentLineBytes = lineBytes; + line = ""; + lineBytes = 0; + + if (currentLine.length === 0) { + const items: SseParserItem[] = []; + if (dataLines.length > 0) { + items.push( + Object.freeze({ + kind: "EVENT", + eventType: eventType.length > 0 ? eventType : "message", + data: dataLines.join("\n"), + id: lastEventId, + hasExplicitId, + }), + ); + } + clearBlock(); + return success(Object.freeze(items)); + } + + if (currentLine.startsWith(":")) return success(COMMENT_ITEMS); + + blockBytes += currentLineBytes + terminatorBytes; + if (blockBytes > resolved.maxEventBytes) { + return fail("EVENT_TOO_LARGE"); + } + if (blockBytes > resolved.maxIncompleteBufferBytes) { + return fail("EVENT_TOO_LARGE"); + } + + const separator = currentLine.indexOf(":"); + const field = + separator === -1 + ? currentLine + : currentLine.slice(0, separator); + let value = + separator === -1 ? "" : currentLine.slice(separator + 1); + if (value.startsWith(" ")) value = value.slice(1); + + if (field === "data") { + dataLines.push(value); + return success(NO_ITEMS); + } + if (field === "event") { + eventType = value; + return success(NO_ITEMS); + } + if (field === "id") { + if (!value.includes("\0")) { + lastEventId = value; + hasExplicitId = true; + } + return success(NO_ITEMS); + } + if (field === "retry" && /^\d+$/u.test(value)) { + const retryMs = Number(value); + if ( + Number.isSafeInteger(retryMs) && + retryMs <= resolved.maxRetryMs + ) { + return success( + Object.freeze([ + Object.freeze({ kind: "RETRY" as const, retryMs }), + ]), + ); + } + } + return success(NO_ITEMS); + } + + function clearBlock(): void { + blockBytes = 0; + dataLines = []; + eventType = ""; + hasExplicitId = false; + } + + function fail( + kind: + | "EVENT_TOO_LARGE" + | "MALFORMED_EVENT" + | "QUEUE_OVERFLOW", + ): RealtimeResult { + state = "FAILED"; + line = ""; + lineBytes = 0; + undrained = ""; + clearBlock(); + return realtimeFailure(kind, "DECODE"); + } + + return Object.freeze({ push, drain, finish }); +} + +function resolveLimits( + input: Partial, +): SseParserLimits { + const limits = { + maxLineBytes: + input.maxLineBytes ?? SSE_PARSER_CEILINGS.maxLineBytes, + maxEventBytes: + input.maxEventBytes ?? SSE_PARSER_CEILINGS.maxEventBytes, + maxIncompleteBufferBytes: + input.maxIncompleteBufferBytes ?? + SSE_PARSER_CEILINGS.maxIncompleteBufferBytes, + maxChunkBytes: + input.maxChunkBytes ?? SSE_PARSER_CEILINGS.maxChunkBytes, + maxItemsPerChunk: + input.maxItemsPerChunk ?? + SSE_PARSER_CEILINGS.maxItemsPerChunk, + maxRetryMs: input.maxRetryMs ?? SSE_PARSER_CEILINGS.maxRetryMs, + }; + if ( + !positiveInteger(limits.maxLineBytes) || + limits.maxLineBytes > SSE_PARSER_CEILINGS.maxLineBytes || + !positiveInteger(limits.maxEventBytes) || + limits.maxEventBytes > SSE_PARSER_CEILINGS.maxEventBytes || + !positiveInteger(limits.maxIncompleteBufferBytes) || + limits.maxIncompleteBufferBytes > + SSE_PARSER_CEILINGS.maxIncompleteBufferBytes || + limits.maxIncompleteBufferBytes < limits.maxEventBytes || + !positiveInteger(limits.maxChunkBytes) || + limits.maxChunkBytes > SSE_PARSER_CEILINGS.maxChunkBytes || + limits.maxChunkBytes < limits.maxEventBytes || + !positiveInteger(limits.maxItemsPerChunk) || + limits.maxItemsPerChunk > + SSE_PARSER_CEILINGS.maxItemsPerChunk || + !positiveInteger(limits.maxRetryMs) || + limits.maxRetryMs > SSE_PARSER_CEILINGS.maxRetryMs + ) { + throw new TypeError("Invalid SSE parser limits."); + } + return Object.freeze(limits); +} + +function success(value: Value): RealtimeResult { + return realtimeSuccess(value); +} + +function positiveInteger(value: number): boolean { + return Number.isSafeInteger(value) && value > 0; +} diff --git a/tests/unit/realtime/sse-parser.test.ts b/tests/unit/realtime/sse-parser.test.ts index 3e14defe..beaabf72 100644 --- a/tests/unit/realtime/sse-parser.test.ts +++ b/tests/unit/realtime/sse-parser.test.ts @@ -4,6 +4,7 @@ import { createIncrementalSseParser, type SseParserItem, } from "../../../src/adapters/realtime/sse/sse-parser.ts"; +import { createIncrementalSseParser as createReferenceSseParser } from "./sse-parser-reference.ts"; const encoder = new TextEncoder(); @@ -60,6 +61,13 @@ describe("incremental SSE parser", () => { hasExplicitId: false, }, ]); + + // Also when the first BOM is split across reads. Node's streaming + // TextDecoder (ignoreBOM: false) drops both BOMs then; the parser must not. + const split = createIncrementalSseParser(); + const bytes = encoder.encode("data: x\n\n"); + expect(split.push(bytes.subarray(0, 1))).toEqual({ ok: true, value: [] }); + expect(split.push(bytes.subarray(1))).toEqual({ ok: true, value: [] }); }); it("preserves standard inherited ID state while marking direct IDs", () => { @@ -407,3 +415,94 @@ describe("incremental SSE parser", () => { }); }); }); + +describe("byte-level SSE parser against the round-2 text parser", () => { + const asciiPieces = [ + "data", "data:", "data: ", "event: ", "id: ", "id:", "retry: ", + ":", "x", " ", "abc", "12", "\0", "\r", "\n", "\r\n", "\n\n", "\r\r", + ]; + const multibytePieces = [...asciiPieces, "é", "한글", "😀", ""]; + + type Create = typeof createIncrementalSseParser; + + // mulberry32: a small seeded PRNG, so a failing seed can be replayed. + function randomSource(seed: number): () => number { + let state = seed; + return () => { + state = (state + 0x6d2b79f5) | 0; + let value = Math.imul(state ^ (state >>> 15), 1 | state); + value ^= value + Math.imul(value ^ (value >>> 7), 61 | value); + return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296; + }; + } + + function run( + create: Create, + limits: Parameters[0], + chunks: readonly Uint8Array[], + skipDrain: readonly boolean[], + ): unknown[] { + const parser = create(limits); + const log: unknown[] = []; + for (const [index, chunk] of chunks.entries()) { + let result = parser.push(chunk); + log.push(result); + while (!skipDrain[index] && result.ok && result.value.length > 0) { + result = parser.drain(); + log.push(result); + } + if (!result.ok) return log; + } + log.push(parser.finish()); + return log; + } + + it.each([ + ["ASCII", asciiPieces], + ["multibyte", multibytePieces], + ])("matches on random %s streams and read splits", (_, pieces) => { + for (let seed = 1; seed <= 1_500; seed += 1) { + const next = randomSource(seed); + const pick = (max: number) => Math.floor(next() * max); + let text = next() < 0.3 ? "" : ""; + for (let count = 1 + pick(40); count > 0; count -= 1) { + text += pieces[pick(pieces.length)]; + } + // Node's streaming TextDecoder, which the reference relies on, drops a + // U+FEFF right after the first character when that character is split + // across reads (see the BOM test above). Keep that input out. + const first = String.fromCodePoint(text.codePointAt(0) ?? 0x20); + while (text.startsWith("", first.length)) { + text = first + text.slice(first.length + 1); + } + let bytes = encoder.encode(text); + // Cutting the stream can end it inside a character. + if (next() < 0.2) bytes = bytes.subarray(0, pick(bytes.length + 1)); + + const chunks: Uint8Array[] = []; + const maxRead = 1 + pick(next() < 0.5 ? 6 : bytes.length + 1); + for (let index = 0; index < bytes.length; ) { + const size = 1 + pick(maxRead); + chunks.push(bytes.subarray(index, index + size)); + index += size; + } + const skipDrain = chunks.map(() => next() < 0.05); + + const maxEventBytes = 4 + pick(60); + const limits = + next() < 0.3 + ? {} + : { + maxLineBytes: 4 + pick(40), + maxEventBytes, + maxIncompleteBufferBytes: maxEventBytes + pick(20), + maxItemsPerChunk: 1 + pick(4), + }; + + expect( + run(createIncrementalSseParser, limits, chunks, skipDrain), + `seed ${seed}`, + ).toEqual(run(createReferenceSseParser, limits, chunks, skipDrain)); + } + }); +});