Files
clean-architecture-frontend…/tests/unit/realtime/sse-parser.test.ts
T
DongHyeonkaandClaude Opus 5.5 aac7a32862 perf(realtime): split SSE lines on bytes and decode each line once (R3-5)
The parser now finds CR/LF with Uint8Array.indexOf before decoding, so a
line's UTF-8 byte count comes from byte offsets instead of re-encoding the
text. Each complete line is decoded once by a streaming fatal TextDecoder;
a line that continues into the next read is stream-decoded, and the bytes
of a character split across reads count once it completes, as before.

Behavior kept (the push() input was already Uint8Array; no seam change in
fetch-sse-connection.ts, public API unchanged):
- CR / LF / CRLF, CR and LF split across reads, multi-line data
- one leading BOM stripped (now done by the parser, decoder uses
  ignoreBOM: true, so a BOM at a later line start stays text)
- line / event / incomplete-buffer byte limits, same failure point
- drain() and the per-call item cap; kept bytes are copied
- a pending CR or the item cap waits while only part of a character is
  left, as the text-based parser did
- error codes (MALFORMED_EVENT / EVENT_TOO_LARGE / QUEUE_OVERFLOW / CLOSED)

Differences only on invalid UTF-8: the old parser decoded a whole chunk
first, so malformed bytes won over another error later in that chunk and
malformed bytes past the item cap failed in push(). Now the first
problem in stream order wins and those bytes fail in drain(). Both
still fail closed.

Fixed: Node's streaming TextDecoder (ignoreBOM: false) drops a U+FEFF
right after the stream's first character when that character is split
across reads, e.g. [EF] [BB BF EF BB BF] decoded to "" instead of
"". The old parser inherited that; the new one keeps the text.

Tests: existing parser and connection tests unchanged and passing. Added
a split-BOM case and a differential test against the round-2 parser
(tests/unit/realtime/sse-parser-reference.ts) over 2x1,500 seeded random
ASCII / multibyte streams with random read splits, random small limits,
truncated streams and skipped drains. 300,000 seeds also matched in a
manual run.

Benchmark (manual, not committed; Node v24.14.0, one push of one chunk,
best of 150 runs, host load average ~11-14 so absolute ms are noisy):

| input (one chunk)              | original e59994a | round 2 | new     | new vs original | new vs round 2 |
|--------------------------------|------------------|---------|---------|-----------------|----------------|
| ASCII, 249 KB, 209 events      | 103.72 ms        | 0.48 ms | 0.30 ms | 348x            | 1.6x           |
| Korean/emoji, 249 KB, 209 ev.  | 23.30 ms         | 0.85 ms | 0.44 ms | 53x             | 1.9x           |
| ASCII, 234 KB, 4 x 60 KB lines | 176.45 ms        | 0.61 ms | 0.34 ms | 513x            | 1.8x           |
| Korean/emoji, 234 KB, 4 x 60KB | 25.28 ms         | 1.10 ms | 0.65 ms | 39x             | 1.7x           |

Across five runs, multibyte new vs original stayed between 19x and 70x
(target: 10x or more).

Realtime bundle: 39903 / 40000 gzip bytes (baseline 39685, +218).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-24 15:31:29 +09:00

509 lines
15 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { describe, expect, it } from "vitest";
import {
createIncrementalSseParser,
type SseParserItem,
} from "../../../src/adapters/realtime/sse/sse-parser.ts";
import { createIncrementalSseParser as createReferenceSseParser } from "./sse-parser-reference.ts";
const encoder = new TextEncoder();
function pushText(
parser: ReturnType<typeof createIncrementalSseParser>,
text: string,
): readonly SseParserItem[] {
const result = parser.push(encoder.encode(text));
if (!result.ok) throw new Error(result.error.kind);
return result.value;
}
describe("incremental SSE parser", () => {
it("handles BOM, chunk boundaries, CR/LF/CRLF, comments and multi-line data", () => {
const parser = createIncrementalSseParser();
const chunks = [
"\uFEFF: ready\r",
"\nretry: 2500\revent: resource.updated\n",
"id: cursor-1\r\ndata: first\rdata:second\n\n",
];
const items = chunks.flatMap((chunk) => pushText(parser, chunk));
expect(items).toEqual([
{ kind: "COMMENT" },
{ kind: "RETRY", retryMs: 2_500 },
{
kind: "EVENT",
eventType: "resource.updated",
data: "first\nsecond",
id: "cursor-1",
hasExplicitId: true,
},
]);
expect(parser.finish()).toEqual({
ok: true,
value: {
items: [],
incompleteEventDiscarded: false,
},
});
});
it("strips exactly one leading BOM", () => {
const parser = createIncrementalSseParser();
// The second BOM belongs to the field name, so the line is ignored.
expect(
pushText(parser, "\uFEFF\uFEFFdata: x\n\ndata: y\n\n"),
).toEqual([
{
kind: "EVENT",
eventType: "message",
data: "y",
id: null,
hasExplicitId: false,
},
]);
// Also when the first BOM is split across reads. Node's streaming
// TextDecoder (ignoreBOM: false) drops both BOMs then; the parser must not.
const split = createIncrementalSseParser();
const bytes = encoder.encode("data: x\n\n");
expect(split.push(bytes.subarray(0, 1))).toEqual({ ok: true, value: [] });
expect(split.push(bytes.subarray(1))).toEqual({ ok: true, value: [] });
});
it("preserves standard inherited ID state while marking direct IDs", () => {
const parser = createIncrementalSseParser();
const items = pushText(
parser,
"id: cursor-a\ndata: one\n\ndata: two\n\ndata:\n\n",
);
expect(items).toEqual([
{
kind: "EVENT",
eventType: "message",
data: "one",
id: "cursor-a",
hasExplicitId: true,
},
{
kind: "EVENT",
eventType: "message",
data: "two",
id: "cursor-a",
hasExplicitId: false,
},
{
kind: "EVENT",
eventType: "message",
data: "",
id: "cursor-a",
hasExplicitId: false,
},
]);
});
it("ignores invalid ID and retry fields without inventing a cursor", () => {
const parser = createIncrementalSseParser();
expect(
pushText(
parser,
"id: invalid\u0000cursor\nretry: 99999\ndata: value\n\n",
),
).toEqual([
{
kind: "EVENT",
eventType: "message",
data: "value",
id: null,
hasExplicitId: false,
},
]);
});
it("discards an event that was not terminated by a blank line", () => {
const parser = createIncrementalSseParser();
expect(pushText(parser, "data: incomplete\n")).toEqual([]);
expect(parser.finish()).toEqual({
ok: true,
value: {
items: [],
incompleteEventDiscarded: true,
},
});
expect(parser.push(encoder.encode("data: late\n\n"))).toEqual({
ok: false,
error: {
kind: "CLOSED",
operation: "DECODE",
retryable: false,
},
});
});
it("fails closed on malformed UTF-8 and parser ceilings", () => {
const malformed = createIncrementalSseParser();
expect(
malformed.push(new Uint8Array([0xc3, 0x28])),
).toEqual({
ok: false,
error: {
kind: "MALFORMED_EVENT",
operation: "DECODE",
retryable: false,
},
});
const longLine = createIncrementalSseParser({
maxLineBytes: 4,
maxEventBytes: 8,
maxIncompleteBufferBytes: 8,
maxRetryMs: 100,
});
expect(longLine.push(encoder.encode("data:"))).toEqual({
ok: false,
error: {
kind: "EVENT_TOO_LARGE",
operation: "DECODE",
retryable: false,
},
});
const largeEvent = createIncrementalSseParser({
maxLineBytes: 16,
maxEventBytes: 8,
maxIncompleteBufferBytes: 16,
maxRetryMs: 100,
});
expect(largeEvent.push(encoder.encode("data:abc\n"))).toEqual({
ok: false,
error: {
kind: "EVENT_TOO_LARGE",
operation: "DECODE",
retryable: false,
},
});
const oversizedChunk = createIncrementalSseParser({
maxChunkBytes: 65_536,
});
expect(
oversizedChunk.push(new Uint8Array(65_537)),
).toEqual({
ok: false,
error: {
kind: "EVENT_TOO_LARGE",
operation: "DECODE",
retryable: false,
},
});
});
it("hands items past the per-chunk cap to drain() instead of failing", () => {
const parser = createIncrementalSseParser({ maxItemsPerChunk: 1 });
const comment = { kind: "COMMENT" };
expect(pushText(parser, ":\r:\r\n:\ndata: a\r")).toEqual([comment]);
// The retained text resumes between CR and LF.
expect(parser.drain()).toEqual({ ok: true, value: [comment] });
expect(parser.drain()).toEqual({ ok: true, value: [comment] });
expect(parser.drain()).toEqual({ ok: true, value: [] });
expect(parser.drain()).toEqual({ ok: true, value: [] });
expect(pushText(parser, "\n\n")).toEqual([
{
kind: "EVENT",
eventType: "message",
data: "a",
id: null,
hasExplicitId: false,
},
]);
// A lone CR followed by another complete line still honours the cap.
const loneCarriageReturn = createIncrementalSseParser({
maxItemsPerChunk: 1,
});
expect(pushText(loneCarriageReturn, ":\r:\n")).toEqual([comment]);
expect(loneCarriageReturn.drain()).toEqual({
ok: true,
value: [comment],
});
// Undrained items are a caller bug: fail closed instead of buffering.
const undrained = createIncrementalSseParser({ maxItemsPerChunk: 1 });
expect(pushText(undrained, ":\n:\n")).toEqual([comment]);
const overflow = {
ok: false,
error: {
kind: "QUEUE_OVERFLOW",
operation: "DECODE",
retryable: false,
},
};
expect(undrained.push(encoder.encode(":\n"))).toEqual(overflow);
const unfinished = createIncrementalSseParser({ maxItemsPerChunk: 1 });
expect(pushText(unfinished, ":\n:\n")).toEqual([comment]);
expect(unfinished.finish()).toEqual(overflow);
});
it("parses the same items whether bytes arrive at once or one by one", () => {
const stream = encoder.encode(
"event: a\r\ndata: 한글\rdata: 😀 é\r\n\r\n" +
": note\rid: c-1\ndata: xy\r\rretry: 10\n" +
"data: last\n\n",
);
const whole = createIncrementalSseParser();
const wholeResult = whole.push(stream);
if (!wholeResult.ok) throw new Error(wholeResult.error.kind);
const split = createIncrementalSseParser();
const splitItems: SseParserItem[] = [];
for (let index = 0; index < stream.length; index += 1) {
const pushed = split.push(stream.subarray(index, index + 1));
if (!pushed.ok) throw new Error(pushed.error.kind);
splitItems.push(...pushed.value);
}
const expected: SseParserItem[] = [
{
kind: "EVENT",
eventType: "a",
data: "한글\n😀 é",
id: null,
hasExplicitId: false,
},
{ kind: "COMMENT" },
{
kind: "EVENT",
eventType: "message",
data: "xy",
id: "c-1",
hasExplicitId: true,
},
{ kind: "RETRY", retryMs: 10 },
{
kind: "EVENT",
eventType: "message",
data: "last",
id: "c-1",
hasExplicitId: false,
},
];
expect(wholeResult.value).toEqual(expected);
expect(splitItems).toEqual(expected);
});
it("dispatches a trailing CR-terminated blank line only at finish", () => {
const parser = createIncrementalSseParser();
expect(pushText(parser, "data: a\rdata: b\r\r")).toEqual([]);
expect(parser.finish()).toEqual({
ok: true,
value: {
items: [
{
kind: "EVENT",
eventType: "message",
data: "a\nb",
id: null,
hasExplicitId: false,
},
],
incompleteEventDiscarded: false,
},
});
});
it("counts UTF-8 bytes, not UTF-16 units, against line and event limits", () => {
const limits = {
maxLineBytes: 9,
maxEventBytes: 11,
maxIncompleteBufferBytes: 11,
};
const tooLarge = {
ok: false,
error: {
kind: "EVENT_TOO_LARGE",
operation: "DECODE",
retryable: false,
},
};
// "data:😀" is 7 UTF-16 units but 9 bytes; the CRLF makes the event 11.
expect(
createIncrementalSseParser(limits).push(
encoder.encode("data:😀\r\n\r\n"),
),
).toMatchObject({ ok: true, value: [{ data: "😀" }] });
expect(
createIncrementalSseParser(limits).push(
encoder.encode("data:😀a"),
),
).toEqual(tooLarge);
expect(
createIncrementalSseParser(limits).push(
encoder.encode("data:한\r\ndata:\r\n"),
),
).toEqual(tooLarge);
// The same limit holds when the multibyte character is split across reads.
const split = createIncrementalSseParser(limits);
const bytes = encoder.encode("data:한한");
expect(split.push(bytes.subarray(0, 6))).toEqual({
ok: true,
value: [],
});
expect(split.push(bytes.subarray(6))).toEqual(tooLarge);
});
it("accepts a maximum-size line across many reads and rejects one byte more", () => {
const maxLineBytes = 32 * 1_024;
const payload = "é".repeat((maxLineBytes - 6) / 2);
const exact = encoder.encode(`data: ${payload}\n\n`);
// 1,000-byte reads split some two-byte "é" characters.
const parser = createIncrementalSseParser({ maxLineBytes });
const items: SseParserItem[] = [];
for (let index = 0; index < exact.length; index += 1_000) {
const pushed = parser.push(exact.subarray(index, index + 1_000));
if (!pushed.ok) throw new Error(pushed.error.kind);
items.push(...pushed.value);
}
expect(items).toEqual([
{
kind: "EVENT",
eventType: "message",
data: payload,
id: null,
hasExplicitId: false,
},
]);
const over = createIncrementalSseParser({ maxLineBytes });
const overBytes = encoder.encode(`data: ${payload}a`);
expect(over.push(overBytes.subarray(0, 20_000))).toEqual({
ok: true,
value: [],
});
expect(over.push(overBytes.subarray(20_000))).toEqual({
ok: false,
error: {
kind: "EVENT_TOO_LARGE",
operation: "DECODE",
retryable: false,
},
});
});
it("parses a full-size chunk of many events", () => {
const events = Array.from({ length: 200 }, (_, index) =>
JSON.stringify({ index, value: "v".repeat(1_200) }),
);
const chunk = encoder.encode(
events
.map((data, index) => `id: c-${index}\r\ndata: ${data}\r\n\r\n`)
.join(""),
);
expect(chunk.byteLength).toBeGreaterThan(240 * 1_024);
expect(chunk.byteLength).toBeLessThanOrEqual(256 * 1_024);
const pushed = createIncrementalSseParser().push(chunk);
if (!pushed.ok) throw new Error(pushed.error.kind);
expect(pushed.value).toHaveLength(200);
expect(pushed.value[199]).toEqual({
kind: "EVENT",
eventType: "message",
data: events[199],
id: "c-199",
hasExplicitId: true,
});
});
});
describe("byte-level SSE parser against the round-2 text parser", () => {
const asciiPieces = [
"data", "data:", "data: ", "event: ", "id: ", "id:", "retry: ",
":", "x", " ", "abc", "12", "\0", "\r", "\n", "\r\n", "\n\n", "\r\r",
];
const multibytePieces = [...asciiPieces, "é", "한글", "😀", ""];
type Create = typeof createIncrementalSseParser;
// mulberry32: a small seeded PRNG, so a failing seed can be replayed.
function randomSource(seed: number): () => number {
let state = seed;
return () => {
state = (state + 0x6d2b79f5) | 0;
let value = Math.imul(state ^ (state >>> 15), 1 | state);
value ^= value + Math.imul(value ^ (value >>> 7), 61 | value);
return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296;
};
}
function run(
create: Create,
limits: Parameters<Create>[0],
chunks: readonly Uint8Array[],
skipDrain: readonly boolean[],
): unknown[] {
const parser = create(limits);
const log: unknown[] = [];
for (const [index, chunk] of chunks.entries()) {
let result = parser.push(chunk);
log.push(result);
while (!skipDrain[index] && result.ok && result.value.length > 0) {
result = parser.drain();
log.push(result);
}
if (!result.ok) return log;
}
log.push(parser.finish());
return log;
}
it.each([
["ASCII", asciiPieces],
["multibyte", multibytePieces],
])("matches on random %s streams and read splits", (_, pieces) => {
for (let seed = 1; seed <= 1_500; seed += 1) {
const next = randomSource(seed);
const pick = (max: number) => Math.floor(next() * max);
let text = next() < 0.3 ? "" : "";
for (let count = 1 + pick(40); count > 0; count -= 1) {
text += pieces[pick(pieces.length)];
}
// Node's streaming TextDecoder, which the reference relies on, drops a
// U+FEFF right after the first character when that character is split
// across reads (see the BOM test above). Keep that input out.
const first = String.fromCodePoint(text.codePointAt(0) ?? 0x20);
while (text.startsWith("", first.length)) {
text = first + text.slice(first.length + 1);
}
let bytes = encoder.encode(text);
// Cutting the stream can end it inside a character.
if (next() < 0.2) bytes = bytes.subarray(0, pick(bytes.length + 1));
const chunks: Uint8Array[] = [];
const maxRead = 1 + pick(next() < 0.5 ? 6 : bytes.length + 1);
for (let index = 0; index < bytes.length; ) {
const size = 1 + pick(maxRead);
chunks.push(bytes.subarray(index, index + size));
index += size;
}
const skipDrain = chunks.map(() => next() < 0.05);
const maxEventBytes = 4 + pick(60);
const limits =
next() < 0.3
? {}
: {
maxLineBytes: 4 + pick(40),
maxEventBytes,
maxIncompleteBufferBytes: maxEventBytes + pick(20),
maxItemsPerChunk: 1 + pick(4),
};
expect(
run(createIncrementalSseParser, limits, chunks, skipDrain),
`seed ${seed}`,
).toEqual(run(createReferenceSseParser, limits, chunks, skipDrain));
}
});
});