The parser now finds CR/LF with Uint8Array.indexOf before decoding, so a
line's UTF-8 byte count comes from byte offsets instead of re-encoding the
text. Each complete line is decoded once by a streaming fatal TextDecoder;
a line that continues into the next read is stream-decoded, and the bytes
of a character split across reads count once it completes, as before.
Behavior kept (the push() input was already Uint8Array; no seam change in
fetch-sse-connection.ts, public API unchanged):
- CR / LF / CRLF, CR and LF split across reads, multi-line data
- one leading BOM stripped (now done by the parser, decoder uses
ignoreBOM: true, so a BOM at a later line start stays text)
- line / event / incomplete-buffer byte limits, same failure point
- drain() and the per-call item cap; kept bytes are copied
- a pending CR or the item cap waits while only part of a character is
left, as the text-based parser did
- error codes (MALFORMED_EVENT / EVENT_TOO_LARGE / QUEUE_OVERFLOW / CLOSED)
Differences only on invalid UTF-8: the old parser decoded a whole chunk
first, so malformed bytes won over another error later in that chunk and
malformed bytes past the item cap failed in push(). Now the first
problem in stream order wins and those bytes fail in drain(). Both
still fail closed.
Fixed: Node's streaming TextDecoder (ignoreBOM: false) drops a U+FEFF
right after the stream's first character when that character is split
across reads, e.g. [EF] [BB BF EF BB BF] decoded to "" instead of
"". The old parser inherited that; the new one keeps the text.
Tests: existing parser and connection tests unchanged and passing. Added
a split-BOM case and a differential test against the round-2 parser
(tests/unit/realtime/sse-parser-reference.ts) over 2x1,500 seeded random
ASCII / multibyte streams with random read splits, random small limits,
truncated streams and skipped drains. 300,000 seeds also matched in a
manual run.
Benchmark (manual, not committed; Node v24.14.0, one push of one chunk,
best of 150 runs, host load average ~11-14 so absolute ms are noisy):
| input (one chunk) | original e59994a | round 2 | new | new vs original | new vs round 2 |
|--------------------------------|------------------|---------|---------|-----------------|----------------|
| ASCII, 249 KB, 209 events | 103.72 ms | 0.48 ms | 0.30 ms | 348x | 1.6x |
| Korean/emoji, 249 KB, 209 ev. | 23.30 ms | 0.85 ms | 0.44 ms | 53x | 1.9x |
| ASCII, 234 KB, 4 x 60 KB lines | 176.45 ms | 0.61 ms | 0.34 ms | 513x | 1.8x |
| Korean/emoji, 234 KB, 4 x 60KB | 25.28 ms | 1.10 ms | 0.65 ms | 39x | 1.7x |
Across five runs, multibyte new vs original stayed between 19x and 70x
(target: 10x or more).
Realtime bundle: 39903 / 40000 gzip bytes (baseline 39685, +218).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
365 lines
10 KiB
TypeScript
365 lines
10 KiB
TypeScript
// The round-2 SSE parser (decode each chunk, then re-encode every segment to
|
|
// count its bytes), kept only as the oracle for the differential test.
|
|
import type { RealtimeResult } from "../../../src/application/ports/realtime/shared.ts";
|
|
import {
|
|
realtimeFailure,
|
|
realtimeSuccess,
|
|
} from "../../../src/adapters/realtime/result.ts";
|
|
|
|
export const SSE_PARSER_CEILINGS = Object.freeze({
|
|
maxLineBytes: 64 * 1_024,
|
|
maxEventBytes: 64 * 1_024,
|
|
maxIncompleteBufferBytes: 128 * 1_024,
|
|
maxChunkBytes: 256 * 1_024,
|
|
maxItemsPerChunk: 256,
|
|
maxRetryMs: 60_000,
|
|
});
|
|
|
|
const LINE_BREAK = /[\r\n]/gu;
|
|
const NO_ITEMS: readonly SseParserItem[] = Object.freeze([]);
|
|
const COMMENT_ITEMS: readonly SseParserItem[] = Object.freeze([
|
|
Object.freeze({ kind: "COMMENT" as const }),
|
|
]);
|
|
|
|
export type SseParserLimits = Readonly<{
|
|
maxLineBytes: number;
|
|
maxEventBytes: number;
|
|
maxIncompleteBufferBytes: number;
|
|
maxChunkBytes: number;
|
|
maxItemsPerChunk: number;
|
|
maxRetryMs: number;
|
|
}>;
|
|
|
|
export type ParsedSseEvent = Readonly<{
|
|
kind: "EVENT";
|
|
eventType: string;
|
|
data: string;
|
|
/**
|
|
* Standard SSE last-event-ID state. Consumers that require cursor-after-
|
|
* effect must additionally require `hasExplicitId` and commit independently.
|
|
*/
|
|
id: string | null;
|
|
hasExplicitId: boolean;
|
|
}>;
|
|
|
|
export type SseParserItem =
|
|
| ParsedSseEvent
|
|
| Readonly<{ kind: "COMMENT" }>
|
|
| Readonly<{ kind: "RETRY"; retryMs: number }>;
|
|
|
|
export type SseParserFinish = Readonly<{
|
|
items: readonly SseParserItem[];
|
|
incompleteEventDiscarded: boolean;
|
|
}>;
|
|
|
|
export type IncrementalSseParser = Readonly<{
|
|
/**
|
|
* Returns at most `maxItemsPerChunk` items. Text past that cap is kept and
|
|
* must be taken with `drain()` until it returns no items; pushing or
|
|
* finishing before then fails with QUEUE_OVERFLOW.
|
|
*/
|
|
push(chunk: Uint8Array): RealtimeResult<readonly SseParserItem[]>;
|
|
drain(): RealtimeResult<readonly SseParserItem[]>;
|
|
finish(): RealtimeResult<SseParserFinish>;
|
|
}>;
|
|
|
|
export function createIncrementalSseParser(
|
|
limits: Partial<SseParserLimits> = {},
|
|
): IncrementalSseParser {
|
|
const resolved = resolveLimits(limits);
|
|
// ignoreBOM: false makes the decoder strip the one leading BOM the
|
|
// stream grammar allows; a second one is ordinary text.
|
|
const decoder = new TextDecoder("utf-8", {
|
|
fatal: true,
|
|
ignoreBOM: false,
|
|
});
|
|
const encoder = new TextEncoder();
|
|
|
|
let state: "OPEN" | "FAILED" | "FINISHED" = "OPEN";
|
|
let pendingCarriageReturn = false;
|
|
let line = "";
|
|
let lineBytes = 0;
|
|
let undrained = "";
|
|
let blockBytes = 0;
|
|
let dataLines: string[] = [];
|
|
let eventType = "";
|
|
let lastEventId: string | null = null;
|
|
let hasExplicitId = false;
|
|
|
|
function push(
|
|
chunk: Uint8Array,
|
|
): RealtimeResult<readonly SseParserItem[]> {
|
|
if (state !== "OPEN") {
|
|
return realtimeFailure("CLOSED", "DECODE");
|
|
}
|
|
if (!(chunk instanceof Uint8Array)) {
|
|
return fail("MALFORMED_EVENT");
|
|
}
|
|
if (chunk.byteLength > resolved.maxChunkBytes) {
|
|
return fail("EVENT_TOO_LARGE");
|
|
}
|
|
if (undrained.length > 0) return fail("QUEUE_OVERFLOW");
|
|
let text: string;
|
|
try {
|
|
text = decoder.decode(chunk, { stream: true });
|
|
} catch {
|
|
return fail("MALFORMED_EVENT");
|
|
}
|
|
return consumeText(text);
|
|
}
|
|
|
|
function drain(): RealtimeResult<readonly SseParserItem[]> {
|
|
if (state !== "OPEN") {
|
|
return realtimeFailure("CLOSED", "DECODE");
|
|
}
|
|
const text = undrained;
|
|
undrained = "";
|
|
return consumeText(text);
|
|
}
|
|
|
|
function finish(): RealtimeResult<SseParserFinish> {
|
|
if (state !== "OPEN") {
|
|
return realtimeFailure("CLOSED", "DECODE");
|
|
}
|
|
if (undrained.length > 0) return fail("QUEUE_OVERFLOW");
|
|
let tail: string;
|
|
try {
|
|
tail = decoder.decode();
|
|
} catch {
|
|
return fail("MALFORMED_EVENT");
|
|
}
|
|
const consumed = consumeText(tail);
|
|
if (!consumed.ok) return consumed;
|
|
const items = [...consumed.value];
|
|
if (pendingCarriageReturn) {
|
|
pendingCarriageReturn = false;
|
|
const processed = processLine(1);
|
|
if (!processed.ok) return processed;
|
|
items.push(...processed.value);
|
|
}
|
|
if (
|
|
undrained.length > 0 ||
|
|
items.length > resolved.maxItemsPerChunk
|
|
) {
|
|
return fail("QUEUE_OVERFLOW");
|
|
}
|
|
const incompleteEventDiscarded =
|
|
lineBytes > 0 ||
|
|
blockBytes > 0 ||
|
|
dataLines.length > 0 ||
|
|
eventType.length > 0 ||
|
|
hasExplicitId;
|
|
clearBlock();
|
|
line = "";
|
|
lineBytes = 0;
|
|
state = "FINISHED";
|
|
return success(
|
|
Object.freeze({
|
|
items: Object.freeze(items),
|
|
incompleteEventDiscarded,
|
|
}),
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Works a line at a time: find the next CR/LF, append the text before it
|
|
* and count its UTF-8 bytes once. Byte counts only grow within a line, so
|
|
* checking the limits per segment fails exactly where a per-character check
|
|
* would. Stops at `maxItemsPerChunk` items and keeps the rest for `drain()`.
|
|
*/
|
|
function consumeText(
|
|
text: string,
|
|
): RealtimeResult<readonly SseParserItem[]> {
|
|
const items: SseParserItem[] = [];
|
|
let index = 0;
|
|
|
|
while (index < text.length) {
|
|
if (items.length >= resolved.maxItemsPerChunk) {
|
|
undrained = text.slice(index);
|
|
break;
|
|
}
|
|
if (pendingCarriageReturn) {
|
|
pendingCarriageReturn = false;
|
|
const lineFeed = text.charCodeAt(index) === 0x0a;
|
|
const processed = processLine(lineFeed ? 2 : 1);
|
|
if (!processed.ok) return processed;
|
|
items.push(...processed.value);
|
|
if (lineFeed) index += 1;
|
|
continue;
|
|
}
|
|
|
|
LINE_BREAK.lastIndex = index;
|
|
const lineBreak = LINE_BREAK.exec(text)?.index ?? text.length;
|
|
if (lineBreak > index) {
|
|
const segment = text.slice(index, lineBreak);
|
|
line += segment;
|
|
lineBytes += encoder.encode(segment).byteLength;
|
|
if (
|
|
lineBytes > resolved.maxLineBytes ||
|
|
lineBytes + blockBytes > resolved.maxIncompleteBufferBytes
|
|
) {
|
|
return fail("EVENT_TOO_LARGE");
|
|
}
|
|
}
|
|
if (lineBreak === text.length) break;
|
|
|
|
index = lineBreak + 1;
|
|
if (text.charCodeAt(lineBreak) === 0x0d) {
|
|
pendingCarriageReturn = true;
|
|
continue;
|
|
}
|
|
const processed = processLine(1);
|
|
if (!processed.ok) return processed;
|
|
items.push(...processed.value);
|
|
}
|
|
return success(Object.freeze(items));
|
|
}
|
|
|
|
function processLine(
|
|
terminatorBytes: number,
|
|
): RealtimeResult<readonly SseParserItem[]> {
|
|
const currentLine = line;
|
|
const currentLineBytes = lineBytes;
|
|
line = "";
|
|
lineBytes = 0;
|
|
|
|
if (currentLine.length === 0) {
|
|
const items: SseParserItem[] = [];
|
|
if (dataLines.length > 0) {
|
|
items.push(
|
|
Object.freeze({
|
|
kind: "EVENT",
|
|
eventType: eventType.length > 0 ? eventType : "message",
|
|
data: dataLines.join("\n"),
|
|
id: lastEventId,
|
|
hasExplicitId,
|
|
}),
|
|
);
|
|
}
|
|
clearBlock();
|
|
return success(Object.freeze(items));
|
|
}
|
|
|
|
if (currentLine.startsWith(":")) return success(COMMENT_ITEMS);
|
|
|
|
blockBytes += currentLineBytes + terminatorBytes;
|
|
if (blockBytes > resolved.maxEventBytes) {
|
|
return fail("EVENT_TOO_LARGE");
|
|
}
|
|
if (blockBytes > resolved.maxIncompleteBufferBytes) {
|
|
return fail("EVENT_TOO_LARGE");
|
|
}
|
|
|
|
const separator = currentLine.indexOf(":");
|
|
const field =
|
|
separator === -1
|
|
? currentLine
|
|
: currentLine.slice(0, separator);
|
|
let value =
|
|
separator === -1 ? "" : currentLine.slice(separator + 1);
|
|
if (value.startsWith(" ")) value = value.slice(1);
|
|
|
|
if (field === "data") {
|
|
dataLines.push(value);
|
|
return success(NO_ITEMS);
|
|
}
|
|
if (field === "event") {
|
|
eventType = value;
|
|
return success(NO_ITEMS);
|
|
}
|
|
if (field === "id") {
|
|
if (!value.includes("\0")) {
|
|
lastEventId = value;
|
|
hasExplicitId = true;
|
|
}
|
|
return success(NO_ITEMS);
|
|
}
|
|
if (field === "retry" && /^\d+$/u.test(value)) {
|
|
const retryMs = Number(value);
|
|
if (
|
|
Number.isSafeInteger(retryMs) &&
|
|
retryMs <= resolved.maxRetryMs
|
|
) {
|
|
return success(
|
|
Object.freeze([
|
|
Object.freeze({ kind: "RETRY" as const, retryMs }),
|
|
]),
|
|
);
|
|
}
|
|
}
|
|
return success(NO_ITEMS);
|
|
}
|
|
|
|
function clearBlock(): void {
|
|
blockBytes = 0;
|
|
dataLines = [];
|
|
eventType = "";
|
|
hasExplicitId = false;
|
|
}
|
|
|
|
function fail(
|
|
kind:
|
|
| "EVENT_TOO_LARGE"
|
|
| "MALFORMED_EVENT"
|
|
| "QUEUE_OVERFLOW",
|
|
): RealtimeResult<never> {
|
|
state = "FAILED";
|
|
line = "";
|
|
lineBytes = 0;
|
|
undrained = "";
|
|
clearBlock();
|
|
return realtimeFailure(kind, "DECODE");
|
|
}
|
|
|
|
return Object.freeze({ push, drain, finish });
|
|
}
|
|
|
|
function resolveLimits(
|
|
input: Partial<SseParserLimits>,
|
|
): SseParserLimits {
|
|
const limits = {
|
|
maxLineBytes:
|
|
input.maxLineBytes ?? SSE_PARSER_CEILINGS.maxLineBytes,
|
|
maxEventBytes:
|
|
input.maxEventBytes ?? SSE_PARSER_CEILINGS.maxEventBytes,
|
|
maxIncompleteBufferBytes:
|
|
input.maxIncompleteBufferBytes ??
|
|
SSE_PARSER_CEILINGS.maxIncompleteBufferBytes,
|
|
maxChunkBytes:
|
|
input.maxChunkBytes ?? SSE_PARSER_CEILINGS.maxChunkBytes,
|
|
maxItemsPerChunk:
|
|
input.maxItemsPerChunk ??
|
|
SSE_PARSER_CEILINGS.maxItemsPerChunk,
|
|
maxRetryMs: input.maxRetryMs ?? SSE_PARSER_CEILINGS.maxRetryMs,
|
|
};
|
|
if (
|
|
!positiveInteger(limits.maxLineBytes) ||
|
|
limits.maxLineBytes > SSE_PARSER_CEILINGS.maxLineBytes ||
|
|
!positiveInteger(limits.maxEventBytes) ||
|
|
limits.maxEventBytes > SSE_PARSER_CEILINGS.maxEventBytes ||
|
|
!positiveInteger(limits.maxIncompleteBufferBytes) ||
|
|
limits.maxIncompleteBufferBytes >
|
|
SSE_PARSER_CEILINGS.maxIncompleteBufferBytes ||
|
|
limits.maxIncompleteBufferBytes < limits.maxEventBytes ||
|
|
!positiveInteger(limits.maxChunkBytes) ||
|
|
limits.maxChunkBytes > SSE_PARSER_CEILINGS.maxChunkBytes ||
|
|
limits.maxChunkBytes < limits.maxEventBytes ||
|
|
!positiveInteger(limits.maxItemsPerChunk) ||
|
|
limits.maxItemsPerChunk >
|
|
SSE_PARSER_CEILINGS.maxItemsPerChunk ||
|
|
!positiveInteger(limits.maxRetryMs) ||
|
|
limits.maxRetryMs > SSE_PARSER_CEILINGS.maxRetryMs
|
|
) {
|
|
throw new TypeError("Invalid SSE parser limits.");
|
|
}
|
|
return Object.freeze(limits);
|
|
}
|
|
|
|
function success<Value>(value: Value): RealtimeResult<Value> {
|
|
return realtimeSuccess(value);
|
|
}
|
|
|
|
function positiveInteger(value: number): boolean {
|
|
return Number.isSafeInteger(value) && value > 0;
|
|
}
|