A single reader chunk with more than maxItemsPerChunk (256) items used to fail the stream with non-retryable QUEUE_OVERFLOW, killing normal replay bursts even though maxChunkBytes already bounds memory. The parser now stops at the cap, keeps the remaining decoded text, and returns it through a new additive drain() call; fetch-sse-connection drains after every push before reading again. Pushing or finishing with undrained text still fails closed (QUEUE_OVERFLOW), so buffered text stays bounded by one chunk. A lone CR followed by another line now re-checks the cap before continuing. COMMENT and empty item lists reuse frozen constants. The old parser test that expected QUEUE_OVERFLOW for 3 comments with a cap of 2 encoded the bug and is replaced. RED: new parser drain test and a 300-event single-chunk connection test failed with QUEUE_OVERFLOW before the change. Gates: check:architecture, check:types:app/test, lint, check:adapter-inventory, test:unit, test:component all pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
394 lines
11 KiB
TypeScript
394 lines
11 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
||
|
||
import {
|
||
createIncrementalSseParser,
|
||
type SseParserItem,
|
||
} from "../../../src/adapters/realtime/sse/sse-parser.ts";
|
||
|
||
const encoder = new TextEncoder();
|
||
|
||
function pushText(
|
||
parser: ReturnType<typeof createIncrementalSseParser>,
|
||
text: string,
|
||
): readonly SseParserItem[] {
|
||
const result = parser.push(encoder.encode(text));
|
||
if (!result.ok) throw new Error(result.error.kind);
|
||
return result.value;
|
||
}
|
||
|
||
describe("incremental SSE parser", () => {
|
||
it("handles BOM, chunk boundaries, CR/LF/CRLF, comments and multi-line data", () => {
|
||
const parser = createIncrementalSseParser();
|
||
const chunks = [
|
||
"\uFEFF: ready\r",
|
||
"\nretry: 2500\revent: resource.updated\n",
|
||
"id: cursor-1\r\ndata: first\rdata:second\n\n",
|
||
];
|
||
const items = chunks.flatMap((chunk) => pushText(parser, chunk));
|
||
|
||
expect(items).toEqual([
|
||
{ kind: "COMMENT" },
|
||
{ kind: "RETRY", retryMs: 2_500 },
|
||
{
|
||
kind: "EVENT",
|
||
eventType: "resource.updated",
|
||
data: "first\nsecond",
|
||
id: "cursor-1",
|
||
hasExplicitId: true,
|
||
},
|
||
]);
|
||
expect(parser.finish()).toEqual({
|
||
ok: true,
|
||
value: {
|
||
items: [],
|
||
incompleteEventDiscarded: false,
|
||
},
|
||
});
|
||
});
|
||
|
||
it("preserves standard inherited ID state while marking direct IDs", () => {
|
||
const parser = createIncrementalSseParser();
|
||
const items = pushText(
|
||
parser,
|
||
"id: cursor-a\ndata: one\n\ndata: two\n\ndata:\n\n",
|
||
);
|
||
|
||
expect(items).toEqual([
|
||
{
|
||
kind: "EVENT",
|
||
eventType: "message",
|
||
data: "one",
|
||
id: "cursor-a",
|
||
hasExplicitId: true,
|
||
},
|
||
{
|
||
kind: "EVENT",
|
||
eventType: "message",
|
||
data: "two",
|
||
id: "cursor-a",
|
||
hasExplicitId: false,
|
||
},
|
||
{
|
||
kind: "EVENT",
|
||
eventType: "message",
|
||
data: "",
|
||
id: "cursor-a",
|
||
hasExplicitId: false,
|
||
},
|
||
]);
|
||
});
|
||
|
||
it("ignores invalid ID and retry fields without inventing a cursor", () => {
|
||
const parser = createIncrementalSseParser();
|
||
expect(
|
||
pushText(
|
||
parser,
|
||
"id: invalid\u0000cursor\nretry: 99999\ndata: value\n\n",
|
||
),
|
||
).toEqual([
|
||
{
|
||
kind: "EVENT",
|
||
eventType: "message",
|
||
data: "value",
|
||
id: null,
|
||
hasExplicitId: false,
|
||
},
|
||
]);
|
||
});
|
||
|
||
it("discards an event that was not terminated by a blank line", () => {
|
||
const parser = createIncrementalSseParser();
|
||
expect(pushText(parser, "data: incomplete\n")).toEqual([]);
|
||
expect(parser.finish()).toEqual({
|
||
ok: true,
|
||
value: {
|
||
items: [],
|
||
incompleteEventDiscarded: true,
|
||
},
|
||
});
|
||
expect(parser.push(encoder.encode("data: late\n\n"))).toEqual({
|
||
ok: false,
|
||
error: {
|
||
kind: "CLOSED",
|
||
operation: "DECODE",
|
||
retryable: false,
|
||
},
|
||
});
|
||
});
|
||
|
||
it("fails closed on malformed UTF-8 and parser ceilings", () => {
|
||
const malformed = createIncrementalSseParser();
|
||
expect(
|
||
malformed.push(new Uint8Array([0xc3, 0x28])),
|
||
).toEqual({
|
||
ok: false,
|
||
error: {
|
||
kind: "MALFORMED_EVENT",
|
||
operation: "DECODE",
|
||
retryable: false,
|
||
},
|
||
});
|
||
|
||
const longLine = createIncrementalSseParser({
|
||
maxLineBytes: 4,
|
||
maxEventBytes: 8,
|
||
maxIncompleteBufferBytes: 8,
|
||
maxRetryMs: 100,
|
||
});
|
||
expect(longLine.push(encoder.encode("data:"))).toEqual({
|
||
ok: false,
|
||
error: {
|
||
kind: "EVENT_TOO_LARGE",
|
||
operation: "DECODE",
|
||
retryable: false,
|
||
},
|
||
});
|
||
|
||
const largeEvent = createIncrementalSseParser({
|
||
maxLineBytes: 16,
|
||
maxEventBytes: 8,
|
||
maxIncompleteBufferBytes: 16,
|
||
maxRetryMs: 100,
|
||
});
|
||
expect(largeEvent.push(encoder.encode("data:abc\n"))).toEqual({
|
||
ok: false,
|
||
error: {
|
||
kind: "EVENT_TOO_LARGE",
|
||
operation: "DECODE",
|
||
retryable: false,
|
||
},
|
||
});
|
||
|
||
const oversizedChunk = createIncrementalSseParser({
|
||
maxChunkBytes: 65_536,
|
||
});
|
||
expect(
|
||
oversizedChunk.push(new Uint8Array(65_537)),
|
||
).toEqual({
|
||
ok: false,
|
||
error: {
|
||
kind: "EVENT_TOO_LARGE",
|
||
operation: "DECODE",
|
||
retryable: false,
|
||
},
|
||
});
|
||
});
|
||
|
||
it("hands items past the per-chunk cap to drain() instead of failing", () => {
|
||
const parser = createIncrementalSseParser({ maxItemsPerChunk: 1 });
|
||
const comment = { kind: "COMMENT" };
|
||
expect(pushText(parser, ":\r:\r\n:\ndata: a\r")).toEqual([comment]);
|
||
// The retained text resumes between CR and LF.
|
||
expect(parser.drain()).toEqual({ ok: true, value: [comment] });
|
||
expect(parser.drain()).toEqual({ ok: true, value: [comment] });
|
||
expect(parser.drain()).toEqual({ ok: true, value: [] });
|
||
expect(parser.drain()).toEqual({ ok: true, value: [] });
|
||
expect(pushText(parser, "\n\n")).toEqual([
|
||
{
|
||
kind: "EVENT",
|
||
eventType: "message",
|
||
data: "a",
|
||
id: null,
|
||
hasExplicitId: false,
|
||
},
|
||
]);
|
||
|
||
// A lone CR followed by another complete line still honours the cap.
|
||
const loneCarriageReturn = createIncrementalSseParser({
|
||
maxItemsPerChunk: 1,
|
||
});
|
||
expect(pushText(loneCarriageReturn, ":\r:\n")).toEqual([comment]);
|
||
expect(loneCarriageReturn.drain()).toEqual({
|
||
ok: true,
|
||
value: [comment],
|
||
});
|
||
|
||
// Undrained items are a caller bug: fail closed instead of buffering.
|
||
const undrained = createIncrementalSseParser({ maxItemsPerChunk: 1 });
|
||
expect(pushText(undrained, ":\n:\n")).toEqual([comment]);
|
||
const overflow = {
|
||
ok: false,
|
||
error: {
|
||
kind: "QUEUE_OVERFLOW",
|
||
operation: "DECODE",
|
||
retryable: false,
|
||
},
|
||
};
|
||
expect(undrained.push(encoder.encode(":\n"))).toEqual(overflow);
|
||
const unfinished = createIncrementalSseParser({ maxItemsPerChunk: 1 });
|
||
expect(pushText(unfinished, ":\n:\n")).toEqual([comment]);
|
||
expect(unfinished.finish()).toEqual(overflow);
|
||
});
|
||
|
||
it("parses the same items whether bytes arrive at once or one by one", () => {
|
||
const stream = encoder.encode(
|
||
"event: a\r\ndata: 한글\rdata: 😀 é\r\n\r\n" +
|
||
": note\rid: c-1\ndata: xy\r\rretry: 10\n" +
|
||
"data: last\n\n",
|
||
);
|
||
const whole = createIncrementalSseParser();
|
||
const wholeResult = whole.push(stream);
|
||
if (!wholeResult.ok) throw new Error(wholeResult.error.kind);
|
||
|
||
const split = createIncrementalSseParser();
|
||
const splitItems: SseParserItem[] = [];
|
||
for (let index = 0; index < stream.length; index += 1) {
|
||
const pushed = split.push(stream.subarray(index, index + 1));
|
||
if (!pushed.ok) throw new Error(pushed.error.kind);
|
||
splitItems.push(...pushed.value);
|
||
}
|
||
|
||
const expected: SseParserItem[] = [
|
||
{
|
||
kind: "EVENT",
|
||
eventType: "a",
|
||
data: "한글\n😀 é",
|
||
id: null,
|
||
hasExplicitId: false,
|
||
},
|
||
{ kind: "COMMENT" },
|
||
{
|
||
kind: "EVENT",
|
||
eventType: "message",
|
||
data: "xy",
|
||
id: "c-1",
|
||
hasExplicitId: true,
|
||
},
|
||
{ kind: "RETRY", retryMs: 10 },
|
||
{
|
||
kind: "EVENT",
|
||
eventType: "message",
|
||
data: "last",
|
||
id: "c-1",
|
||
hasExplicitId: false,
|
||
},
|
||
];
|
||
expect(wholeResult.value).toEqual(expected);
|
||
expect(splitItems).toEqual(expected);
|
||
});
|
||
|
||
it("dispatches a trailing CR-terminated blank line only at finish", () => {
|
||
const parser = createIncrementalSseParser();
|
||
expect(pushText(parser, "data: a\rdata: b\r\r")).toEqual([]);
|
||
expect(parser.finish()).toEqual({
|
||
ok: true,
|
||
value: {
|
||
items: [
|
||
{
|
||
kind: "EVENT",
|
||
eventType: "message",
|
||
data: "a\nb",
|
||
id: null,
|
||
hasExplicitId: false,
|
||
},
|
||
],
|
||
incompleteEventDiscarded: false,
|
||
},
|
||
});
|
||
});
|
||
|
||
it("counts UTF-8 bytes, not UTF-16 units, against line and event limits", () => {
|
||
const limits = {
|
||
maxLineBytes: 9,
|
||
maxEventBytes: 11,
|
||
maxIncompleteBufferBytes: 11,
|
||
};
|
||
const tooLarge = {
|
||
ok: false,
|
||
error: {
|
||
kind: "EVENT_TOO_LARGE",
|
||
operation: "DECODE",
|
||
retryable: false,
|
||
},
|
||
};
|
||
|
||
// "data:😀" is 7 UTF-16 units but 9 bytes; the CRLF makes the event 11.
|
||
expect(
|
||
createIncrementalSseParser(limits).push(
|
||
encoder.encode("data:😀\r\n\r\n"),
|
||
),
|
||
).toMatchObject({ ok: true, value: [{ data: "😀" }] });
|
||
expect(
|
||
createIncrementalSseParser(limits).push(
|
||
encoder.encode("data:😀a"),
|
||
),
|
||
).toEqual(tooLarge);
|
||
expect(
|
||
createIncrementalSseParser(limits).push(
|
||
encoder.encode("data:한\r\ndata:\r\n"),
|
||
),
|
||
).toEqual(tooLarge);
|
||
|
||
// The same limit holds when the multibyte character is split across reads.
|
||
const split = createIncrementalSseParser(limits);
|
||
const bytes = encoder.encode("data:한한");
|
||
expect(split.push(bytes.subarray(0, 6))).toEqual({
|
||
ok: true,
|
||
value: [],
|
||
});
|
||
expect(split.push(bytes.subarray(6))).toEqual(tooLarge);
|
||
});
|
||
|
||
it("accepts a maximum-size line across many reads and rejects one byte more", () => {
|
||
const maxLineBytes = 32 * 1_024;
|
||
const payload = "é".repeat((maxLineBytes - 6) / 2);
|
||
const exact = encoder.encode(`data: ${payload}\n\n`);
|
||
// 1,000-byte reads split some two-byte "é" characters.
|
||
const parser = createIncrementalSseParser({ maxLineBytes });
|
||
const items: SseParserItem[] = [];
|
||
for (let index = 0; index < exact.length; index += 1_000) {
|
||
const pushed = parser.push(exact.subarray(index, index + 1_000));
|
||
if (!pushed.ok) throw new Error(pushed.error.kind);
|
||
items.push(...pushed.value);
|
||
}
|
||
expect(items).toEqual([
|
||
{
|
||
kind: "EVENT",
|
||
eventType: "message",
|
||
data: payload,
|
||
id: null,
|
||
hasExplicitId: false,
|
||
},
|
||
]);
|
||
|
||
const over = createIncrementalSseParser({ maxLineBytes });
|
||
const overBytes = encoder.encode(`data: ${payload}a`);
|
||
expect(over.push(overBytes.subarray(0, 20_000))).toEqual({
|
||
ok: true,
|
||
value: [],
|
||
});
|
||
expect(over.push(overBytes.subarray(20_000))).toEqual({
|
||
ok: false,
|
||
error: {
|
||
kind: "EVENT_TOO_LARGE",
|
||
operation: "DECODE",
|
||
retryable: false,
|
||
},
|
||
});
|
||
});
|
||
|
||
it("parses a full-size chunk of many events", () => {
|
||
const events = Array.from({ length: 200 }, (_, index) =>
|
||
JSON.stringify({ index, value: "v".repeat(1_200) }),
|
||
);
|
||
const chunk = encoder.encode(
|
||
events
|
||
.map((data, index) => `id: c-${index}\r\ndata: ${data}\r\n\r\n`)
|
||
.join(""),
|
||
);
|
||
expect(chunk.byteLength).toBeGreaterThan(240 * 1_024);
|
||
expect(chunk.byteLength).toBeLessThanOrEqual(256 * 1_024);
|
||
|
||
const pushed = createIncrementalSseParser().push(chunk);
|
||
if (!pushed.ok) throw new Error(pushed.error.kind);
|
||
expect(pushed.value).toHaveLength(200);
|
||
expect(pushed.value[199]).toEqual({
|
||
kind: "EVENT",
|
||
eventType: "message",
|
||
data: events[199],
|
||
id: "c-199",
|
||
hasExplicitId: true,
|
||
});
|
||
});
|
||
});
|