501 lines
15 KiB
Python
501 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
from helpers import read_json, run_cli
|
|
|
|
|
|
def sample_bytes() -> bytes:
|
|
return (
|
|
"# 문서\r\n"
|
|
"\r\n"
|
|
"첫 문단은 분할 전후에 그대로 남아야 합니다.\r\n"
|
|
"\r\n"
|
|
"## 코드\r\n"
|
|
"\r\n"
|
|
"```python\r\n"
|
|
"## 이것은 제목이 아니다\r\n"
|
|
"```not-a-closing-fence\r\n"
|
|
"print('계속 같은 fence 안')\r\n"
|
|
"```\r\n"
|
|
"\r\n"
|
|
"## 끝\r\n"
|
|
"\r\n"
|
|
"마지막 문단입니다.\r\n"
|
|
).encode("utf-8")
|
|
|
|
|
|
def test_split_and_input_reassembly_are_byte_identical(tmp_path: Path) -> None:
|
|
document = tmp_path / "source.md"
|
|
document.write_bytes(sample_bytes())
|
|
chunks = tmp_path / "chunks"
|
|
split = run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
"--max-chars",
|
|
55,
|
|
)
|
|
assert split.returncode == 0, split.stderr
|
|
manifest = read_json(chunks / "manifest.json")
|
|
assert manifest["self_check"] is True
|
|
assert manifest["source"]["sha256"] == hashlib.sha256(sample_bytes()).hexdigest()
|
|
assert [item["index"] for item in manifest["chunks"]] == list(
|
|
range(1, len(manifest["chunks"]) + 1)
|
|
)
|
|
|
|
fenced_piece = [
|
|
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
|
for item in manifest["chunks"]
|
|
if "```not-a-closing-fence" in (chunks / item["input_file"]).read_text(encoding="utf-8")
|
|
]
|
|
assert len(fenced_piece) == 1
|
|
assert "print('계속 같은 fence 안')" in fenced_piece[0]
|
|
assert fenced_piece[0].rstrip().endswith("```")
|
|
|
|
output = tmp_path / "round-trip.md"
|
|
assembled = run_cli(
|
|
"reassemble_document.py",
|
|
"--manifest",
|
|
chunks / "manifest.json",
|
|
"--output",
|
|
output,
|
|
)
|
|
assert assembled.returncode == 0, assembled.stderr
|
|
assert output.read_bytes() == sample_bytes()
|
|
|
|
|
|
def test_reassemble_rewritten_chunks_and_reject_tampered_input(tmp_path: Path) -> None:
|
|
document = tmp_path / "source.md"
|
|
document.write_text("# 제목\n\n첫 문단\n\n## 다음\n\n둘째 문단\n", encoding="utf-8")
|
|
chunks = tmp_path / "chunks"
|
|
split = run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
"--max-chars",
|
|
20,
|
|
)
|
|
assert split.returncode == 0, split.stderr
|
|
manifest = read_json(chunks / "manifest.json")
|
|
expected: list[str] = []
|
|
for item in manifest["chunks"]:
|
|
source = (chunks / item["input_file"]).read_text(encoding="utf-8")
|
|
rewritten = f"[청크 {item['index']}]\n{source}"
|
|
(chunks / item["rewritten_file"]).write_text(rewritten, encoding="utf-8")
|
|
expected.append(rewritten)
|
|
|
|
output = tmp_path / "rewritten.md"
|
|
assembled = run_cli(
|
|
"reassemble_document.py",
|
|
"--manifest",
|
|
chunks / "manifest.json",
|
|
"--output",
|
|
output,
|
|
"--source",
|
|
"rewritten",
|
|
)
|
|
assert assembled.returncode == 0, assembled.stderr
|
|
assert output.read_text(encoding="utf-8") == "".join(expected)
|
|
|
|
first = chunks / manifest["chunks"][0]["input_file"]
|
|
first.write_text(first.read_text(encoding="utf-8") + "tampered", encoding="utf-8")
|
|
rejected_output = tmp_path / "must-not-exist.md"
|
|
rejected = run_cli(
|
|
"reassemble_document.py",
|
|
"--manifest",
|
|
chunks / "manifest.json",
|
|
"--output",
|
|
rejected_output,
|
|
"--source",
|
|
"rewritten",
|
|
)
|
|
assert rejected.returncode == 2
|
|
assert not rejected_output.exists()
|
|
|
|
|
|
def test_reassemble_rejects_manifest_path_escape(tmp_path: Path) -> None:
|
|
document = tmp_path / "source.md"
|
|
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
|
|
chunks = tmp_path / "chunks"
|
|
assert run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
"--max-chars",
|
|
50,
|
|
).returncode == 0
|
|
manifest_path = chunks / "manifest.json"
|
|
manifest = read_json(manifest_path)
|
|
outside = tmp_path / "outside.md"
|
|
shutil.copyfile(chunks / manifest["chunks"][0]["input_file"], outside)
|
|
manifest["chunks"][0]["input_file"] = "../outside.md"
|
|
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
|
result = run_cli(
|
|
"reassemble_document.py",
|
|
"--manifest",
|
|
manifest_path,
|
|
"--output",
|
|
tmp_path / "output.md",
|
|
)
|
|
assert result.returncode == 2
|
|
assert "벗어납니다" in result.stderr
|
|
|
|
|
|
def test_reassemble_applies_chunk_manifest_schema(tmp_path: Path) -> None:
|
|
document = tmp_path / "source.md"
|
|
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
|
|
chunks = tmp_path / "chunks"
|
|
assert run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
).returncode == 0
|
|
manifest_path = chunks / "manifest.json"
|
|
manifest = read_json(manifest_path)
|
|
manifest["chunks"][0]["boundary_reason"] = "INVALID"
|
|
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
|
result = run_cli(
|
|
"reassemble_document.py",
|
|
"--manifest",
|
|
manifest_path,
|
|
"--output",
|
|
tmp_path / "output.md",
|
|
)
|
|
assert result.returncode == 2
|
|
assert "schema 위반" in result.stderr
|
|
|
|
|
|
def test_split_keeps_indented_list_fence_atomic(tmp_path: Path) -> None:
|
|
document = tmp_path / "nested.md"
|
|
nested = (
|
|
"# 제목\n\n- 항목\n\n"
|
|
" ```python\n"
|
|
" first = 1\n"
|
|
"\n"
|
|
" second = 2\n"
|
|
" ```\n\n"
|
|
"## 다음\n\n본문\n"
|
|
)
|
|
document.write_text(nested, encoding="utf-8")
|
|
chunks = tmp_path / "nested-chunks"
|
|
result = run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
"--max-chars",
|
|
40,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
manifest = read_json(chunks / "manifest.json")
|
|
pieces = [
|
|
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
|
for item in manifest["chunks"]
|
|
]
|
|
containing_code = [piece for piece in pieces if "first = 1" in piece or "second = 2" in piece]
|
|
assert len(containing_code) == 1
|
|
assert "first = 1" in containing_code[0] and "second = 2" in containing_code[0]
|
|
|
|
|
|
def test_indented_backticks_inside_top_level_fence_do_not_close_it(tmp_path: Path) -> None:
|
|
document = tmp_path / "top-level-fence.md"
|
|
text = (
|
|
"# 제목\n\n"
|
|
"```text\n"
|
|
"line\n"
|
|
" ```\n"
|
|
"\n"
|
|
"## 이것도 코드 내용\n"
|
|
"secret\n"
|
|
"```\n\n"
|
|
"## 실제 다음 절\n\n본문\n"
|
|
)
|
|
document.write_text(text, encoding="utf-8")
|
|
chunks = tmp_path / "top-level-chunks"
|
|
result = run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
"--max-chars",
|
|
38,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
manifest = read_json(chunks / "manifest.json")
|
|
pieces = [
|
|
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
|
for item in manifest["chunks"]
|
|
]
|
|
code_pieces = [piece for piece in pieces if "secret" in piece or "이것도 코드 내용" in piece]
|
|
assert len(code_pieces) == 1
|
|
assert " ```" in code_pieces[0] and "secret" in code_pieces[0]
|
|
|
|
|
|
def test_same_line_list_and_blockquote_fences_stay_atomic(tmp_path: Path) -> None:
|
|
document = tmp_path / "container-fences.md"
|
|
text = (
|
|
"# 제목\n\n"
|
|
"- ```python\n"
|
|
" first = 1\n\n"
|
|
" ## 목록 안 코드 제목\n"
|
|
" second = 2\n"
|
|
" ```\n\n"
|
|
"> ```text\n"
|
|
"> quote line\n"
|
|
">\n"
|
|
"> ## 인용 안 코드 제목\n"
|
|
"> ```\n\n"
|
|
"## 실제 다음 절\n\n본문\n"
|
|
)
|
|
document.write_text(text, encoding="utf-8")
|
|
chunks = tmp_path / "container-chunks"
|
|
result = run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
"--max-chars",
|
|
35,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
manifest = read_json(chunks / "manifest.json")
|
|
pieces = [
|
|
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
|
for item in manifest["chunks"]
|
|
]
|
|
assert len([piece for piece in pieces if "first = 1" in piece or "second = 2" in piece]) == 1
|
|
assert len([piece for piece in pieces if "quote line" in piece or "인용 안 코드 제목" in piece]) == 1
|
|
|
|
|
|
def test_nested_list_heading_is_not_used_as_a_top_level_split_boundary(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
document = tmp_path / "nested-list-heading.md"
|
|
text = (
|
|
"# 제목\n\n"
|
|
"도입 설명은 청크를 채우기 위해 충분한 길이로 이어진다.\n"
|
|
"- 목록 항목은 다음 설명과 소제목을 소유한다.\n"
|
|
" ## 목록 내부 소제목\n"
|
|
" 이 내용은 목록 항목에서 분리되면 의미가 달라진다.\n"
|
|
"## 실제 절\n"
|
|
"본문이다.\n"
|
|
)
|
|
document.write_text(text, encoding="utf-8")
|
|
chunks = tmp_path / "nested-list-heading-chunks"
|
|
result = run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
"--max-chars",
|
|
95,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
manifest = read_json(chunks / "manifest.json")
|
|
pieces = [
|
|
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
|
for item in manifest["chunks"]
|
|
]
|
|
nested_piece = next(piece for piece in pieces if "목록 내부 소제목" in piece)
|
|
assert "- 목록 항목" in nested_piece
|
|
assert not any(piece.startswith(" ## 목록 내부 소제목") for piece in pieces)
|
|
|
|
|
|
def test_comment_token_inside_html_attribute_does_not_make_document_atomic(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
document = tmp_path / "inline-html-attribute.md"
|
|
document.write_text(
|
|
'<span title="<!--">보이는 설명</span>\n\n## 다음\n본문이다.\n',
|
|
encoding="utf-8",
|
|
)
|
|
chunks = tmp_path / "inline-html-attribute-chunks"
|
|
result = run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
"--max-chars",
|
|
40,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
manifest = read_json(chunks / "manifest.json")
|
|
assert len(manifest["chunks"]) >= 2
|
|
assert all(item["boundary_reason"] != "oversize_atomic_block" for item in manifest["chunks"])
|
|
|
|
|
|
def test_multiline_html_comment_stays_atomic(tmp_path: Path) -> None:
|
|
document = tmp_path / "comment.md"
|
|
text = (
|
|
"# 제목\n\n<!--\nhidden start\n\n## hidden heading\nhidden end\n-->\n\n"
|
|
"## 실제 절\n\n본문\n"
|
|
)
|
|
document.write_text(text, encoding="utf-8")
|
|
chunks = tmp_path / "comment-chunks"
|
|
result = run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
"--max-chars",
|
|
35,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
manifest = read_json(chunks / "manifest.json")
|
|
pieces = [
|
|
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
|
for item in manifest["chunks"]
|
|
]
|
|
comment_pieces = [piece for piece in pieces if "hidden start" in piece or "hidden end" in piece]
|
|
assert len(comment_pieces) == 1
|
|
|
|
|
|
def test_html_comment_marker_in_fence_info_does_not_hide_fence(tmp_path: Path) -> None:
|
|
document = tmp_path / "fence-info-comment.md"
|
|
text = (
|
|
"# 제목\n\n"
|
|
"```html <!--\n"
|
|
"<section>\n\n"
|
|
"## 여전히 코드인 제목\n"
|
|
"</section>\n"
|
|
"-->\n"
|
|
"```\n\n"
|
|
"## 실제 절\n\n본문\n"
|
|
)
|
|
document.write_text(text, encoding="utf-8")
|
|
chunks = tmp_path / "fence-info-comment-chunks"
|
|
result = run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
"--max-chars",
|
|
38,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
manifest = read_json(chunks / "manifest.json")
|
|
pieces = [
|
|
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
|
for item in manifest["chunks"]
|
|
]
|
|
code_pieces = [
|
|
piece
|
|
for piece in pieces
|
|
if "<section>" in piece or "여전히 코드인 제목" in piece
|
|
]
|
|
assert len(code_pieces) == 1
|
|
assert code_pieces[0].startswith("```html <!--")
|
|
assert "-->\n```" in code_pieces[0]
|
|
|
|
|
|
def test_output_dangling_symlinks_are_not_followed_without_force(tmp_path: Path) -> None:
|
|
document = tmp_path / "source.md"
|
|
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
|
|
outside_dir = tmp_path / "outside-dir"
|
|
split_link = tmp_path / "split-link"
|
|
split_link.symlink_to(outside_dir, target_is_directory=True)
|
|
split = run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
split_link,
|
|
)
|
|
assert split.returncode == 2
|
|
assert split_link.is_symlink()
|
|
assert not outside_dir.exists()
|
|
|
|
chunks = tmp_path / "chunks"
|
|
assert run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
).returncode == 0
|
|
outside_file = tmp_path / "outside.md"
|
|
output_link = tmp_path / "output-link.md"
|
|
output_link.symlink_to(outside_file)
|
|
assembled = run_cli(
|
|
"reassemble_document.py",
|
|
"--manifest",
|
|
chunks / "manifest.json",
|
|
"--output",
|
|
output_link,
|
|
)
|
|
assert assembled.returncode == 2
|
|
assert output_link.is_symlink()
|
|
assert not outside_file.exists()
|
|
|
|
|
|
def test_force_reassemble_cannot_overwrite_manifest_or_chunks(tmp_path: Path) -> None:
|
|
document = tmp_path / "source.md"
|
|
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
|
|
chunks = tmp_path / "chunks"
|
|
assert run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
).returncode == 0
|
|
manifest_path = chunks / "manifest.json"
|
|
manifest_before = manifest_path.read_bytes()
|
|
manifest = read_json(manifest_path)
|
|
input_path = chunks / manifest["chunks"][0]["input_file"]
|
|
input_before = input_path.read_bytes()
|
|
for protected in (manifest_path, input_path, document):
|
|
result = run_cli(
|
|
"reassemble_document.py",
|
|
"--manifest",
|
|
manifest_path,
|
|
"--output",
|
|
protected,
|
|
"--force",
|
|
)
|
|
assert result.returncode == 2
|
|
assert "덮을 수 없습니다" in result.stderr
|
|
assert manifest_path.read_bytes() == manifest_before
|
|
assert input_path.read_bytes() == input_before
|
|
|
|
|
|
def test_thematic_break_is_a_safe_split_boundary(tmp_path: Path) -> None:
|
|
document = tmp_path / "thematic.md"
|
|
document.write_text("가" * 30 + "\n* * *\n" + "나" * 30 + "\n", encoding="utf-8")
|
|
chunks = tmp_path / "thematic-chunks"
|
|
result = run_cli(
|
|
"split_document.py",
|
|
"--document",
|
|
document,
|
|
"--output-dir",
|
|
chunks,
|
|
"--max-chars",
|
|
40,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
manifest = read_json(chunks / "manifest.json")
|
|
assert len(manifest["chunks"]) == 2
|
|
assert all(
|
|
item["boundary_reason"] != "oversize_atomic_block"
|
|
for item in manifest["chunks"]
|
|
)
|