from __future__ import annotations
import hashlib
import json
import shutil
from pathlib import Path
from helpers import read_json, run_cli
def sample_bytes() -> bytes:
return (
"# 문서\r\n"
"\r\n"
"첫 문단은 분할 전후에 그대로 남아야 합니다.\r\n"
"\r\n"
"## 코드\r\n"
"\r\n"
"```python\r\n"
"## 이것은 제목이 아니다\r\n"
"```not-a-closing-fence\r\n"
"print('계속 같은 fence 안')\r\n"
"```\r\n"
"\r\n"
"## 끝\r\n"
"\r\n"
"마지막 문단입니다.\r\n"
).encode("utf-8")
def test_split_and_input_reassembly_are_byte_identical(tmp_path: Path) -> None:
document = tmp_path / "source.md"
document.write_bytes(sample_bytes())
chunks = tmp_path / "chunks"
split = run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
"--max-chars",
55,
)
assert split.returncode == 0, split.stderr
manifest = read_json(chunks / "manifest.json")
assert manifest["self_check"] is True
assert manifest["source"]["sha256"] == hashlib.sha256(sample_bytes()).hexdigest()
assert [item["index"] for item in manifest["chunks"]] == list(
range(1, len(manifest["chunks"]) + 1)
)
fenced_piece = [
(chunks / item["input_file"]).read_text(encoding="utf-8")
for item in manifest["chunks"]
if "```not-a-closing-fence" in (chunks / item["input_file"]).read_text(encoding="utf-8")
]
assert len(fenced_piece) == 1
assert "print('계속 같은 fence 안')" in fenced_piece[0]
assert fenced_piece[0].rstrip().endswith("```")
output = tmp_path / "round-trip.md"
assembled = run_cli(
"reassemble_document.py",
"--manifest",
chunks / "manifest.json",
"--output",
output,
)
assert assembled.returncode == 0, assembled.stderr
assert output.read_bytes() == sample_bytes()
def test_reassemble_rewritten_chunks_and_reject_tampered_input(tmp_path: Path) -> None:
document = tmp_path / "source.md"
document.write_text("# 제목\n\n첫 문단\n\n## 다음\n\n둘째 문단\n", encoding="utf-8")
chunks = tmp_path / "chunks"
split = run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
"--max-chars",
20,
)
assert split.returncode == 0, split.stderr
manifest = read_json(chunks / "manifest.json")
expected: list[str] = []
for item in manifest["chunks"]:
source = (chunks / item["input_file"]).read_text(encoding="utf-8")
rewritten = f"[청크 {item['index']}]\n{source}"
(chunks / item["rewritten_file"]).write_text(rewritten, encoding="utf-8")
expected.append(rewritten)
output = tmp_path / "rewritten.md"
assembled = run_cli(
"reassemble_document.py",
"--manifest",
chunks / "manifest.json",
"--output",
output,
"--source",
"rewritten",
)
assert assembled.returncode == 0, assembled.stderr
assert output.read_text(encoding="utf-8") == "".join(expected)
first = chunks / manifest["chunks"][0]["input_file"]
first.write_text(first.read_text(encoding="utf-8") + "tampered", encoding="utf-8")
rejected_output = tmp_path / "must-not-exist.md"
rejected = run_cli(
"reassemble_document.py",
"--manifest",
chunks / "manifest.json",
"--output",
rejected_output,
"--source",
"rewritten",
)
assert rejected.returncode == 2
assert not rejected_output.exists()
def test_reassemble_rejects_manifest_path_escape(tmp_path: Path) -> None:
document = tmp_path / "source.md"
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
chunks = tmp_path / "chunks"
assert run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
"--max-chars",
50,
).returncode == 0
manifest_path = chunks / "manifest.json"
manifest = read_json(manifest_path)
outside = tmp_path / "outside.md"
shutil.copyfile(chunks / manifest["chunks"][0]["input_file"], outside)
manifest["chunks"][0]["input_file"] = "../outside.md"
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
result = run_cli(
"reassemble_document.py",
"--manifest",
manifest_path,
"--output",
tmp_path / "output.md",
)
assert result.returncode == 2
assert "벗어납니다" in result.stderr
def test_reassemble_applies_chunk_manifest_schema(tmp_path: Path) -> None:
document = tmp_path / "source.md"
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
chunks = tmp_path / "chunks"
assert run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
).returncode == 0
manifest_path = chunks / "manifest.json"
manifest = read_json(manifest_path)
manifest["chunks"][0]["boundary_reason"] = "INVALID"
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
result = run_cli(
"reassemble_document.py",
"--manifest",
manifest_path,
"--output",
tmp_path / "output.md",
)
assert result.returncode == 2
assert "schema 위반" in result.stderr
def test_split_keeps_indented_list_fence_atomic(tmp_path: Path) -> None:
document = tmp_path / "nested.md"
nested = (
"# 제목\n\n- 항목\n\n"
" ```python\n"
" first = 1\n"
"\n"
" second = 2\n"
" ```\n\n"
"## 다음\n\n본문\n"
)
document.write_text(nested, encoding="utf-8")
chunks = tmp_path / "nested-chunks"
result = run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
"--max-chars",
40,
)
assert result.returncode == 0, result.stderr
manifest = read_json(chunks / "manifest.json")
pieces = [
(chunks / item["input_file"]).read_text(encoding="utf-8")
for item in manifest["chunks"]
]
containing_code = [piece for piece in pieces if "first = 1" in piece or "second = 2" in piece]
assert len(containing_code) == 1
assert "first = 1" in containing_code[0] and "second = 2" in containing_code[0]
def test_indented_backticks_inside_top_level_fence_do_not_close_it(tmp_path: Path) -> None:
document = tmp_path / "top-level-fence.md"
text = (
"# 제목\n\n"
"```text\n"
"line\n"
" ```\n"
"\n"
"## 이것도 코드 내용\n"
"secret\n"
"```\n\n"
"## 실제 다음 절\n\n본문\n"
)
document.write_text(text, encoding="utf-8")
chunks = tmp_path / "top-level-chunks"
result = run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
"--max-chars",
38,
)
assert result.returncode == 0, result.stderr
manifest = read_json(chunks / "manifest.json")
pieces = [
(chunks / item["input_file"]).read_text(encoding="utf-8")
for item in manifest["chunks"]
]
code_pieces = [piece for piece in pieces if "secret" in piece or "이것도 코드 내용" in piece]
assert len(code_pieces) == 1
assert " ```" in code_pieces[0] and "secret" in code_pieces[0]
def test_same_line_list_and_blockquote_fences_stay_atomic(tmp_path: Path) -> None:
document = tmp_path / "container-fences.md"
text = (
"# 제목\n\n"
"- ```python\n"
" first = 1\n\n"
" ## 목록 안 코드 제목\n"
" second = 2\n"
" ```\n\n"
"> ```text\n"
"> quote line\n"
">\n"
"> ## 인용 안 코드 제목\n"
"> ```\n\n"
"## 실제 다음 절\n\n본문\n"
)
document.write_text(text, encoding="utf-8")
chunks = tmp_path / "container-chunks"
result = run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
"--max-chars",
35,
)
assert result.returncode == 0, result.stderr
manifest = read_json(chunks / "manifest.json")
pieces = [
(chunks / item["input_file"]).read_text(encoding="utf-8")
for item in manifest["chunks"]
]
assert len([piece for piece in pieces if "first = 1" in piece or "second = 2" in piece]) == 1
assert len([piece for piece in pieces if "quote line" in piece or "인용 안 코드 제목" in piece]) == 1
def test_nested_list_heading_is_not_used_as_a_top_level_split_boundary(
tmp_path: Path,
) -> None:
document = tmp_path / "nested-list-heading.md"
text = (
"# 제목\n\n"
"도입 설명은 청크를 채우기 위해 충분한 길이로 이어진다.\n"
"- 목록 항목은 다음 설명과 소제목을 소유한다.\n"
" ## 목록 내부 소제목\n"
" 이 내용은 목록 항목에서 분리되면 의미가 달라진다.\n"
"## 실제 절\n"
"본문이다.\n"
)
document.write_text(text, encoding="utf-8")
chunks = tmp_path / "nested-list-heading-chunks"
result = run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
"--max-chars",
95,
)
assert result.returncode == 0, result.stderr
manifest = read_json(chunks / "manifest.json")
pieces = [
(chunks / item["input_file"]).read_text(encoding="utf-8")
for item in manifest["chunks"]
]
nested_piece = next(piece for piece in pieces if "목록 내부 소제목" in piece)
assert "- 목록 항목" in nested_piece
assert not any(piece.startswith(" ## 목록 내부 소제목") for piece in pieces)
def test_comment_token_inside_html_attribute_does_not_make_document_atomic(
tmp_path: Path,
) -> None:
document = tmp_path / "inline-html-attribute.md"
document.write_text(
'보이는 설명\n\n## 다음\n본문이다.\n',
encoding="utf-8",
)
chunks = tmp_path / "inline-html-attribute-chunks"
result = run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
"--max-chars",
40,
)
assert result.returncode == 0, result.stderr
manifest = read_json(chunks / "manifest.json")
assert len(manifest["chunks"]) >= 2
assert all(item["boundary_reason"] != "oversize_atomic_block" for item in manifest["chunks"])
def test_multiline_html_comment_stays_atomic(tmp_path: Path) -> None:
document = tmp_path / "comment.md"
text = (
"# 제목\n\n\n\n"
"## 실제 절\n\n본문\n"
)
document.write_text(text, encoding="utf-8")
chunks = tmp_path / "comment-chunks"
result = run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
"--max-chars",
35,
)
assert result.returncode == 0, result.stderr
manifest = read_json(chunks / "manifest.json")
pieces = [
(chunks / item["input_file"]).read_text(encoding="utf-8")
for item in manifest["chunks"]
]
comment_pieces = [piece for piece in pieces if "hidden start" in piece or "hidden end" in piece]
assert len(comment_pieces) == 1
def test_html_comment_marker_in_fence_info_does_not_hide_fence(tmp_path: Path) -> None:
document = tmp_path / "fence-info-comment.md"
text = (
"# 제목\n\n"
"```html \n"
"```\n\n"
"## 실제 절\n\n본문\n"
)
document.write_text(text, encoding="utf-8")
chunks = tmp_path / "fence-info-comment-chunks"
result = run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
"--max-chars",
38,
)
assert result.returncode == 0, result.stderr
manifest = read_json(chunks / "manifest.json")
pieces = [
(chunks / item["input_file"]).read_text(encoding="utf-8")
for item in manifest["chunks"]
]
code_pieces = [
piece
for piece in pieces
if "" in piece or "여전히 코드인 제목" in piece
]
assert len(code_pieces) == 1
assert code_pieces[0].startswith("```html \n```" in code_pieces[0]
def test_output_dangling_symlinks_are_not_followed_without_force(tmp_path: Path) -> None:
document = tmp_path / "source.md"
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
outside_dir = tmp_path / "outside-dir"
split_link = tmp_path / "split-link"
split_link.symlink_to(outside_dir, target_is_directory=True)
split = run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
split_link,
)
assert split.returncode == 2
assert split_link.is_symlink()
assert not outside_dir.exists()
chunks = tmp_path / "chunks"
assert run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
).returncode == 0
outside_file = tmp_path / "outside.md"
output_link = tmp_path / "output-link.md"
output_link.symlink_to(outside_file)
assembled = run_cli(
"reassemble_document.py",
"--manifest",
chunks / "manifest.json",
"--output",
output_link,
)
assert assembled.returncode == 2
assert output_link.is_symlink()
assert not outside_file.exists()
def test_force_reassemble_cannot_overwrite_manifest_or_chunks(tmp_path: Path) -> None:
document = tmp_path / "source.md"
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
chunks = tmp_path / "chunks"
assert run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
).returncode == 0
manifest_path = chunks / "manifest.json"
manifest_before = manifest_path.read_bytes()
manifest = read_json(manifest_path)
input_path = chunks / manifest["chunks"][0]["input_file"]
input_before = input_path.read_bytes()
for protected in (manifest_path, input_path, document):
result = run_cli(
"reassemble_document.py",
"--manifest",
manifest_path,
"--output",
protected,
"--force",
)
assert result.returncode == 2
assert "덮을 수 없습니다" in result.stderr
assert manifest_path.read_bytes() == manifest_before
assert input_path.read_bytes() == input_before
def test_thematic_break_is_a_safe_split_boundary(tmp_path: Path) -> None:
document = tmp_path / "thematic.md"
document.write_text("가" * 30 + "\n* * *\n" + "나" * 30 + "\n", encoding="utf-8")
chunks = tmp_path / "thematic-chunks"
result = run_cli(
"split_document.py",
"--document",
document,
"--output-dir",
chunks,
"--max-chars",
40,
)
assert result.returncode == 0, result.stderr
manifest = read_json(chunks / "manifest.json")
assert len(manifest["chunks"]) == 2
assert all(
item["boundary_reason"] != "oversize_atomic_block"
for item in manifest["chunks"]
)