init: document-haness 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,500 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from helpers import read_json, run_cli
|
||||
|
||||
|
||||
def sample_bytes() -> bytes:
|
||||
return (
|
||||
"# 문서\r\n"
|
||||
"\r\n"
|
||||
"첫 문단은 분할 전후에 그대로 남아야 합니다.\r\n"
|
||||
"\r\n"
|
||||
"## 코드\r\n"
|
||||
"\r\n"
|
||||
"```python\r\n"
|
||||
"## 이것은 제목이 아니다\r\n"
|
||||
"```not-a-closing-fence\r\n"
|
||||
"print('계속 같은 fence 안')\r\n"
|
||||
"```\r\n"
|
||||
"\r\n"
|
||||
"## 끝\r\n"
|
||||
"\r\n"
|
||||
"마지막 문단입니다.\r\n"
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def test_split_and_input_reassembly_are_byte_identical(tmp_path: Path) -> None:
|
||||
document = tmp_path / "source.md"
|
||||
document.write_bytes(sample_bytes())
|
||||
chunks = tmp_path / "chunks"
|
||||
split = run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
"--max-chars",
|
||||
55,
|
||||
)
|
||||
assert split.returncode == 0, split.stderr
|
||||
manifest = read_json(chunks / "manifest.json")
|
||||
assert manifest["self_check"] is True
|
||||
assert manifest["source"]["sha256"] == hashlib.sha256(sample_bytes()).hexdigest()
|
||||
assert [item["index"] for item in manifest["chunks"]] == list(
|
||||
range(1, len(manifest["chunks"]) + 1)
|
||||
)
|
||||
|
||||
fenced_piece = [
|
||||
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
||||
for item in manifest["chunks"]
|
||||
if "```not-a-closing-fence" in (chunks / item["input_file"]).read_text(encoding="utf-8")
|
||||
]
|
||||
assert len(fenced_piece) == 1
|
||||
assert "print('계속 같은 fence 안')" in fenced_piece[0]
|
||||
assert fenced_piece[0].rstrip().endswith("```")
|
||||
|
||||
output = tmp_path / "round-trip.md"
|
||||
assembled = run_cli(
|
||||
"reassemble_document.py",
|
||||
"--manifest",
|
||||
chunks / "manifest.json",
|
||||
"--output",
|
||||
output,
|
||||
)
|
||||
assert assembled.returncode == 0, assembled.stderr
|
||||
assert output.read_bytes() == sample_bytes()
|
||||
|
||||
|
||||
def test_reassemble_rewritten_chunks_and_reject_tampered_input(tmp_path: Path) -> None:
|
||||
document = tmp_path / "source.md"
|
||||
document.write_text("# 제목\n\n첫 문단\n\n## 다음\n\n둘째 문단\n", encoding="utf-8")
|
||||
chunks = tmp_path / "chunks"
|
||||
split = run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
"--max-chars",
|
||||
20,
|
||||
)
|
||||
assert split.returncode == 0, split.stderr
|
||||
manifest = read_json(chunks / "manifest.json")
|
||||
expected: list[str] = []
|
||||
for item in manifest["chunks"]:
|
||||
source = (chunks / item["input_file"]).read_text(encoding="utf-8")
|
||||
rewritten = f"[청크 {item['index']}]\n{source}"
|
||||
(chunks / item["rewritten_file"]).write_text(rewritten, encoding="utf-8")
|
||||
expected.append(rewritten)
|
||||
|
||||
output = tmp_path / "rewritten.md"
|
||||
assembled = run_cli(
|
||||
"reassemble_document.py",
|
||||
"--manifest",
|
||||
chunks / "manifest.json",
|
||||
"--output",
|
||||
output,
|
||||
"--source",
|
||||
"rewritten",
|
||||
)
|
||||
assert assembled.returncode == 0, assembled.stderr
|
||||
assert output.read_text(encoding="utf-8") == "".join(expected)
|
||||
|
||||
first = chunks / manifest["chunks"][0]["input_file"]
|
||||
first.write_text(first.read_text(encoding="utf-8") + "tampered", encoding="utf-8")
|
||||
rejected_output = tmp_path / "must-not-exist.md"
|
||||
rejected = run_cli(
|
||||
"reassemble_document.py",
|
||||
"--manifest",
|
||||
chunks / "manifest.json",
|
||||
"--output",
|
||||
rejected_output,
|
||||
"--source",
|
||||
"rewritten",
|
||||
)
|
||||
assert rejected.returncode == 2
|
||||
assert not rejected_output.exists()
|
||||
|
||||
|
||||
def test_reassemble_rejects_manifest_path_escape(tmp_path: Path) -> None:
|
||||
document = tmp_path / "source.md"
|
||||
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
|
||||
chunks = tmp_path / "chunks"
|
||||
assert run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
"--max-chars",
|
||||
50,
|
||||
).returncode == 0
|
||||
manifest_path = chunks / "manifest.json"
|
||||
manifest = read_json(manifest_path)
|
||||
outside = tmp_path / "outside.md"
|
||||
shutil.copyfile(chunks / manifest["chunks"][0]["input_file"], outside)
|
||||
manifest["chunks"][0]["input_file"] = "../outside.md"
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
result = run_cli(
|
||||
"reassemble_document.py",
|
||||
"--manifest",
|
||||
manifest_path,
|
||||
"--output",
|
||||
tmp_path / "output.md",
|
||||
)
|
||||
assert result.returncode == 2
|
||||
assert "벗어납니다" in result.stderr
|
||||
|
||||
|
||||
def test_reassemble_applies_chunk_manifest_schema(tmp_path: Path) -> None:
|
||||
document = tmp_path / "source.md"
|
||||
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
|
||||
chunks = tmp_path / "chunks"
|
||||
assert run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
).returncode == 0
|
||||
manifest_path = chunks / "manifest.json"
|
||||
manifest = read_json(manifest_path)
|
||||
manifest["chunks"][0]["boundary_reason"] = "INVALID"
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
result = run_cli(
|
||||
"reassemble_document.py",
|
||||
"--manifest",
|
||||
manifest_path,
|
||||
"--output",
|
||||
tmp_path / "output.md",
|
||||
)
|
||||
assert result.returncode == 2
|
||||
assert "schema 위반" in result.stderr
|
||||
|
||||
|
||||
def test_split_keeps_indented_list_fence_atomic(tmp_path: Path) -> None:
|
||||
document = tmp_path / "nested.md"
|
||||
nested = (
|
||||
"# 제목\n\n- 항목\n\n"
|
||||
" ```python\n"
|
||||
" first = 1\n"
|
||||
"\n"
|
||||
" second = 2\n"
|
||||
" ```\n\n"
|
||||
"## 다음\n\n본문\n"
|
||||
)
|
||||
document.write_text(nested, encoding="utf-8")
|
||||
chunks = tmp_path / "nested-chunks"
|
||||
result = run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
"--max-chars",
|
||||
40,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
manifest = read_json(chunks / "manifest.json")
|
||||
pieces = [
|
||||
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
||||
for item in manifest["chunks"]
|
||||
]
|
||||
containing_code = [piece for piece in pieces if "first = 1" in piece or "second = 2" in piece]
|
||||
assert len(containing_code) == 1
|
||||
assert "first = 1" in containing_code[0] and "second = 2" in containing_code[0]
|
||||
|
||||
|
||||
def test_indented_backticks_inside_top_level_fence_do_not_close_it(tmp_path: Path) -> None:
|
||||
document = tmp_path / "top-level-fence.md"
|
||||
text = (
|
||||
"# 제목\n\n"
|
||||
"```text\n"
|
||||
"line\n"
|
||||
" ```\n"
|
||||
"\n"
|
||||
"## 이것도 코드 내용\n"
|
||||
"secret\n"
|
||||
"```\n\n"
|
||||
"## 실제 다음 절\n\n본문\n"
|
||||
)
|
||||
document.write_text(text, encoding="utf-8")
|
||||
chunks = tmp_path / "top-level-chunks"
|
||||
result = run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
"--max-chars",
|
||||
38,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
manifest = read_json(chunks / "manifest.json")
|
||||
pieces = [
|
||||
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
||||
for item in manifest["chunks"]
|
||||
]
|
||||
code_pieces = [piece for piece in pieces if "secret" in piece or "이것도 코드 내용" in piece]
|
||||
assert len(code_pieces) == 1
|
||||
assert " ```" in code_pieces[0] and "secret" in code_pieces[0]
|
||||
|
||||
|
||||
def test_same_line_list_and_blockquote_fences_stay_atomic(tmp_path: Path) -> None:
|
||||
document = tmp_path / "container-fences.md"
|
||||
text = (
|
||||
"# 제목\n\n"
|
||||
"- ```python\n"
|
||||
" first = 1\n\n"
|
||||
" ## 목록 안 코드 제목\n"
|
||||
" second = 2\n"
|
||||
" ```\n\n"
|
||||
"> ```text\n"
|
||||
"> quote line\n"
|
||||
">\n"
|
||||
"> ## 인용 안 코드 제목\n"
|
||||
"> ```\n\n"
|
||||
"## 실제 다음 절\n\n본문\n"
|
||||
)
|
||||
document.write_text(text, encoding="utf-8")
|
||||
chunks = tmp_path / "container-chunks"
|
||||
result = run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
"--max-chars",
|
||||
35,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
manifest = read_json(chunks / "manifest.json")
|
||||
pieces = [
|
||||
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
||||
for item in manifest["chunks"]
|
||||
]
|
||||
assert len([piece for piece in pieces if "first = 1" in piece or "second = 2" in piece]) == 1
|
||||
assert len([piece for piece in pieces if "quote line" in piece or "인용 안 코드 제목" in piece]) == 1
|
||||
|
||||
|
||||
def test_nested_list_heading_is_not_used_as_a_top_level_split_boundary(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
document = tmp_path / "nested-list-heading.md"
|
||||
text = (
|
||||
"# 제목\n\n"
|
||||
"도입 설명은 청크를 채우기 위해 충분한 길이로 이어진다.\n"
|
||||
"- 목록 항목은 다음 설명과 소제목을 소유한다.\n"
|
||||
" ## 목록 내부 소제목\n"
|
||||
" 이 내용은 목록 항목에서 분리되면 의미가 달라진다.\n"
|
||||
"## 실제 절\n"
|
||||
"본문이다.\n"
|
||||
)
|
||||
document.write_text(text, encoding="utf-8")
|
||||
chunks = tmp_path / "nested-list-heading-chunks"
|
||||
result = run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
"--max-chars",
|
||||
95,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
manifest = read_json(chunks / "manifest.json")
|
||||
pieces = [
|
||||
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
||||
for item in manifest["chunks"]
|
||||
]
|
||||
nested_piece = next(piece for piece in pieces if "목록 내부 소제목" in piece)
|
||||
assert "- 목록 항목" in nested_piece
|
||||
assert not any(piece.startswith(" ## 목록 내부 소제목") for piece in pieces)
|
||||
|
||||
|
||||
def test_comment_token_inside_html_attribute_does_not_make_document_atomic(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
document = tmp_path / "inline-html-attribute.md"
|
||||
document.write_text(
|
||||
'<span title="<!--">보이는 설명</span>\n\n## 다음\n본문이다.\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
chunks = tmp_path / "inline-html-attribute-chunks"
|
||||
result = run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
"--max-chars",
|
||||
40,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
manifest = read_json(chunks / "manifest.json")
|
||||
assert len(manifest["chunks"]) >= 2
|
||||
assert all(item["boundary_reason"] != "oversize_atomic_block" for item in manifest["chunks"])
|
||||
|
||||
|
||||
def test_multiline_html_comment_stays_atomic(tmp_path: Path) -> None:
|
||||
document = tmp_path / "comment.md"
|
||||
text = (
|
||||
"# 제목\n\n<!--\nhidden start\n\n## hidden heading\nhidden end\n-->\n\n"
|
||||
"## 실제 절\n\n본문\n"
|
||||
)
|
||||
document.write_text(text, encoding="utf-8")
|
||||
chunks = tmp_path / "comment-chunks"
|
||||
result = run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
"--max-chars",
|
||||
35,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
manifest = read_json(chunks / "manifest.json")
|
||||
pieces = [
|
||||
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
||||
for item in manifest["chunks"]
|
||||
]
|
||||
comment_pieces = [piece for piece in pieces if "hidden start" in piece or "hidden end" in piece]
|
||||
assert len(comment_pieces) == 1
|
||||
|
||||
|
||||
def test_html_comment_marker_in_fence_info_does_not_hide_fence(tmp_path: Path) -> None:
|
||||
document = tmp_path / "fence-info-comment.md"
|
||||
text = (
|
||||
"# 제목\n\n"
|
||||
"```html <!--\n"
|
||||
"<section>\n\n"
|
||||
"## 여전히 코드인 제목\n"
|
||||
"</section>\n"
|
||||
"-->\n"
|
||||
"```\n\n"
|
||||
"## 실제 절\n\n본문\n"
|
||||
)
|
||||
document.write_text(text, encoding="utf-8")
|
||||
chunks = tmp_path / "fence-info-comment-chunks"
|
||||
result = run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
"--max-chars",
|
||||
38,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
manifest = read_json(chunks / "manifest.json")
|
||||
pieces = [
|
||||
(chunks / item["input_file"]).read_text(encoding="utf-8")
|
||||
for item in manifest["chunks"]
|
||||
]
|
||||
code_pieces = [
|
||||
piece
|
||||
for piece in pieces
|
||||
if "<section>" in piece or "여전히 코드인 제목" in piece
|
||||
]
|
||||
assert len(code_pieces) == 1
|
||||
assert code_pieces[0].startswith("```html <!--")
|
||||
assert "-->\n```" in code_pieces[0]
|
||||
|
||||
|
||||
def test_output_dangling_symlinks_are_not_followed_without_force(tmp_path: Path) -> None:
|
||||
document = tmp_path / "source.md"
|
||||
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
|
||||
outside_dir = tmp_path / "outside-dir"
|
||||
split_link = tmp_path / "split-link"
|
||||
split_link.symlink_to(outside_dir, target_is_directory=True)
|
||||
split = run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
split_link,
|
||||
)
|
||||
assert split.returncode == 2
|
||||
assert split_link.is_symlink()
|
||||
assert not outside_dir.exists()
|
||||
|
||||
chunks = tmp_path / "chunks"
|
||||
assert run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
).returncode == 0
|
||||
outside_file = tmp_path / "outside.md"
|
||||
output_link = tmp_path / "output-link.md"
|
||||
output_link.symlink_to(outside_file)
|
||||
assembled = run_cli(
|
||||
"reassemble_document.py",
|
||||
"--manifest",
|
||||
chunks / "manifest.json",
|
||||
"--output",
|
||||
output_link,
|
||||
)
|
||||
assert assembled.returncode == 2
|
||||
assert output_link.is_symlink()
|
||||
assert not outside_file.exists()
|
||||
|
||||
|
||||
def test_force_reassemble_cannot_overwrite_manifest_or_chunks(tmp_path: Path) -> None:
|
||||
document = tmp_path / "source.md"
|
||||
document.write_text("# 제목\n\n본문\n", encoding="utf-8")
|
||||
chunks = tmp_path / "chunks"
|
||||
assert run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
).returncode == 0
|
||||
manifest_path = chunks / "manifest.json"
|
||||
manifest_before = manifest_path.read_bytes()
|
||||
manifest = read_json(manifest_path)
|
||||
input_path = chunks / manifest["chunks"][0]["input_file"]
|
||||
input_before = input_path.read_bytes()
|
||||
for protected in (manifest_path, input_path, document):
|
||||
result = run_cli(
|
||||
"reassemble_document.py",
|
||||
"--manifest",
|
||||
manifest_path,
|
||||
"--output",
|
||||
protected,
|
||||
"--force",
|
||||
)
|
||||
assert result.returncode == 2
|
||||
assert "덮을 수 없습니다" in result.stderr
|
||||
assert manifest_path.read_bytes() == manifest_before
|
||||
assert input_path.read_bytes() == input_before
|
||||
|
||||
|
||||
def test_thematic_break_is_a_safe_split_boundary(tmp_path: Path) -> None:
|
||||
document = tmp_path / "thematic.md"
|
||||
document.write_text("가" * 30 + "\n* * *\n" + "나" * 30 + "\n", encoding="utf-8")
|
||||
chunks = tmp_path / "thematic-chunks"
|
||||
result = run_cli(
|
||||
"split_document.py",
|
||||
"--document",
|
||||
document,
|
||||
"--output-dir",
|
||||
chunks,
|
||||
"--max-chars",
|
||||
40,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
manifest = read_json(chunks / "manifest.json")
|
||||
assert len(manifest["chunks"]) == 2
|
||||
assert all(
|
||||
item["boundary_reason"] != "oversize_atomic_block"
|
||||
for item in manifest["chunks"]
|
||||
)
|
||||
Reference in New Issue
Block a user