from __future__ import annotations import hashlib import json import shutil from pathlib import Path from helpers import read_json, run_cli def sample_bytes() -> bytes: return ( "# 문서\r\n" "\r\n" "첫 문단은 분할 전후에 그대로 남아야 합니다.\r\n" "\r\n" "## 코드\r\n" "\r\n" "```python\r\n" "## 이것은 제목이 아니다\r\n" "```not-a-closing-fence\r\n" "print('계속 같은 fence 안')\r\n" "```\r\n" "\r\n" "## 끝\r\n" "\r\n" "마지막 문단입니다.\r\n" ).encode("utf-8") def test_split_and_input_reassembly_are_byte_identical(tmp_path: Path) -> None: document = tmp_path / "source.md" document.write_bytes(sample_bytes()) chunks = tmp_path / "chunks" split = run_cli( "split_document.py", "--document", document, "--output-dir", chunks, "--max-chars", 55, ) assert split.returncode == 0, split.stderr manifest = read_json(chunks / "manifest.json") assert manifest["self_check"] is True assert manifest["source"]["sha256"] == hashlib.sha256(sample_bytes()).hexdigest() assert [item["index"] for item in manifest["chunks"]] == list( range(1, len(manifest["chunks"]) + 1) ) fenced_piece = [ (chunks / item["input_file"]).read_text(encoding="utf-8") for item in manifest["chunks"] if "```not-a-closing-fence" in (chunks / item["input_file"]).read_text(encoding="utf-8") ] assert len(fenced_piece) == 1 assert "print('계속 같은 fence 안')" in fenced_piece[0] assert fenced_piece[0].rstrip().endswith("```") output = tmp_path / "round-trip.md" assembled = run_cli( "reassemble_document.py", "--manifest", chunks / "manifest.json", "--output", output, ) assert assembled.returncode == 0, assembled.stderr assert output.read_bytes() == sample_bytes() def test_reassemble_rewritten_chunks_and_reject_tampered_input(tmp_path: Path) -> None: document = tmp_path / "source.md" document.write_text("# 제목\n\n첫 문단\n\n## 다음\n\n둘째 문단\n", encoding="utf-8") chunks = tmp_path / "chunks" split = run_cli( "split_document.py", "--document", document, "--output-dir", chunks, "--max-chars", 20, ) assert split.returncode == 0, split.stderr manifest = read_json(chunks / "manifest.json") expected: list[str] = [] for item in manifest["chunks"]: source = (chunks / item["input_file"]).read_text(encoding="utf-8") rewritten = f"[청크 {item['index']}]\n{source}" (chunks / item["rewritten_file"]).write_text(rewritten, encoding="utf-8") expected.append(rewritten) output = tmp_path / "rewritten.md" assembled = run_cli( "reassemble_document.py", "--manifest", chunks / "manifest.json", "--output", output, "--source", "rewritten", ) assert assembled.returncode == 0, assembled.stderr assert output.read_text(encoding="utf-8") == "".join(expected) first = chunks / manifest["chunks"][0]["input_file"] first.write_text(first.read_text(encoding="utf-8") + "tampered", encoding="utf-8") rejected_output = tmp_path / "must-not-exist.md" rejected = run_cli( "reassemble_document.py", "--manifest", chunks / "manifest.json", "--output", rejected_output, "--source", "rewritten", ) assert rejected.returncode == 2 assert not rejected_output.exists() def test_reassemble_rejects_manifest_path_escape(tmp_path: Path) -> None: document = tmp_path / "source.md" document.write_text("# 제목\n\n본문\n", encoding="utf-8") chunks = tmp_path / "chunks" assert run_cli( "split_document.py", "--document", document, "--output-dir", chunks, "--max-chars", 50, ).returncode == 0 manifest_path = chunks / "manifest.json" manifest = read_json(manifest_path) outside = tmp_path / "outside.md" shutil.copyfile(chunks / manifest["chunks"][0]["input_file"], outside) manifest["chunks"][0]["input_file"] = "../outside.md" manifest_path.write_text(json.dumps(manifest), encoding="utf-8") result = run_cli( "reassemble_document.py", "--manifest", manifest_path, "--output", tmp_path / "output.md", ) assert result.returncode == 2 assert "벗어납니다" in result.stderr def test_reassemble_applies_chunk_manifest_schema(tmp_path: Path) -> None: document = tmp_path / "source.md" document.write_text("# 제목\n\n본문\n", encoding="utf-8") chunks = tmp_path / "chunks" assert run_cli( "split_document.py", "--document", document, "--output-dir", chunks, ).returncode == 0 manifest_path = chunks / "manifest.json" manifest = read_json(manifest_path) manifest["chunks"][0]["boundary_reason"] = "INVALID" manifest_path.write_text(json.dumps(manifest), encoding="utf-8") result = run_cli( "reassemble_document.py", "--manifest", manifest_path, "--output", tmp_path / "output.md", ) assert result.returncode == 2 assert "schema 위반" in result.stderr def test_split_keeps_indented_list_fence_atomic(tmp_path: Path) -> None: document = tmp_path / "nested.md" nested = ( "# 제목\n\n- 항목\n\n" " ```python\n" " first = 1\n" "\n" " second = 2\n" " ```\n\n" "## 다음\n\n본문\n" ) document.write_text(nested, encoding="utf-8") chunks = tmp_path / "nested-chunks" result = run_cli( "split_document.py", "--document", document, "--output-dir", chunks, "--max-chars", 40, ) assert result.returncode == 0, result.stderr manifest = read_json(chunks / "manifest.json") pieces = [ (chunks / item["input_file"]).read_text(encoding="utf-8") for item in manifest["chunks"] ] containing_code = [piece for piece in pieces if "first = 1" in piece or "second = 2" in piece] assert len(containing_code) == 1 assert "first = 1" in containing_code[0] and "second = 2" in containing_code[0] def test_indented_backticks_inside_top_level_fence_do_not_close_it(tmp_path: Path) -> None: document = tmp_path / "top-level-fence.md" text = ( "# 제목\n\n" "```text\n" "line\n" " ```\n" "\n" "## 이것도 코드 내용\n" "secret\n" "```\n\n" "## 실제 다음 절\n\n본문\n" ) document.write_text(text, encoding="utf-8") chunks = tmp_path / "top-level-chunks" result = run_cli( "split_document.py", "--document", document, "--output-dir", chunks, "--max-chars", 38, ) assert result.returncode == 0, result.stderr manifest = read_json(chunks / "manifest.json") pieces = [ (chunks / item["input_file"]).read_text(encoding="utf-8") for item in manifest["chunks"] ] code_pieces = [piece for piece in pieces if "secret" in piece or "이것도 코드 내용" in piece] assert len(code_pieces) == 1 assert " ```" in code_pieces[0] and "secret" in code_pieces[0] def test_same_line_list_and_blockquote_fences_stay_atomic(tmp_path: Path) -> None: document = tmp_path / "container-fences.md" text = ( "# 제목\n\n" "- ```python\n" " first = 1\n\n" " ## 목록 안 코드 제목\n" " second = 2\n" " ```\n\n" "> ```text\n" "> quote line\n" ">\n" "> ## 인용 안 코드 제목\n" "> ```\n\n" "## 실제 다음 절\n\n본문\n" ) document.write_text(text, encoding="utf-8") chunks = tmp_path / "container-chunks" result = run_cli( "split_document.py", "--document", document, "--output-dir", chunks, "--max-chars", 35, ) assert result.returncode == 0, result.stderr manifest = read_json(chunks / "manifest.json") pieces = [ (chunks / item["input_file"]).read_text(encoding="utf-8") for item in manifest["chunks"] ] assert len([piece for piece in pieces if "first = 1" in piece or "second = 2" in piece]) == 1 assert len([piece for piece in pieces if "quote line" in piece or "인용 안 코드 제목" in piece]) == 1 def test_nested_list_heading_is_not_used_as_a_top_level_split_boundary( tmp_path: Path, ) -> None: document = tmp_path / "nested-list-heading.md" text = ( "# 제목\n\n" "도입 설명은 청크를 채우기 위해 충분한 길이로 이어진다.\n" "- 목록 항목은 다음 설명과 소제목을 소유한다.\n" " ## 목록 내부 소제목\n" " 이 내용은 목록 항목에서 분리되면 의미가 달라진다.\n" "## 실제 절\n" "본문이다.\n" ) document.write_text(text, encoding="utf-8") chunks = tmp_path / "nested-list-heading-chunks" result = run_cli( "split_document.py", "--document", document, "--output-dir", chunks, "--max-chars", 95, ) assert result.returncode == 0, result.stderr manifest = read_json(chunks / "manifest.json") pieces = [ (chunks / item["input_file"]).read_text(encoding="utf-8") for item in manifest["chunks"] ] nested_piece = next(piece for piece in pieces if "목록 내부 소제목" in piece) assert "- 목록 항목" in nested_piece assert not any(piece.startswith(" ## 목록 내부 소제목") for piece in pieces) def test_comment_token_inside_html_attribute_does_not_make_document_atomic( tmp_path: Path, ) -> None: document = tmp_path / "inline-html-attribute.md" document.write_text( '보이는 설명\n\n## 다음\n본문이다.\n', encoding="utf-8", ) chunks = tmp_path / "inline-html-attribute-chunks" result = run_cli( "split_document.py", "--document", document, "--output-dir", chunks, "--max-chars", 40, ) assert result.returncode == 0, result.stderr manifest = read_json(chunks / "manifest.json") assert len(manifest["chunks"]) >= 2 assert all(item["boundary_reason"] != "oversize_atomic_block" for item in manifest["chunks"]) def test_multiline_html_comment_stays_atomic(tmp_path: Path) -> None: document = tmp_path / "comment.md" text = ( "# 제목\n\n\n\n" "## 실제 절\n\n본문\n" ) document.write_text(text, encoding="utf-8") chunks = tmp_path / "comment-chunks" result = run_cli( "split_document.py", "--document", document, "--output-dir", chunks, "--max-chars", 35, ) assert result.returncode == 0, result.stderr manifest = read_json(chunks / "manifest.json") pieces = [ (chunks / item["input_file"]).read_text(encoding="utf-8") for item in manifest["chunks"] ] comment_pieces = [piece for piece in pieces if "hidden start" in piece or "hidden end" in piece] assert len(comment_pieces) == 1 def test_html_comment_marker_in_fence_info_does_not_hide_fence(tmp_path: Path) -> None: document = tmp_path / "fence-info-comment.md" text = ( "# 제목\n\n" "```html \n" "```\n\n" "## 실제 절\n\n본문\n" ) document.write_text(text, encoding="utf-8") chunks = tmp_path / "fence-info-comment-chunks" result = run_cli( "split_document.py", "--document", document, "--output-dir", chunks, "--max-chars", 38, ) assert result.returncode == 0, result.stderr manifest = read_json(chunks / "manifest.json") pieces = [ (chunks / item["input_file"]).read_text(encoding="utf-8") for item in manifest["chunks"] ] code_pieces = [ piece for piece in pieces if "
" in piece or "여전히 코드인 제목" in piece ] assert len(code_pieces) == 1 assert code_pieces[0].startswith("```html \n```" in code_pieces[0] def test_output_dangling_symlinks_are_not_followed_without_force(tmp_path: Path) -> None: document = tmp_path / "source.md" document.write_text("# 제목\n\n본문\n", encoding="utf-8") outside_dir = tmp_path / "outside-dir" split_link = tmp_path / "split-link" split_link.symlink_to(outside_dir, target_is_directory=True) split = run_cli( "split_document.py", "--document", document, "--output-dir", split_link, ) assert split.returncode == 2 assert split_link.is_symlink() assert not outside_dir.exists() chunks = tmp_path / "chunks" assert run_cli( "split_document.py", "--document", document, "--output-dir", chunks, ).returncode == 0 outside_file = tmp_path / "outside.md" output_link = tmp_path / "output-link.md" output_link.symlink_to(outside_file) assembled = run_cli( "reassemble_document.py", "--manifest", chunks / "manifest.json", "--output", output_link, ) assert assembled.returncode == 2 assert output_link.is_symlink() assert not outside_file.exists() def test_force_reassemble_cannot_overwrite_manifest_or_chunks(tmp_path: Path) -> None: document = tmp_path / "source.md" document.write_text("# 제목\n\n본문\n", encoding="utf-8") chunks = tmp_path / "chunks" assert run_cli( "split_document.py", "--document", document, "--output-dir", chunks, ).returncode == 0 manifest_path = chunks / "manifest.json" manifest_before = manifest_path.read_bytes() manifest = read_json(manifest_path) input_path = chunks / manifest["chunks"][0]["input_file"] input_before = input_path.read_bytes() for protected in (manifest_path, input_path, document): result = run_cli( "reassemble_document.py", "--manifest", manifest_path, "--output", protected, "--force", ) assert result.returncode == 2 assert "덮을 수 없습니다" in result.stderr assert manifest_path.read_bytes() == manifest_before assert input_path.read_bytes() == input_before def test_thematic_break_is_a_safe_split_boundary(tmp_path: Path) -> None: document = tmp_path / "thematic.md" document.write_text("가" * 30 + "\n* * *\n" + "나" * 30 + "\n", encoding="utf-8") chunks = tmp_path / "thematic-chunks" result = run_cli( "split_document.py", "--document", document, "--output-dir", chunks, "--max-chars", 40, ) assert result.returncode == 0, result.stderr manifest = read_json(chunks / "manifest.json") assert len(manifest["chunks"]) == 2 assert all( item["boundary_reason"] != "oversize_atomic_block" for item in manifest["chunks"] )