init: llm-wiki-haness 하네스 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,142 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
RUNTIME_DIR = REPO_ROOT / "harness/runtime"
|
||||
sys.path.insert(0, str(RUNTIME_DIR))
|
||||
|
||||
import proof_manifest # noqa: E402
|
||||
|
||||
|
||||
class ProofManifestTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary_directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary_directory.cleanup)
|
||||
self.root = Path(self.temporary_directory.name)
|
||||
self.source = self.root / "source.md"
|
||||
self.source_bytes = "첫째 줄\n정확한 인용문\n셋째 줄\n".encode("utf-8")
|
||||
self.source.write_bytes(self.source_bytes)
|
||||
self.quote = "정확한 인용문"
|
||||
self.manifest = {
|
||||
"schema_version": proof_manifest.SCHEMA_VERSION,
|
||||
"run": {"id": "run-20260720-01", "profile": "audit"},
|
||||
"proofs": [
|
||||
{
|
||||
"finding": {"id": "L1-F01", "role": "current_state"},
|
||||
"source": {
|
||||
"path": "source.md",
|
||||
"sha256": hashlib.sha256(self.source_bytes).hexdigest(),
|
||||
"line_start": 2,
|
||||
"line_end": 2,
|
||||
"quote_utf8": self.quote,
|
||||
},
|
||||
"execution": {
|
||||
"argv": ["grep", "-nF", "--", self.quote, "source.md"],
|
||||
"exit_code": 0,
|
||||
"stdout_utf8": self.quote,
|
||||
"stdout_sha256": hashlib.sha256(self.quote.encode("utf-8")).hexdigest(),
|
||||
"exact_match": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
def _run_cli(self, manifest: dict, *extra_args: str) -> subprocess.CompletedProcess[str]:
|
||||
manifest_path = self.root / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8")
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(RUNTIME_DIR / "proof_manifest.py"),
|
||||
str(manifest_path),
|
||||
"--repo-root",
|
||||
str(self.root),
|
||||
*extra_args,
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
def test_valid_unicode_quote_passes_without_implicit_output_file(self) -> None:
|
||||
before = {path.name for path in self.root.iterdir()}
|
||||
result = self._run_cli(self.manifest)
|
||||
after = {path.name for path in self.root.iterdir()}
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stdout)
|
||||
verification = json.loads(result.stdout)["verification"]
|
||||
self.assertEqual(verification["status"], "PASS")
|
||||
self.assertEqual((verification["proof_count"], verification["pass_count"], verification["fail_count"]), (1, 1, 0))
|
||||
self.assertEqual(after - before, {"manifest.json"})
|
||||
|
||||
def test_output_is_written_only_when_explicitly_requested(self) -> None:
|
||||
output = self.root / "verified.json"
|
||||
result = self._run_cli(self.manifest, "--output", str(output))
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stdout)
|
||||
self.assertTrue(output.is_file())
|
||||
self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["verification"]["status"], "PASS")
|
||||
|
||||
def test_fail_closed_cli_cases_return_nonzero(self) -> None:
|
||||
cases: list[tuple[str, dict, str]] = []
|
||||
|
||||
nonexistent = deepcopy(self.manifest)
|
||||
nonexistent["proofs"][0]["source"]["path"] = "missing.md"
|
||||
cases.append(("nonexistent", nonexistent, "SOURCE_NOT_FOUND"))
|
||||
|
||||
bad_source_hash = deepcopy(self.manifest)
|
||||
bad_source_hash["proofs"][0]["source"]["sha256"] = "0" * 64
|
||||
cases.append(("source_hash", bad_source_hash, "SOURCE_HASH_MISMATCH"))
|
||||
|
||||
quote_mismatch = deepcopy(self.manifest)
|
||||
quote_mismatch["proofs"][0]["source"]["quote_utf8"] = "없는 인용문"
|
||||
quote_mismatch["proofs"][0]["execution"]["stdout_utf8"] = "없는 인용문"
|
||||
quote_mismatch["proofs"][0]["execution"]["stdout_sha256"] = hashlib.sha256("없는 인용문".encode("utf-8")).hexdigest()
|
||||
cases.append(("quote", quote_mismatch, "QUOTE_MISMATCH"))
|
||||
|
||||
duplicate = deepcopy(self.manifest)
|
||||
duplicate["proofs"].append(deepcopy(duplicate["proofs"][0]))
|
||||
cases.append(("duplicate", duplicate, "DUPLICATE_FINDING_ROLE"))
|
||||
|
||||
for name, manifest, expected_code in cases:
|
||||
with self.subTest(name=name):
|
||||
result = self._run_cli(manifest)
|
||||
body = json.loads(result.stdout)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(body["status"], "FAIL")
|
||||
self.assertIn(expected_code, {error["code"] for error in body["errors"]})
|
||||
|
||||
def test_stdout_hash_and_exact_match_are_validated(self) -> None:
|
||||
bad = deepcopy(self.manifest)
|
||||
bad["proofs"][0]["execution"]["stdout_sha256"] = "f" * 64
|
||||
bad["proofs"][0]["execution"]["exact_match"] = False
|
||||
|
||||
with self.assertRaises(proof_manifest.ManifestValidationError) as raised:
|
||||
proof_manifest.verify_manifest(bad, self.root.resolve(), {"audit"})
|
||||
|
||||
codes = {issue["code"] for issue in raised.exception.issues}
|
||||
self.assertIn("STDOUT_HASH_MISMATCH", codes)
|
||||
self.assertIn("EXACT_MATCH_FALSE", codes)
|
||||
|
||||
def test_quote_must_be_inside_declared_line_range(self) -> None:
|
||||
bad = deepcopy(self.manifest)
|
||||
bad["proofs"][0]["source"]["line_start"] = 1
|
||||
bad["proofs"][0]["source"]["line_end"] = 1
|
||||
|
||||
with self.assertRaises(proof_manifest.ManifestValidationError) as raised:
|
||||
proof_manifest.verify_manifest(bad, self.root.resolve(), {"audit"})
|
||||
|
||||
self.assertIn("QUOTE_MISMATCH", {issue["code"] for issue in raised.exception.issues})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user