init: llm-wiki-haness 하네스 설계

2026-07-24 14:21:35 +09:00
parent 42bf3db4fd
commit 6c53ded9cb
2436 changed files with 194486 additions and 1 deletions
@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RUNTIME = REPO_ROOT / "harness/runtime"
+sys.path.insert(0, str(RUNTIME))
+
+import proof_manifest  # noqa: E402
+
+
+class ProofHardGateTest(unittest.TestCase):
+    def setUp(self) -> None:
+        temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(temporary.cleanup)
+        base = Path(temporary.name)
+        self.repo = base / "repo"
+        self.run = base / "run"
+        self.repo.mkdir()
+        self.run.mkdir()
+        self.profiles = self.repo / "profiles.json"
+        self.profiles.write_text(
+            json.dumps({"schema_version": "execution-profiles/v1", "profiles": {"audit": {}}}),
+            encoding="utf-8",
+        )
+        source = self.run / "source.md"
+        source.write_text("evidence\n", encoding="utf-8")
+        manifest = {
+            "schema_version": proof_manifest.SCHEMA_VERSION,
+            "run": {"id": "run-hard-gate", "profile": "audit"},
+            "proofs": [{
+                "finding": {"id": "F1", "role": "quote"},
+                "source": {
+                    "namespace": "run",
+                    "path": "source.md",
+                    "sha256": hashlib.sha256(source.read_bytes()).hexdigest(),
+                    "line_start": 1,
+                    "line_end": 1,
+                    "quote_utf8": "evidence",
+                },
+                "execution": {
+                    "argv": ["proof-runner/exact-utf8-v1", "run", "source.md", "1:1"],
+                    "exit_code": 0,
+                    "stdout_utf8": "evidence",
+                    "stdout_sha256": hashlib.sha256(b"evidence").hexdigest(),
+                    "exact_match": True,
+                },
+            }],
+        }
+        verified = proof_manifest.verify_manifest(manifest, self.repo, {"audit"}, run_root=self.run)
+        self.manifest = self.run / "proof-manifest.json"
+        self.manifest.write_bytes(proof_manifest.manifest_bytes(verified))
+        self.sha256 = hashlib.sha256(self.manifest.read_bytes()).hexdigest()
+
+    def _run(self, sha256: str, proof_count: int = 1) -> subprocess.CompletedProcess[str]:
+        return subprocess.run([
+            sys.executable,
+            str(RUNTIME / "proof_hard_gate.py"),
+            str(self.manifest),
+            "--manifest-sha256", sha256,
+            "--proof-count", str(proof_count),
+            "--pass-count", "1",
+            "--fail-count", "0",
+            "--repo-root", str(self.repo),
+            "--run-root", str(self.run),
+            "--profiles", str(self.profiles),
+        ], check=False, capture_output=True, text=True)
+
+    def test_manifest_reference_hash_schema_and_counts_pass(self) -> None:
+        result = self._run(self.sha256)
+        self.assertEqual(result.returncode, 0, result.stdout)
+        body = json.loads(result.stdout)
+        self.assertEqual(body["schema_version"], "proof-hard-gate-result/v1")
+        self.assertEqual(body["status"], "PASS")
+        self.assertEqual(body["proof_count"], 1)
+
+    def test_hash_and_count_mismatch_fail_closed(self) -> None:
+        bad_hash = self._run("0" * 64)
+        self.assertEqual(bad_hash.returncode, 1)
+        self.assertIn("MANIFEST_HASH_MISMATCH", bad_hash.stdout)
+        bad_count = self._run(self.sha256, proof_count=2)
+        self.assertEqual(bad_count.returncode, 1)
+        self.assertIn("MANIFEST_COUNT_MISMATCH", bad_count.stdout)
+
+    def test_manifest_outside_repo_or_run_root_is_rejected(self) -> None:
+        outside = self.repo.parent / "outside"
+        outside.mkdir()
+        moved = outside / "proof-manifest.json"
+        moved.write_bytes(self.manifest.read_bytes())
+        self.manifest = moved
+        result = self._run(hashlib.sha256(moved.read_bytes()).hexdigest())
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("MANIFEST_OUTSIDE_ALLOWED_ROOT", result.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()