init: llm-wiki-haness 하네스 설계

2026-07-24 14:21:35 +09:00
parent 42bf3db4fd
commit 6c53ded9cb
2436 changed files with 194486 additions and 1 deletions
@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import sys
+import tempfile
+import unittest
+
+
+RUNTIME = Path(__file__).resolve().parents[1] / "runtime"
+if str(RUNTIME) not in sys.path:
+    sys.path.insert(0, str(RUNTIME))
+
+import semantic_audit  # noqa: E402
+import semantic_candidate_builder as builder  # noqa: E402
+import semantic_surface_extractor as extractor  # noqa: E402
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+class SemanticAuditTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self.tempdir = tempfile.TemporaryDirectory()
+        self.root = Path(self.tempdir.name)
+        (self.root / "harness/source").mkdir(parents=True)
+        for source in (builder.DEFAULT_ONTOLOGY, extractor.DEFAULT_POLICY, Path("harness/source/execution-profiles.json")):
+            shutil.copyfile(REPO_ROOT / source, self.root / source)
+        (self.root / "raw/branch-notes").mkdir(parents=True)
+        self.path = self.root / "raw/branch-notes/feature-audit.md"
+        self.path.write_text(
+            "---\n"
+            "title: audit fixture\n"
+            "source_type: branch-note\n"
+            "status: verified\n"
+            "semantic_surface_exclusions:\n"
+            "  - branch-contract-packet|fixture\n"
+            "  - scope|fixture\n"
+            "  - decision-evidence|fixture\n"
+            "  - edge-failure-dependency|fixture\n"
+            "  - claims-to-verify|fixture\n"
+            "---\n\n"
+            "<!-- section-id: implementation -->\n"
+            "## 구현\n"
+            "stage owner is mapper\n"
+            "stage owner is adapter\n",
+            encoding="utf-8",
+        )
+        policy = extractor.load_policy(self.root)
+        extraction = extractor.extract_document(self.root, self.path, policy)
+        surface = extraction["surfaces"][0]
+        lines = self.path.read_text(encoding="utf-8").splitlines()
+        assertions = []
+        for index, quote in enumerate(("stage owner is mapper", "stage owner is adapter"), 1):
+            line = lines.index(quote) + 1
+            assertions.append({
+                "assertion_id": f"A{index}",
+                "source_surface": surface["surface_id"],
+                "subject": "stage-7",
+                "predicate": "owns",
+                "object": quote.rsplit(" ", 1)[-1],
+                "condition": "normal-path",
+                "modality": "must",
+                "scope": "branch",
+                "quote": quote,
+                "line_start": line,
+                "line_end": line,
+            })
+        assertion_result = {
+            "schema_version": builder.ASSERTION_SCHEMA,
+            "subject": extraction["path"],
+            "mode": "local",
+            "surface_manifest_sha256": hashlib.sha256(extractor.canonical_json_bytes(extraction)).hexdigest(),
+            "assertions": assertions,
+        }
+        self.candidates = builder.build(self.root, extraction, assertion_result)
+        self.request = semantic_audit.build_verdict_request(self.candidates)
+
+    def tearDown(self) -> None:
+        self.tempdir.cleanup()
+
+    def _result(self, verdict: str, proof: dict[str, str] | None = None) -> dict[str, object]:
+        candidate = self.request["candidates"][0]
+        by_id = {item["assertion_id"]: item for item in self.request["assertions"]}
+        left, right = by_id[candidate["assertion_a"]], by_id[candidate["assertion_b"]]
+        return {
+            "schema_version": semantic_audit.AUDIT_RESULT_SCHEMA,
+            "request_sha256": hashlib.sha256(semantic_audit.canonical_json_bytes(self.request)).hexdigest(),
+            "subject": self.request["subject"],
+            "mode": self.request["mode"],
+            "auditor": {"contract_version": "semantic-coherence/v1", "model_id": "test-model", "run_id": "run-1"},
+            "verdicts": [{
+                "candidate_id": candidate["candidate_id"],
+                "verdict": verdict,
+                "rationale": "fixture verdict",
+                "evidence_a": {"quote": left["quote"], "line_start": left["line_start"], "line_end": left["line_end"]},
+                "evidence_b": {"quote": right["quote"], "line_start": right["line_start"], "line_end": right["line_end"]},
+                "proof_manifest": proof,
+            }],
+        }
+
+    def _proof(self) -> dict[str, str]:
+        candidate = self.request["candidates"][0]
+        by_id = {item["assertion_id"]: item for item in self.request["assertions"]}
+        source_bytes = self.path.read_bytes()
+        proofs = []
+        for role, key in (("assertion_a", "assertion_a"), ("assertion_b", "assertion_b")):
+            assertion = by_id[candidate[key]]
+            quote = assertion["quote"]
+            proofs.append({
+                "finding": {"id": candidate["candidate_id"], "role": role},
+                "source": {
+                    "path": self.path.relative_to(self.root).as_posix(),
+                    "sha256": hashlib.sha256(source_bytes).hexdigest(),
+                    "line_start": assertion["line_start"],
+                    "line_end": assertion["line_end"],
+                    "quote_utf8": quote,
+                },
+                "execution": {
+                    "argv": ["proof-runner", candidate["candidate_id"], role],
+                    "exit_code": 0,
+                    "stdout_utf8": quote,
+                    "stdout_sha256": hashlib.sha256(quote.encode("utf-8")).hexdigest(),
+                    "exact_match": True,
+                },
+            })
+        manifest = {
+            "schema_version": "proof-manifest/v1",
+            "run": {"id": "semantic-run-1", "profile": "audit"},
+            "proofs": proofs,
+        }
+        verified = __import__("proof_manifest").verify_manifest(manifest, self.root, {"audit"})
+        path = self.root / "proof.json"
+        path.write_text(json.dumps(verified, ensure_ascii=False, sort_keys=True), encoding="utf-8")
+        return {"namespace": "repo", "path": "proof.json", "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
+
+    def test_positive_verdict_passes_without_finding_proof(self) -> None:
+        validated = semantic_audit.validate_result(self.root, self.request, self._result("COMPLEMENTARY"))
+        self.assertEqual(validated["status"], "PASS")
+        self.assertEqual(validated["coverage"]["processed_pairs"], 1)
+
+    def test_verified_contradiction_and_explicit_blocking_win(self) -> None:
+        validated = semantic_audit.validate_result(self.root, self.request, self._result("CONTRADICTION", self._proof()))
+        self.assertEqual(validated["status"], "FAIL")
+        self.assertEqual(validated["counts"]["blocking"], 1)
+        explicit_request = semantic_audit.build_verdict_request(
+            self.candidates,
+            explicit_blocking=[{"code": "TYPED_BLOCK", "message": "deterministic blocker"}],
+        )
+        positive = self._result("CONSISTENT")
+        positive["request_sha256"] = hashlib.sha256(semantic_audit.canonical_json_bytes(explicit_request)).hexdigest()
+        validated = semantic_audit.validate_result(self.root, explicit_request, positive)
+        self.assertEqual(validated["status"], "FAIL")
+        self.assertEqual(validated["counts"]["blocking"], 1)
+
+    def test_unverified_hub_finding_is_dropped_and_blocks_pass(self) -> None:
+        policy_path = self.root / extractor.DEFAULT_POLICY
+        policy_document = json.loads(policy_path.read_text(encoding="utf-8"))
+        policy_document["documents"]["branch-note"]["mode"] = "hub"
+        policy_path.write_text(json.dumps(policy_document), encoding="utf-8")
+        extraction = extractor.extract_document(self.root, self.path, extractor.load_policy(self.root))
+        assertion_result = {
+            "schema_version": builder.ASSERTION_SCHEMA,
+            "subject": extraction["path"],
+            "mode": "hub",
+            "surface_manifest_sha256": hashlib.sha256(extractor.canonical_json_bytes(extraction)).hexdigest(),
+            "assertions": self.request["assertions"],
+        }
+        hub_candidates = builder.build(self.root, extraction, assertion_result)
+        request = semantic_audit.build_verdict_request(hub_candidates)
+        result = self._result("CONTRADICTION", {"namespace": "repo", "path": "missing.json", "sha256": "0" * 64})
+        result["mode"] = "hub"
+        result["request_sha256"] = hashlib.sha256(semantic_audit.canonical_json_bytes(request)).hexdigest()
+        validated = semantic_audit.validate_result(self.root, request, result)
+        self.assertEqual(validated["status"], "FAIL")
+        self.assertEqual(validated["counts"]["dropped_pairs"], 1)
+        self.assertEqual(validated["counts"]["verified_findings"], 0)
+
+    def test_unverified_local_finding_is_dropped_and_blocks_certificate(self) -> None:
+        result = self._result("CONTRADICTION", {"namespace": "repo", "path": "missing.json", "sha256": "0" * 64})
+        validated = semantic_audit.validate_result(self.root, self.request, result)
+        self.assertEqual(validated["status"], "FAIL")
+        self.assertEqual(validated["counts"]["dropped_pairs"], 1)
+        self.assertEqual(validated["counts"]["verified_findings"], 0)
+
+    def test_forged_request_or_stale_document_cannot_be_certified(self) -> None:
+        forged = json.loads(json.dumps(self.request))
+        forged["candidates"][0]["rule_ids"] = ["C7"]
+        result = self._result("CONSISTENT")
+        result["request_sha256"] = hashlib.sha256(semantic_audit.canonical_json_bytes(forged)).hexdigest()
+        with self.assertRaises(semantic_audit.SemanticAuditError):
+            semantic_audit.validate_result(self.root, forged, result)
+
+        self.path.write_text(self.path.read_text(encoding="utf-8") + "changed\n", encoding="utf-8")
+        with self.assertRaises(semantic_audit.SemanticAuditError):
+            semantic_audit.validate_result(self.root, self.request, self._result("CONSISTENT"))
+
+
+if __name__ == "__main__":
+    unittest.main()