203 lines
9.7 KiB
Python
203 lines
9.7 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
|
|
RUNTIME = Path(__file__).resolve().parents[1] / "runtime"
|
|
if str(RUNTIME) not in sys.path:
|
|
sys.path.insert(0, str(RUNTIME))
|
|
|
|
import semantic_audit # noqa: E402
|
|
import semantic_candidate_builder as builder # noqa: E402
|
|
import semantic_surface_extractor as extractor # noqa: E402
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
class SemanticAuditTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tempdir = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tempdir.name)
|
|
(self.root / "harness/source").mkdir(parents=True)
|
|
for source in (builder.DEFAULT_ONTOLOGY, extractor.DEFAULT_POLICY, Path("harness/source/execution-profiles.json")):
|
|
shutil.copyfile(REPO_ROOT / source, self.root / source)
|
|
(self.root / "raw/branch-notes").mkdir(parents=True)
|
|
self.path = self.root / "raw/branch-notes/feature-audit.md"
|
|
self.path.write_text(
|
|
"---\n"
|
|
"title: audit fixture\n"
|
|
"source_type: branch-note\n"
|
|
"status: verified\n"
|
|
"semantic_surface_exclusions:\n"
|
|
" - branch-contract-packet|fixture\n"
|
|
" - scope|fixture\n"
|
|
" - decision-evidence|fixture\n"
|
|
" - edge-failure-dependency|fixture\n"
|
|
" - claims-to-verify|fixture\n"
|
|
"---\n\n"
|
|
"<!-- section-id: implementation -->\n"
|
|
"## 구현\n"
|
|
"stage owner is mapper\n"
|
|
"stage owner is adapter\n",
|
|
encoding="utf-8",
|
|
)
|
|
policy = extractor.load_policy(self.root)
|
|
extraction = extractor.extract_document(self.root, self.path, policy)
|
|
surface = extraction["surfaces"][0]
|
|
lines = self.path.read_text(encoding="utf-8").splitlines()
|
|
assertions = []
|
|
for index, quote in enumerate(("stage owner is mapper", "stage owner is adapter"), 1):
|
|
line = lines.index(quote) + 1
|
|
assertions.append({
|
|
"assertion_id": f"A{index}",
|
|
"source_surface": surface["surface_id"],
|
|
"subject": "stage-7",
|
|
"predicate": "owns",
|
|
"object": quote.rsplit(" ", 1)[-1],
|
|
"condition": "normal-path",
|
|
"modality": "must",
|
|
"scope": "branch",
|
|
"quote": quote,
|
|
"line_start": line,
|
|
"line_end": line,
|
|
})
|
|
assertion_result = {
|
|
"schema_version": builder.ASSERTION_SCHEMA,
|
|
"subject": extraction["path"],
|
|
"mode": "local",
|
|
"surface_manifest_sha256": hashlib.sha256(extractor.canonical_json_bytes(extraction)).hexdigest(),
|
|
"assertions": assertions,
|
|
}
|
|
self.candidates = builder.build(self.root, extraction, assertion_result)
|
|
self.request = semantic_audit.build_verdict_request(self.candidates)
|
|
|
|
def tearDown(self) -> None:
|
|
self.tempdir.cleanup()
|
|
|
|
def _result(self, verdict: str, proof: dict[str, str] | None = None) -> dict[str, object]:
|
|
candidate = self.request["candidates"][0]
|
|
by_id = {item["assertion_id"]: item for item in self.request["assertions"]}
|
|
left, right = by_id[candidate["assertion_a"]], by_id[candidate["assertion_b"]]
|
|
return {
|
|
"schema_version": semantic_audit.AUDIT_RESULT_SCHEMA,
|
|
"request_sha256": hashlib.sha256(semantic_audit.canonical_json_bytes(self.request)).hexdigest(),
|
|
"subject": self.request["subject"],
|
|
"mode": self.request["mode"],
|
|
"auditor": {"contract_version": "semantic-coherence/v1", "model_id": "test-model", "run_id": "run-1"},
|
|
"verdicts": [{
|
|
"candidate_id": candidate["candidate_id"],
|
|
"verdict": verdict,
|
|
"rationale": "fixture verdict",
|
|
"evidence_a": {"quote": left["quote"], "line_start": left["line_start"], "line_end": left["line_end"]},
|
|
"evidence_b": {"quote": right["quote"], "line_start": right["line_start"], "line_end": right["line_end"]},
|
|
"proof_manifest": proof,
|
|
}],
|
|
}
|
|
|
|
def _proof(self) -> dict[str, str]:
|
|
candidate = self.request["candidates"][0]
|
|
by_id = {item["assertion_id"]: item for item in self.request["assertions"]}
|
|
source_bytes = self.path.read_bytes()
|
|
proofs = []
|
|
for role, key in (("assertion_a", "assertion_a"), ("assertion_b", "assertion_b")):
|
|
assertion = by_id[candidate[key]]
|
|
quote = assertion["quote"]
|
|
proofs.append({
|
|
"finding": {"id": candidate["candidate_id"], "role": role},
|
|
"source": {
|
|
"path": self.path.relative_to(self.root).as_posix(),
|
|
"sha256": hashlib.sha256(source_bytes).hexdigest(),
|
|
"line_start": assertion["line_start"],
|
|
"line_end": assertion["line_end"],
|
|
"quote_utf8": quote,
|
|
},
|
|
"execution": {
|
|
"argv": ["proof-runner", candidate["candidate_id"], role],
|
|
"exit_code": 0,
|
|
"stdout_utf8": quote,
|
|
"stdout_sha256": hashlib.sha256(quote.encode("utf-8")).hexdigest(),
|
|
"exact_match": True,
|
|
},
|
|
})
|
|
manifest = {
|
|
"schema_version": "proof-manifest/v1",
|
|
"run": {"id": "semantic-run-1", "profile": "audit"},
|
|
"proofs": proofs,
|
|
}
|
|
verified = __import__("proof_manifest").verify_manifest(manifest, self.root, {"audit"})
|
|
path = self.root / "proof.json"
|
|
path.write_text(json.dumps(verified, ensure_ascii=False, sort_keys=True), encoding="utf-8")
|
|
return {"namespace": "repo", "path": "proof.json", "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
|
|
|
|
def test_positive_verdict_passes_without_finding_proof(self) -> None:
|
|
validated = semantic_audit.validate_result(self.root, self.request, self._result("COMPLEMENTARY"))
|
|
self.assertEqual(validated["status"], "PASS")
|
|
self.assertEqual(validated["coverage"]["processed_pairs"], 1)
|
|
|
|
def test_verified_contradiction_and_explicit_blocking_win(self) -> None:
|
|
validated = semantic_audit.validate_result(self.root, self.request, self._result("CONTRADICTION", self._proof()))
|
|
self.assertEqual(validated["status"], "FAIL")
|
|
self.assertEqual(validated["counts"]["blocking"], 1)
|
|
explicit_request = semantic_audit.build_verdict_request(
|
|
self.candidates,
|
|
explicit_blocking=[{"code": "TYPED_BLOCK", "message": "deterministic blocker"}],
|
|
)
|
|
positive = self._result("CONSISTENT")
|
|
positive["request_sha256"] = hashlib.sha256(semantic_audit.canonical_json_bytes(explicit_request)).hexdigest()
|
|
validated = semantic_audit.validate_result(self.root, explicit_request, positive)
|
|
self.assertEqual(validated["status"], "FAIL")
|
|
self.assertEqual(validated["counts"]["blocking"], 1)
|
|
|
|
def test_unverified_hub_finding_is_dropped_and_blocks_pass(self) -> None:
|
|
policy_path = self.root / extractor.DEFAULT_POLICY
|
|
policy_document = json.loads(policy_path.read_text(encoding="utf-8"))
|
|
policy_document["documents"]["branch-note"]["mode"] = "hub"
|
|
policy_path.write_text(json.dumps(policy_document), encoding="utf-8")
|
|
extraction = extractor.extract_document(self.root, self.path, extractor.load_policy(self.root))
|
|
assertion_result = {
|
|
"schema_version": builder.ASSERTION_SCHEMA,
|
|
"subject": extraction["path"],
|
|
"mode": "hub",
|
|
"surface_manifest_sha256": hashlib.sha256(extractor.canonical_json_bytes(extraction)).hexdigest(),
|
|
"assertions": self.request["assertions"],
|
|
}
|
|
hub_candidates = builder.build(self.root, extraction, assertion_result)
|
|
request = semantic_audit.build_verdict_request(hub_candidates)
|
|
result = self._result("CONTRADICTION", {"namespace": "repo", "path": "missing.json", "sha256": "0" * 64})
|
|
result["mode"] = "hub"
|
|
result["request_sha256"] = hashlib.sha256(semantic_audit.canonical_json_bytes(request)).hexdigest()
|
|
validated = semantic_audit.validate_result(self.root, request, result)
|
|
self.assertEqual(validated["status"], "FAIL")
|
|
self.assertEqual(validated["counts"]["dropped_pairs"], 1)
|
|
self.assertEqual(validated["counts"]["verified_findings"], 0)
|
|
|
|
def test_unverified_local_finding_is_dropped_and_blocks_certificate(self) -> None:
|
|
result = self._result("CONTRADICTION", {"namespace": "repo", "path": "missing.json", "sha256": "0" * 64})
|
|
validated = semantic_audit.validate_result(self.root, self.request, result)
|
|
self.assertEqual(validated["status"], "FAIL")
|
|
self.assertEqual(validated["counts"]["dropped_pairs"], 1)
|
|
self.assertEqual(validated["counts"]["verified_findings"], 0)
|
|
|
|
def test_forged_request_or_stale_document_cannot_be_certified(self) -> None:
|
|
forged = json.loads(json.dumps(self.request))
|
|
forged["candidates"][0]["rule_ids"] = ["C7"]
|
|
result = self._result("CONSISTENT")
|
|
result["request_sha256"] = hashlib.sha256(semantic_audit.canonical_json_bytes(forged)).hexdigest()
|
|
with self.assertRaises(semantic_audit.SemanticAuditError):
|
|
semantic_audit.validate_result(self.root, forged, result)
|
|
|
|
self.path.write_text(self.path.read_text(encoding="utf-8") + "changed\n", encoding="utf-8")
|
|
with self.assertRaises(semantic_audit.SemanticAuditError):
|
|
semantic_audit.validate_result(self.root, self.request, self._result("CONSISTENT"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|