init: llm-wiki-haness 하네스 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,372 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build semantic auditor requests and validate grounded verdict results.
|
||||
|
||||
This runtime deliberately does not infer assertions or verdicts. It binds the
|
||||
auditor's work to deterministic surface/candidate bytes, revalidates proof
|
||||
manifests, and applies the blocking policy from the ontology.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
import proof_manifest
|
||||
import semantic_candidate_builder
|
||||
import semantic_surface_extractor
|
||||
|
||||
|
||||
ASSERTION_REQUEST_SCHEMA = "semantic-assertion-request/v1"
|
||||
VERDICT_REQUEST_SCHEMA = "semantic-verdict-request/v1"
|
||||
AUDIT_RESULT_SCHEMA = "semantic-audit-result/v1"
|
||||
VALIDATED_RESULT_SCHEMA = "semantic-audit-validation-result/v1"
|
||||
HEX_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
class SemanticAuditError(ValueError):
|
||||
"""Audit request/result bytes violate the deterministic contract."""
|
||||
|
||||
|
||||
def canonical_json_bytes(value: Any) -> bytes:
|
||||
return semantic_surface_extractor.canonical_json_bytes(value)
|
||||
|
||||
|
||||
def _sha(value: Any) -> str:
|
||||
return hashlib.sha256(canonical_json_bytes(value)).hexdigest()
|
||||
|
||||
|
||||
def build_assertion_request(extraction: Mapping[str, Any], ontology: Mapping[str, Any]) -> dict[str, Any]:
|
||||
coverage = extraction.get("coverage", {})
|
||||
if extraction.get("findings") and any(item.get("severity") == "error" for item in extraction["findings"]):
|
||||
raise SemanticAuditError("cannot request assertions for uncovered semantic surfaces")
|
||||
if coverage.get("eligible_surface_blocks") != coverage.get("extracted_surface_blocks", 0) + coverage.get("explicitly_excluded_blocks", 0):
|
||||
raise SemanticAuditError("surface coverage is incomplete")
|
||||
return {
|
||||
"schema_version": ASSERTION_REQUEST_SCHEMA,
|
||||
"subject": extraction["path"],
|
||||
"mode": extraction["mode"],
|
||||
"document_sha256": extraction["document_sha256"],
|
||||
"surface_manifest_sha256": _sha(extraction),
|
||||
"ontology_sha256": _sha(ontology),
|
||||
"predicate_ontology": list(ontology["predicates"]),
|
||||
"surfaces": list(extraction["surfaces"]),
|
||||
"explicitly_excluded": list(extraction["excluded"]),
|
||||
"output_schema": semantic_candidate_builder.ASSERTION_SCHEMA,
|
||||
}
|
||||
|
||||
|
||||
def build_verdict_request(candidate_result: Mapping[str, Any], *, explicit_blocking: Iterable[Mapping[str, Any]] = ()) -> dict[str, Any]:
|
||||
if candidate_result.get("schema_version") != semantic_candidate_builder.RESULT_SCHEMA or candidate_result.get("status") != "PASS":
|
||||
raise SemanticAuditError("candidate result must be semantic-candidate-result/v1 PASS")
|
||||
blocking: list[dict[str, str]] = []
|
||||
for index, item in enumerate(explicit_blocking):
|
||||
if not isinstance(item, Mapping) or set(item) != {"code", "message"}:
|
||||
raise SemanticAuditError(f"explicit_blocking[{index}] must contain code and message")
|
||||
code, message = item.get("code"), item.get("message")
|
||||
if not isinstance(code, str) or not code or not isinstance(message, str) or not message:
|
||||
raise SemanticAuditError(f"explicit_blocking[{index}] fields must be non-empty strings")
|
||||
blocking.append({"code": code, "message": message})
|
||||
return {
|
||||
"schema_version": VERDICT_REQUEST_SCHEMA,
|
||||
"subject": candidate_result["subject"],
|
||||
"mode": candidate_result["mode"],
|
||||
"document_sha256": candidate_result["document_sha256"],
|
||||
"candidate_manifest_sha256": _sha(candidate_result),
|
||||
"ontology_sha256": candidate_result["ontology_sha256"],
|
||||
"coverage": dict(candidate_result["coverage"]),
|
||||
"assertions": list(candidate_result["assertions"]),
|
||||
"candidates": list(candidate_result["candidates"]),
|
||||
"explicit_blocking": sorted(blocking, key=lambda item: (item["code"], item["message"])),
|
||||
"output_schema": AUDIT_RESULT_SCHEMA,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_reference(root: Path, reference: Mapping[str, Any], run_root: Path | None) -> tuple[Path, str]:
|
||||
if set(reference) != {"namespace", "path", "sha256"}:
|
||||
raise SemanticAuditError("proof_manifest must contain namespace, path, and sha256")
|
||||
namespace, value, expected = reference.get("namespace"), reference.get("path"), reference.get("sha256")
|
||||
if namespace not in {"repo", "run"}:
|
||||
raise SemanticAuditError("proof manifest namespace must be repo or run")
|
||||
base = root if namespace == "repo" else run_root
|
||||
if base is None:
|
||||
raise SemanticAuditError("run proof manifest requires run_root")
|
||||
if not isinstance(value, str) or not value or "\\" in value or Path(value).is_absolute():
|
||||
raise SemanticAuditError("proof manifest path must be relative POSIX")
|
||||
path = (base / value).resolve()
|
||||
try:
|
||||
path.relative_to(base.resolve())
|
||||
except ValueError as exc:
|
||||
raise SemanticAuditError("proof manifest path escapes its namespace") from exc
|
||||
if not path.is_file():
|
||||
raise SemanticAuditError("proof manifest does not exist")
|
||||
if not isinstance(expected, str) or not HEX_SHA256.fullmatch(expected):
|
||||
raise SemanticAuditError("proof manifest sha256 is invalid")
|
||||
observed = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
if observed != expected:
|
||||
raise SemanticAuditError("proof manifest hash mismatch")
|
||||
return path, expected
|
||||
|
||||
|
||||
def _verify_evidence(verdict: Mapping[str, Any], candidate: Mapping[str, Any], assertions: Mapping[str, Mapping[str, Any]]) -> None:
|
||||
for key, assertion_key in (("evidence_a", "assertion_a"), ("evidence_b", "assertion_b")):
|
||||
evidence = verdict.get(key)
|
||||
assertion = assertions[candidate[assertion_key]]
|
||||
if not isinstance(evidence, Mapping) or set(evidence) != {"quote", "line_start", "line_end"}:
|
||||
raise SemanticAuditError(f"{key} must contain quote and exact line range")
|
||||
if (
|
||||
evidence.get("quote") != assertion["quote"]
|
||||
or evidence.get("line_start") != assertion["line_start"]
|
||||
or evidence.get("line_end") != assertion["line_end"]
|
||||
):
|
||||
raise SemanticAuditError(f"{key} does not match its grounded assertion")
|
||||
|
||||
|
||||
def _verify_negative_proof(
|
||||
root: Path,
|
||||
run_root: Path | None,
|
||||
profiles: set[str],
|
||||
verdict: Mapping[str, Any],
|
||||
candidate: Mapping[str, Any],
|
||||
assertions: Mapping[str, Mapping[str, Any]],
|
||||
) -> str:
|
||||
reference = verdict.get("proof_manifest")
|
||||
if not isinstance(reference, Mapping):
|
||||
raise SemanticAuditError("negative semantic verdict requires proof_manifest")
|
||||
path, digest = _resolve_reference(root, reference, run_root)
|
||||
try:
|
||||
manifest = json.loads(path.read_text(encoding="utf-8"))
|
||||
verified = proof_manifest.verify_manifest(manifest, root, profiles, run_root=run_root)
|
||||
except (OSError, UnicodeError, json.JSONDecodeError, proof_manifest.ManifestValidationError) as exc:
|
||||
raise SemanticAuditError(f"proof manifest revalidation failed: {exc}") from exc
|
||||
if verified["verification"]["status"] != "PASS" or verified["verification"]["fail_count"] != 0:
|
||||
raise SemanticAuditError("proof manifest is not PASS")
|
||||
expected = {
|
||||
(candidate["candidate_id"], "assertion_a", assertions[candidate["assertion_a"]]["quote"]),
|
||||
(candidate["candidate_id"], "assertion_b", assertions[candidate["assertion_b"]]["quote"]),
|
||||
}
|
||||
observed = {
|
||||
(item["finding"]["id"], item["finding"]["role"], item["source"]["quote_utf8"])
|
||||
for item in verified["proofs"]
|
||||
}
|
||||
if not expected.issubset(observed):
|
||||
raise SemanticAuditError("proof manifest does not bind both candidate assertions")
|
||||
return digest
|
||||
|
||||
|
||||
def validate_result(
|
||||
root: Path,
|
||||
request: Mapping[str, Any],
|
||||
result: Any,
|
||||
*,
|
||||
ontology_path: Path = semantic_candidate_builder.DEFAULT_ONTOLOGY,
|
||||
profiles_path: Path = proof_manifest.DEFAULT_PROFILES,
|
||||
run_root: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
root = root.resolve(strict=True)
|
||||
ontology = semantic_candidate_builder.load_ontology(root, ontology_path)
|
||||
request_fields = {
|
||||
"schema_version", "subject", "mode", "document_sha256", "candidate_manifest_sha256",
|
||||
"ontology_sha256", "coverage", "assertions", "candidates", "explicit_blocking", "output_schema",
|
||||
}
|
||||
if not isinstance(request, Mapping) or set(request) != request_fields or request.get("schema_version") != VERDICT_REQUEST_SCHEMA:
|
||||
raise SemanticAuditError(f"expected {VERDICT_REQUEST_SCHEMA}")
|
||||
subject = request.get("subject")
|
||||
if not isinstance(subject, str) or not subject or Path(subject).is_absolute() or ".." in Path(subject).parts:
|
||||
raise SemanticAuditError("verdict request subject must be a canonical repo-relative path")
|
||||
document = Path(os.path.abspath(root / subject))
|
||||
try:
|
||||
document.relative_to(root)
|
||||
document.resolve(strict=True).relative_to(root)
|
||||
except (ValueError, FileNotFoundError) as exc:
|
||||
raise SemanticAuditError("verdict request subject escapes repository") from exc
|
||||
if not document.is_file():
|
||||
raise SemanticAuditError("verdict request subject does not exist")
|
||||
current_document_sha = hashlib.sha256(document.read_bytes()).hexdigest()
|
||||
if request.get("document_sha256") != current_document_sha:
|
||||
raise SemanticAuditError("verdict request is stale for current document bytes")
|
||||
policy = semantic_surface_extractor.load_policy(root)
|
||||
extraction = semantic_surface_extractor.extract_document(root, document, policy)
|
||||
if extraction["mode"] != request.get("mode"):
|
||||
raise SemanticAuditError("verdict request mode differs from current document policy")
|
||||
assertion_result = {
|
||||
"schema_version": semantic_candidate_builder.ASSERTION_SCHEMA,
|
||||
"subject": subject,
|
||||
"mode": request["mode"],
|
||||
"surface_manifest_sha256": _sha(extraction),
|
||||
"assertions": request.get("assertions"),
|
||||
}
|
||||
rebuilt = semantic_candidate_builder.build(root, extraction, assertion_result, ontology_path)
|
||||
for field in ("ontology_sha256", "coverage", "assertions", "candidates"):
|
||||
if request.get(field) != rebuilt[field]:
|
||||
raise SemanticAuditError(f"verdict request {field} differs from deterministic reconstruction")
|
||||
if request.get("candidate_manifest_sha256") != _sha(rebuilt):
|
||||
raise SemanticAuditError("verdict request candidate manifest hash is stale or forged")
|
||||
if request.get("output_schema") != AUDIT_RESULT_SCHEMA:
|
||||
raise SemanticAuditError("verdict request output schema mismatch")
|
||||
required = {"schema_version", "request_sha256", "subject", "mode", "auditor", "verdicts"}
|
||||
if not isinstance(result, dict) or set(result) != required or result.get("schema_version") != AUDIT_RESULT_SCHEMA:
|
||||
raise SemanticAuditError(f"audit result must contain exact {AUDIT_RESULT_SCHEMA} fields")
|
||||
if result.get("request_sha256") != _sha(request):
|
||||
raise SemanticAuditError("audit result is not bound to current verdict request")
|
||||
if result.get("subject") != request.get("subject") or result.get("mode") != request.get("mode"):
|
||||
raise SemanticAuditError("audit result subject/mode mismatch")
|
||||
auditor = result.get("auditor")
|
||||
if not isinstance(auditor, dict) or set(auditor) != {"contract_version", "model_id", "run_id"}:
|
||||
raise SemanticAuditError("auditor must contain contract_version/model_id/run_id")
|
||||
if auditor.get("contract_version") != ontology["auditor_contract_version"]:
|
||||
raise SemanticAuditError("auditor contract version mismatch")
|
||||
if any(not isinstance(auditor.get(key), str) or not auditor[key] for key in ("model_id", "run_id")):
|
||||
raise SemanticAuditError("auditor model_id/run_id must be non-empty")
|
||||
raw_verdicts = result.get("verdicts")
|
||||
if not isinstance(raw_verdicts, list):
|
||||
raise SemanticAuditError("verdicts must be an array")
|
||||
candidates = {item["candidate_id"]: item for item in request["candidates"]}
|
||||
assertions = {item["assertion_id"]: item for item in request["assertions"]}
|
||||
seen: set[str] = set()
|
||||
verified_findings: list[dict[str, Any]] = []
|
||||
dropped: list[dict[str, Any]] = []
|
||||
positive = {"CONSISTENT", "COMPLEMENTARY", "CONTEXTUAL_VARIANT"}
|
||||
negative = {"AMBIGUOUS_AUTHORITY", "RESTATEMENT_DRIFT", "CONTRADICTION"}
|
||||
profiles_source = profiles_path if profiles_path.is_absolute() else root / profiles_path
|
||||
profiles = proof_manifest.load_allowed_profiles(profiles_source.resolve(strict=True))
|
||||
for index, item in enumerate(raw_verdicts):
|
||||
fields = {"candidate_id", "verdict", "rationale", "evidence_a", "evidence_b", "proof_manifest"}
|
||||
if not isinstance(item, dict) or set(item) != fields:
|
||||
raise SemanticAuditError(f"verdicts[{index}] has missing or unknown fields")
|
||||
candidate_id = item.get("candidate_id")
|
||||
verdict = item.get("verdict")
|
||||
if candidate_id not in candidates or candidate_id in seen:
|
||||
raise SemanticAuditError(f"verdicts[{index}] has unknown or duplicate candidate_id")
|
||||
seen.add(candidate_id)
|
||||
if verdict not in ontology["verdicts"]:
|
||||
raise SemanticAuditError(f"verdicts[{index}] is outside exact verdict set")
|
||||
if not isinstance(item.get("rationale"), str) or not item["rationale"].strip():
|
||||
raise SemanticAuditError(f"verdicts[{index}].rationale must be non-empty")
|
||||
candidate = candidates[candidate_id]
|
||||
_verify_evidence(item, candidate, assertions)
|
||||
if verdict in positive:
|
||||
if item.get("proof_manifest") is not None:
|
||||
raise SemanticAuditError("passing verdict must not claim a finding proof")
|
||||
continue
|
||||
try:
|
||||
proof_sha = _verify_negative_proof(root, run_root, profiles, item, candidate, assertions)
|
||||
except SemanticAuditError as exc:
|
||||
dropped.append({"candidate_id": candidate_id, "verdict": verdict, "reason": str(exc)})
|
||||
continue
|
||||
verified_findings.append({
|
||||
"candidate_id": candidate_id,
|
||||
"verdict": verdict,
|
||||
"rationale": item["rationale"],
|
||||
"evidence_a": dict(item["evidence_a"]),
|
||||
"evidence_b": dict(item["evidence_b"]),
|
||||
"proof_manifest_sha256": proof_sha,
|
||||
})
|
||||
missing = sorted(set(candidates) - seen)
|
||||
if missing:
|
||||
raise SemanticAuditError(f"candidate verdict coverage is incomplete: {missing}")
|
||||
mode = str(request["mode"])
|
||||
explicit = list(request.get("explicit_blocking", []))
|
||||
blocking = len(explicit) + sum(
|
||||
item["verdict"] == "CONTRADICTION" or (mode == "hub" and item["verdict"] == "AMBIGUOUS_AUTHORITY")
|
||||
for item in verified_findings
|
||||
)
|
||||
readiness = sum(item["verdict"] == "RESTATEMENT_DRIFT" for item in verified_findings)
|
||||
# A negative judgment without replayable proof is not evidence of a
|
||||
# contradiction, but it is also not a certifiable clean audit. Treat
|
||||
# dropped candidates as fail-closed in every mode so a local certificate
|
||||
# cannot hide an auditor-raised contradiction merely because its proof
|
||||
# reference was omitted or stale.
|
||||
status = "PASS" if blocking == 0 and readiness == 0 and not dropped else "FAIL"
|
||||
return {
|
||||
"schema_version": VALIDATED_RESULT_SCHEMA,
|
||||
"status": status,
|
||||
"subject": request["subject"],
|
||||
"mode": mode,
|
||||
"document_sha256": request["document_sha256"],
|
||||
"request_sha256": _sha(request),
|
||||
"ontology_sha256": request["ontology_sha256"],
|
||||
"coverage": {
|
||||
"eligible_surfaces": request["coverage"]["eligible_surfaces"],
|
||||
"processed_surfaces": request["coverage"]["processed_surfaces"],
|
||||
"candidate_pairs": len(candidates),
|
||||
"processed_pairs": len(seen),
|
||||
"dropped_pairs": len(dropped),
|
||||
},
|
||||
"findings": verified_findings,
|
||||
"dropped": dropped,
|
||||
"explicit_blocking": explicit,
|
||||
"counts": {
|
||||
"blocking": blocking,
|
||||
"readiness_blocking": readiness,
|
||||
"verified_findings": len(verified_findings),
|
||||
"dropped_pairs": len(dropped),
|
||||
},
|
||||
"auditor": dict(auditor),
|
||||
}
|
||||
|
||||
|
||||
def _load(path: Path) -> Any:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _one_extraction(value: Any) -> Mapping[str, Any]:
|
||||
if isinstance(value, dict) and {"path", "mode", "surfaces", "coverage"}.issubset(value):
|
||||
return value
|
||||
documents = value.get("documents") if isinstance(value, dict) else None
|
||||
if not isinstance(documents, list) or len(documents) != 1 or not isinstance(documents[0], dict):
|
||||
raise SemanticAuditError("assertion request input must contain exactly one extracted document")
|
||||
return documents[0]
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("phase", choices=("assertion-request", "verdict-request", "validate"))
|
||||
parser.add_argument("input", type=Path)
|
||||
parser.add_argument("result", type=Path, nargs="?")
|
||||
parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[2])
|
||||
parser.add_argument("--run-root", type=Path)
|
||||
parser.add_argument("--explicit-blocking", type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
root = args.root.resolve(strict=True)
|
||||
if args.phase == "assertion-request":
|
||||
extraction = _one_extraction(_load(args.input))
|
||||
ontology = semantic_candidate_builder.load_ontology(root)
|
||||
output = build_assertion_request(extraction, ontology)
|
||||
elif args.phase == "verdict-request":
|
||||
blocking: Iterable[Mapping[str, Any]] = ()
|
||||
if args.explicit_blocking is not None:
|
||||
value = _load(args.explicit_blocking)
|
||||
if not isinstance(value, list):
|
||||
raise SemanticAuditError("explicit blocking input must be an array")
|
||||
blocking = value
|
||||
output = build_verdict_request(_load(args.input), explicit_blocking=blocking)
|
||||
else:
|
||||
if args.result is None:
|
||||
raise SemanticAuditError("validate requires request and result paths")
|
||||
output = validate_result(root, _load(args.input), _load(args.result), run_root=args.run_root)
|
||||
exit_code = 0 if output.get("status", "PASS") == "PASS" else 1
|
||||
except (
|
||||
SemanticAuditError,
|
||||
semantic_candidate_builder.SemanticCandidateError,
|
||||
semantic_surface_extractor.SemanticSurfaceError,
|
||||
proof_manifest.ManifestValidationError,
|
||||
OSError,
|
||||
UnicodeError,
|
||||
json.JSONDecodeError,
|
||||
) as exc:
|
||||
output = {"schema_version": VALIDATED_RESULT_SCHEMA, "status": "ERROR", "errors": [{"code": "SEMANTIC_AUDIT_ERROR", "message": str(exc)}]}
|
||||
exit_code = 2
|
||||
json.dump(output, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
sys.stdout.write("\n")
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user