#!/usr/bin/env python3 """Evaluate the 70-case typed and semantic consistency regression corpus. The deterministic half is executed against the real typed-contract checker. The semantic half remains release-blocking until three truthful live auditor runs are recorded; fixture completeness is never reported as an LLM result. """ from __future__ import annotations import argparse from collections import Counter from datetime import datetime import hashlib import json from pathlib import Path import re import shutil import sys import tempfile from typing import Any, Iterable, Mapping import contract_projection import semantic_candidate_builder import semantic_surface_extractor import typed_contract_check DEFAULT_ROOT = Path(__file__).resolve().parents[2] DEFAULT_MANIFEST = Path("harness/tests/fixtures/semantic-consistency/manifest.json") RESULT_SCHEMA = "semantic-regression-result/v1" MANIFEST_SCHEMA = "semantic-consistency-corpus/v1" FIXTURE_SCHEMA = "semantic-regression-fixture/v1" RUN_SCHEMA = "semantic-evaluation-run/v1" TYPES = ("A4", "E1", "DELEG", "D7", "A1", "HUB") DETERMINISTIC_TYPES = frozenset({"A4", "E1", "DELEG", "D7"}) SEMANTIC_TYPES = frozenset({"A1", "HUB"}) EXPECTED_CODES = { "A4": "MANUAL_ARTIFACT_SCHEMA_RESTATEMENT", "E1": "DUPLICATE_CONCERN_OWNER", "DELEG": "UNACCEPTED_DELEGATION", "D7": "GENERATED_CONTRACT_PROJECTION_DRIFT", "A1": "CONTRADICTION", "HUB": "CONTRADICTION", } CASE_ID_RE = re.compile(r"^(A4|E1|DELEG|D7|A1|HUB)-\d{3}$") HEX_SHA256 = re.compile(r"^[0-9a-f]{64}$") AUDITOR_PROMPT = Path("harness/source/agents/bodies/wiki-semantic-coherence-auditor.md") class SemanticRegressionError(ValueError): """The corpus schema, path set, or evaluation record is unusable.""" def _safe_path(root: Path, value: Any, location: str) -> Path: if not isinstance(value, str) or not value or "\\" in value: raise SemanticRegressionError(f"{location}: expected repo-relative POSIX path") relative = Path(value) if relative.is_absolute() or ".." in relative.parts: raise SemanticRegressionError(f"{location}: path escapes repository") path = (root / relative).resolve() try: path.relative_to(root) except ValueError as exc: raise SemanticRegressionError(f"{location}: path escapes repository") from exc if not path.is_file(): raise SemanticRegressionError(f"{location}: file does not exist: {value}") return path def _load_json(path: Path, schema: str) -> dict[str, Any]: try: document = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError) as exc: raise SemanticRegressionError(f"cannot read {path}: {exc}") from exc if not isinstance(document, dict) or document.get("schema_version") != schema: raise SemanticRegressionError(f"{path}: expected {schema}") return document def load_manifest(root: Path, manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]: root = root.resolve(strict=True) source = manifest_path if manifest_path.is_absolute() else root / manifest_path manifest = _load_json(source, MANIFEST_SCHEMA) cases = manifest.get("cases") if manifest.get("case_count") != 70 or not isinstance(cases, list) or len(cases) != 70: raise SemanticRegressionError("manifest must declare exactly 70 cases") thresholds = manifest.get("thresholds") expected_thresholds = { "deterministic_recall": 1.0, "deterministic_false_negatives": 0, "deterministic_negative_false_positives": 0, "semantic_critical_high_recall": 1.0, "semantic_overall_recall": 0.95, "semantic_precision": 0.90, "semantic_dropped_pairs": 0, "live_runs": 3, } if thresholds != expected_thresholds: raise SemanticRegressionError("manifest thresholds do not match the WP17 release contract") seen: set[str] = set() distribution: Counter[str] = Counter() required = { "case_id", "type", "severity", "documents", "surface_a", "surface_b", "expected", "counterexample", "provenance", } for index, case in enumerate(cases): if not isinstance(case, dict) or set(case) != required: raise SemanticRegressionError(f"cases[{index}] must contain exactly {sorted(required)}") case_id = case["case_id"] case_type = case["type"] if not isinstance(case_id, str) or not CASE_ID_RE.fullmatch(case_id): raise SemanticRegressionError(f"cases[{index}].case_id is invalid") if case_id in seen: raise SemanticRegressionError(f"duplicate case id: {case_id}") seen.add(case_id) if case_type not in TYPES or not case_id.startswith(f"{case_type}-"): raise SemanticRegressionError(f"{case_id}: type does not match case id") distribution[case_type] += 1 if case.get("severity") not in {"Critical", "High", "Medium", "Low"}: raise SemanticRegressionError(f"{case_id}: invalid severity") if case.get("expected") != EXPECTED_CODES[case_type]: raise SemanticRegressionError(f"{case_id}: unexpected expected code") if case.get("provenance") != "design-fixture": raise SemanticRegressionError(f"{case_id}: provenance must be design-fixture") documents = case.get("documents") if not isinstance(documents, list) or len(documents) != 1: raise SemanticRegressionError(f"{case_id}: documents must name one positive fixture") for field in ("surface_a", "surface_b"): surface = case.get(field) if ( not isinstance(surface, dict) or set(surface) != {"section", "quote"} or not all(isinstance(surface[key], str) and surface[key] for key in surface) ): raise SemanticRegressionError(f"{case_id}: invalid {field}") for path_index, value in enumerate(documents): _safe_path(root, value, f"{case_id}.documents[{path_index}]") _safe_path(root, case["counterexample"], f"{case_id}.counterexample") if set(distribution) != set(TYPES) or any(distribution[item] == 0 for item in TYPES): raise SemanticRegressionError("all six case types must be represented") declared_distribution = manifest.get("type_distribution") if declared_distribution != {name: distribution[name] for name in TYPES}: raise SemanticRegressionError("type_distribution does not match cases") runs = manifest.get("evaluation_runs") if not isinstance(runs, list) or len(runs) != 3: raise SemanticRegressionError("evaluation_runs must name exactly three records") for index, value in enumerate(runs): _safe_path(root, value, f"evaluation_runs[{index}]") return manifest def _fixture(root: Path, value: str, case_id: str, polarity: str) -> dict[str, Any]: path = _safe_path(root, value, f"{case_id}.{polarity}") fixture = _load_json(path, FIXTURE_SCHEMA) if fixture.get("case_id") != case_id or fixture.get("polarity") != polarity: raise SemanticRegressionError(f"{path}: case_id/polarity mismatch") files = fixture.get("files") if not isinstance(files, dict) or not files: raise SemanticRegressionError(f"{path}: files must be a non-empty object") for relative, content in files.items(): if not isinstance(content, str): raise SemanticRegressionError(f"{path}: fixture contents must be strings") candidate = Path(relative) if candidate.is_absolute() or ".." in candidate.parts or "\\" in relative: raise SemanticRegressionError(f"{path}: unsafe fixture path {relative}") if not isinstance(fixture.get("materialize_projections", False), bool): raise SemanticRegressionError(f"{path}: materialize_projections must be boolean") mutations = fixture.get("mutations", []) if not isinstance(mutations, list): raise SemanticRegressionError(f"{path}: mutations must be an array") for mutation in mutations: if not isinstance(mutation, dict) or set(mutation) != {"path", "old", "new"}: raise SemanticRegressionError(f"{path}: invalid mutation") if not all(isinstance(mutation[key], str) for key in mutation): raise SemanticRegressionError(f"{path}: mutation values must be strings") return fixture def _materialize(root: Path, fixture: Mapping[str, Any]) -> Path: stage = Path(tempfile.mkdtemp(prefix="semantic-regression-")) for relative, content in fixture["files"].items(): path = stage / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content, encoding="utf-8") schema = stage / typed_contract_check.DEFAULT_SCHEMA schema.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(root / typed_contract_check.DEFAULT_SCHEMA, schema) if fixture.get("materialize_projections"): updates, result = contract_projection.build_updates(stage) if result["status"] == "FAIL": raise SemanticRegressionError( "fixture projection precondition failed: " + ",".join(sorted({item["code"] for item in result["findings"]})) ) for path, text in updates.items(): path.write_text(text, encoding="utf-8") for mutation in fixture.get("mutations", []): path = stage / mutation["path"] if not path.is_file(): raise SemanticRegressionError(f"mutation target does not exist: {mutation['path']}") text = path.read_text(encoding="utf-8") if mutation["old"] not in text: raise SemanticRegressionError(f"mutation source not found: {mutation['old']!r}") path.write_text(text.replace(mutation["old"], mutation["new"], 1), encoding="utf-8") return stage def evaluate_deterministic(root: Path, cases: Iterable[Mapping[str, Any]]) -> dict[str, Any]: by_type: dict[str, dict[str, int | float]] = {} findings: list[dict[str, Any]] = [] total_positive = true_positive = false_negative = negative_fp = 0 for case in cases: if case["type"] not in DETERMINISTIC_TYPES: continue case_id = str(case["case_id"]) positive = _fixture(root, case["documents"][0], case_id, "positive") negative = _fixture(root, case["counterexample"], case_id, "negative") metrics = by_type.setdefault(case["type"], {"cases": 0, "true_positive": 0, "false_negative": 0, "negative_false_positive": 0}) metrics["cases"] += 1 total_positive += 1 positive_stage = _materialize(root, positive) negative_stage = _materialize(root, negative) try: positive_codes = {item["code"] for item in typed_contract_check.check(positive_stage)["findings"]} negative_result = typed_contract_check.check(negative_stage) finally: shutil.rmtree(positive_stage) shutil.rmtree(negative_stage) if case["expected"] in positive_codes: true_positive += 1 metrics["true_positive"] += 1 else: false_negative += 1 metrics["false_negative"] += 1 findings.append({ "code": "DETERMINISTIC_FALSE_NEGATIVE", "case_id": case_id, "message": f"expected {case['expected']}, observed {sorted(positive_codes)}", }) if negative_result["status"] != "PASS": negative_fp += 1 metrics["negative_false_positive"] += 1 findings.append({ "code": "DETERMINISTIC_FALSE_POSITIVE", "case_id": case_id, "message": f"counterexample findings: {[item['code'] for item in negative_result['findings']]}", }) for metrics in by_type.values(): count = int(metrics["cases"]) metrics["recall"] = int(metrics["true_positive"]) / count if count else 0.0 recall = true_positive / total_positive if total_positive else 0.0 status = "PASS" if recall == 1.0 and false_negative == 0 and negative_fp == 0 else "FAIL" return { "status": status, "case_count": total_positive, "metrics": { "recall": recall, "true_positive": true_positive, "false_negative": false_negative, "negative_false_positive": negative_fp, }, "by_type": by_type, "findings": findings, } def validate_semantic_corpus(root: Path, cases: Iterable[Mapping[str, Any]]) -> dict[str, Any]: findings: list[dict[str, Any]] = [] count = 0 high_critical = 0 for case in cases: if case["type"] not in SEMANTIC_TYPES: continue count += 1 high_critical += case["severity"] in {"Critical", "High"} positive = _fixture(root, case["documents"][0], case["case_id"], "positive") negative = _fixture(root, case["counterexample"], case["case_id"], "negative") positive_text = "\n".join(positive["files"].values()) negative_text = "\n".join(negative["files"].values()) for name, surface in (("surface_a", case["surface_a"]), ("surface_b", case["surface_b"])): if surface["quote"] not in positive_text: findings.append({ "code": "SEMANTIC_PAIR_DROPPED", "case_id": case["case_id"], "message": f"{name} quote is absent from positive fixture", }) if case["surface_a"]["quote"] not in negative_text or case["surface_b"]["quote"] in negative_text: # The counterexample must retain the authority claim but replace # the contradictory claim with a compatible variant. findings.append({ "code": "INVALID_SEMANTIC_COUNTEREXAMPLE", "case_id": case["case_id"], "message": "counterexample does not preserve A while replacing B", }) return { "status": "READY" if not findings else "FAIL", "case_count": count, "critical_high_count": high_critical, "dropped_pairs": sum(item["code"] == "SEMANTIC_PAIR_DROPPED" for item in findings), "findings": findings, } def _median(values: list[float]) -> float: ordered = sorted(values) return ordered[len(ordered) // 2] def evaluate_live_runs( semantic_cases: Iterable[Mapping[str, Any]], runs: Iterable[Mapping[str, Any]], ) -> dict[str, Any]: cases = list(semantic_cases) run_list = list(runs) not_run = [run for run in run_list if run.get("status") == "NOT_RUN"] if not_run: return { "status": "NOT_RUN", "required_runs": 3, "completed_runs": len(run_list) - len(not_run), "metrics": None, "findings": [{ "code": "LIVE_EVALUATION_NOT_RUN", "run_id": str(run.get("run_id", "")), "message": str(run.get("reason", "live semantic evaluation has not run")), } for run in not_run], } if len(run_list) != 3: raise SemanticRegressionError("live evaluation requires exactly three runs") case_map = {case["case_id"]: case for case in cases} per_run: list[dict[str, Any]] = [] findings: list[dict[str, Any]] = [] for run in run_list: if run.get("status") != "COMPLETED": raise SemanticRegressionError(f"unsupported evaluation status: {run.get('status')}") for field in ("model_id", "auditor_contract_version", "ontology_sha256", "prompt_sha256"): if not isinstance(run.get(field), str) or not run[field]: raise SemanticRegressionError(f"{run.get('run_id')}: {field} is required") if not HEX_SHA256.fullmatch(str(run["ontology_sha256"])) or not HEX_SHA256.fullmatch(str(run["prompt_sha256"])): raise SemanticRegressionError(f"{run.get('run_id')}: ontology/prompt sha256 is invalid") executed_at = run.get("executed_at") if not isinstance(executed_at, str) or not executed_at: raise SemanticRegressionError(f"{run.get('run_id')}: executed_at is required") try: datetime.fromisoformat(executed_at) except ValueError as exc: raise SemanticRegressionError(f"{run.get('run_id')}: executed_at is not ISO-8601") from exc predictions = run.get("predictions") if not isinstance(predictions, list): raise SemanticRegressionError(f"{run.get('run_id')}: predictions must be an array") by_case = {item.get("case_id"): item for item in predictions if isinstance(item, dict)} if set(by_case) != set(case_map) or len(predictions) != len(case_map): raise SemanticRegressionError(f"{run.get('run_id')}: predictions must cover every semantic case exactly once") tp = fn = fp = dropped = critical_high_tp = critical_high_total = 0 for case_id, case in case_map.items(): item = by_case[case_id] if set(item) != {"case_id", "positive", "counterexample", "dropped"}: raise SemanticRegressionError(f"{run.get('run_id')}/{case_id}: invalid prediction fields") if item["dropped"]: dropped += 1 positive_hit = item["positive"] == case["expected"] tp += positive_hit fn += not positive_hit fp += item["counterexample"] not in {None, "CONSISTENT", "CONTEXTUAL_VARIANT", "COMPLEMENTARY"} if case["severity"] in {"Critical", "High"}: critical_high_total += 1 critical_high_tp += positive_hit recall = tp / len(case_map) if case_map else 0.0 critical_high_recall = critical_high_tp / critical_high_total if critical_high_total else 0.0 precision = tp / (tp + fp) if tp + fp else 0.0 per_run.append({ "run_id": run["run_id"], "recall": recall, "critical_high_recall": critical_high_recall, "precision": precision, "dropped_pairs": dropped, "true_positive": tp, "false_negative": fn, "false_positive": fp, }) if critical_high_recall != 1.0: findings.append({"code": "SEMANTIC_CRITICAL_HIGH_RECALL_FAILED", "run_id": run["run_id"], "message": str(critical_high_recall)}) if dropped: findings.append({"code": "SEMANTIC_PAIR_DROPPED", "run_id": run["run_id"], "message": str(dropped)}) median_recall = _median([item["recall"] for item in per_run]) median_precision = _median([item["precision"] for item in per_run]) if median_recall < 0.95: findings.append({"code": "SEMANTIC_RECALL_BELOW_THRESHOLD", "message": str(median_recall)}) if median_precision < 0.90: findings.append({"code": "SEMANTIC_PRECISION_BELOW_THRESHOLD", "message": str(median_precision)}) return { "status": "PASS" if not findings else "FAIL", "required_runs": 3, "completed_runs": 3, "metrics": { "median_overall_recall": median_recall, "median_precision": median_precision, "runs": per_run, }, "findings": findings, } def check(root: Path, manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]: root = root.resolve(strict=True) manifest = load_manifest(root, manifest_path) cases = manifest["cases"] deterministic = evaluate_deterministic(root, cases) semantic_corpus = validate_semantic_corpus(root, cases) runs = [ _load_json(_safe_path(root, value, f"evaluation_runs[{index}]"), RUN_SCHEMA) for index, value in enumerate(manifest["evaluation_runs"]) ] ontology = semantic_candidate_builder.load_ontology(root) expected_ontology_sha = hashlib.sha256( semantic_surface_extractor.canonical_json_bytes(ontology) ).hexdigest() expected_prompt_sha = hashlib.sha256((root / AUDITOR_PROMPT).read_bytes()).hexdigest() for run in runs: if run.get("status") == "COMPLETED" and ( run.get("auditor_contract_version") != ontology["auditor_contract_version"] or run.get("ontology_sha256") != expected_ontology_sha or run.get("prompt_sha256") != expected_prompt_sha ): raise SemanticRegressionError( f"{run.get('run_id')}: live evaluation contract, ontology, or prompt is stale" ) live = evaluate_live_runs((case for case in cases if case["type"] in SEMANTIC_TYPES), runs) findings = [*deterministic["findings"], *semantic_corpus["findings"], *live["findings"]] status = ( "PASS" if deterministic["status"] == "PASS" and semantic_corpus["status"] == "READY" and live["status"] == "PASS" else "FAIL" ) return { "schema_version": RESULT_SCHEMA, "status": status, "case_count": len(cases), "type_distribution": manifest["type_distribution"], "deterministic": deterministic, "semantic_corpus": semantic_corpus, "live_evaluation": live, "findings": findings, } def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", type=Path, default=DEFAULT_ROOT) parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) parser.add_argument("--check", action="store_true", required=True) args = parser.parse_args(argv) try: result = check(args.root, args.manifest) exit_code = 0 if result["status"] == "PASS" else 1 except (SemanticRegressionError, typed_contract_check.TypedContractError, OSError, UnicodeError, json.JSONDecodeError) as exc: result = { "schema_version": RESULT_SCHEMA, "status": "ERROR", "errors": [{"code": "SEMANTIC_REGRESSION_ERROR", "message": str(exc)}], } exit_code = 2 json.dump(result, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True) sys.stdout.write("\n") return exit_code if __name__ == "__main__": raise SystemExit(main())