from __future__ import annotations from contextlib import redirect_stdout from datetime import datetime, timezone import hashlib import io import json from pathlib import Path import shutil import sys import tempfile import unittest ROOT = Path(__file__).resolve().parents[2] RUNTIME = ROOT / "harness/runtime" sys.path.insert(0, str(RUNTIME)) import release_gate # noqa: E402 import semantic_regression # noqa: E402 class SemanticRegressionTest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.manifest = semantic_regression.load_manifest(ROOT) cls.result = semantic_regression.check(ROOT) def test_manifest_has_exactly_70_resolvable_provenanced_cases_and_all_types(self) -> None: cases = self.manifest["cases"] self.assertEqual(len(cases), 70) self.assertEqual( self.manifest["type_distribution"], {"A4": 12, "E1": 12, "DELEG": 12, "D7": 12, "A1": 11, "HUB": 11}, ) self.assertEqual({case["type"] for case in cases}, set(semantic_regression.TYPES)) self.assertEqual({case["provenance"] for case in cases}, {"design-fixture"}) self.assertIn("not historical audit findings", self.manifest["corpus_origin"]) for case in cases: self.assertEqual(len(case["documents"]), 1) positive = ROOT / case["documents"][0] negative = ROOT / case["counterexample"] self.assertTrue(positive.is_file(), positive) self.assertTrue(negative.is_file(), negative) self.assertNotEqual(positive, negative) self.assertEqual(json.loads(positive.read_text(encoding="utf-8"))["polarity"], "positive") self.assertEqual(json.loads(negative.read_text(encoding="utf-8"))["polarity"], "negative") def test_deterministic_cases_replay_with_full_recall_no_fn_and_no_negative_fp(self) -> None: deterministic = self.result["deterministic"] self.assertEqual(deterministic["status"], "PASS", deterministic) self.assertEqual(deterministic["case_count"], 48) self.assertEqual(deterministic["metrics"]["recall"], 1.0) self.assertEqual(deterministic["metrics"]["false_negative"], 0) self.assertEqual(deterministic["metrics"]["negative_false_positive"], 0) for case_type in semantic_regression.DETERMINISTIC_TYPES: self.assertEqual(deterministic["by_type"][case_type]["recall"], 1.0) def test_semantic_fixture_pairs_are_ready_and_none_are_dropped(self) -> None: semantic = self.result["semantic_corpus"] self.assertEqual(semantic["status"], "READY", semantic) self.assertEqual(semantic["case_count"], 22) self.assertEqual(semantic["critical_high_count"], 14) self.assertEqual(semantic["dropped_pairs"], 0) def _completed_runs(self) -> tuple[list[dict], list[dict]]: cases = [case for case in self.manifest["cases"] if case["type"] in semantic_regression.SEMANTIC_TYPES] ontology = semantic_regression.semantic_candidate_builder.load_ontology(ROOT) ontology_sha = hashlib.sha256( semantic_regression.semantic_surface_extractor.canonical_json_bytes(ontology) ).hexdigest() prompt_sha = hashlib.sha256((ROOT / semantic_regression.AUDITOR_PROMPT).read_bytes()).hexdigest() predictions = [ { "case_id": case["case_id"], "positive": case["expected"], "counterexample": "CONSISTENT", "dropped": False, } for case in cases ] runs = [ { "schema_version": semantic_regression.RUN_SCHEMA, "run_id": f"run-{index}", "status": "COMPLETED", "model_id": "test-model", "auditor_contract_version": ontology["auditor_contract_version"], "ontology_sha256": ontology_sha, "prompt_sha256": prompt_sha, "executed_at": datetime.now(timezone.utc).isoformat(), "predictions": json.loads(json.dumps(predictions)), } for index in range(1, 4) ] return cases, runs def test_semantic_metric_thresholds_are_machine_evaluated(self) -> None: cases, runs = self._completed_runs() passed = semantic_regression.evaluate_live_runs(cases, runs) self.assertEqual(passed["status"], "PASS", passed) self.assertEqual(passed["metrics"]["median_overall_recall"], 1.0) self.assertEqual(passed["metrics"]["median_precision"], 1.0) critical = next(case for case in cases if case["severity"] == "Critical") for run in runs: for prediction in run["predictions"]: if prediction["case_id"] == critical["case_id"]: prediction["positive"] = "CONSISTENT" prediction["dropped"] = True for prediction in run["predictions"][:3]: prediction["counterexample"] = "LOCAL_SEMANTIC_CONTRADICTION" failed = semantic_regression.evaluate_live_runs(cases, runs) codes = {item["code"] for item in failed["findings"]} self.assertEqual(failed["status"], "FAIL") self.assertTrue( { "SEMANTIC_CRITICAL_HIGH_RECALL_FAILED", "SEMANTIC_PRECISION_BELOW_THRESHOLD", "SEMANTIC_PAIR_DROPPED", } <= codes, codes, ) def test_three_truthful_live_runs_pass_release_thresholds(self) -> None: self.assertEqual(self.result["schema_version"], "semantic-regression-result/v1") self.assertEqual(self.result["status"], "PASS") self.assertEqual(self.result["deterministic"]["status"], "PASS") self.assertEqual(self.result["live_evaluation"]["status"], "PASS") self.assertEqual(self.result["live_evaluation"]["completed_runs"], 3) self.assertEqual( {item["code"] for item in self.result["findings"]}, set(), ) output = io.StringIO() with redirect_stdout(output): exit_code = semantic_regression.main(["--root", str(ROOT), "--check"]) self.assertEqual(exit_code, 0) self.assertEqual(json.loads(output.getvalue())["live_evaluation"]["status"], "PASS") def test_schema_or_io_failure_returns_exit_two(self) -> None: with tempfile.TemporaryDirectory() as directory: output = io.StringIO() with redirect_stdout(output): exit_code = semantic_regression.main( ["--root", directory, "--manifest", "missing.json", "--check"] ) self.assertEqual(exit_code, 2) self.assertEqual(json.loads(output.getvalue())["status"], "ERROR") def test_three_completed_threshold_passing_runs_allow_exit_zero(self) -> None: cases, runs = self._completed_runs() del cases with tempfile.TemporaryDirectory() as directory: root = Path(directory) fixture_source = ROOT / "harness/tests/fixtures/semantic-consistency" fixture_target = root / "harness/tests/fixtures/semantic-consistency" fixture_target.parent.mkdir(parents=True) shutil.copytree(fixture_source, fixture_target) schema = root / "harness/source/typed-contracts.json" schema.parent.mkdir(parents=True) shutil.copy2(ROOT / "harness/source/typed-contracts.json", schema) ontology = root / semantic_regression.semantic_candidate_builder.DEFAULT_ONTOLOGY ontology.parent.mkdir(parents=True, exist_ok=True) shutil.copy2( ROOT / semantic_regression.semantic_candidate_builder.DEFAULT_ONTOLOGY, ontology, ) prompt = root / semantic_regression.AUDITOR_PROMPT prompt.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(ROOT / semantic_regression.AUDITOR_PROMPT, prompt) for run in runs: path = fixture_target / "evaluation-runs" / f"{run['run_id']}.json" path.write_text(json.dumps(run, indent=2) + "\n", encoding="utf-8") output = io.StringIO() with redirect_stdout(output): exit_code = semantic_regression.main(["--root", str(root), "--check"]) result = json.loads(output.getvalue()) self.assertEqual(exit_code, 0, result) self.assertEqual(result["status"], "PASS") self.assertEqual(result["live_evaluation"]["status"], "PASS") def test_release_gate_uses_semantic_regression_command_at_r2(self) -> None: commands = {command.name: command for command in release_gate.default_commands(ROOT)} command = commands["semantic_regression_gate"] self.assertEqual(command.release, 2) self.assertEqual( command.argv[1:], ("harness/runtime/semantic_regression.py", "--root", str(ROOT), "--check"), ) if __name__ == "__main__": unittest.main()