195 lines
8.9 KiB
Python
195 lines
8.9 KiB
Python
from __future__ import annotations
|
|
|
|
from contextlib import redirect_stdout
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
RUNTIME = ROOT / "harness/runtime"
|
|
sys.path.insert(0, str(RUNTIME))
|
|
import release_gate # noqa: E402
|
|
import semantic_regression # noqa: E402
|
|
|
|
|
|
class SemanticRegressionTest(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls) -> None:
|
|
cls.manifest = semantic_regression.load_manifest(ROOT)
|
|
cls.result = semantic_regression.check(ROOT)
|
|
|
|
def test_manifest_has_exactly_70_resolvable_provenanced_cases_and_all_types(self) -> None:
|
|
cases = self.manifest["cases"]
|
|
self.assertEqual(len(cases), 70)
|
|
self.assertEqual(
|
|
self.manifest["type_distribution"],
|
|
{"A4": 12, "E1": 12, "DELEG": 12, "D7": 12, "A1": 11, "HUB": 11},
|
|
)
|
|
self.assertEqual({case["type"] for case in cases}, set(semantic_regression.TYPES))
|
|
self.assertEqual({case["provenance"] for case in cases}, {"design-fixture"})
|
|
self.assertIn("not historical audit findings", self.manifest["corpus_origin"])
|
|
for case in cases:
|
|
self.assertEqual(len(case["documents"]), 1)
|
|
positive = ROOT / case["documents"][0]
|
|
negative = ROOT / case["counterexample"]
|
|
self.assertTrue(positive.is_file(), positive)
|
|
self.assertTrue(negative.is_file(), negative)
|
|
self.assertNotEqual(positive, negative)
|
|
self.assertEqual(json.loads(positive.read_text(encoding="utf-8"))["polarity"], "positive")
|
|
self.assertEqual(json.loads(negative.read_text(encoding="utf-8"))["polarity"], "negative")
|
|
|
|
def test_deterministic_cases_replay_with_full_recall_no_fn_and_no_negative_fp(self) -> None:
|
|
deterministic = self.result["deterministic"]
|
|
self.assertEqual(deterministic["status"], "PASS", deterministic)
|
|
self.assertEqual(deterministic["case_count"], 48)
|
|
self.assertEqual(deterministic["metrics"]["recall"], 1.0)
|
|
self.assertEqual(deterministic["metrics"]["false_negative"], 0)
|
|
self.assertEqual(deterministic["metrics"]["negative_false_positive"], 0)
|
|
for case_type in semantic_regression.DETERMINISTIC_TYPES:
|
|
self.assertEqual(deterministic["by_type"][case_type]["recall"], 1.0)
|
|
|
|
def test_semantic_fixture_pairs_are_ready_and_none_are_dropped(self) -> None:
|
|
semantic = self.result["semantic_corpus"]
|
|
self.assertEqual(semantic["status"], "READY", semantic)
|
|
self.assertEqual(semantic["case_count"], 22)
|
|
self.assertEqual(semantic["critical_high_count"], 14)
|
|
self.assertEqual(semantic["dropped_pairs"], 0)
|
|
|
|
def _completed_runs(self) -> tuple[list[dict], list[dict]]:
|
|
cases = [case for case in self.manifest["cases"] if case["type"] in semantic_regression.SEMANTIC_TYPES]
|
|
ontology = semantic_regression.semantic_candidate_builder.load_ontology(ROOT)
|
|
ontology_sha = hashlib.sha256(
|
|
semantic_regression.semantic_surface_extractor.canonical_json_bytes(ontology)
|
|
).hexdigest()
|
|
prompt_sha = hashlib.sha256((ROOT / semantic_regression.AUDITOR_PROMPT).read_bytes()).hexdigest()
|
|
predictions = [
|
|
{
|
|
"case_id": case["case_id"],
|
|
"positive": case["expected"],
|
|
"counterexample": "CONSISTENT",
|
|
"dropped": False,
|
|
}
|
|
for case in cases
|
|
]
|
|
runs = [
|
|
{
|
|
"schema_version": semantic_regression.RUN_SCHEMA,
|
|
"run_id": f"run-{index}",
|
|
"status": "COMPLETED",
|
|
"model_id": "test-model",
|
|
"auditor_contract_version": ontology["auditor_contract_version"],
|
|
"ontology_sha256": ontology_sha,
|
|
"prompt_sha256": prompt_sha,
|
|
"executed_at": datetime.now(timezone.utc).isoformat(),
|
|
"predictions": json.loads(json.dumps(predictions)),
|
|
}
|
|
for index in range(1, 4)
|
|
]
|
|
return cases, runs
|
|
|
|
def test_semantic_metric_thresholds_are_machine_evaluated(self) -> None:
|
|
cases, runs = self._completed_runs()
|
|
passed = semantic_regression.evaluate_live_runs(cases, runs)
|
|
self.assertEqual(passed["status"], "PASS", passed)
|
|
self.assertEqual(passed["metrics"]["median_overall_recall"], 1.0)
|
|
self.assertEqual(passed["metrics"]["median_precision"], 1.0)
|
|
|
|
critical = next(case for case in cases if case["severity"] == "Critical")
|
|
for run in runs:
|
|
for prediction in run["predictions"]:
|
|
if prediction["case_id"] == critical["case_id"]:
|
|
prediction["positive"] = "CONSISTENT"
|
|
prediction["dropped"] = True
|
|
for prediction in run["predictions"][:3]:
|
|
prediction["counterexample"] = "LOCAL_SEMANTIC_CONTRADICTION"
|
|
failed = semantic_regression.evaluate_live_runs(cases, runs)
|
|
codes = {item["code"] for item in failed["findings"]}
|
|
self.assertEqual(failed["status"], "FAIL")
|
|
self.assertTrue(
|
|
{
|
|
"SEMANTIC_CRITICAL_HIGH_RECALL_FAILED",
|
|
"SEMANTIC_PRECISION_BELOW_THRESHOLD",
|
|
"SEMANTIC_PAIR_DROPPED",
|
|
}
|
|
<= codes,
|
|
codes,
|
|
)
|
|
|
|
def test_three_truthful_live_runs_pass_release_thresholds(self) -> None:
|
|
self.assertEqual(self.result["schema_version"], "semantic-regression-result/v1")
|
|
self.assertEqual(self.result["status"], "PASS")
|
|
self.assertEqual(self.result["deterministic"]["status"], "PASS")
|
|
self.assertEqual(self.result["live_evaluation"]["status"], "PASS")
|
|
self.assertEqual(self.result["live_evaluation"]["completed_runs"], 3)
|
|
self.assertEqual(
|
|
{item["code"] for item in self.result["findings"]},
|
|
set(),
|
|
)
|
|
output = io.StringIO()
|
|
with redirect_stdout(output):
|
|
exit_code = semantic_regression.main(["--root", str(ROOT), "--check"])
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertEqual(json.loads(output.getvalue())["live_evaluation"]["status"], "PASS")
|
|
|
|
def test_schema_or_io_failure_returns_exit_two(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
output = io.StringIO()
|
|
with redirect_stdout(output):
|
|
exit_code = semantic_regression.main(
|
|
["--root", directory, "--manifest", "missing.json", "--check"]
|
|
)
|
|
self.assertEqual(exit_code, 2)
|
|
self.assertEqual(json.loads(output.getvalue())["status"], "ERROR")
|
|
|
|
def test_three_completed_threshold_passing_runs_allow_exit_zero(self) -> None:
|
|
cases, runs = self._completed_runs()
|
|
del cases
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
fixture_source = ROOT / "harness/tests/fixtures/semantic-consistency"
|
|
fixture_target = root / "harness/tests/fixtures/semantic-consistency"
|
|
fixture_target.parent.mkdir(parents=True)
|
|
shutil.copytree(fixture_source, fixture_target)
|
|
schema = root / "harness/source/typed-contracts.json"
|
|
schema.parent.mkdir(parents=True)
|
|
shutil.copy2(ROOT / "harness/source/typed-contracts.json", schema)
|
|
ontology = root / semantic_regression.semantic_candidate_builder.DEFAULT_ONTOLOGY
|
|
ontology.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(
|
|
ROOT / semantic_regression.semantic_candidate_builder.DEFAULT_ONTOLOGY,
|
|
ontology,
|
|
)
|
|
prompt = root / semantic_regression.AUDITOR_PROMPT
|
|
prompt.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(ROOT / semantic_regression.AUDITOR_PROMPT, prompt)
|
|
for run in runs:
|
|
path = fixture_target / "evaluation-runs" / f"{run['run_id']}.json"
|
|
path.write_text(json.dumps(run, indent=2) + "\n", encoding="utf-8")
|
|
output = io.StringIO()
|
|
with redirect_stdout(output):
|
|
exit_code = semantic_regression.main(["--root", str(root), "--check"])
|
|
result = json.loads(output.getvalue())
|
|
self.assertEqual(exit_code, 0, result)
|
|
self.assertEqual(result["status"], "PASS")
|
|
self.assertEqual(result["live_evaluation"]["status"], "PASS")
|
|
|
|
def test_release_gate_uses_semantic_regression_command_at_r2(self) -> None:
|
|
commands = {command.name: command for command in release_gate.default_commands(ROOT)}
|
|
command = commands["semantic_regression_gate"]
|
|
self.assertEqual(command.release, 2)
|
|
self.assertEqual(
|
|
command.argv[1:],
|
|
("harness/runtime/semantic_regression.py", "--root", str(ROOT), "--check"),
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|