init: llm-wiki-haness 하네스 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,194 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import redirect_stdout
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
RUNTIME = ROOT / "harness/runtime"
|
||||
sys.path.insert(0, str(RUNTIME))
|
||||
import release_gate # noqa: E402
|
||||
import semantic_regression # noqa: E402
|
||||
|
||||
|
||||
class SemanticRegressionTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.manifest = semantic_regression.load_manifest(ROOT)
|
||||
cls.result = semantic_regression.check(ROOT)
|
||||
|
||||
def test_manifest_has_exactly_70_resolvable_provenanced_cases_and_all_types(self) -> None:
|
||||
cases = self.manifest["cases"]
|
||||
self.assertEqual(len(cases), 70)
|
||||
self.assertEqual(
|
||||
self.manifest["type_distribution"],
|
||||
{"A4": 12, "E1": 12, "DELEG": 12, "D7": 12, "A1": 11, "HUB": 11},
|
||||
)
|
||||
self.assertEqual({case["type"] for case in cases}, set(semantic_regression.TYPES))
|
||||
self.assertEqual({case["provenance"] for case in cases}, {"design-fixture"})
|
||||
self.assertIn("not historical audit findings", self.manifest["corpus_origin"])
|
||||
for case in cases:
|
||||
self.assertEqual(len(case["documents"]), 1)
|
||||
positive = ROOT / case["documents"][0]
|
||||
negative = ROOT / case["counterexample"]
|
||||
self.assertTrue(positive.is_file(), positive)
|
||||
self.assertTrue(negative.is_file(), negative)
|
||||
self.assertNotEqual(positive, negative)
|
||||
self.assertEqual(json.loads(positive.read_text(encoding="utf-8"))["polarity"], "positive")
|
||||
self.assertEqual(json.loads(negative.read_text(encoding="utf-8"))["polarity"], "negative")
|
||||
|
||||
def test_deterministic_cases_replay_with_full_recall_no_fn_and_no_negative_fp(self) -> None:
|
||||
deterministic = self.result["deterministic"]
|
||||
self.assertEqual(deterministic["status"], "PASS", deterministic)
|
||||
self.assertEqual(deterministic["case_count"], 48)
|
||||
self.assertEqual(deterministic["metrics"]["recall"], 1.0)
|
||||
self.assertEqual(deterministic["metrics"]["false_negative"], 0)
|
||||
self.assertEqual(deterministic["metrics"]["negative_false_positive"], 0)
|
||||
for case_type in semantic_regression.DETERMINISTIC_TYPES:
|
||||
self.assertEqual(deterministic["by_type"][case_type]["recall"], 1.0)
|
||||
|
||||
def test_semantic_fixture_pairs_are_ready_and_none_are_dropped(self) -> None:
|
||||
semantic = self.result["semantic_corpus"]
|
||||
self.assertEqual(semantic["status"], "READY", semantic)
|
||||
self.assertEqual(semantic["case_count"], 22)
|
||||
self.assertEqual(semantic["critical_high_count"], 14)
|
||||
self.assertEqual(semantic["dropped_pairs"], 0)
|
||||
|
||||
def _completed_runs(self) -> tuple[list[dict], list[dict]]:
|
||||
cases = [case for case in self.manifest["cases"] if case["type"] in semantic_regression.SEMANTIC_TYPES]
|
||||
ontology = semantic_regression.semantic_candidate_builder.load_ontology(ROOT)
|
||||
ontology_sha = hashlib.sha256(
|
||||
semantic_regression.semantic_surface_extractor.canonical_json_bytes(ontology)
|
||||
).hexdigest()
|
||||
prompt_sha = hashlib.sha256((ROOT / semantic_regression.AUDITOR_PROMPT).read_bytes()).hexdigest()
|
||||
predictions = [
|
||||
{
|
||||
"case_id": case["case_id"],
|
||||
"positive": case["expected"],
|
||||
"counterexample": "CONSISTENT",
|
||||
"dropped": False,
|
||||
}
|
||||
for case in cases
|
||||
]
|
||||
runs = [
|
||||
{
|
||||
"schema_version": semantic_regression.RUN_SCHEMA,
|
||||
"run_id": f"run-{index}",
|
||||
"status": "COMPLETED",
|
||||
"model_id": "test-model",
|
||||
"auditor_contract_version": ontology["auditor_contract_version"],
|
||||
"ontology_sha256": ontology_sha,
|
||||
"prompt_sha256": prompt_sha,
|
||||
"executed_at": datetime.now(timezone.utc).isoformat(),
|
||||
"predictions": json.loads(json.dumps(predictions)),
|
||||
}
|
||||
for index in range(1, 4)
|
||||
]
|
||||
return cases, runs
|
||||
|
||||
def test_semantic_metric_thresholds_are_machine_evaluated(self) -> None:
|
||||
cases, runs = self._completed_runs()
|
||||
passed = semantic_regression.evaluate_live_runs(cases, runs)
|
||||
self.assertEqual(passed["status"], "PASS", passed)
|
||||
self.assertEqual(passed["metrics"]["median_overall_recall"], 1.0)
|
||||
self.assertEqual(passed["metrics"]["median_precision"], 1.0)
|
||||
|
||||
critical = next(case for case in cases if case["severity"] == "Critical")
|
||||
for run in runs:
|
||||
for prediction in run["predictions"]:
|
||||
if prediction["case_id"] == critical["case_id"]:
|
||||
prediction["positive"] = "CONSISTENT"
|
||||
prediction["dropped"] = True
|
||||
for prediction in run["predictions"][:3]:
|
||||
prediction["counterexample"] = "LOCAL_SEMANTIC_CONTRADICTION"
|
||||
failed = semantic_regression.evaluate_live_runs(cases, runs)
|
||||
codes = {item["code"] for item in failed["findings"]}
|
||||
self.assertEqual(failed["status"], "FAIL")
|
||||
self.assertTrue(
|
||||
{
|
||||
"SEMANTIC_CRITICAL_HIGH_RECALL_FAILED",
|
||||
"SEMANTIC_PRECISION_BELOW_THRESHOLD",
|
||||
"SEMANTIC_PAIR_DROPPED",
|
||||
}
|
||||
<= codes,
|
||||
codes,
|
||||
)
|
||||
|
||||
def test_three_truthful_live_runs_pass_release_thresholds(self) -> None:
|
||||
self.assertEqual(self.result["schema_version"], "semantic-regression-result/v1")
|
||||
self.assertEqual(self.result["status"], "PASS")
|
||||
self.assertEqual(self.result["deterministic"]["status"], "PASS")
|
||||
self.assertEqual(self.result["live_evaluation"]["status"], "PASS")
|
||||
self.assertEqual(self.result["live_evaluation"]["completed_runs"], 3)
|
||||
self.assertEqual(
|
||||
{item["code"] for item in self.result["findings"]},
|
||||
set(),
|
||||
)
|
||||
output = io.StringIO()
|
||||
with redirect_stdout(output):
|
||||
exit_code = semantic_regression.main(["--root", str(ROOT), "--check"])
|
||||
self.assertEqual(exit_code, 0)
|
||||
self.assertEqual(json.loads(output.getvalue())["live_evaluation"]["status"], "PASS")
|
||||
|
||||
def test_schema_or_io_failure_returns_exit_two(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = io.StringIO()
|
||||
with redirect_stdout(output):
|
||||
exit_code = semantic_regression.main(
|
||||
["--root", directory, "--manifest", "missing.json", "--check"]
|
||||
)
|
||||
self.assertEqual(exit_code, 2)
|
||||
self.assertEqual(json.loads(output.getvalue())["status"], "ERROR")
|
||||
|
||||
def test_three_completed_threshold_passing_runs_allow_exit_zero(self) -> None:
|
||||
cases, runs = self._completed_runs()
|
||||
del cases
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
fixture_source = ROOT / "harness/tests/fixtures/semantic-consistency"
|
||||
fixture_target = root / "harness/tests/fixtures/semantic-consistency"
|
||||
fixture_target.parent.mkdir(parents=True)
|
||||
shutil.copytree(fixture_source, fixture_target)
|
||||
schema = root / "harness/source/typed-contracts.json"
|
||||
schema.parent.mkdir(parents=True)
|
||||
shutil.copy2(ROOT / "harness/source/typed-contracts.json", schema)
|
||||
ontology = root / semantic_regression.semantic_candidate_builder.DEFAULT_ONTOLOGY
|
||||
ontology.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(
|
||||
ROOT / semantic_regression.semantic_candidate_builder.DEFAULT_ONTOLOGY,
|
||||
ontology,
|
||||
)
|
||||
prompt = root / semantic_regression.AUDITOR_PROMPT
|
||||
prompt.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(ROOT / semantic_regression.AUDITOR_PROMPT, prompt)
|
||||
for run in runs:
|
||||
path = fixture_target / "evaluation-runs" / f"{run['run_id']}.json"
|
||||
path.write_text(json.dumps(run, indent=2) + "\n", encoding="utf-8")
|
||||
output = io.StringIO()
|
||||
with redirect_stdout(output):
|
||||
exit_code = semantic_regression.main(["--root", str(root), "--check"])
|
||||
result = json.loads(output.getvalue())
|
||||
self.assertEqual(exit_code, 0, result)
|
||||
self.assertEqual(result["status"], "PASS")
|
||||
self.assertEqual(result["live_evaluation"]["status"], "PASS")
|
||||
|
||||
def test_release_gate_uses_semantic_regression_command_at_r2(self) -> None:
|
||||
commands = {command.name: command for command in release_gate.default_commands(ROOT)}
|
||||
command = commands["semantic_regression_gate"]
|
||||
self.assertEqual(command.release, 2)
|
||||
self.assertEqual(
|
||||
command.argv[1:],
|
||||
("harness/runtime/semantic_regression.py", "--root", str(ROOT), "--check"),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user