init: llm-wiki-haness 하네스 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,479 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate the 70-case typed and semantic consistency regression corpus.
|
||||
|
||||
The deterministic half is executed against the real typed-contract checker.
|
||||
The semantic half remains release-blocking until three truthful live auditor
|
||||
runs are recorded; fixture completeness is never reported as an LLM result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
import contract_projection
|
||||
import semantic_candidate_builder
|
||||
import semantic_surface_extractor
|
||||
import typed_contract_check
|
||||
|
||||
|
||||
DEFAULT_ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_MANIFEST = Path("harness/tests/fixtures/semantic-consistency/manifest.json")
|
||||
RESULT_SCHEMA = "semantic-regression-result/v1"
|
||||
MANIFEST_SCHEMA = "semantic-consistency-corpus/v1"
|
||||
FIXTURE_SCHEMA = "semantic-regression-fixture/v1"
|
||||
RUN_SCHEMA = "semantic-evaluation-run/v1"
|
||||
TYPES = ("A4", "E1", "DELEG", "D7", "A1", "HUB")
|
||||
DETERMINISTIC_TYPES = frozenset({"A4", "E1", "DELEG", "D7"})
|
||||
SEMANTIC_TYPES = frozenset({"A1", "HUB"})
|
||||
EXPECTED_CODES = {
|
||||
"A4": "MANUAL_ARTIFACT_SCHEMA_RESTATEMENT",
|
||||
"E1": "DUPLICATE_CONCERN_OWNER",
|
||||
"DELEG": "UNACCEPTED_DELEGATION",
|
||||
"D7": "GENERATED_CONTRACT_PROJECTION_DRIFT",
|
||||
"A1": "CONTRADICTION",
|
||||
"HUB": "CONTRADICTION",
|
||||
}
|
||||
CASE_ID_RE = re.compile(r"^(A4|E1|DELEG|D7|A1|HUB)-\d{3}$")
|
||||
HEX_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
AUDITOR_PROMPT = Path("harness/source/agents/bodies/wiki-semantic-coherence-auditor.md")
|
||||
|
||||
|
||||
class SemanticRegressionError(ValueError):
|
||||
"""The corpus schema, path set, or evaluation record is unusable."""
|
||||
|
||||
|
||||
def _safe_path(root: Path, value: Any, location: str) -> Path:
|
||||
if not isinstance(value, str) or not value or "\\" in value:
|
||||
raise SemanticRegressionError(f"{location}: expected repo-relative POSIX path")
|
||||
relative = Path(value)
|
||||
if relative.is_absolute() or ".." in relative.parts:
|
||||
raise SemanticRegressionError(f"{location}: path escapes repository")
|
||||
path = (root / relative).resolve()
|
||||
try:
|
||||
path.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise SemanticRegressionError(f"{location}: path escapes repository") from exc
|
||||
if not path.is_file():
|
||||
raise SemanticRegressionError(f"{location}: file does not exist: {value}")
|
||||
return path
|
||||
|
||||
|
||||
def _load_json(path: Path, schema: str) -> dict[str, Any]:
|
||||
try:
|
||||
document = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise SemanticRegressionError(f"cannot read {path}: {exc}") from exc
|
||||
if not isinstance(document, dict) or document.get("schema_version") != schema:
|
||||
raise SemanticRegressionError(f"{path}: expected {schema}")
|
||||
return document
|
||||
|
||||
|
||||
def load_manifest(root: Path, manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]:
|
||||
root = root.resolve(strict=True)
|
||||
source = manifest_path if manifest_path.is_absolute() else root / manifest_path
|
||||
manifest = _load_json(source, MANIFEST_SCHEMA)
|
||||
cases = manifest.get("cases")
|
||||
if manifest.get("case_count") != 70 or not isinstance(cases, list) or len(cases) != 70:
|
||||
raise SemanticRegressionError("manifest must declare exactly 70 cases")
|
||||
thresholds = manifest.get("thresholds")
|
||||
expected_thresholds = {
|
||||
"deterministic_recall": 1.0,
|
||||
"deterministic_false_negatives": 0,
|
||||
"deterministic_negative_false_positives": 0,
|
||||
"semantic_critical_high_recall": 1.0,
|
||||
"semantic_overall_recall": 0.95,
|
||||
"semantic_precision": 0.90,
|
||||
"semantic_dropped_pairs": 0,
|
||||
"live_runs": 3,
|
||||
}
|
||||
if thresholds != expected_thresholds:
|
||||
raise SemanticRegressionError("manifest thresholds do not match the WP17 release contract")
|
||||
seen: set[str] = set()
|
||||
distribution: Counter[str] = Counter()
|
||||
required = {
|
||||
"case_id",
|
||||
"type",
|
||||
"severity",
|
||||
"documents",
|
||||
"surface_a",
|
||||
"surface_b",
|
||||
"expected",
|
||||
"counterexample",
|
||||
"provenance",
|
||||
}
|
||||
for index, case in enumerate(cases):
|
||||
if not isinstance(case, dict) or set(case) != required:
|
||||
raise SemanticRegressionError(f"cases[{index}] must contain exactly {sorted(required)}")
|
||||
case_id = case["case_id"]
|
||||
case_type = case["type"]
|
||||
if not isinstance(case_id, str) or not CASE_ID_RE.fullmatch(case_id):
|
||||
raise SemanticRegressionError(f"cases[{index}].case_id is invalid")
|
||||
if case_id in seen:
|
||||
raise SemanticRegressionError(f"duplicate case id: {case_id}")
|
||||
seen.add(case_id)
|
||||
if case_type not in TYPES or not case_id.startswith(f"{case_type}-"):
|
||||
raise SemanticRegressionError(f"{case_id}: type does not match case id")
|
||||
distribution[case_type] += 1
|
||||
if case.get("severity") not in {"Critical", "High", "Medium", "Low"}:
|
||||
raise SemanticRegressionError(f"{case_id}: invalid severity")
|
||||
if case.get("expected") != EXPECTED_CODES[case_type]:
|
||||
raise SemanticRegressionError(f"{case_id}: unexpected expected code")
|
||||
if case.get("provenance") != "design-fixture":
|
||||
raise SemanticRegressionError(f"{case_id}: provenance must be design-fixture")
|
||||
documents = case.get("documents")
|
||||
if not isinstance(documents, list) or len(documents) != 1:
|
||||
raise SemanticRegressionError(f"{case_id}: documents must name one positive fixture")
|
||||
for field in ("surface_a", "surface_b"):
|
||||
surface = case.get(field)
|
||||
if (
|
||||
not isinstance(surface, dict)
|
||||
or set(surface) != {"section", "quote"}
|
||||
or not all(isinstance(surface[key], str) and surface[key] for key in surface)
|
||||
):
|
||||
raise SemanticRegressionError(f"{case_id}: invalid {field}")
|
||||
for path_index, value in enumerate(documents):
|
||||
_safe_path(root, value, f"{case_id}.documents[{path_index}]")
|
||||
_safe_path(root, case["counterexample"], f"{case_id}.counterexample")
|
||||
if set(distribution) != set(TYPES) or any(distribution[item] == 0 for item in TYPES):
|
||||
raise SemanticRegressionError("all six case types must be represented")
|
||||
declared_distribution = manifest.get("type_distribution")
|
||||
if declared_distribution != {name: distribution[name] for name in TYPES}:
|
||||
raise SemanticRegressionError("type_distribution does not match cases")
|
||||
runs = manifest.get("evaluation_runs")
|
||||
if not isinstance(runs, list) or len(runs) != 3:
|
||||
raise SemanticRegressionError("evaluation_runs must name exactly three records")
|
||||
for index, value in enumerate(runs):
|
||||
_safe_path(root, value, f"evaluation_runs[{index}]")
|
||||
return manifest
|
||||
|
||||
|
||||
def _fixture(root: Path, value: str, case_id: str, polarity: str) -> dict[str, Any]:
|
||||
path = _safe_path(root, value, f"{case_id}.{polarity}")
|
||||
fixture = _load_json(path, FIXTURE_SCHEMA)
|
||||
if fixture.get("case_id") != case_id or fixture.get("polarity") != polarity:
|
||||
raise SemanticRegressionError(f"{path}: case_id/polarity mismatch")
|
||||
files = fixture.get("files")
|
||||
if not isinstance(files, dict) or not files:
|
||||
raise SemanticRegressionError(f"{path}: files must be a non-empty object")
|
||||
for relative, content in files.items():
|
||||
if not isinstance(content, str):
|
||||
raise SemanticRegressionError(f"{path}: fixture contents must be strings")
|
||||
candidate = Path(relative)
|
||||
if candidate.is_absolute() or ".." in candidate.parts or "\\" in relative:
|
||||
raise SemanticRegressionError(f"{path}: unsafe fixture path {relative}")
|
||||
if not isinstance(fixture.get("materialize_projections", False), bool):
|
||||
raise SemanticRegressionError(f"{path}: materialize_projections must be boolean")
|
||||
mutations = fixture.get("mutations", [])
|
||||
if not isinstance(mutations, list):
|
||||
raise SemanticRegressionError(f"{path}: mutations must be an array")
|
||||
for mutation in mutations:
|
||||
if not isinstance(mutation, dict) or set(mutation) != {"path", "old", "new"}:
|
||||
raise SemanticRegressionError(f"{path}: invalid mutation")
|
||||
if not all(isinstance(mutation[key], str) for key in mutation):
|
||||
raise SemanticRegressionError(f"{path}: mutation values must be strings")
|
||||
return fixture
|
||||
|
||||
|
||||
def _materialize(root: Path, fixture: Mapping[str, Any]) -> Path:
|
||||
stage = Path(tempfile.mkdtemp(prefix="semantic-regression-"))
|
||||
for relative, content in fixture["files"].items():
|
||||
path = stage / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
schema = stage / typed_contract_check.DEFAULT_SCHEMA
|
||||
schema.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(root / typed_contract_check.DEFAULT_SCHEMA, schema)
|
||||
if fixture.get("materialize_projections"):
|
||||
updates, result = contract_projection.build_updates(stage)
|
||||
if result["status"] == "FAIL":
|
||||
raise SemanticRegressionError(
|
||||
"fixture projection precondition failed: "
|
||||
+ ",".join(sorted({item["code"] for item in result["findings"]}))
|
||||
)
|
||||
for path, text in updates.items():
|
||||
path.write_text(text, encoding="utf-8")
|
||||
for mutation in fixture.get("mutations", []):
|
||||
path = stage / mutation["path"]
|
||||
if not path.is_file():
|
||||
raise SemanticRegressionError(f"mutation target does not exist: {mutation['path']}")
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if mutation["old"] not in text:
|
||||
raise SemanticRegressionError(f"mutation source not found: {mutation['old']!r}")
|
||||
path.write_text(text.replace(mutation["old"], mutation["new"], 1), encoding="utf-8")
|
||||
return stage
|
||||
|
||||
|
||||
def evaluate_deterministic(root: Path, cases: Iterable[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
by_type: dict[str, dict[str, int | float]] = {}
|
||||
findings: list[dict[str, Any]] = []
|
||||
total_positive = true_positive = false_negative = negative_fp = 0
|
||||
for case in cases:
|
||||
if case["type"] not in DETERMINISTIC_TYPES:
|
||||
continue
|
||||
case_id = str(case["case_id"])
|
||||
positive = _fixture(root, case["documents"][0], case_id, "positive")
|
||||
negative = _fixture(root, case["counterexample"], case_id, "negative")
|
||||
metrics = by_type.setdefault(case["type"], {"cases": 0, "true_positive": 0, "false_negative": 0, "negative_false_positive": 0})
|
||||
metrics["cases"] += 1
|
||||
total_positive += 1
|
||||
positive_stage = _materialize(root, positive)
|
||||
negative_stage = _materialize(root, negative)
|
||||
try:
|
||||
positive_codes = {item["code"] for item in typed_contract_check.check(positive_stage)["findings"]}
|
||||
negative_result = typed_contract_check.check(negative_stage)
|
||||
finally:
|
||||
shutil.rmtree(positive_stage)
|
||||
shutil.rmtree(negative_stage)
|
||||
if case["expected"] in positive_codes:
|
||||
true_positive += 1
|
||||
metrics["true_positive"] += 1
|
||||
else:
|
||||
false_negative += 1
|
||||
metrics["false_negative"] += 1
|
||||
findings.append({
|
||||
"code": "DETERMINISTIC_FALSE_NEGATIVE",
|
||||
"case_id": case_id,
|
||||
"message": f"expected {case['expected']}, observed {sorted(positive_codes)}",
|
||||
})
|
||||
if negative_result["status"] != "PASS":
|
||||
negative_fp += 1
|
||||
metrics["negative_false_positive"] += 1
|
||||
findings.append({
|
||||
"code": "DETERMINISTIC_FALSE_POSITIVE",
|
||||
"case_id": case_id,
|
||||
"message": f"counterexample findings: {[item['code'] for item in negative_result['findings']]}",
|
||||
})
|
||||
for metrics in by_type.values():
|
||||
count = int(metrics["cases"])
|
||||
metrics["recall"] = int(metrics["true_positive"]) / count if count else 0.0
|
||||
recall = true_positive / total_positive if total_positive else 0.0
|
||||
status = "PASS" if recall == 1.0 and false_negative == 0 and negative_fp == 0 else "FAIL"
|
||||
return {
|
||||
"status": status,
|
||||
"case_count": total_positive,
|
||||
"metrics": {
|
||||
"recall": recall,
|
||||
"true_positive": true_positive,
|
||||
"false_negative": false_negative,
|
||||
"negative_false_positive": negative_fp,
|
||||
},
|
||||
"by_type": by_type,
|
||||
"findings": findings,
|
||||
}
|
||||
|
||||
|
||||
def validate_semantic_corpus(root: Path, cases: Iterable[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
count = 0
|
||||
high_critical = 0
|
||||
for case in cases:
|
||||
if case["type"] not in SEMANTIC_TYPES:
|
||||
continue
|
||||
count += 1
|
||||
high_critical += case["severity"] in {"Critical", "High"}
|
||||
positive = _fixture(root, case["documents"][0], case["case_id"], "positive")
|
||||
negative = _fixture(root, case["counterexample"], case["case_id"], "negative")
|
||||
positive_text = "\n".join(positive["files"].values())
|
||||
negative_text = "\n".join(negative["files"].values())
|
||||
for name, surface in (("surface_a", case["surface_a"]), ("surface_b", case["surface_b"])):
|
||||
if surface["quote"] not in positive_text:
|
||||
findings.append({
|
||||
"code": "SEMANTIC_PAIR_DROPPED",
|
||||
"case_id": case["case_id"],
|
||||
"message": f"{name} quote is absent from positive fixture",
|
||||
})
|
||||
if case["surface_a"]["quote"] not in negative_text or case["surface_b"]["quote"] in negative_text:
|
||||
# The counterexample must retain the authority claim but replace
|
||||
# the contradictory claim with a compatible variant.
|
||||
findings.append({
|
||||
"code": "INVALID_SEMANTIC_COUNTEREXAMPLE",
|
||||
"case_id": case["case_id"],
|
||||
"message": "counterexample does not preserve A while replacing B",
|
||||
})
|
||||
return {
|
||||
"status": "READY" if not findings else "FAIL",
|
||||
"case_count": count,
|
||||
"critical_high_count": high_critical,
|
||||
"dropped_pairs": sum(item["code"] == "SEMANTIC_PAIR_DROPPED" for item in findings),
|
||||
"findings": findings,
|
||||
}
|
||||
|
||||
|
||||
def _median(values: list[float]) -> float:
|
||||
ordered = sorted(values)
|
||||
return ordered[len(ordered) // 2]
|
||||
|
||||
|
||||
def evaluate_live_runs(
|
||||
semantic_cases: Iterable[Mapping[str, Any]],
|
||||
runs: Iterable[Mapping[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
cases = list(semantic_cases)
|
||||
run_list = list(runs)
|
||||
not_run = [run for run in run_list if run.get("status") == "NOT_RUN"]
|
||||
if not_run:
|
||||
return {
|
||||
"status": "NOT_RUN",
|
||||
"required_runs": 3,
|
||||
"completed_runs": len(run_list) - len(not_run),
|
||||
"metrics": None,
|
||||
"findings": [{
|
||||
"code": "LIVE_EVALUATION_NOT_RUN",
|
||||
"run_id": str(run.get("run_id", "")),
|
||||
"message": str(run.get("reason", "live semantic evaluation has not run")),
|
||||
} for run in not_run],
|
||||
}
|
||||
if len(run_list) != 3:
|
||||
raise SemanticRegressionError("live evaluation requires exactly three runs")
|
||||
case_map = {case["case_id"]: case for case in cases}
|
||||
per_run: list[dict[str, Any]] = []
|
||||
findings: list[dict[str, Any]] = []
|
||||
for run in run_list:
|
||||
if run.get("status") != "COMPLETED":
|
||||
raise SemanticRegressionError(f"unsupported evaluation status: {run.get('status')}")
|
||||
for field in ("model_id", "auditor_contract_version", "ontology_sha256", "prompt_sha256"):
|
||||
if not isinstance(run.get(field), str) or not run[field]:
|
||||
raise SemanticRegressionError(f"{run.get('run_id')}: {field} is required")
|
||||
if not HEX_SHA256.fullmatch(str(run["ontology_sha256"])) or not HEX_SHA256.fullmatch(str(run["prompt_sha256"])):
|
||||
raise SemanticRegressionError(f"{run.get('run_id')}: ontology/prompt sha256 is invalid")
|
||||
executed_at = run.get("executed_at")
|
||||
if not isinstance(executed_at, str) or not executed_at:
|
||||
raise SemanticRegressionError(f"{run.get('run_id')}: executed_at is required")
|
||||
try:
|
||||
datetime.fromisoformat(executed_at)
|
||||
except ValueError as exc:
|
||||
raise SemanticRegressionError(f"{run.get('run_id')}: executed_at is not ISO-8601") from exc
|
||||
predictions = run.get("predictions")
|
||||
if not isinstance(predictions, list):
|
||||
raise SemanticRegressionError(f"{run.get('run_id')}: predictions must be an array")
|
||||
by_case = {item.get("case_id"): item for item in predictions if isinstance(item, dict)}
|
||||
if set(by_case) != set(case_map) or len(predictions) != len(case_map):
|
||||
raise SemanticRegressionError(f"{run.get('run_id')}: predictions must cover every semantic case exactly once")
|
||||
tp = fn = fp = dropped = critical_high_tp = critical_high_total = 0
|
||||
for case_id, case in case_map.items():
|
||||
item = by_case[case_id]
|
||||
if set(item) != {"case_id", "positive", "counterexample", "dropped"}:
|
||||
raise SemanticRegressionError(f"{run.get('run_id')}/{case_id}: invalid prediction fields")
|
||||
if item["dropped"]:
|
||||
dropped += 1
|
||||
positive_hit = item["positive"] == case["expected"]
|
||||
tp += positive_hit
|
||||
fn += not positive_hit
|
||||
fp += item["counterexample"] not in {None, "CONSISTENT", "CONTEXTUAL_VARIANT", "COMPLEMENTARY"}
|
||||
if case["severity"] in {"Critical", "High"}:
|
||||
critical_high_total += 1
|
||||
critical_high_tp += positive_hit
|
||||
recall = tp / len(case_map) if case_map else 0.0
|
||||
critical_high_recall = critical_high_tp / critical_high_total if critical_high_total else 0.0
|
||||
precision = tp / (tp + fp) if tp + fp else 0.0
|
||||
per_run.append({
|
||||
"run_id": run["run_id"],
|
||||
"recall": recall,
|
||||
"critical_high_recall": critical_high_recall,
|
||||
"precision": precision,
|
||||
"dropped_pairs": dropped,
|
||||
"true_positive": tp,
|
||||
"false_negative": fn,
|
||||
"false_positive": fp,
|
||||
})
|
||||
if critical_high_recall != 1.0:
|
||||
findings.append({"code": "SEMANTIC_CRITICAL_HIGH_RECALL_FAILED", "run_id": run["run_id"], "message": str(critical_high_recall)})
|
||||
if dropped:
|
||||
findings.append({"code": "SEMANTIC_PAIR_DROPPED", "run_id": run["run_id"], "message": str(dropped)})
|
||||
median_recall = _median([item["recall"] for item in per_run])
|
||||
median_precision = _median([item["precision"] for item in per_run])
|
||||
if median_recall < 0.95:
|
||||
findings.append({"code": "SEMANTIC_RECALL_BELOW_THRESHOLD", "message": str(median_recall)})
|
||||
if median_precision < 0.90:
|
||||
findings.append({"code": "SEMANTIC_PRECISION_BELOW_THRESHOLD", "message": str(median_precision)})
|
||||
return {
|
||||
"status": "PASS" if not findings else "FAIL",
|
||||
"required_runs": 3,
|
||||
"completed_runs": 3,
|
||||
"metrics": {
|
||||
"median_overall_recall": median_recall,
|
||||
"median_precision": median_precision,
|
||||
"runs": per_run,
|
||||
},
|
||||
"findings": findings,
|
||||
}
|
||||
|
||||
|
||||
def check(root: Path, manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]:
|
||||
root = root.resolve(strict=True)
|
||||
manifest = load_manifest(root, manifest_path)
|
||||
cases = manifest["cases"]
|
||||
deterministic = evaluate_deterministic(root, cases)
|
||||
semantic_corpus = validate_semantic_corpus(root, cases)
|
||||
runs = [
|
||||
_load_json(_safe_path(root, value, f"evaluation_runs[{index}]"), RUN_SCHEMA)
|
||||
for index, value in enumerate(manifest["evaluation_runs"])
|
||||
]
|
||||
ontology = semantic_candidate_builder.load_ontology(root)
|
||||
expected_ontology_sha = hashlib.sha256(
|
||||
semantic_surface_extractor.canonical_json_bytes(ontology)
|
||||
).hexdigest()
|
||||
expected_prompt_sha = hashlib.sha256((root / AUDITOR_PROMPT).read_bytes()).hexdigest()
|
||||
for run in runs:
|
||||
if run.get("status") == "COMPLETED" and (
|
||||
run.get("auditor_contract_version") != ontology["auditor_contract_version"]
|
||||
or run.get("ontology_sha256") != expected_ontology_sha
|
||||
or run.get("prompt_sha256") != expected_prompt_sha
|
||||
):
|
||||
raise SemanticRegressionError(
|
||||
f"{run.get('run_id')}: live evaluation contract, ontology, or prompt is stale"
|
||||
)
|
||||
live = evaluate_live_runs((case for case in cases if case["type"] in SEMANTIC_TYPES), runs)
|
||||
findings = [*deterministic["findings"], *semantic_corpus["findings"], *live["findings"]]
|
||||
status = (
|
||||
"PASS"
|
||||
if deterministic["status"] == "PASS"
|
||||
and semantic_corpus["status"] == "READY"
|
||||
and live["status"] == "PASS"
|
||||
else "FAIL"
|
||||
)
|
||||
return {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"status": status,
|
||||
"case_count": len(cases),
|
||||
"type_distribution": manifest["type_distribution"],
|
||||
"deterministic": deterministic,
|
||||
"semantic_corpus": semantic_corpus,
|
||||
"live_evaluation": live,
|
||||
"findings": findings,
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
|
||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
||||
parser.add_argument("--check", action="store_true", required=True)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
result = check(args.root, args.manifest)
|
||||
exit_code = 0 if result["status"] == "PASS" else 1
|
||||
except (SemanticRegressionError, typed_contract_check.TypedContractError, OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
result = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"status": "ERROR",
|
||||
"errors": [{"code": "SEMANTIC_REGRESSION_ERROR", "message": str(exc)}],
|
||||
}
|
||||
exit_code = 2
|
||||
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
sys.stdout.write("\n")
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user