init: llm-wiki-haness 하네스 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
RUNTIME = REPO_ROOT / "harness/runtime"
|
||||
|
||||
|
||||
class ProofRunnerTest(unittest.TestCase):
|
||||
def test_cli_builds_fixed_verified_manifest(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / "source.md").write_text("첫 줄\n고정 인용\n", encoding="utf-8")
|
||||
request = root / "request.json"
|
||||
request.write_text(json.dumps({
|
||||
"schema_version": "proof-request/v1",
|
||||
"run": {"id": "run-1", "profile": "audit"},
|
||||
"proofs": [{"finding": {"id": "L1-F01", "role": "current_state"}, "source": {"path": "source.md", "quote_utf8": "고정 인용"}}],
|
||||
}, ensure_ascii=False), encoding="utf-8")
|
||||
output = root / "proof.json"
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(RUNTIME / "proof_runner.py"), str(request), "--repo-root", str(root), "--output", str(output)],
|
||||
check=False, capture_output=True, text=True,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stdout)
|
||||
run_result = json.loads(result.stdout)
|
||||
self.assertEqual(run_result["schema_version"], "proof-runner-result/v1")
|
||||
self.assertEqual(run_result["status"], "PASS")
|
||||
manifest = json.loads(output.read_text(encoding="utf-8"))
|
||||
self.assertEqual(manifest["verification"]["status"], "PASS")
|
||||
self.assertEqual(manifest["proofs"][0]["execution"]["argv"][0], "proof-runner/exact-utf8-v1")
|
||||
self.assertEqual(run_result["manifest"]["sha256"], hashlib.sha256(output.read_bytes()).hexdigest())
|
||||
summary = root / "proof-summary.md"
|
||||
self.assertTrue(summary.is_file())
|
||||
summary_text = summary.read_text(encoding="utf-8")
|
||||
self.assertIn(run_result["manifest"]["sha256"], summary_text)
|
||||
self.assertIn("- PASS: 1", summary_text)
|
||||
self.assertIn("- FAIL: 0", summary_text)
|
||||
|
||||
def test_ambiguous_quote_fails_without_output(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / "source.md").write_text("same\nsame\n", encoding="utf-8")
|
||||
request = root / "request.json"
|
||||
request.write_text(json.dumps({"schema_version": "proof-request/v1", "run": {"id": "run-1", "profile": "audit"}, "proofs": [{"finding": {"id": "F1", "role": "quote"}, "source": {"path": "source.md", "quote_utf8": "same"}}]}), encoding="utf-8")
|
||||
output = root / "proof.json"
|
||||
result = subprocess.run([sys.executable, str(RUNTIME / "proof_runner.py"), str(request), "--repo-root", str(root), "--output", str(output)], check=False, capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertFalse(output.exists())
|
||||
self.assertFalse((root / "proof-summary.md").exists())
|
||||
self.assertIn("AMBIGUOUS_QUOTE", result.stdout)
|
||||
|
||||
def test_atomic_summary_failure_rolls_back_manifest(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / "source.md").write_text("proof\n", encoding="utf-8")
|
||||
request = root / "request.json"
|
||||
request.write_text(json.dumps({
|
||||
"schema_version": "proof-request/v1",
|
||||
"run": {"id": "run-1", "profile": "audit"},
|
||||
"proofs": [{"finding": {"id": "F1", "role": "quote"}, "source": {"path": "source.md", "quote_utf8": "proof"}}],
|
||||
}), encoding="utf-8")
|
||||
output = root / "proof-manifest.json"
|
||||
summary = root / "occupied"
|
||||
summary.mkdir()
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(RUNTIME / "proof_runner.py"), str(request), "--repo-root", str(root), "--output", str(output), "--summary-output", str(summary)],
|
||||
check=False, capture_output=True, text=True,
|
||||
)
|
||||
self.assertEqual(result.returncode, 2, result.stdout)
|
||||
self.assertFalse(output.exists())
|
||||
|
||||
def test_run_namespace_is_confined_to_explicit_run_root(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
base = Path(directory)
|
||||
repo = base / "repo"
|
||||
run = base / "run"
|
||||
repo.mkdir()
|
||||
run.mkdir()
|
||||
(run / "source.md").write_text("격리 인용\n", encoding="utf-8")
|
||||
request = run / "request.json"
|
||||
request.write_text(json.dumps({
|
||||
"schema_version": "proof-request/v1",
|
||||
"run": {"id": "run-namespace", "profile": "audit"},
|
||||
"proofs": [{
|
||||
"finding": {"id": "F1", "role": "quote"},
|
||||
"source": {"namespace": "run", "path": "source.md", "quote_utf8": "격리 인용"},
|
||||
}],
|
||||
}, ensure_ascii=False), encoding="utf-8")
|
||||
output = run / "proof-manifest.json"
|
||||
command = [
|
||||
sys.executable,
|
||||
str(RUNTIME / "proof_runner.py"),
|
||||
str(request),
|
||||
"--repo-root",
|
||||
str(repo),
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
|
||||
missing_root = subprocess.run(command, check=False, capture_output=True, text=True)
|
||||
self.assertEqual(missing_root.returncode, 2)
|
||||
self.assertIn("RUN_ROOT_REQUIRED", missing_root.stdout)
|
||||
|
||||
passed = subprocess.run([*command, "--run-root", str(run)], check=False, capture_output=True, text=True)
|
||||
self.assertEqual(passed.returncode, 0, passed.stdout)
|
||||
manifest = json.loads(output.read_text(encoding="utf-8"))
|
||||
self.assertEqual(manifest["proofs"][0]["source"]["namespace"], "run")
|
||||
self.assertEqual(manifest["proofs"][0]["source"]["path"], "source.md")
|
||||
|
||||
def test_run_namespace_rejects_escape_and_output_outside_allowed_roots(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
base = Path(directory)
|
||||
repo = base / "repo"
|
||||
run = base / "run"
|
||||
outside = base / "outside"
|
||||
repo.mkdir()
|
||||
run.mkdir()
|
||||
outside.mkdir()
|
||||
(base / "secret.md").write_text("secret\n", encoding="utf-8")
|
||||
request = run / "request.json"
|
||||
request.write_text(json.dumps({
|
||||
"schema_version": "proof-request/v1",
|
||||
"run": {"id": "run-escape", "profile": "audit"},
|
||||
"proofs": [{
|
||||
"finding": {"id": "F1", "role": "quote"},
|
||||
"source": {"namespace": "run", "path": "../secret.md", "quote_utf8": "secret"},
|
||||
}],
|
||||
}), encoding="utf-8")
|
||||
escaped = subprocess.run([
|
||||
sys.executable, str(RUNTIME / "proof_runner.py"), str(request),
|
||||
"--repo-root", str(repo), "--run-root", str(run), "--output", str(run / "proof.json"),
|
||||
], check=False, capture_output=True, text=True)
|
||||
self.assertEqual(escaped.returncode, 2)
|
||||
self.assertIn("SOURCE_OUTSIDE_NAMESPACE", escaped.stdout)
|
||||
|
||||
request_body = json.loads(request.read_text(encoding="utf-8"))
|
||||
(run / "source.md").write_text("proof\n", encoding="utf-8")
|
||||
request_body["proofs"][0]["source"] = {"namespace": "run", "path": "source.md", "quote_utf8": "proof"}
|
||||
request.write_text(json.dumps(request_body), encoding="utf-8")
|
||||
rejected_output = subprocess.run([
|
||||
sys.executable, str(RUNTIME / "proof_runner.py"), str(request),
|
||||
"--repo-root", str(repo), "--run-root", str(run), "--output", str(outside / "proof.json"),
|
||||
], check=False, capture_output=True, text=True)
|
||||
self.assertEqual(rejected_output.returncode, 2)
|
||||
self.assertIn("OUTPUT_OUTSIDE_ALLOWED_ROOT", rejected_output.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user