143 lines
5.9 KiB
Python
143 lines
5.9 KiB
Python
from __future__ import annotations
|
|
|
|
from copy import deepcopy
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
RUNTIME_DIR = REPO_ROOT / "harness/runtime"
|
|
sys.path.insert(0, str(RUNTIME_DIR))
|
|
|
|
import proof_manifest # noqa: E402
|
|
|
|
|
|
class ProofManifestTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temporary_directory = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temporary_directory.cleanup)
|
|
self.root = Path(self.temporary_directory.name)
|
|
self.source = self.root / "source.md"
|
|
self.source_bytes = "첫째 줄\n정확한 인용문\n셋째 줄\n".encode("utf-8")
|
|
self.source.write_bytes(self.source_bytes)
|
|
self.quote = "정확한 인용문"
|
|
self.manifest = {
|
|
"schema_version": proof_manifest.SCHEMA_VERSION,
|
|
"run": {"id": "run-20260720-01", "profile": "audit"},
|
|
"proofs": [
|
|
{
|
|
"finding": {"id": "L1-F01", "role": "current_state"},
|
|
"source": {
|
|
"path": "source.md",
|
|
"sha256": hashlib.sha256(self.source_bytes).hexdigest(),
|
|
"line_start": 2,
|
|
"line_end": 2,
|
|
"quote_utf8": self.quote,
|
|
},
|
|
"execution": {
|
|
"argv": ["grep", "-nF", "--", self.quote, "source.md"],
|
|
"exit_code": 0,
|
|
"stdout_utf8": self.quote,
|
|
"stdout_sha256": hashlib.sha256(self.quote.encode("utf-8")).hexdigest(),
|
|
"exact_match": True,
|
|
},
|
|
}
|
|
],
|
|
}
|
|
|
|
def _run_cli(self, manifest: dict, *extra_args: str) -> subprocess.CompletedProcess[str]:
|
|
manifest_path = self.root / "manifest.json"
|
|
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8")
|
|
return subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(RUNTIME_DIR / "proof_manifest.py"),
|
|
str(manifest_path),
|
|
"--repo-root",
|
|
str(self.root),
|
|
*extra_args,
|
|
],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
def test_valid_unicode_quote_passes_without_implicit_output_file(self) -> None:
|
|
before = {path.name for path in self.root.iterdir()}
|
|
result = self._run_cli(self.manifest)
|
|
after = {path.name for path in self.root.iterdir()}
|
|
|
|
self.assertEqual(result.returncode, 0, result.stdout)
|
|
verification = json.loads(result.stdout)["verification"]
|
|
self.assertEqual(verification["status"], "PASS")
|
|
self.assertEqual((verification["proof_count"], verification["pass_count"], verification["fail_count"]), (1, 1, 0))
|
|
self.assertEqual(after - before, {"manifest.json"})
|
|
|
|
def test_output_is_written_only_when_explicitly_requested(self) -> None:
|
|
output = self.root / "verified.json"
|
|
result = self._run_cli(self.manifest, "--output", str(output))
|
|
|
|
self.assertEqual(result.returncode, 0, result.stdout)
|
|
self.assertTrue(output.is_file())
|
|
self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["verification"]["status"], "PASS")
|
|
|
|
def test_fail_closed_cli_cases_return_nonzero(self) -> None:
|
|
cases: list[tuple[str, dict, str]] = []
|
|
|
|
nonexistent = deepcopy(self.manifest)
|
|
nonexistent["proofs"][0]["source"]["path"] = "missing.md"
|
|
cases.append(("nonexistent", nonexistent, "SOURCE_NOT_FOUND"))
|
|
|
|
bad_source_hash = deepcopy(self.manifest)
|
|
bad_source_hash["proofs"][0]["source"]["sha256"] = "0" * 64
|
|
cases.append(("source_hash", bad_source_hash, "SOURCE_HASH_MISMATCH"))
|
|
|
|
quote_mismatch = deepcopy(self.manifest)
|
|
quote_mismatch["proofs"][0]["source"]["quote_utf8"] = "없는 인용문"
|
|
quote_mismatch["proofs"][0]["execution"]["stdout_utf8"] = "없는 인용문"
|
|
quote_mismatch["proofs"][0]["execution"]["stdout_sha256"] = hashlib.sha256("없는 인용문".encode("utf-8")).hexdigest()
|
|
cases.append(("quote", quote_mismatch, "QUOTE_MISMATCH"))
|
|
|
|
duplicate = deepcopy(self.manifest)
|
|
duplicate["proofs"].append(deepcopy(duplicate["proofs"][0]))
|
|
cases.append(("duplicate", duplicate, "DUPLICATE_FINDING_ROLE"))
|
|
|
|
for name, manifest, expected_code in cases:
|
|
with self.subTest(name=name):
|
|
result = self._run_cli(manifest)
|
|
body = json.loads(result.stdout)
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertEqual(body["status"], "FAIL")
|
|
self.assertIn(expected_code, {error["code"] for error in body["errors"]})
|
|
|
|
def test_stdout_hash_and_exact_match_are_validated(self) -> None:
|
|
bad = deepcopy(self.manifest)
|
|
bad["proofs"][0]["execution"]["stdout_sha256"] = "f" * 64
|
|
bad["proofs"][0]["execution"]["exact_match"] = False
|
|
|
|
with self.assertRaises(proof_manifest.ManifestValidationError) as raised:
|
|
proof_manifest.verify_manifest(bad, self.root.resolve(), {"audit"})
|
|
|
|
codes = {issue["code"] for issue in raised.exception.issues}
|
|
self.assertIn("STDOUT_HASH_MISMATCH", codes)
|
|
self.assertIn("EXACT_MATCH_FALSE", codes)
|
|
|
|
def test_quote_must_be_inside_declared_line_range(self) -> None:
|
|
bad = deepcopy(self.manifest)
|
|
bad["proofs"][0]["source"]["line_start"] = 1
|
|
bad["proofs"][0]["source"]["line_end"] = 1
|
|
|
|
with self.assertRaises(proof_manifest.ManifestValidationError) as raised:
|
|
proof_manifest.verify_manifest(bad, self.root.resolve(), {"audit"})
|
|
|
|
self.assertIn("QUOTE_MISMATCH", {issue["code"] for issue in raised.exception.issues})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|