init: llm-wiki-haness 하네스 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create fixed exact-quote proofs and verify them without executing request argv."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import proof_manifest
|
||||
from fs_transaction import replace_many
|
||||
|
||||
|
||||
REQUEST_SCHEMA = "proof-request/v1"
|
||||
DEFAULT_ROOT = Path(__file__).resolve().parents[2]
|
||||
RESULT_SCHEMA = "proof-runner-result/v1"
|
||||
|
||||
|
||||
class ProofRequestError(ValueError):
|
||||
def __init__(self, code: str, message: str, location: str = "") -> None:
|
||||
self.code = code
|
||||
self.location = location
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
def _source_path(
|
||||
repo_root: Path,
|
||||
run_root: Path | None,
|
||||
source: dict[str, Any],
|
||||
) -> tuple[str, str, Path]:
|
||||
namespace = source.get("namespace", "repo")
|
||||
if namespace not in {"repo", "run"}:
|
||||
raise ProofRequestError("INVALID_SOURCE_NAMESPACE", "source.namespace must be repo or run")
|
||||
value = source.get("path")
|
||||
if not isinstance(value, str) or not value or "\\" in value:
|
||||
raise ProofRequestError("INVALID_SOURCE_PATH", "source.path must be a namespace-relative POSIX path")
|
||||
candidate = Path(value)
|
||||
if candidate.is_absolute():
|
||||
raise ProofRequestError("SOURCE_OUTSIDE_NAMESPACE", "absolute source path is not allowed", value)
|
||||
base = repo_root if namespace == "repo" else run_root
|
||||
if base is None:
|
||||
raise ProofRequestError("RUN_ROOT_REQUIRED", "run namespace requires --run-root", value)
|
||||
resolved = (base / candidate).resolve()
|
||||
try:
|
||||
resolved.relative_to(base)
|
||||
except ValueError as exc:
|
||||
raise ProofRequestError("SOURCE_OUTSIDE_NAMESPACE", f"source path escapes {namespace} root", value) from exc
|
||||
if not resolved.is_file():
|
||||
raise ProofRequestError("SOURCE_NOT_FOUND", "source file does not exist", value)
|
||||
return namespace, candidate.as_posix(), resolved
|
||||
|
||||
|
||||
def _line_range(text: str, quote: str, source: dict[str, Any], location: str) -> tuple[int, int]:
|
||||
start_value = source.get("line_start")
|
||||
end_value = source.get("line_end")
|
||||
if (start_value is None) != (end_value is None):
|
||||
raise ProofRequestError("INCOMPLETE_LINE_RANGE", "line_start and line_end must be supplied together", location)
|
||||
if start_value is not None:
|
||||
if (
|
||||
not isinstance(start_value, int)
|
||||
or isinstance(start_value, bool)
|
||||
or not isinstance(end_value, int)
|
||||
or isinstance(end_value, bool)
|
||||
or start_value < 1
|
||||
or end_value < start_value
|
||||
):
|
||||
raise ProofRequestError("INVALID_LINE_RANGE", "line range must contain positive ordered integers", location)
|
||||
lines = text.splitlines(keepends=True)
|
||||
if end_value > len(lines) or quote not in "".join(lines[start_value - 1 : end_value]):
|
||||
raise ProofRequestError("QUOTE_MISMATCH", "quote is not inside the requested line range", location)
|
||||
return start_value, end_value
|
||||
|
||||
positions: list[int] = []
|
||||
cursor = text.find(quote)
|
||||
while cursor >= 0:
|
||||
positions.append(cursor)
|
||||
cursor = text.find(quote, cursor + 1)
|
||||
if not positions:
|
||||
raise ProofRequestError("QUOTE_MISMATCH", "expected quote was not found", location)
|
||||
if len(positions) != 1:
|
||||
raise ProofRequestError("AMBIGUOUS_QUOTE", "quote occurs more than once; provide a line range", location)
|
||||
start_index = positions[0]
|
||||
end_index = start_index + len(quote) - 1
|
||||
return text.count("\n", 0, start_index) + 1, text.count("\n", 0, end_index) + 1
|
||||
|
||||
|
||||
def build_manifest(request: Any, root: Path, run_root: Path | None = None) -> dict[str, Any]:
|
||||
if not isinstance(request, dict) or request.get("schema_version") != REQUEST_SCHEMA:
|
||||
raise ProofRequestError("REQUEST_SCHEMA_MISMATCH", f"expected {REQUEST_SCHEMA}")
|
||||
run = request.get("run")
|
||||
proofs = request.get("proofs")
|
||||
if not isinstance(run, dict) or set(run) != {"id", "profile"}:
|
||||
raise ProofRequestError("INVALID_RUN", "run must contain only id and profile")
|
||||
if not isinstance(proofs, list) or not proofs:
|
||||
raise ProofRequestError("INVALID_PROOFS", "proofs must be a non-empty array")
|
||||
generated: list[dict[str, Any]] = []
|
||||
for index, item in enumerate(proofs):
|
||||
location = f"proofs[{index}]"
|
||||
if not isinstance(item, dict) or set(item) != {"finding", "source"}:
|
||||
raise ProofRequestError("INVALID_PROOF_REQUEST", "proof must contain finding and source", location)
|
||||
finding = item.get("finding")
|
||||
source = item.get("source")
|
||||
if not isinstance(finding, dict) or set(finding) != {"id", "role"} or not isinstance(source, dict):
|
||||
raise ProofRequestError("INVALID_PROOF_REQUEST", "invalid finding/source object", location)
|
||||
allowed_source = {"namespace", "path", "quote_utf8", "line_start", "line_end"}
|
||||
if set(source) - allowed_source or not {"path", "quote_utf8"}.issubset(source):
|
||||
raise ProofRequestError("INVALID_PROOF_REQUEST", "invalid source fields", location)
|
||||
namespace, relative, resolved = _source_path(root, run_root, source)
|
||||
quote = source.get("quote_utf8")
|
||||
if not isinstance(quote, str) or not quote:
|
||||
raise ProofRequestError("INVALID_QUOTE", "quote_utf8 must be non-empty", location)
|
||||
source_bytes = resolved.read_bytes()
|
||||
try:
|
||||
source_text = source_bytes.decode("utf-8", errors="strict")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ProofRequestError("SOURCE_NOT_UTF8", str(exc), relative) from exc
|
||||
line_start, line_end = _line_range(source_text, quote, source, location)
|
||||
generated.append(
|
||||
{
|
||||
"finding": {"id": finding.get("id"), "role": finding.get("role")},
|
||||
"source": {
|
||||
"namespace": namespace,
|
||||
"path": relative,
|
||||
"sha256": hashlib.sha256(source_bytes).hexdigest(),
|
||||
"line_start": line_start,
|
||||
"line_end": line_end,
|
||||
"quote_utf8": quote,
|
||||
},
|
||||
"execution": {
|
||||
"argv": ["proof-runner/exact-utf8-v1", namespace, relative, f"{line_start}:{line_end}"],
|
||||
"exit_code": 0,
|
||||
"stdout_utf8": quote,
|
||||
"stdout_sha256": hashlib.sha256(quote.encode("utf-8")).hexdigest(),
|
||||
"exact_match": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
return {"schema_version": proof_manifest.SCHEMA_VERSION, "run": dict(run), "proofs": generated}
|
||||
|
||||
|
||||
def _failure(exc: Exception) -> dict[str, Any]:
|
||||
if isinstance(exc, proof_manifest.ManifestValidationError):
|
||||
errors = exc.issues
|
||||
else:
|
||||
errors = [{"code": getattr(exc, "code", "IO_OR_JSON_ERROR"), "location": getattr(exc, "location", ""), "message": str(exc)}]
|
||||
return {"schema_version": proof_manifest.RESULT_SCHEMA_VERSION, "status": "FAIL", "errors": errors}
|
||||
|
||||
|
||||
def _display_path(path: Path, root: Path) -> str:
|
||||
resolved = path.resolve()
|
||||
try:
|
||||
return resolved.relative_to(root).as_posix()
|
||||
except ValueError:
|
||||
return resolved.as_posix()
|
||||
|
||||
|
||||
def _constrained_output(path: Path, repo_root: Path, run_root: Path | None) -> Path:
|
||||
output = path.resolve()
|
||||
roots = [repo_root, *( [run_root] if run_root is not None else [] )]
|
||||
if not any(output.is_relative_to(root) for root in roots):
|
||||
raise ProofRequestError(
|
||||
"OUTPUT_OUTSIDE_ALLOWED_ROOT",
|
||||
"proof output must be under --repo-root or --run-root",
|
||||
output.as_posix(),
|
||||
)
|
||||
if not output.parent.is_dir():
|
||||
raise ProofRequestError("OUTPUT_PARENT_NOT_FOUND", "proof output parent does not exist", output.parent.as_posix())
|
||||
return output
|
||||
|
||||
|
||||
def render_report_summary(manifest_path: str, manifest_sha256: str, proof_count: int) -> str:
|
||||
"""Render the compact proof section consumed by report workflows."""
|
||||
return (
|
||||
"## 증명 결과\n\n"
|
||||
f"- Manifest: `{manifest_path}`\n"
|
||||
f"- Manifest SHA-256: `{manifest_sha256}`\n"
|
||||
f"- Proof: {proof_count}\n"
|
||||
f"- PASS: {proof_count}\n"
|
||||
"- FAIL: 0\n"
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("request", type=Path, help="proof-request/v1 JSON")
|
||||
parser.add_argument("--repo-root", type=Path, default=DEFAULT_ROOT)
|
||||
parser.add_argument("--run-root", type=Path, help="isolated root for source.namespace=run and run artifacts")
|
||||
parser.add_argument("--profiles", type=Path, default=proof_manifest.DEFAULT_PROFILES)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--summary-output",
|
||||
type=Path,
|
||||
help="compact Markdown proof summary (default: proof-summary.md beside --output)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
root = args.repo_root.resolve(strict=True)
|
||||
run_root = args.run_root.resolve(strict=True) if args.run_root is not None else None
|
||||
if run_root is not None and not run_root.is_dir():
|
||||
raise ProofRequestError("INVALID_RUN_ROOT", "--run-root must be a directory", str(run_root))
|
||||
with args.request.open("r", encoding="utf-8") as stream:
|
||||
request = json.load(stream)
|
||||
manifest = build_manifest(request, root, run_root)
|
||||
profiles = proof_manifest.load_allowed_profiles(args.profiles.resolve(strict=True))
|
||||
verified = proof_manifest.verify_manifest(manifest, root, profiles, run_root=run_root)
|
||||
manifest_content = proof_manifest.manifest_bytes(verified)
|
||||
manifest_hash = hashlib.sha256(manifest_content).hexdigest()
|
||||
output = _constrained_output(args.output, root, run_root)
|
||||
summary_output = _constrained_output(args.summary_output or args.output.with_name("proof-summary.md"), root, run_root)
|
||||
if output == summary_output:
|
||||
raise ProofRequestError("OUTPUT_PATH_COLLISION", "manifest and summary outputs must differ", str(output))
|
||||
manifest_display = _display_path(output, root)
|
||||
summary_content = render_report_summary(
|
||||
manifest_display,
|
||||
manifest_hash,
|
||||
verified["verification"]["proof_count"],
|
||||
).encode("utf-8")
|
||||
replace_many({output: manifest_content, summary_output: summary_content})
|
||||
result = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"status": "PASS",
|
||||
"manifest": {"path": manifest_display, "sha256": manifest_hash},
|
||||
"report_summary": {"path": _display_path(summary_output, root)},
|
||||
"proof_count": verified["verification"]["proof_count"],
|
||||
"pass_count": verified["verification"]["pass_count"],
|
||||
"fail_count": verified["verification"]["fail_count"],
|
||||
}
|
||||
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
sys.stdout.write("\n")
|
||||
return 0
|
||||
except (ProofRequestError, proof_manifest.ManifestValidationError, OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
json.dump(_failure(exc), sys.stdout, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
sys.stdout.write("\n")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user