864 lines
33 KiB
Python
864 lines
33 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import shutil
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from helpers import (
|
|
FIXTURES,
|
|
init_run,
|
|
install_good_contracts,
|
|
read_json,
|
|
run_cli,
|
|
write_evidence,
|
|
write_json,
|
|
write_reviews,
|
|
)
|
|
|
|
|
|
def transition(run_dir: Path, status: str) -> None:
|
|
result = run_cli("update_run.py", "--run-dir", run_dir, "--status", status, "--reason", "test")
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
|
|
def lint_final(run_dir: Path, document_name: str = "final.md") -> None:
|
|
arguments: list[object] = [
|
|
"lint_document.py",
|
|
"--document",
|
|
run_dir / document_name,
|
|
"--logic-map",
|
|
run_dir / "04_logic_map.json",
|
|
"--term-ledger",
|
|
run_dir / "05_term_ledger.json",
|
|
"--reader-contract",
|
|
run_dir / "02_reader_contract.json",
|
|
"--output",
|
|
run_dir / "08_lint.json",
|
|
]
|
|
manifest = read_json(run_dir / "00_run.json")
|
|
if manifest["mode"] in {"write", "revise"}:
|
|
arguments.extend(["--draft-baseline", run_dir / "07_draft.md"])
|
|
if manifest["mode"] == "revise":
|
|
arguments.extend(["--baseline", manifest["inputs"]["draft"]["resolved_path"]])
|
|
if manifest["route_hint"] == "deep":
|
|
arguments.extend(["--fail-on", "warning"])
|
|
result = run_cli(*arguments)
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
|
|
def blocking_finding(finding_id: str = "R-BLOCK") -> dict[str, str]:
|
|
return {
|
|
"id": finding_id,
|
|
"severity": "high",
|
|
"location": "원리 절 1문단",
|
|
"reader_impact": "핵심 인과를 잘못 이해할 수 있다.",
|
|
"suggestion": "현재 계약 안에서 원인과 결과의 연결을 보완한다.",
|
|
"evidence": "원인 설명 없이 해결책이 먼저 제시된다.",
|
|
"violated_rule": "quality-rubric.md#hard-gate",
|
|
"owner": "doc-drafter",
|
|
}
|
|
|
|
|
|
def make_light_complete(tmp_path: Path) -> Path:
|
|
run_dir = init_run(tmp_path, route="light")
|
|
install_good_contracts(run_dir)
|
|
shutil.copyfile(FIXTURES / "good" / "document.md", run_dir / "final.md")
|
|
lint_final(run_dir)
|
|
for status in ("planned", "drafted", "finalized"):
|
|
transition(run_dir, status)
|
|
return run_dir
|
|
|
|
|
|
def test_offline_e2e_light_reaches_verified(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 0, result.stderr
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
manifest = read_json(run_dir / "00_run.json")
|
|
assert report["verdict"] == "pass"
|
|
assert report["document_verdict"] == "pass"
|
|
assert manifest["status"] == "verified"
|
|
assert report["summary"]["omissions"] == manifest["omissions"]
|
|
lint_report = read_json(run_dir / "08_lint.json")
|
|
assert report["summary"]["lint"]["finding_counts"] == lint_report["summary"]
|
|
assert report["summary"]["lint"]["finding_rule_ids"] == list(
|
|
dict.fromkeys(item["rule_id"] for item in lint_report["findings"])
|
|
)
|
|
assert report["summary"]["lint"]["fidelity"] == lint_report["fidelity"]
|
|
assert report["summary"]["lint"]["limitations"] == lint_report["limitations"]
|
|
assert report["summary"]["lint"]["rules_sha256"] == lint_report["rules_sha256"]
|
|
|
|
|
|
def test_verifier_rejects_invented_lint_findings_even_with_matching_aggregates(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
lint_report = read_json(run_dir / "08_lint.json")
|
|
duplicate_rule = "DOC-P001"
|
|
for line in (2, 3):
|
|
lint_report["findings"].append(
|
|
{
|
|
"rule_id": duplicate_rule,
|
|
"severity": "warning",
|
|
"message": f"중복 rule 집계 검증 {line}",
|
|
"path": str(run_dir / "final.md"),
|
|
"line": line,
|
|
"column": 1,
|
|
"section_id": None,
|
|
"context": None,
|
|
}
|
|
)
|
|
lint_report["summary"]["warnings"] += 2
|
|
lint_report["summary"]["total"] += 2
|
|
write_json(run_dir / "08_lint.json", lint_report)
|
|
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
assert report["summary"]["lint"] is None
|
|
assert any(
|
|
item["id"] == "lint-canonical-replay" and item["status"] == "fail"
|
|
for item in report["checks"]
|
|
)
|
|
|
|
|
|
def test_forged_empty_pass_lint_cannot_hide_current_document_errors(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
bad_document = "# 첫 제목\n\n# 둘째 제목\n\nTODO: 설명을 완성한다.\n"
|
|
(run_dir / "07_draft.md").write_text(bad_document, encoding="utf-8")
|
|
(run_dir / "final.md").write_text(bad_document, encoding="utf-8")
|
|
digest = hashlib.sha256(bad_document.encode("utf-8")).hexdigest()
|
|
lint_report = read_json(run_dir / "08_lint.json")
|
|
lint_report["document"]["sha256"] = digest
|
|
lint_report["fidelity"]["draft_baseline"]["sha256"] = digest
|
|
lint_report["fidelity"]["finalization_change_rate"] = 0.0
|
|
lint_report["findings"] = []
|
|
lint_report["summary"] = {"errors": 0, "warnings": 0, "info": 0, "total": 0}
|
|
lint_report["verdict"] = "pass"
|
|
write_json(run_dir / "08_lint.json", lint_report)
|
|
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
replay = next(
|
|
item for item in report["checks"] if item["id"] == "lint-canonical-replay"
|
|
)
|
|
assert replay["status"] == "fail"
|
|
assert "DOC-H002" in replay["message"]
|
|
assert "DOC-M001" in replay["message"]
|
|
|
|
|
|
def test_lint_summary_and_verdict_are_recomputed_from_findings(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
lint_report = read_json(run_dir / "08_lint.json")
|
|
lint_report["summary"]["errors"] = 1
|
|
lint_report["summary"]["total"] = 1
|
|
write_json(run_dir / "08_lint.json", lint_report)
|
|
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
assert report["document_verdict"] == "not_evaluated"
|
|
assert report["summary"]["lint"] is None
|
|
assert any(
|
|
item["id"] == "lint-semantics" and item["status"] == "fail"
|
|
for item in report["checks"]
|
|
)
|
|
|
|
|
|
def test_missing_required_artifact_fails_and_holds(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
(run_dir / "05_term_ledger.json").unlink()
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
assert report["verdict"] == "fail"
|
|
assert read_json(run_dir / "00_run.json")["status"] == "hold_for_review"
|
|
|
|
|
|
def test_omissions_reject_duplicates_required_or_present_artifacts(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
manifest = read_json(run_dir / "00_run.json")
|
|
manifest["omissions"].append(dict(manifest["omissions"][0]))
|
|
manifest["omissions"].append(
|
|
{"artifact": "07_draft.md", "reason": "필수 draft를 생략했다고 잘못 선언"}
|
|
)
|
|
write_json(run_dir / "00_run.json", manifest)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
failure = next(item for item in report["checks"] if item["id"] == "omissions-valid")
|
|
assert failure["status"] == "fail"
|
|
assert "duplicate=" in failure["message"]
|
|
assert "required=" in failure["message"]
|
|
assert "present=" in failure["message"]
|
|
assert report["summary"]["omissions"] == manifest["omissions"]
|
|
|
|
|
|
def test_every_missing_optional_artifact_needs_an_omission_reason(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
manifest = read_json(run_dir / "00_run.json")
|
|
manifest["omissions"] = [
|
|
item
|
|
for item in manifest["omissions"]
|
|
if item["artifact"] != "08_logic_review.json"
|
|
]
|
|
write_json(run_dir / "00_run.json", manifest)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
failure = next(
|
|
item
|
|
for item in read_json(run_dir / "09_final_report.json")["checks"]
|
|
if item["id"] == "omissions-complete"
|
|
)
|
|
assert failure["status"] == "fail"
|
|
assert "08_logic_review.json" in failure["message"]
|
|
|
|
|
|
def test_unknown_omission_artifact_is_a_schema_input_error(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
manifest = read_json(run_dir / "00_run.json")
|
|
manifest["omissions"].append(
|
|
{"artifact": "08_combined_review.json", "reason": "알 수 없는 합성 파일"}
|
|
)
|
|
write_json(run_dir / "00_run.json", manifest)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 2
|
|
assert read_json(run_dir / "09_final_report.json")["verdict"] == "input_error"
|
|
|
|
|
|
def test_review_mode_does_not_require_final_document(tmp_path: Path) -> None:
|
|
run_dir = init_run(tmp_path, route="light", mode="review", with_draft=True)
|
|
install_good_contracts(run_dir)
|
|
write_reviews(run_dir)
|
|
lint_final(run_dir, "07_draft.md")
|
|
transition(run_dir, "planned")
|
|
transition(run_dir, "reviewed")
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 0, result.stderr
|
|
assert not (run_dir / "final.md").exists()
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
assert report["document_verdict"] == "pass"
|
|
assert read_json(run_dir / "00_run.json")["status"] == "reviewed"
|
|
|
|
|
|
def test_standard_review_keeps_route_evidence_requirement(tmp_path: Path) -> None:
|
|
run_dir = init_run(tmp_path, route="standard", mode="review", with_draft=True)
|
|
install_good_contracts(run_dir)
|
|
write_evidence(run_dir)
|
|
write_reviews(run_dir)
|
|
lint_final(run_dir, "07_draft.md")
|
|
for status in ("evidence_ready", "planned", "reviewed"):
|
|
transition(run_dir, status)
|
|
(run_dir / "03_evidence_map.json").unlink()
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
assert any(
|
|
item["id"] == "artifact-03_evidence_map.json" and item["status"] == "fail"
|
|
for item in read_json(run_dir / "09_final_report.json")["checks"]
|
|
)
|
|
|
|
|
|
def test_review_must_record_exact_current_draft_path_hash_and_filename_type(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
run_dir = init_run(tmp_path, route="light", mode="review", with_draft=True)
|
|
install_good_contracts(run_dir)
|
|
write_reviews(run_dir)
|
|
lint_final(run_dir, "07_draft.md")
|
|
for status in ("planned", "reviewed"):
|
|
transition(run_dir, status)
|
|
logic_review = read_json(run_dir / "08_logic_review.json")
|
|
logic_review["review_type"] = "reader"
|
|
logic_review["document"]["path"] = "different.md"
|
|
logic_review["document"]["sha256"] = "0" * 64
|
|
write_json(run_dir / "08_logic_review.json", logic_review)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
failed = {
|
|
item["id"]
|
|
for item in read_json(run_dir / "09_final_report.json")["checks"]
|
|
if item["status"] == "fail"
|
|
}
|
|
assert "review-type-08_logic_review.json" in failed
|
|
assert "review-document-08_logic_review.json" in failed
|
|
assert read_json(run_dir / "09_final_report.json")["document_verdict"] == "not_evaluated"
|
|
assert all(
|
|
item["artifact"] != "08_logic_review.json"
|
|
for item in read_json(run_dir / "09_final_report.json")["summary"]["reviews"]
|
|
)
|
|
|
|
|
|
def test_review_nested_field_types_are_checked_by_schema(tmp_path: Path) -> None:
|
|
run_dir = init_run(tmp_path, route="light", mode="review", with_draft=True)
|
|
install_good_contracts(run_dir)
|
|
write_reviews(run_dir)
|
|
lint_final(run_dir, "07_draft.md")
|
|
for status in ("planned", "reviewed"):
|
|
transition(run_dir, status)
|
|
logic_review = read_json(run_dir / "08_logic_review.json")
|
|
logic_review["findings"] = [
|
|
{
|
|
"id": "R1",
|
|
"severity": "high",
|
|
"location": 12,
|
|
"reader_impact": 34,
|
|
"suggestion": 56,
|
|
}
|
|
]
|
|
write_json(run_dir / "08_logic_review.json", logic_review)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
assert any(
|
|
item["id"] == "schema-08_logic_review.json" and item["status"] == "fail"
|
|
for item in read_json(run_dir / "09_final_report.json")["checks"]
|
|
)
|
|
|
|
|
|
def test_review_optional_finding_fields_and_final_summary_use_exact_contract(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
run_dir = init_run(tmp_path, route="standard")
|
|
install_good_contracts(run_dir)
|
|
write_evidence(run_dir)
|
|
write_reviews(run_dir)
|
|
logic_review = read_json(run_dir / "08_logic_review.json")
|
|
logic_review["findings"] = [
|
|
{
|
|
"id": "R-LOW",
|
|
"severity": "low",
|
|
"location": "검증과 한계 절",
|
|
"reader_impact": "빠른 독자가 한 번 더 읽게 된다.",
|
|
"suggestion": "짧은 연결 문장을 보완한다.",
|
|
"evidence": "두 문장의 관계가 암시적으로만 드러난다.",
|
|
"violated_rule": "quality-rubric.md#reader_fit",
|
|
"owner": "doc-drafter",
|
|
}
|
|
]
|
|
write_json(run_dir / "08_logic_review.json", logic_review)
|
|
shutil.copyfile(FIXTURES / "good" / "document.md", run_dir / "final.md")
|
|
lint_final(run_dir)
|
|
for status in ("evidence_ready", "planned", "drafted", "reviewed", "finalized"):
|
|
transition(run_dir, status)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 0, result.stderr
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
logic_summary = next(
|
|
item
|
|
for item in report["summary"]["reviews"]
|
|
if item["review_type"] == "logic"
|
|
)
|
|
assert logic_summary["finding_counts"] == {
|
|
"critical": 0,
|
|
"high": 0,
|
|
"medium": 0,
|
|
"low": 1,
|
|
"total": 1,
|
|
}
|
|
assert logic_summary["finding_ids"] == ["R-LOW"]
|
|
|
|
|
|
def test_review_finding_rejects_finalizer_as_owner(tmp_path: Path) -> None:
|
|
run_dir = init_run(tmp_path, route="light", mode="review", with_draft=True)
|
|
install_good_contracts(run_dir)
|
|
write_reviews(run_dir)
|
|
lint_final(run_dir, "07_draft.md")
|
|
for status in ("planned", "reviewed"):
|
|
transition(run_dir, status)
|
|
logic_review = read_json(run_dir / "08_logic_review.json")
|
|
logic_review["findings"] = [
|
|
{
|
|
"id": "R-OWNER",
|
|
"severity": "low",
|
|
"location": "검증과 한계 절",
|
|
"reader_impact": "수정 책임이 final 단계로 잘못 전달된다.",
|
|
"suggestion": "Phase 3의 draft owner에게 반환한다.",
|
|
"owner": "doc-finalizer",
|
|
}
|
|
]
|
|
write_json(run_dir / "08_logic_review.json", logic_review)
|
|
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
|
|
assert result.returncode == 1
|
|
assert any(
|
|
item["id"] == "schema-08_logic_review.json" and item["status"] == "fail"
|
|
for item in read_json(run_dir / "09_final_report.json")["checks"]
|
|
)
|
|
|
|
|
|
def test_review_rejects_undeclared_optional_field_alias(tmp_path: Path) -> None:
|
|
run_dir = init_run(tmp_path, route="light", mode="review", with_draft=True)
|
|
install_good_contracts(run_dir)
|
|
write_reviews(run_dir)
|
|
lint_final(run_dir, "07_draft.md")
|
|
for status in ("planned", "reviewed"):
|
|
transition(run_dir, status)
|
|
logic_review = read_json(run_dir / "08_logic_review.json")
|
|
logic_review["findings"] = [
|
|
{
|
|
"id": "R-ALIAS",
|
|
"severity": "low",
|
|
"location": "문제 절",
|
|
"reader_impact": "표현이 모호하다.",
|
|
"suggestion": "연결을 명시한다.",
|
|
"observed_evidence": "schema에 없는 별칭",
|
|
}
|
|
]
|
|
write_json(run_dir / "08_logic_review.json", logic_review)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
assert any(
|
|
item["id"] == "schema-08_logic_review.json" and item["status"] == "fail"
|
|
for item in read_json(run_dir / "09_final_report.json")["checks"]
|
|
)
|
|
|
|
|
|
def test_review_pass_rejects_blocking_finding(tmp_path: Path) -> None:
|
|
run_dir = init_run(tmp_path, route="light", mode="review", with_draft=True)
|
|
install_good_contracts(run_dir)
|
|
write_reviews(run_dir)
|
|
lint_final(run_dir, "07_draft.md")
|
|
for status in ("planned", "reviewed"):
|
|
transition(run_dir, status)
|
|
logic_review = read_json(run_dir / "08_logic_review.json")
|
|
logic_review["findings"] = [blocking_finding("R-PASS-BLOCK")]
|
|
write_json(run_dir / "08_logic_review.json", logic_review)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
assert report["document_verdict"] == "not_evaluated"
|
|
assert any(
|
|
item["id"] == "schema-08_logic_review.json" and item["status"] == "fail"
|
|
for item in report["checks"]
|
|
)
|
|
|
|
|
|
def test_review_mode_reports_revise_without_failing_successful_diagnostics(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
brief_path = tmp_path / "request.md"
|
|
brief_path.write_text("기존 문서의 문제를 검토해 주세요.\n", encoding="utf-8")
|
|
original_path = tmp_path / "original.md"
|
|
original_path.write_text(
|
|
(FIXTURES / "good" / "document.md").read_text(encoding="utf-8")
|
|
+ "\nTODO: 독자 연결을 보완한다.\n",
|
|
encoding="utf-8",
|
|
)
|
|
initialized = run_cli(
|
|
"init_run.py",
|
|
"--brief",
|
|
brief_path,
|
|
"--draft",
|
|
original_path,
|
|
"--kind",
|
|
"explanation",
|
|
"--route",
|
|
"light",
|
|
"--mode",
|
|
"review",
|
|
"--workspace",
|
|
tmp_path / "workspace",
|
|
"--date",
|
|
"2026-07-23",
|
|
)
|
|
assert initialized.returncode == 0, initialized.stderr
|
|
run_dir = Path(initialized.stdout.split("\t", 1)[0])
|
|
install_good_contracts(run_dir)
|
|
draft_path = run_dir / "07_draft.md"
|
|
shutil.copyfile(original_path, draft_path)
|
|
write_reviews(run_dir)
|
|
logic_review = read_json(run_dir / "08_logic_review.json")
|
|
logic_review["verdict"] = "revise"
|
|
logic_review["findings"] = [blocking_finding()]
|
|
write_json(run_dir / "08_logic_review.json", logic_review)
|
|
lint_result = run_cli(
|
|
"lint_document.py",
|
|
"--document",
|
|
draft_path,
|
|
"--logic-map",
|
|
run_dir / "04_logic_map.json",
|
|
"--term-ledger",
|
|
run_dir / "05_term_ledger.json",
|
|
"--reader-contract",
|
|
run_dir / "02_reader_contract.json",
|
|
"--output",
|
|
run_dir / "08_lint.json",
|
|
)
|
|
assert lint_result.returncode == 1
|
|
for status in ("planned", "reviewed"):
|
|
transition(run_dir, status)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 0, result.stderr
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
assert report["verdict"] == "pass"
|
|
assert report["document_verdict"] == "revise"
|
|
assert report["summary"]["lint"]["verdict"] == "fail"
|
|
lint_report = read_json(run_dir / "08_lint.json")
|
|
assert report["summary"]["lint"]["finding_counts"] == lint_report["summary"]
|
|
assert report["summary"]["lint"]["finding_rule_ids"] == list(
|
|
dict.fromkeys(item["rule_id"] for item in lint_report["findings"])
|
|
)
|
|
assert read_json(run_dir / "00_run.json")["status"] == "reviewed"
|
|
assert all(
|
|
item["status"] == "pass"
|
|
for item in report["checks"]
|
|
if item["id"] in {"lint-verdict", "review-verdict-08_logic_review.json"}
|
|
)
|
|
|
|
|
|
def test_review_mode_hold_is_execution_failure_and_not_evaluated(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
run_dir = init_run(tmp_path, route="light", mode="review", with_draft=True)
|
|
install_good_contracts(run_dir)
|
|
write_reviews(run_dir)
|
|
lint_final(run_dir, "07_draft.md")
|
|
for status in ("planned", "reviewed"):
|
|
transition(run_dir, status)
|
|
logic_review = read_json(run_dir / "08_logic_review.json")
|
|
logic_review["verdict"] = "hold_for_review"
|
|
finding = blocking_finding("R-HOLD")
|
|
finding["evidence"] = "필수 source 파일에 접근할 수 없다."
|
|
finding["owner"] = "doc-evidence-curator"
|
|
logic_review["findings"] = [finding]
|
|
write_json(run_dir / "08_logic_review.json", logic_review)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
assert report["verdict"] == "fail"
|
|
assert report["document_verdict"] == "not_evaluated"
|
|
assert read_json(run_dir / "00_run.json")["status"] == "hold_for_review"
|
|
|
|
|
|
def test_standard_evidence_contract_rejects_unbound_load_bearing_claim(tmp_path: Path) -> None:
|
|
run_dir = init_run(tmp_path, route="standard")
|
|
install_good_contracts(run_dir)
|
|
write_evidence(run_dir)
|
|
write_reviews(run_dir)
|
|
shutil.copyfile(FIXTURES / "good" / "document.md", run_dir / "final.md")
|
|
lint_final(run_dir)
|
|
for status in ("evidence_ready", "planned", "drafted", "reviewed", "finalized"):
|
|
transition(run_dir, status)
|
|
evidence = read_json(run_dir / "03_evidence_map.json")
|
|
evidence["claims"][0]["source_ids"] = []
|
|
write_json(run_dir / "03_evidence_map.json", evidence)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
assert any(
|
|
item["id"] in {"schema-03_evidence_map.json", "evidence-contract"}
|
|
and item["status"] == "fail"
|
|
for item in report["checks"]
|
|
)
|
|
|
|
|
|
def test_source_hash_drift_is_rejected(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
manifest = read_json(run_dir / "00_run.json")
|
|
Path(manifest["inputs"]["brief"]["resolved_path"]).write_text("변조됨\n", encoding="utf-8")
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
assert any(
|
|
item["id"] == "hash-sources" and item["status"] == "fail"
|
|
for item in read_json(run_dir / "09_final_report.json")["checks"]
|
|
)
|
|
|
|
|
|
def test_run_input_summary_must_match_sources_inventory(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
manifest = read_json(run_dir / "00_run.json")
|
|
manifest["inputs"]["source_count"] = 1
|
|
manifest["inputs"]["brief_sha256"] = "0" * 64
|
|
write_json(run_dir / "00_run.json", manifest)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
assert any(
|
|
item["id"] == "hash-input-inventory" and item["status"] == "fail"
|
|
for item in read_json(run_dir / "09_final_report.json")["checks"]
|
|
)
|
|
|
|
|
|
def test_nested_lint_types_are_checked_by_schema(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
lint_report = read_json(run_dir / "08_lint.json")
|
|
lint_report["summary"]["errors"] = "0"
|
|
write_json(run_dir / "08_lint.json", lint_report)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
failed = {
|
|
item["id"]
|
|
for item in read_json(run_dir / "09_final_report.json")["checks"]
|
|
if item["status"] == "fail"
|
|
}
|
|
assert "schema-08_lint.json" in failed
|
|
|
|
|
|
def test_lint_contract_input_hash_drift_is_rejected(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
logic = read_json(run_dir / "04_logic_map.json")
|
|
logic["closure"] += " 추가 문장"
|
|
write_json(run_dir / "04_logic_map.json", logic)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
assert any(
|
|
item["id"] == "lint-input-hash-04_logic_map.json"
|
|
and item["status"] == "fail"
|
|
for item in read_json(run_dir / "09_final_report.json")["checks"]
|
|
)
|
|
|
|
|
|
def test_stale_lint_rules_fingerprint_is_not_summarized(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
lint_report = read_json(run_dir / "08_lint.json")
|
|
lint_report["rules_sha256"] = "0" * 64
|
|
write_json(run_dir / "08_lint.json", lint_report)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
assert report["summary"]["lint"] is None
|
|
assert report["document_verdict"] == "not_evaluated"
|
|
assert any(
|
|
item["id"] == "lint-rules-sha256" and item["status"] == "fail"
|
|
for item in report["checks"]
|
|
)
|
|
|
|
|
|
def test_verify_output_is_canonical_and_cannot_overwrite_run_inputs(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
manifest_path = run_dir / "00_run.json"
|
|
before = manifest_path.read_bytes()
|
|
result = run_cli(
|
|
"verify_run.py",
|
|
"--run-dir",
|
|
run_dir,
|
|
"--output",
|
|
manifest_path,
|
|
)
|
|
assert result.returncode == 2
|
|
assert manifest_path.read_bytes() == before
|
|
assert read_json(manifest_path)["status"] == "finalized"
|
|
|
|
custom = tmp_path / "custom-report.json"
|
|
result = run_cli(
|
|
"verify_run.py",
|
|
"--run-dir",
|
|
run_dir,
|
|
"--output",
|
|
custom,
|
|
)
|
|
assert result.returncode == 2
|
|
assert not custom.exists()
|
|
assert read_json(manifest_path)["status"] == "finalized"
|
|
|
|
|
|
def test_verify_existing_canonical_output_requires_tool_ownership(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
output = run_dir / "09_final_report.json"
|
|
output.write_text('{"tool":"lint_document","sentinel":true}\n', encoding="utf-8")
|
|
before = output.read_bytes()
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 2
|
|
assert output.read_bytes() == before
|
|
assert read_json(run_dir / "00_run.json")["status"] == "finalized"
|
|
|
|
|
|
def test_verify_malformed_same_tool_output_is_not_overwritten(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
output = run_dir / "09_final_report.json"
|
|
output.write_text('{"tool":"verify_run","sentinel":true}\n', encoding="utf-8")
|
|
before = output.read_bytes()
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 2
|
|
assert output.read_bytes() == before
|
|
assert read_json(run_dir / "00_run.json")["status"] == "finalized"
|
|
|
|
|
|
def test_verify_publish_rejects_entry_created_after_preflight(
|
|
tmp_path: Path, monkeypatch
|
|
) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
first = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert first.returncode == 0, first.stderr
|
|
output = run_dir / "09_final_report.json"
|
|
output.unlink()
|
|
|
|
canonical = (
|
|
Path(__file__).resolve().parents[1]
|
|
/ "skills"
|
|
/ "technical-doc-flow"
|
|
/ "scripts"
|
|
/ "verify_run.py"
|
|
)
|
|
sys.path.insert(0, str(canonical.parent))
|
|
spec = importlib.util.spec_from_file_location("conditional_verify_run", canonical)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
original_publish = module.publish_report_json
|
|
|
|
def publish_after_race(prepared, value):
|
|
prepared.path.write_bytes(b"unrelated verifier sentinel\n")
|
|
return original_publish(prepared, value)
|
|
|
|
monkeypatch.setattr(module, "publish_report_json", publish_after_race)
|
|
|
|
result = module.main(["--run-dir", str(run_dir)])
|
|
|
|
assert result == 2
|
|
assert output.read_bytes() == b"unrelated verifier sentinel\n"
|
|
assert read_json(run_dir / "00_run.json")["status"] == "verified"
|
|
|
|
|
|
def test_verify_can_conditionally_refresh_owned_report(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
|
|
first = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
second = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
|
|
assert first.returncode == 0, first.stderr
|
|
assert second.returncode == 0, second.stderr
|
|
assert read_json(run_dir / "09_final_report.json")["tool"] == "verify_run"
|
|
assert read_json(run_dir / "00_run.json")["status"] == "verified"
|
|
|
|
|
|
def test_verify_rejects_symlinked_manifest_without_writing_external_target(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
manifest_path = run_dir / "00_run.json"
|
|
external = tmp_path / "outside-manifest.json"
|
|
manifest_path.replace(external)
|
|
manifest_path.symlink_to(external)
|
|
before = external.read_bytes()
|
|
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 2
|
|
assert external.read_bytes() == before
|
|
assert read_json(external)["status"] == "finalized"
|
|
|
|
|
|
def test_verify_detects_document_change_during_verified_transition(
|
|
tmp_path: Path, monkeypatch
|
|
) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
canonical = (
|
|
Path(__file__).resolve().parents[1]
|
|
/ "skills"
|
|
/ "technical-doc-flow"
|
|
/ "scripts"
|
|
/ "verify_run.py"
|
|
)
|
|
sys.path.insert(0, str(canonical.parent))
|
|
spec = importlib.util.spec_from_file_location("canonical_verify_run", canonical)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
original_transition = module.transition_manifest
|
|
|
|
def mutate_then_transition(manifest_path, target, *, reason, allow_same=False):
|
|
if target == "verified":
|
|
final_path = run_dir / "final.md"
|
|
final_path.write_text(
|
|
final_path.read_text(encoding="utf-8") + "\n동시 변경\n",
|
|
encoding="utf-8",
|
|
)
|
|
return original_transition(
|
|
manifest_path,
|
|
target,
|
|
reason=reason,
|
|
allow_same=allow_same,
|
|
)
|
|
|
|
monkeypatch.setattr(module, "transition_manifest", mutate_then_transition)
|
|
result = module.main(["--run-dir", str(run_dir)])
|
|
assert result == 1
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
assert report["verdict"] == "fail"
|
|
assert read_json(run_dir / "00_run.json")["status"] == "hold_for_review"
|
|
assert any(
|
|
item["id"] == "verification-snapshot-post-transition"
|
|
and item["status"] == "fail"
|
|
for item in report["checks"]
|
|
)
|
|
|
|
|
|
def test_optional_review_is_validated_and_gates_when_present(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
write_reviews(run_dir)
|
|
manifest = read_json(run_dir / "00_run.json")
|
|
manifest["omissions"] = [
|
|
item
|
|
for item in manifest["omissions"]
|
|
if item["artifact"] not in {"08_logic_review.json", "08_reader_review.json"}
|
|
]
|
|
write_json(run_dir / "00_run.json", manifest)
|
|
logic_review = read_json(run_dir / "08_logic_review.json")
|
|
logic_review["verdict"] = "revise"
|
|
logic_review["findings"] = [blocking_finding("R-OPTIONAL")]
|
|
write_json(run_dir / "08_logic_review.json", logic_review)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
assert any(
|
|
item["id"] == "review-verdict-08_logic_review.json"
|
|
and item["status"] == "fail"
|
|
for item in read_json(run_dir / "09_final_report.json")["checks"]
|
|
)
|
|
|
|
|
|
def test_runtime_contract_override_is_schema_validated(tmp_path: Path) -> None:
|
|
run_dir = make_light_complete(tmp_path)
|
|
contract_path = (
|
|
Path(__file__).resolve().parents[1]
|
|
/ "skills"
|
|
/ "technical-doc-flow"
|
|
/ "config"
|
|
/ "runtime-contract.json"
|
|
)
|
|
contract = read_json(contract_path)
|
|
contract["agents"] = [123]
|
|
invalid_contract = tmp_path / "runtime-contract.json"
|
|
write_json(invalid_contract, contract)
|
|
result = run_cli(
|
|
"verify_run.py", "--run-dir", run_dir, "--contract", invalid_contract
|
|
)
|
|
assert result.returncode == 2
|
|
assert read_json(run_dir / "09_final_report.json")["verdict"] == "input_error"
|
|
|
|
|
|
def test_non_pass_independent_review_blocks_standard_run(tmp_path: Path) -> None:
|
|
run_dir = init_run(tmp_path, route="standard")
|
|
install_good_contracts(run_dir)
|
|
write_evidence(run_dir)
|
|
write_reviews(run_dir)
|
|
shutil.copyfile(FIXTURES / "good" / "document.md", run_dir / "final.md")
|
|
lint_final(run_dir)
|
|
for status in ("evidence_ready", "planned", "drafted", "reviewed", "finalized"):
|
|
transition(run_dir, status)
|
|
logic_review = read_json(run_dir / "08_logic_review.json")
|
|
logic_review["verdict"] = "revise"
|
|
logic_review["findings"] = [blocking_finding("R-REVISE")]
|
|
write_json(run_dir / "08_logic_review.json", logic_review)
|
|
result = run_cli("verify_run.py", "--run-dir", run_dir)
|
|
assert result.returncode == 1
|
|
report = read_json(run_dir / "09_final_report.json")
|
|
assert any(
|
|
item["id"] == "review-verdict-08_logic_review.json" and item["status"] == "fail"
|
|
for item in report["checks"]
|
|
)
|
|
assert "Phase 3" in next(
|
|
item["message"]
|
|
for item in report["checks"]
|
|
if item["id"] == "review-verdict-08_logic_review.json"
|
|
)
|
|
assert report["document_verdict"] == "revise"
|
|
assert read_json(run_dir / "00_run.json")["status"] == "hold_for_review"
|