Files
llm-wiki/.claude/hooks/test_wiki_claim_gate.py

429 lines
19 KiB
Python

#!/usr/bin/env python3
"""wiki_claim_gate.py 회귀 고정 테스트 — check_markdown_write 행동 동치 (refactor 전후 동일)."""
import importlib.util
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
_SPEC = importlib.util.spec_from_file_location(
"wcg", str(Path(__file__).with_name("wiki_claim_gate.py")))
wcg = importlib.util.module_from_spec(_SPEC)
_SPEC.loader.exec_module(wcg)
CLAIMS_TABLE = (
"## Claims Extracted\n"
"| Claim ID | Claim | Evidence quote | Strength | Applies to | Does not prove |\n"
"|---|---|---|---|---|---|\n"
"| C1 | x | q | company-case-study | a | b |\n"
)
USAGE = "## Usage Boundaries\n- x\n"
DEM = (
"## Decision Evidence Map\n"
"| Decision ID | Decision | Supporting Claims | Evidence Strength | Open Risk |\n"
"|---|---|---|---|---|\n"
"| D1 | x | C1 | company-case-study | none |\n"
)
CTV = "## 검증해야 할 주장 / Claims To Verify\n- v\n"
class TestSourceNote(unittest.TestCase):
def test_missing_claims_table_blocks(self):
f = wcg.check_markdown_write("raw/official-docs/x.md", "# t\n" + USAGE)
self.assertTrue(any("Claims Extracted" in m for m in f))
def test_complete_source_note_passes(self):
f = wcg.check_markdown_write("raw/official-docs/x.md", "# t\n" + CLAIMS_TABLE + USAGE)
self.assertEqual(f, [])
def test_company_blog_same_rule(self):
f = wcg.check_markdown_write("raw/company-tech-blogs/x.md", "# t\n본문")
self.assertTrue(any("Claims Extracted" in m for m in f))
class TestBranchNote(unittest.TestCase):
def test_missing_dem_blocks(self):
f = wcg.check_markdown_write("raw/branch-notes/feature-x.md", "# t\n" + CTV)
self.assertTrue(any("Decision Evidence Map" in m for m in f))
def test_complete_branch_note_passes(self):
f = wcg.check_markdown_write("raw/branch-notes/feature-x.md", "# t\n" + DEM + CTV)
self.assertEqual(f, [])
def test_officially_supported_without_strength_blocks(self):
body = "# t\n" + DEM + CTV + "\n이 기능은 officially supported 된다.\n"
f = wcg.check_markdown_write("raw/branch-notes/feature-x.md", body)
self.assertTrue(any("official" in m.lower() for m in f))
def test_officially_supported_with_strength_passes(self):
body = "# t\n" + DEM + CTV + "\nofficially supported (official-vendor-doc).\n"
f = wcg.check_markdown_write("raw/branch-notes/feature-x.md", body)
self.assertEqual(f, [])
class TestConceptNote(unittest.TestCase):
def test_missing_claim_backed_blocks(self):
f = wcg.check_markdown_write("wiki/concepts/x.md", "# t\n본문")
self.assertTrue(any("Claim-backed Knowledge" in m for m in f))
class TestSpecReport(unittest.TestCase):
def test_complete_verdict_without_traceability_blocks(self):
f = wcg.check_markdown_write("docs/superpowers/specs/2026-01-01-x-report.md",
"Verdict: COMPLETE\n근거 없음")
self.assertTrue(any("traceability" in m.lower() for m in f))
INVEST_SOURCES = (
"## 출처 / Sources\n"
"| # | 제목 | 출처 등급 | URL | 발행/조사일 |\n"
"|---|---|---|---|---|\n"
"| S1 | x | official | url | 2025 |\n"
)
INVEST_QUOTES = '## 핵심 인용 / Key quotes (verbatim)\n> [S1] "원문"\n'
INVEST_CLAIMS = (
"## Claims Extracted / 추출된 주장\n"
"| Claim ID | Claim | Evidence quote | Strength | 적용 조건 | 증명 못 하는 것 |\n"
"|---|---|---|---|---|---|\n"
'| C1 | x | [S1] "q" | official | a | b |\n'
)
class TestInvestResearch(unittest.TestCase):
def test_missing_claims_blocks(self):
f = wcg.check_markdown_write("raw/invest-research/2026-06-08-x.md",
"# t\n" + INVEST_SOURCES + INVEST_QUOTES)
self.assertTrue(any("Claims Extracted" in m for m in f))
def test_missing_sources_blocks(self):
f = wcg.check_markdown_write("raw/invest-research/2026-06-08-x.md",
"# t\n" + INVEST_QUOTES + INVEST_CLAIMS)
self.assertTrue(any("출처" in m or "Sources" in m for m in f))
def test_missing_verbatim_blocks(self):
f = wcg.check_markdown_write("raw/invest-research/2026-06-08-x.md",
"# t\n" + INVEST_SOURCES + INVEST_CLAIMS)
self.assertTrue(any("핵심 인용" in m for m in f))
def test_complete_passes(self):
f = wcg.check_markdown_write("raw/invest-research/2026-06-08-x.md",
"# t\n" + INVEST_SOURCES + INVEST_QUOTES + INVEST_CLAIMS)
self.assertEqual(f, [])
class TestUnrelatedPath(unittest.TestCase):
def test_non_gated_path_passes(self):
f = wcg.check_markdown_write("raw/lectures/x.md", "# anything\n")
self.assertEqual(f, [])
def test_non_md_passes(self):
f = wcg.check_markdown_write("raw/official-docs/x.txt", "anything")
self.assertEqual(f, [])
import tempfile as _tf
PROJ_OK = "# t\n## 실제 구현 내용 (`actually-implemented`)\n- x\n## Sources\n- `[[raw/x]]`\n"
class TestWikiProjectsGate(unittest.TestCase):
"""P1-7: wiki/projects 증거 구조 게이트 + named-hub 면제."""
def test_missing_sections_blocks(self):
f = wcg.check_markdown_write("wiki/projects/ca-tmpl/x.md", "# t\n본문만")
self.assertTrue(any("실제 구현 내용" in m for m in f))
self.assertTrue(any("Sources" in m for m in f))
def test_complete_passes(self):
f = wcg.check_markdown_write("wiki/projects/ca-tmpl/x.md", PROJ_OK)
self.assertEqual(f, [])
def test_named_hub_exempt(self):
with _tf.TemporaryDirectory() as d:
root = Path(d)
(root / "wiki" / "projects" / "myproj").mkdir(parents=True)
f = wcg.check_markdown_write("wiki/projects/myproj.md", "# hub\nMOC만", root=root)
self.assertEqual(f, [])
def _mk_canonical(root, rel, status):
p = root / (rel + ".md")
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(f"---\ntitle: x\nstatus: {status}\n---\n# t\n", encoding="utf-8")
class TestDerivedGate(unittest.TestCase):
"""P1-7/8: 파생 산출물 canonical 경유 + 원천 status 게이트."""
def test_no_sources_section_blocks(self):
f = wcg.check_markdown_write("wiki/blog/x.md", "# t\n본문")
self.assertTrue(any("Sources" in m for m in f))
def test_no_canonical_link_blocks(self):
f = wcg.check_markdown_write("wiki/interview/x.md", "# t\n## Sources\n- 외부 링크만\n")
self.assertTrue(any("canonical wikilink" in m for m in f))
def test_portfolio_requires_projects_link(self):
f = wcg.check_markdown_write(
"wiki/portfolio/x.md", "# t\n## Sources\n- [[wiki/concepts/a]]\n")
self.assertTrue(any("wiki/projects" in m for m in f))
def test_draft_canonical_source_blocks(self):
with _tf.TemporaryDirectory() as d:
root = Path(d)
_mk_canonical(root, "wiki/concepts/a", "draft")
f = wcg.check_markdown_write(
"wiki/blog/x.md", "# t\n## Sources\n- [[wiki/concepts/a]]\n", root=root)
self.assertTrue(any("status 게이트" in m for m in f))
def test_reviewed_canonical_source_passes(self):
with _tf.TemporaryDirectory() as d:
root = Path(d)
_mk_canonical(root, "wiki/concepts/a", "reviewed")
f = wcg.check_markdown_write(
"wiki/blog/x.md", "# t\n## Sources\n- [[wiki/concepts/a]]\n", root=root)
self.assertEqual(f, [])
def test_mixed_one_draft_blocks(self):
with _tf.TemporaryDirectory() as d:
root = Path(d)
_mk_canonical(root, "wiki/concepts/a", "verified")
_mk_canonical(root, "wiki/projects/b", "draft")
f = wcg.check_markdown_write(
"wiki/interview/x.md",
"# t\n## Sources\n- [[wiki/concepts/a]]\n- [[wiki/projects/b]]\n", root=root)
self.assertTrue(any("wiki/projects/b" in m for m in f))
def test_explainer_status_exempt(self):
with _tf.TemporaryDirectory() as d:
root = Path(d)
_mk_canonical(root, "wiki/concepts/a", "draft")
f = wcg.check_markdown_write(
"wiki/explainer/x.md", "# t\n## Sources\n- [[wiki/concepts/a]]\n", root=root)
self.assertEqual(f, []) # explainer 는 canonical 경유만, status 면제
INVEST_DAILY_BASE = (
"# t\n## 고정 체크리스트 (매일 동일)\n"
"| 자산군 | 핵심 지표 | 값 / 방향 | 출처 | 조사시점 |\n"
"|---|---|---|---|---|\n"
"{rows}\n"
"## 출처 / Sources (deep-research 조사 기록)\n- x\n"
)
class TestInvestDailyGate(unittest.TestCase):
"""P1-7: invest-daily 수치행 출처/조사시점 강제 (빈 행은 허용)."""
def test_value_without_source_blocks(self):
text = INVEST_DAILY_BASE.format(rows="| 금리 | 미 10Y | 4.4% ↑ | | |")
f = wcg.check_markdown_write("raw/invest-daily/2026-06-10.md", text)
self.assertTrue(any("출처 비어있음" in m for m in f))
def test_value_without_time_blocks(self):
text = INVEST_DAILY_BASE.format(rows="| 금리 | 미 10Y | 4.4% ↑ | [x](https://a) | |")
f = wcg.check_markdown_write("raw/invest-daily/2026-06-10.md", text)
self.assertTrue(any("조사시점 비어있음" in m for m in f))
def test_empty_row_allowed(self):
text = INVEST_DAILY_BASE.format(rows="| 금리 | 미 10Y | | | |")
f = wcg.check_markdown_write("raw/invest-daily/2026-06-10.md", text)
self.assertEqual(f, [])
def test_complete_row_passes(self):
text = INVEST_DAILY_BASE.format(
rows="| 금리 | 미 10Y | 4.4% ↑ | [FRED](https://a) | 2026-06-10 09:00 KST |")
f = wcg.check_markdown_write("raw/invest-daily/2026-06-10.md", text)
self.assertEqual(f, [])
def test_missing_sections_blocks(self):
f = wcg.check_markdown_write("raw/invest-daily/2026-06-10.md", "# t\n본문만")
self.assertTrue(any("고정 체크리스트" in m for m in f))
def _run_main_stop(message):
ev = {"hook_event_name": "Stop", "last_assistant_message": message}
return _sp.run(["python3", _GATE, "--main-stop"], input=_json.dumps(ev),
capture_output=True, text=True)
class TestMainStopGate(unittest.TestCase):
"""P1-9: Claude main agent Stop — fenced wiki-stats 만 검증 (COMPLETE trap 미적용)."""
def test_imbalanced_stats_blocks(self):
msg = "끝.\n```wiki-stats\nagent: branch-spec\nfound: 9\nprocessed: 7\ndropped: 0\n```"
self.assertEqual(_run_main_stop(msg).returncode, 2)
def test_balanced_stats_allows(self):
msg = "끝.\n```wiki-stats\nagent: branch-spec\nfound: 9\nprocessed: 7\ndropped: 2\ndropped_reason: 2 deferred\n```"
self.assertEqual(_run_main_stop(msg).returncode, 0)
def test_complete_without_traceability_allows(self):
# main agent 의 메타 대화 ("Verdict: COMPLETE" 인용) 는 차단하지 않는다.
self.assertEqual(_run_main_stop("훅이 Verdict: COMPLETE 를 검사한다").returncode, 0)
def test_malformed_verdict_block_allows(self):
# wiki-verdict 는 main-stop 검증 범위 밖 (judge 는 항상 subagent).
msg = "예시:\n```wiki-verdict\nagent: x\nverdict: ready\nblocking: 3\nshould_fix: 0\nadvisory: 0\n```"
self.assertEqual(_run_main_stop(msg).returncode, 0)
def test_no_marker_allows(self):
self.assertEqual(_run_main_stop("일반 응답").returncode, 0)
import json as _json
import subprocess as _sp
_GATE = str(Path(__file__).with_name("wiki_claim_gate.py"))
def _run_stop(message):
ev = {"hook_event_name": "SubagentStop", "last_assistant_message": message}
return _sp.run(["python3", _GATE], input=_json.dumps(ev), capture_output=True, text=True)
class TestSubagentStopVerdict(unittest.TestCase):
def test_malformed_verdict_blocks(self):
msg = "리뷰 끝.\n```wiki-verdict\nagent: branch-depth-auditor\nverdict: ready\nblocking: 3\nshould_fix: 0\nadvisory: 0\n```"
self.assertEqual(_run_stop(msg).returncode, 2)
def test_valid_verdict_allows(self):
msg = "리뷰 끝.\n```wiki-verdict\nagent: branch-depth-auditor\nverdict: ready\nblocking: 0\nshould_fix: 0\nadvisory: 0\n```"
self.assertEqual(_run_stop(msg).returncode, 0)
def test_no_marker_allows(self):
self.assertEqual(_run_stop("그냥 일반 subagent 출력, 마커 없음").returncode, 0)
def test_worker_status_done_allows(self):
# wiki-source-summarizer 성공 출력은 `**Status:** DONE` — branch-traceability 와 무관.
# bare DONE 으로 Decision Evidence Map / UNSUPPORTED_DECISION 을 요구하면 안 됨.
msg = "raw 자료 생성 완료.\n## Claims Extracted\n| Claim ID | ... |\n\n**Status:** DONE"
self.assertEqual(_run_stop(msg).returncode, 0)
def test_audit_verdict_complete_without_traceability_blocks(self):
# 감사/리뷰 완료 주장(Verdict: COMPLETE)은 여전히 traceability 누락 시 차단.
msg = "# Audit\n**Verdict:** COMPLETE\n근거 없음"
self.assertEqual(_run_stop(msg).returncode, 2)
def _run_stop_as(agent_type, message):
ev = {"hook_event_name": "SubagentStop", "agent_type": agent_type,
"last_assistant_message": message}
return _sp.run(["python3", _GATE], input=_json.dumps(ev), capture_output=True, text=True)
class TestSubagentStopAgentTypeScoping(unittest.TestCase):
"""P0-1: 위키 출력 계약은 WIKI_AGENT_TYPES 에만 적용 — 범용 subagent 오차단 방지.
(실측 재현 2026-06-10: Explore 가 'Verdict: COMPLETE' 한 마디로 차단 → 이탈 재시도)"""
def test_non_wiki_agent_complete_allows(self):
r = _run_stop_as("Explore", "PROBE OK — Verdict: COMPLETE")
self.assertEqual(r.returncode, 0)
def test_non_wiki_agent_malformed_verdict_block_allows(self):
# 보고서에 인용된 (모순된) 예시 블록도 범용 에이전트에선 차단 사유가 아님.
msg = "감사 예시:\n```wiki-verdict\nagent: branch-depth-auditor\nverdict: ready\nblocking: 3\nshould_fix: 0\nadvisory: 0\n```"
self.assertEqual(_run_stop_as("general-purpose", msg).returncode, 0)
def test_wiki_agent_complete_without_traceability_blocks(self):
msg = "# Audit\n**Verdict:** COMPLETE\n근거 없음"
self.assertEqual(_run_stop_as("wiki-research-lane", msg).returncode, 2)
def test_wiki_agent_malformed_verdict_blocks(self):
msg = "리뷰 끝.\n```wiki-verdict\nagent: branch-depth-auditor\nverdict: ready\nblocking: 3\nshould_fix: 0\nadvisory: 0\n```"
self.assertEqual(_run_stop_as("branch-depth-auditor", msg).returncode, 2)
def test_missing_agent_type_still_validates(self):
# 타 플랫폼(Gemini AfterAgent 등) — agent_type 부재 시 기존 보수적 검증 유지.
msg = "# Audit\n**Verdict:** COMPLETE\n근거 없음"
self.assertEqual(_run_stop(msg).returncode, 2)
class TestRetryRevalidation(unittest.TestCase):
"""P2-22: stop_hook_active(재시도)에도 위키 에이전트 스키마 위반은 계속 차단."""
def _run_retry(self, agent_type, message):
ev = {"hook_event_name": "SubagentStop", "agent_type": agent_type,
"stop_hook_active": True, "last_assistant_message": message}
return _sp.run(["python3", _GATE], input=_json.dumps(ev),
capture_output=True, text=True)
def test_retry_malformed_verdict_still_blocks(self):
msg = "리뷰.\n```wiki-verdict\nagent: branch-depth-auditor\nverdict: ready\nblocking: 3\nshould_fix: 0\nadvisory: 0\n```"
self.assertEqual(self._run_retry("branch-depth-auditor", msg).returncode, 2)
def test_retry_valid_passes(self):
msg = "리뷰.\n```wiki-verdict\nagent: branch-depth-auditor\nverdict: not-ready\nblocking: 3\nshould_fix: 0\nadvisory: 0\n```"
self.assertEqual(self._run_retry("branch-depth-auditor", msg).returncode, 0)
def test_retry_non_wiki_agent_still_scoped_out(self):
self.assertEqual(self._run_retry("Explore", "Verdict: COMPLETE").returncode, 0)
def test_main_stop_retry_keeps_one_retry(self):
# main_stop_gate 는 one-retry 유지 (대화 흐름 보호).
ev = {"hook_event_name": "Stop", "stop_hook_active": True,
"last_assistant_message": "x\n```wiki-stats\nagent: a\nfound: 2\nprocessed: 1\ndropped: 0\n```"}
r = _sp.run(["python3", _GATE, "--main-stop"], input=_json.dumps(ev),
capture_output=True, text=True)
self.assertEqual(r.returncode, 0)
def _run_gate(event, *extra_args):
return _sp.run(["python3", _GATE, *extra_args], input=_json.dumps(event),
capture_output=True, text=True)
class TestAntigravityMode(unittest.TestCase):
def _write_event(self, rel, content):
return {"hook_event_name": "PreToolUse", "tool_name": "Write",
"tool_input": {"file_path": rel, "content": content}}
def test_deny_emits_decision_json_exit0(self):
ev = self._write_event("raw/branch-notes/x.md", "# t\n본문만, DEM 없음")
r = _run_gate(ev, "--antigravity")
self.assertEqual(r.returncode, 0) # antigravity: exit 0, deny via JSON
out = _json.loads(r.stdout)
self.assertEqual(out["decision"], "deny")
self.assertIn("reason", out)
def test_allow_emits_decision_json(self):
body = ("# t\n## Decision Evidence Map\n"
"| Decision ID | Decision | Supporting Claims | Evidence Strength | Open Risk |\n"
"|---|---|---|---|---|\n| D1 | x | C1 | company-case-study | none |\n"
"## 검증해야 할 주장 / Claims To Verify\n- v\n")
ev = self._write_event("raw/branch-notes/x.md", body)
r = _run_gate(ev, "--antigravity")
self.assertEqual(r.returncode, 0)
self.assertEqual(_json.loads(r.stdout)["decision"], "allow")
def test_afteragent_prompt_response_verdict_deny(self):
# Gemini AfterAgent: 에이전트 출력은 prompt_response, 이벤트명 AfterAgent
ev = {"hook_event_name": "AfterAgent",
"prompt_response": "x\n```wiki-verdict\nagent: branch-depth-auditor\nverdict: ready\nblocking: 3\nshould_fix: 0\nadvisory: 0\n```"}
r = _run_gate(ev, "--antigravity")
self.assertEqual(r.returncode, 0)
self.assertEqual(_json.loads(r.stdout)["decision"], "deny")
def test_non_antigravity_still_exit2(self):
# 회귀: --antigravity 없으면 Claude exit-code 규약 그대로
ev = self._write_event("raw/branch-notes/x.md", "# t\n본문만, DEM 없음")
self.assertEqual(_run_gate(ev).returncode, 2)
class TestSubagentStopStats(unittest.TestCase):
def test_imbalanced_stats_blocks(self):
msg = "x\n```wiki-stats\nagent: coverage-auditor\nfound: 12\nprocessed: 10\ndropped: 0\n```"
self.assertEqual(_run_stop(msg).returncode, 2)
def test_balanced_stats_allows(self):
msg = "x\n```wiki-stats\nagent: coverage-auditor\nfound: 10\nprocessed: 10\ndropped: 0\n```"
self.assertEqual(_run_stop(msg).returncode, 0)
def test_dropped_without_reason_blocks(self):
msg = "x\n```wiki-stats\nagent: coverage-auditor\nfound: 12\nprocessed: 10\ndropped: 2\n```"
self.assertEqual(_run_stop(msg).returncode, 2)
if __name__ == "__main__":
unittest.main(verbosity=2)