132 lines
5.8 KiB
Python
132 lines
5.8 KiB
Python
"""extract.py(quote-verifier·funnel·fallback) + vote.py(블록 렌더) 테스트 — MockBackend."""
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from deep_research.backends.mock import MockBackend
|
|
from deep_research.extract import (FileExtraction, Quote, extract_file,
|
|
numbered_content, render_digest, verify_quotes)
|
|
from deep_research.vote import FindingVote, VoteResult, render_vote_block
|
|
|
|
|
|
@pytest.fixture
|
|
def sample(tmp_path: Path) -> Path:
|
|
p = tmp_path / "note.md"
|
|
p.write_text("alpha line\nbeta line with policy X\ngamma line\n", encoding="utf-8")
|
|
return p
|
|
|
|
|
|
# ---------- quote-verifier (결정론 trust boundary) ----------
|
|
|
|
def test_verify_pass_corrected_dropped(sample):
|
|
ext = FileExtraction(relevant=True, summary="s", facts=[], quotes=[
|
|
Quote(line=2, quote="beta line with policy X"), # PASS
|
|
Quote(line=1, quote="gamma line"), # 다른 라인 → CORRECTED(3)
|
|
Quote(line=2, quote="완전 위조된 인용"), # DROPPED
|
|
])
|
|
v = verify_quotes(sample, ext)
|
|
assert (v["pass"], v["corrected"], v["dropped"]) == (1, 1, 1)
|
|
assert any(q.line == 3 and q.quote == "gamma line" for q in v["kept"])
|
|
assert all("위조" not in q.quote for q in v["kept"]) # 위조 인용은 digest 에 못 들어감
|
|
|
|
|
|
def test_numbered_content_truncation():
|
|
text = "\n".join(f"line-{i}" for i in range(1000))
|
|
numbered, truncated = numbered_content(text, limit=200)
|
|
assert truncated and numbered.startswith("1\tline-0")
|
|
|
|
|
|
# ---------- extract_file: fallback 사다리 + engine funnel ----------
|
|
|
|
def _ok(relevant=True):
|
|
return FileExtraction(relevant=relevant, summary="요약", facts=["사실1"],
|
|
quotes=[Quote(line=2, quote="beta line with policy X")])
|
|
|
|
|
|
def test_first_backend_wins(sample):
|
|
b1, b2 = MockBackend({"extract": _ok()}), MockBackend({"extract": _ok()})
|
|
r = asyncio.run(extract_file([("codex", b1), ("antigravity", b2)], "q", sample, 8,
|
|
asyncio.Semaphore(1)))
|
|
assert r["engine"] == "codex" and b2.calls == []
|
|
assert r["verify"]["pass"] == 1
|
|
|
|
|
|
def test_fallback_ladder(sample):
|
|
b1 = MockBackend({}, default=None) # codex 실패
|
|
b2 = MockBackend({"extract": _ok()}) # agy 성공
|
|
r = asyncio.run(extract_file([("codex", b1), ("antigravity", b2)], "q", sample, 8,
|
|
asyncio.Semaphore(1)))
|
|
assert r["engine"] == "antigravity" # silent engine swap 아님 — engine 기록
|
|
|
|
|
|
def test_all_fail_recorded(sample):
|
|
b = MockBackend({}, default=None)
|
|
r = asyncio.run(extract_file([("codex", b)], "q", sample, 8, asyncio.Semaphore(1)))
|
|
assert "error" in r
|
|
|
|
|
|
# ---------- digest 렌더: funnel 균형 ----------
|
|
|
|
def test_digest_funnel(sample):
|
|
ok = {"path": str(sample), "engine": "codex", "truncated": False, "relevant": True,
|
|
"summary": "s", "facts": ["f"],
|
|
"verify": {"pass": 1, "corrected": 0, "dropped": 1,
|
|
"kept": [Quote(line=2, quote="beta line with policy X")]}}
|
|
bad = {"path": "x.md", "engine": None, "error": "전 엔진 실패"}
|
|
d = render_digest("q", [ok, bad])
|
|
assert "found: 2" in d and "processed: 1" in d and "dropped: 1" in d
|
|
assert "dropped_reason" in d
|
|
assert f"{sample}:2" in d # file:line 포인터
|
|
assert "폐기 1" in d # 위조 인용 수 가시화
|
|
|
|
|
|
def test_digest_zero_quote_marker(sample):
|
|
"""인용 전멸(전부 DROPPED) 파일의 facts 는 미검증 주장 — 마커 필수 (계명 2)."""
|
|
ok = {"path": str(sample), "engine": "codex", "truncated": False, "relevant": True,
|
|
"summary": "s", "facts": ["근거 없는 주장"],
|
|
"verify": {"pass": 0, "corrected": 0, "dropped": 2, "kept": []}}
|
|
d = render_digest("q", [ok])
|
|
assert "검증 인용 0건" in d and "미검증 주장" in d
|
|
# 인용이 1건이라도 살아 있으면 마커 없음
|
|
ok["verify"] = {"pass": 1, "corrected": 0, "dropped": 1,
|
|
"kept": [Quote(line=2, quote="beta line with policy X")]}
|
|
assert "미검증 주장" not in render_digest("q", [ok])
|
|
|
|
|
|
# ---------- vote: wiki-verdict 블록 렌더 (wiki_quorum 호환) ----------
|
|
|
|
def _parse_with_wiki_rules(block: str):
|
|
"""wiki_quorum.py 의 실제 파서로 파싱 (엔진 불가지 계약 검증용)."""
|
|
import importlib.util
|
|
import sys
|
|
hooks = Path(__file__).resolve().parents[3] / ".claude" / "hooks"
|
|
sys.path.insert(0, str(hooks))
|
|
spec = importlib.util.spec_from_file_location("wr", hooks / "wiki_rules.py")
|
|
wr = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(wr)
|
|
return wr.parse_verdict_block(block)
|
|
|
|
|
|
def test_vote_block_quorum_compatible():
|
|
r = VoteResult(votes=[
|
|
FindingVote(id="4.1.1", action="KEEP", reason="ok"),
|
|
FindingVote(id="finding with space", action="REJECT", reason="bad"),
|
|
])
|
|
block = render_vote_block("codex", r)
|
|
assert "agent: codex-adversarial-vote" in block
|
|
assert "finding: 4.1.1 action: KEEP" in block
|
|
assert "finding: finding-with-space action: REJECT" in block # 공백 정규화
|
|
parsed = _parse_with_wiki_rules(block)
|
|
assert parsed and len(parsed["findings"]) == 2
|
|
|
|
|
|
def test_vote_block_truncation_note():
|
|
"""findings 절단 시 표 자체에 누락 사실 기록 (no-silent-caps) + quorum 파서 비파괴."""
|
|
r = VoteResult(votes=[FindingVote(id="1", action="KEEP", reason="ok")])
|
|
block = render_vote_block("codex", r, truncated_chars=99_999)
|
|
assert "잘린 finding 은 이 표에서 누락" in block
|
|
parsed = _parse_with_wiki_rules(block) # 노트는 fence 밖 — 파싱 영향 없음
|
|
assert parsed and len(parsed["findings"]) == 1
|
|
assert "누락" not in render_vote_block("codex", r) # 미절단 시 노트 없음
|