init: llm-wiki-haness 하네스 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,153 @@
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from deep_research.schemas import Scope, Angle, Search, SearchResult, Extract, Claim, Verdict, Report
|
||||
from deep_research.core import norm_url, Deduper, rank_claims, tally, build_synth_blocks, build_stats
|
||||
from deep_research.config import Config
|
||||
|
||||
|
||||
def test_scope_requires_min_3_angles():
|
||||
with pytest.raises(ValidationError):
|
||||
Scope(question="q", summary="s", angles=[Angle(label="a", query="x")])
|
||||
|
||||
|
||||
def test_scope_accepts_5_angles():
|
||||
angles = [Angle(label=f"a{i}", query="x") for i in range(5)]
|
||||
s = Scope(question="q", summary="s", angles=angles)
|
||||
assert len(s.angles) == 5
|
||||
|
||||
|
||||
def test_search_relevance_enum_enforced():
|
||||
with pytest.raises(ValidationError):
|
||||
SearchResult(url="http://a", title="t", relevance="bogus")
|
||||
|
||||
|
||||
def test_json_schema_export_works():
|
||||
schema = Scope.model_json_schema()
|
||||
assert "angles" in schema["properties"]
|
||||
assert schema["properties"]["angles"]["minItems"] == 3
|
||||
assert schema["properties"]["angles"]["maxItems"] == 6
|
||||
|
||||
|
||||
def test_norm_url_strips_www_and_trailing_slash():
|
||||
assert norm_url("http://www.Example.com/Path/") == "example.com/path"
|
||||
|
||||
|
||||
def test_norm_url_bare_host():
|
||||
assert norm_url("https://example.com") == "example.com"
|
||||
|
||||
|
||||
def test_norm_url_fallback_on_garbage():
|
||||
assert norm_url("not a url") == "not a url"
|
||||
|
||||
|
||||
def test_deduper_filters_exact_dupes():
|
||||
d = Deduper(Config(MAX_FETCH=15))
|
||||
r = [{"url": "http://a.com/x", "title": "t", "relevance": "high"}]
|
||||
assert len(d.filter_novel("angle1", r)) == 1
|
||||
# same normalized url from another angle -> dup
|
||||
r2 = [{"url": "http://www.a.com/x/", "title": "t2", "relevance": "high"}]
|
||||
assert d.filter_novel("angle2", r2) == []
|
||||
assert len(d.dupes) == 1
|
||||
|
||||
|
||||
def test_deduper_budget_drops_medium_low_when_slots_exhausted():
|
||||
d = Deduper(Config(MAX_FETCH=1))
|
||||
first = [{"url": "http://a.com/1", "title": "t", "relevance": "high"}]
|
||||
d.filter_novel("a1", first) # consumes the only slot
|
||||
more = [
|
||||
{"url": "http://b.com/2", "title": "t", "relevance": "medium"},
|
||||
{"url": "http://c.com/3", "title": "t", "relevance": "low"},
|
||||
]
|
||||
assert d.filter_novel("a2", more) == []
|
||||
assert len(d.budget_dropped) == 2
|
||||
|
||||
|
||||
def test_deduper_high_passes_even_when_slots_exhausted():
|
||||
# 원본: high(rank 0)는 budget 조건(rank>=1)에 안 걸려 slot<=0 이어도 통과
|
||||
d = Deduper(Config(MAX_FETCH=1))
|
||||
d.filter_novel("a1", [{"url": "http://a.com/1", "title": "t", "relevance": "high"}])
|
||||
high = [{"url": "http://d.com/4", "title": "t", "relevance": "high"}]
|
||||
assert len(d.filter_novel("a2", high)) == 1
|
||||
assert d.fetch_slots == -1
|
||||
|
||||
|
||||
def _claim(imp, qual, name="c"):
|
||||
return {"claim": name, "importance": imp, "sourceQuality": qual}
|
||||
|
||||
|
||||
def test_rank_claims_orders_by_importance_then_quality():
|
||||
claims = [
|
||||
_claim("tangential", "primary", "t-prim"),
|
||||
_claim("central", "blog", "c-blog"),
|
||||
_claim("central", "primary", "c-prim"),
|
||||
]
|
||||
ranked = rank_claims(claims, max_verify=25)
|
||||
assert [c["claim"] for c in ranked] == ["c-prim", "c-blog", "t-prim"]
|
||||
|
||||
|
||||
def test_rank_claims_truncates_to_max():
|
||||
claims = [_claim("central", "primary", f"c{i}") for i in range(30)]
|
||||
assert len(rank_claims(claims, max_verify=25)) == 25
|
||||
|
||||
|
||||
def test_tally_survives_2_valid_0_refute():
|
||||
v = [{"refuted": False}, {"refuted": False}, {"refuted": False}]
|
||||
t = tally(v, votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is True and t["refutedVotes"] == 0
|
||||
|
||||
|
||||
def test_tally_killed_2_refute():
|
||||
v = [{"refuted": True}, {"refuted": True}, {"refuted": False}]
|
||||
t = tally(v, votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is False and t["refutedVotes"] == 2
|
||||
|
||||
|
||||
def test_tally_all_abstain_does_not_survive():
|
||||
# ⚠️ 거짓 생존 차단: all-None -> refuted=0 이지만 valid<2 라 미생존
|
||||
t = tally([None, None, None], votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is False and t["abstained"] == 3
|
||||
|
||||
|
||||
def test_tally_one_valid_two_abstain_does_not_survive():
|
||||
t = tally([{"refuted": False}, None, None], votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is False # valid(1) < 2
|
||||
|
||||
|
||||
def test_tally_boundary_one_refute_two_valid_survives():
|
||||
t = tally([{"refuted": True}, {"refuted": False}], votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is True # 1 refute < 2 required, quorum met
|
||||
|
||||
|
||||
def test_tally_boundary_two_refute_two_valid_killed():
|
||||
t = tally([{"refuted": True}, {"refuted": True}], votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is False # 2 refute == 2 required, strict < fails
|
||||
|
||||
|
||||
def test_build_synth_blocks_includes_vote_and_source():
|
||||
confirmed = [{
|
||||
"claim": "X causes Y", "quote": "q", "sourceUrl": "http://a", "sourceQuality": "primary",
|
||||
"valid": [{"refuted": False, "confidence": "high", "evidence": "e"}],
|
||||
"refutedVotes": 0,
|
||||
}]
|
||||
killed = [{
|
||||
"claim": "Z", "sourceUrl": "http://b", "valid": [{"refuted": True}], "refutedVotes": 1,
|
||||
}]
|
||||
block, killed_block = build_synth_blocks(confirmed, killed)
|
||||
assert "X causes Y" in block and "1-0" in block
|
||||
assert "Refuted claims" in killed_block and "Z" in killed_block
|
||||
|
||||
|
||||
def test_build_synth_blocks_no_killed():
|
||||
block, killed_block = build_synth_blocks([{
|
||||
"claim": "X", "quote": "q", "sourceUrl": "http://a", "sourceQuality": "primary",
|
||||
"valid": [{"refuted": False, "confidence": "high", "evidence": "e"}], "refutedVotes": 0,
|
||||
}], [])
|
||||
assert killed_block == ""
|
||||
|
||||
|
||||
def test_build_stats_agent_calls_formula():
|
||||
s = build_stats(angles=5, sources=12, claims=20, voted=18, confirmed=10, killed=8,
|
||||
after_synth=6, dupes=3, budget_dropped=2, votes_per_claim=3)
|
||||
# 1 + angles + sources + voted*votes_per_claim + 1
|
||||
assert s["agentCalls"] == 1 + 5 + 12 + 18 * 3 + 1
|
||||
assert s["confirmed"] == 10 and s["afterSynthesis"] == 6
|
||||
@@ -0,0 +1,131 @@
|
||||
"""extract.py(quote-verifier·funnel·fallback) + vote.py(블록 렌더) 테스트 — MockBackend."""
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deep_research.backends.mock import MockBackend
|
||||
from deep_research.extract import (FileExtraction, Quote, extract_file,
|
||||
numbered_content, render_digest, verify_quotes)
|
||||
from deep_research.vote import FindingVote, VoteResult, render_vote_block
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample(tmp_path: Path) -> Path:
|
||||
p = tmp_path / "note.md"
|
||||
p.write_text("alpha line\nbeta line with policy X\ngamma line\n", encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
# ---------- quote-verifier (결정론 trust boundary) ----------
|
||||
|
||||
def test_verify_pass_corrected_dropped(sample):
|
||||
ext = FileExtraction(relevant=True, summary="s", facts=[], quotes=[
|
||||
Quote(line=2, quote="beta line with policy X"), # PASS
|
||||
Quote(line=1, quote="gamma line"), # 다른 라인 → CORRECTED(3)
|
||||
Quote(line=2, quote="완전 위조된 인용"), # DROPPED
|
||||
])
|
||||
v = verify_quotes(sample, ext)
|
||||
assert (v["pass"], v["corrected"], v["dropped"]) == (1, 1, 1)
|
||||
assert any(q.line == 3 and q.quote == "gamma line" for q in v["kept"])
|
||||
assert all("위조" not in q.quote for q in v["kept"]) # 위조 인용은 digest 에 못 들어감
|
||||
|
||||
|
||||
def test_numbered_content_truncation():
|
||||
text = "\n".join(f"line-{i}" for i in range(1000))
|
||||
numbered, truncated = numbered_content(text, limit=200)
|
||||
assert truncated and numbered.startswith("1\tline-0")
|
||||
|
||||
|
||||
# ---------- extract_file: fallback 사다리 + engine funnel ----------
|
||||
|
||||
def _ok(relevant=True):
|
||||
return FileExtraction(relevant=relevant, summary="요약", facts=["사실1"],
|
||||
quotes=[Quote(line=2, quote="beta line with policy X")])
|
||||
|
||||
|
||||
def test_first_backend_wins(sample):
|
||||
b1, b2 = MockBackend({"extract": _ok()}), MockBackend({"extract": _ok()})
|
||||
r = asyncio.run(extract_file([("codex", b1), ("antigravity", b2)], "q", sample, 8,
|
||||
asyncio.Semaphore(1)))
|
||||
assert r["engine"] == "codex" and b2.calls == []
|
||||
assert r["verify"]["pass"] == 1
|
||||
|
||||
|
||||
def test_fallback_ladder(sample):
|
||||
b1 = MockBackend({}, default=None) # codex 실패
|
||||
b2 = MockBackend({"extract": _ok()}) # agy 성공
|
||||
r = asyncio.run(extract_file([("codex", b1), ("antigravity", b2)], "q", sample, 8,
|
||||
asyncio.Semaphore(1)))
|
||||
assert r["engine"] == "antigravity" # silent engine swap 아님 — engine 기록
|
||||
|
||||
|
||||
def test_all_fail_recorded(sample):
|
||||
b = MockBackend({}, default=None)
|
||||
r = asyncio.run(extract_file([("codex", b)], "q", sample, 8, asyncio.Semaphore(1)))
|
||||
assert "error" in r
|
||||
|
||||
|
||||
# ---------- digest 렌더: funnel 균형 ----------
|
||||
|
||||
def test_digest_funnel(sample):
|
||||
ok = {"path": str(sample), "engine": "codex", "truncated": False, "relevant": True,
|
||||
"summary": "s", "facts": ["f"],
|
||||
"verify": {"pass": 1, "corrected": 0, "dropped": 1,
|
||||
"kept": [Quote(line=2, quote="beta line with policy X")]}}
|
||||
bad = {"path": "x.md", "engine": None, "error": "전 엔진 실패"}
|
||||
d = render_digest("q", [ok, bad])
|
||||
assert "found: 2" in d and "processed: 1" in d and "dropped: 1" in d
|
||||
assert "dropped_reason" in d
|
||||
assert f"{sample}:2" in d # file:line 포인터
|
||||
assert "폐기 1" in d # 위조 인용 수 가시화
|
||||
|
||||
|
||||
def test_digest_zero_quote_marker(sample):
|
||||
"""인용 전멸(전부 DROPPED) 파일의 facts 는 미검증 주장 — 마커 필수 (계명 2)."""
|
||||
ok = {"path": str(sample), "engine": "codex", "truncated": False, "relevant": True,
|
||||
"summary": "s", "facts": ["근거 없는 주장"],
|
||||
"verify": {"pass": 0, "corrected": 0, "dropped": 2, "kept": []}}
|
||||
d = render_digest("q", [ok])
|
||||
assert "검증 인용 0건" in d and "미검증 주장" in d
|
||||
# 인용이 1건이라도 살아 있으면 마커 없음
|
||||
ok["verify"] = {"pass": 1, "corrected": 0, "dropped": 1,
|
||||
"kept": [Quote(line=2, quote="beta line with policy X")]}
|
||||
assert "미검증 주장" not in render_digest("q", [ok])
|
||||
|
||||
|
||||
# ---------- vote: wiki-verdict 블록 렌더 (wiki_quorum 호환) ----------
|
||||
|
||||
def _parse_with_wiki_rules(block: str):
|
||||
"""wiki_quorum.py 의 실제 파서로 파싱 (엔진 불가지 계약 검증용)."""
|
||||
import importlib.util
|
||||
import sys
|
||||
hooks = Path(__file__).resolve().parents[3] / ".claude" / "hooks"
|
||||
sys.path.insert(0, str(hooks))
|
||||
spec = importlib.util.spec_from_file_location("wr", hooks / "wiki_rules.py")
|
||||
wr = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(wr)
|
||||
return wr.parse_verdict_block(block)
|
||||
|
||||
|
||||
def test_vote_block_quorum_compatible():
|
||||
r = VoteResult(votes=[
|
||||
FindingVote(id="4.1.1", action="KEEP", reason="ok"),
|
||||
FindingVote(id="finding with space", action="REJECT", reason="bad"),
|
||||
])
|
||||
block = render_vote_block("codex", r)
|
||||
assert "agent: codex-adversarial-vote" in block
|
||||
assert "finding: 4.1.1 action: KEEP" in block
|
||||
assert "finding: finding-with-space action: REJECT" in block # 공백 정규화
|
||||
parsed = _parse_with_wiki_rules(block)
|
||||
assert parsed and len(parsed["findings"]) == 2
|
||||
|
||||
|
||||
def test_vote_block_truncation_note():
|
||||
"""findings 절단 시 표 자체에 누락 사실 기록 (no-silent-caps) + quorum 파서 비파괴."""
|
||||
r = VoteResult(votes=[FindingVote(id="1", action="KEEP", reason="ok")])
|
||||
block = render_vote_block("codex", r, truncated_chars=99_999)
|
||||
assert "잘린 finding 은 이 표에서 누락" in block
|
||||
parsed = _parse_with_wiki_rules(block) # 노트는 fence 밖 — 파싱 영향 없음
|
||||
assert parsed and len(parsed["findings"]) == 1
|
||||
assert "누락" not in render_vote_block("codex", r) # 미절단 시 노트 없음
|
||||
@@ -0,0 +1,136 @@
|
||||
import pytest
|
||||
from deep_research.pipeline import run_research
|
||||
from deep_research.backends.mock import MockBackend
|
||||
from deep_research.config import Config
|
||||
from deep_research.schemas import Scope, Angle, Search, SearchResult, Extract, Claim, Verdict, Report
|
||||
|
||||
CFG = Config(MAX_FETCH=15, MAX_VERIFY_CLAIMS=25)
|
||||
|
||||
|
||||
def _scope():
|
||||
return Scope(question="Q", summary="s",
|
||||
angles=[Angle(label=f"a{i}", query=f"q{i}") for i in range(3)])
|
||||
|
||||
|
||||
def _search(prompt, label):
|
||||
# 각 각도마다 고유 URL 1개
|
||||
n = label.split(":")[1]
|
||||
return Search(results=[SearchResult(url=f"http://{n}.com/x", title="t", relevance="high")])
|
||||
|
||||
|
||||
def _extract(prompt, label):
|
||||
return Extract(sourceQuality="primary",
|
||||
claims=[Claim(claim="C-" + label, quote="q", importance="central")])
|
||||
|
||||
|
||||
def _verdict_pass(prompt, label):
|
||||
return Verdict(refuted=False, evidence="e", confidence="high")
|
||||
|
||||
|
||||
def _report():
|
||||
return Report(summary="done", findings=[], caveats="none")
|
||||
|
||||
|
||||
async def test_happy_path_returns_report_and_stats():
|
||||
backend = MockBackend({
|
||||
"scope": _scope(),
|
||||
"search:": _search,
|
||||
"fetch:": _extract,
|
||||
"v": _verdict_pass,
|
||||
"synthesize": _report(),
|
||||
})
|
||||
out = await run_research("Q", backend, config=CFG)
|
||||
assert out["summary"] == "done"
|
||||
assert out["stats"]["confirmed"] == 3 # 3 각도 × 1 claim, 전부 생존
|
||||
assert out["stats"]["angles"] == 3
|
||||
|
||||
|
||||
async def test_empty_question_returns_error():
|
||||
out = await run_research(" ", MockBackend({}), config=CFG)
|
||||
assert "error" in out
|
||||
|
||||
|
||||
async def test_no_claims_degenerate():
|
||||
backend = MockBackend({
|
||||
"scope": _scope(),
|
||||
"search:": _search,
|
||||
"fetch:": lambda p, l: Extract(sourceQuality="unreliable", claims=[]),
|
||||
})
|
||||
out = await run_research("Q", backend, config=CFG)
|
||||
assert out["findings"] == [] and out["stats"]["claims"] == 0
|
||||
|
||||
|
||||
async def test_all_refuted_degenerate():
|
||||
backend = MockBackend({
|
||||
"scope": _scope(),
|
||||
"search:": _search,
|
||||
"fetch:": _extract,
|
||||
"v": lambda p, l: Verdict(refuted=True, evidence="e", confidence="high"),
|
||||
})
|
||||
out = await run_research("Q", backend, config=CFG)
|
||||
assert out["findings"] == [] and out["stats"]["confirmed"] == 0
|
||||
assert len(out["refuted"]) == 3
|
||||
|
||||
|
||||
async def test_synth_failure_salvages_confirmed():
|
||||
backend = MockBackend({
|
||||
"scope": _scope(),
|
||||
"search:": _search,
|
||||
"fetch:": _extract,
|
||||
"v": _verdict_pass,
|
||||
"synthesize": None, # 합성 실패
|
||||
})
|
||||
out = await run_research("Q", backend, config=CFG)
|
||||
assert out["findings"] == [] and len(out["confirmed"]) == 3
|
||||
|
||||
|
||||
# ── Task 9: report.py ──────────────────────────────────────────────────────
|
||||
from deep_research.report import to_markdown
|
||||
|
||||
|
||||
def test_to_markdown_renders_summary_and_stats():
|
||||
result = {
|
||||
"question": "Q", "summary": "ans",
|
||||
"findings": [{"claim": "F1", "confidence": "high", "sources": ["http://a"], "evidence": "e"}],
|
||||
"caveats": "c", "refuted": [], "sources": [],
|
||||
"stats": {"angles": 5, "confirmed": 1},
|
||||
}
|
||||
md = to_markdown(result)
|
||||
assert "# Deep Research" in md and "ans" in md and "F1" in md
|
||||
|
||||
|
||||
# ── Task 10: backends/codex.py ────────────────────────────────────────────
|
||||
from deep_research.backends.codex import extract_final_json
|
||||
|
||||
|
||||
def test_extract_final_json_from_jsonl():
|
||||
jsonl = "\n".join([
|
||||
'{"type":"reasoning","text":"thinking"}',
|
||||
'{"type":"web_search","query":"x"}',
|
||||
'{"type":"agent_message","text":"{\\"question\\":\\"Q\\",\\"summary\\":\\"s\\",\\"angles\\":[]}"}',
|
||||
])
|
||||
obj = extract_final_json(jsonl)
|
||||
assert obj["summary"] == "s"
|
||||
|
||||
|
||||
def test_extract_final_json_returns_none_when_absent():
|
||||
assert extract_final_json('{"type":"reasoning","text":"only"}') is None
|
||||
|
||||
|
||||
# ── Task 11: backends/antigravity.py ─────────────────────────────────────
|
||||
from deep_research.backends.antigravity import extract_json_block
|
||||
|
||||
|
||||
def test_extract_json_block_from_fenced():
|
||||
text = 'prelude\n```json\n{"refuted": false, "evidence": "e", "confidence": "high"}\n```\ntrailing'
|
||||
obj = extract_json_block(text)
|
||||
assert obj["confidence"] == "high"
|
||||
|
||||
|
||||
def test_extract_json_block_bare_object():
|
||||
obj = extract_json_block('noise {"refuted": true, "evidence": "e", "confidence": "low"} more')
|
||||
assert obj["refuted"] is True
|
||||
|
||||
|
||||
def test_extract_json_block_none_when_absent():
|
||||
assert extract_json_block("no json here") is None
|
||||
Reference in New Issue
Block a user