init: llm-wiki-haness 하네스 설계

2026-07-24 14:21:35 +09:00
parent 42bf3db4fd
commit 6c53ded9cb
2436 changed files with 194486 additions and 1 deletions
@@ -0,0 +1,136 @@
+import pytest
+from deep_research.pipeline import run_research
+from deep_research.backends.mock import MockBackend
+from deep_research.config import Config
+from deep_research.schemas import Scope, Angle, Search, SearchResult, Extract, Claim, Verdict, Report
+
+CFG = Config(MAX_FETCH=15, MAX_VERIFY_CLAIMS=25)
+
+
+def _scope():
+    return Scope(question="Q", summary="s",
+                 angles=[Angle(label=f"a{i}", query=f"q{i}") for i in range(3)])
+
+
+def _search(prompt, label):
+    # 각 각도마다 고유 URL 1개
+    n = label.split(":")[1]
+    return Search(results=[SearchResult(url=f"http://{n}.com/x", title="t", relevance="high")])
+
+
+def _extract(prompt, label):
+    return Extract(sourceQuality="primary",
+                   claims=[Claim(claim="C-" + label, quote="q", importance="central")])
+
+
+def _verdict_pass(prompt, label):
+    return Verdict(refuted=False, evidence="e", confidence="high")
+
+
+def _report():
+    return Report(summary="done", findings=[], caveats="none")
+
+
+async def test_happy_path_returns_report_and_stats():
+    backend = MockBackend({
+        "scope": _scope(),
+        "search:": _search,
+        "fetch:": _extract,
+        "v": _verdict_pass,
+        "synthesize": _report(),
+    })
+    out = await run_research("Q", backend, config=CFG)
+    assert out["summary"] == "done"
+    assert out["stats"]["confirmed"] == 3   # 3 각도 × 1 claim, 전부 생존
+    assert out["stats"]["angles"] == 3
+
+
+async def test_empty_question_returns_error():
+    out = await run_research("   ", MockBackend({}), config=CFG)
+    assert "error" in out
+
+
+async def test_no_claims_degenerate():
+    backend = MockBackend({
+        "scope": _scope(),
+        "search:": _search,
+        "fetch:": lambda p, l: Extract(sourceQuality="unreliable", claims=[]),
+    })
+    out = await run_research("Q", backend, config=CFG)
+    assert out["findings"] == [] and out["stats"]["claims"] == 0
+
+
+async def test_all_refuted_degenerate():
+    backend = MockBackend({
+        "scope": _scope(),
+        "search:": _search,
+        "fetch:": _extract,
+        "v": lambda p, l: Verdict(refuted=True, evidence="e", confidence="high"),
+    })
+    out = await run_research("Q", backend, config=CFG)
+    assert out["findings"] == [] and out["stats"]["confirmed"] == 0
+    assert len(out["refuted"]) == 3
+
+
+async def test_synth_failure_salvages_confirmed():
+    backend = MockBackend({
+        "scope": _scope(),
+        "search:": _search,
+        "fetch:": _extract,
+        "v": _verdict_pass,
+        "synthesize": None,   # 합성 실패
+    })
+    out = await run_research("Q", backend, config=CFG)
+    assert out["findings"] == [] and len(out["confirmed"]) == 3
+
+
+# ── Task 9: report.py ──────────────────────────────────────────────────────
+from deep_research.report import to_markdown
+
+
+def test_to_markdown_renders_summary_and_stats():
+    result = {
+        "question": "Q", "summary": "ans",
+        "findings": [{"claim": "F1", "confidence": "high", "sources": ["http://a"], "evidence": "e"}],
+        "caveats": "c", "refuted": [], "sources": [],
+        "stats": {"angles": 5, "confirmed": 1},
+    }
+    md = to_markdown(result)
+    assert "# Deep Research" in md and "ans" in md and "F1" in md
+
+
+# ── Task 10: backends/codex.py ────────────────────────────────────────────
+from deep_research.backends.codex import extract_final_json
+
+
+def test_extract_final_json_from_jsonl():
+    jsonl = "\n".join([
+        '{"type":"reasoning","text":"thinking"}',
+        '{"type":"web_search","query":"x"}',
+        '{"type":"agent_message","text":"{\\"question\\":\\"Q\\",\\"summary\\":\\"s\\",\\"angles\\":[]}"}',
+    ])
+    obj = extract_final_json(jsonl)
+    assert obj["summary"] == "s"
+
+
+def test_extract_final_json_returns_none_when_absent():
+    assert extract_final_json('{"type":"reasoning","text":"only"}') is None
+
+
+# ── Task 11: backends/antigravity.py ─────────────────────────────────────
+from deep_research.backends.antigravity import extract_json_block
+
+
+def test_extract_json_block_from_fenced():
+    text = 'prelude\n```json\n{"refuted": false, "evidence": "e", "confidence": "high"}\n```\ntrailing'
+    obj = extract_json_block(text)
+    assert obj["confidence"] == "high"
+
+
+def test_extract_json_block_bare_object():
+    obj = extract_json_block('noise {"refuted": true, "evidence": "e", "confidence": "low"} more')
+    assert obj["refuted"] is True
+
+
+def test_extract_json_block_none_when_absent():
+    assert extract_json_block("no json here") is None