init: llm-wiki-haness 하네스 설계

This commit is contained in:
DongHyeonka
2026-07-24 14:21:35 +09:00
parent 42bf3db4fd
commit 6c53ded9cb
2436 changed files with 194486 additions and 1 deletions
+5
View File
@@ -0,0 +1,5 @@
.venv/
__pycache__/
*.pyc
*.egg-info/
.pytest_cache/
+42
View File
@@ -0,0 +1,42 @@
# deep-research driver
Claude Code 의 deep-research 하네스를 **codex-cli / antigravity-cli** 로 이식한 외부 Python 드라이버.
## 전제
- `codex` 또는 `agy` CLI 설치 + 로그인(구독). 별도 검색 API 키 불필요 — 웹 조사는 각 CLI 네이티브 도구가 수행.
## 동작 방식 (중요)
이 드라이버는 codex/antigravity **안에서 도는 게 아니라 그 CLI 들을 워커로 호출**한다.
당신이 `deep-research` 를 실행하면 → 결정론 제어(dedup·득표 산식)는 이 Python 이 맡고
→ 각 단계의 LLM 작업은 `codex exec` / `agy -p` 를 subprocess 로 ~수십 회 호출해 처리한다.
(Claude Code 의 `/deep-research` 스킬처럼 챗 안에서 트리거하는 구조가 아님.)
## 설치 / 사용
```bash
# 1) 전역 명령으로 설치 (어느 디렉터리에서나 `deep-research`)
pipx install /home/donghyeon/dev/llm-wiki-private/scripts/deep-research
deep-research --backend codex "2026년 한국 전기차 보조금 정책 변화"
deep-research --backend antigravity "..." --json
# 2) pipx 없이: 프로젝트 venv 의 console-script 사용
cd scripts/deep-research && .venv/bin/pip install -e .
.venv/bin/deep-research --backend codex "..."
# 3) 모듈로 직접 실행 (설치 없이)
.venv/bin/python -m deep_research --backend codex "..."
```
## 구조
- `core.py` — 결정론 제어(dedup·예산·3-vote 정족수·랭킹). 원본 JS 와 1:1, 순수함수.
- `backends/` — 플랫폼 어댑터(codex/antigravity/mock).
- 충실도 기준 원본: `deep-research-wf_aecef33f-4cc.js` (spec 참조).
## 테스트
```bash
pip install -e ".[dev]" && pytest
```
MockBackend 로 네트워크 0 상태에서 6페이즈 + 퇴화 경로 3종 검증.
@@ -0,0 +1,43 @@
"""CLI: python -m deep_research --backend {codex|antigravity} "<질문>" [--json]"""
import argparse
import asyncio
import shutil
import sys
from deep_research.config import DEFAULT
from deep_research.pipeline import run_research
from deep_research.report import to_markdown, to_json
def _make_backend(name: str):
if name == "codex":
if shutil.which("codex") is None:
sys.exit("error: `codex` CLI not found on PATH. Install + login (subscription).")
from deep_research.backends.codex import CodexBackend
return CodexBackend()
if name == "antigravity":
if shutil.which("agy") is None:
sys.exit("error: `agy` CLI not found on PATH. Install + login (subscription).")
from deep_research.backends.antigravity import AntigravityBackend
return AntigravityBackend(retries=DEFAULT.ANTIGRAVITY_RETRIES)
sys.exit(f"error: unknown backend '{name}'")
def main(argv=None):
ap = argparse.ArgumentParser(prog="deep-research")
ap.add_argument("question")
ap.add_argument("--backend", required=True, choices=["codex", "antigravity"])
ap.add_argument("--json", action="store_true", help="emit raw JSON instead of markdown")
ap.add_argument("--concurrency", type=int, default=DEFAULT.CONCURRENCY)
args = ap.parse_args(argv)
if args.concurrency < 1:
sys.exit("error: --concurrency must be >= 1")
backend = _make_backend(args.backend)
cfg = DEFAULT.__class__(CONCURRENCY=args.concurrency)
result = asyncio.run(run_research(args.question, backend, config=cfg))
print(to_json(result) if args.json else to_markdown(result))
if __name__ == "__main__":
main()
@@ -0,0 +1,72 @@
"""Antigravity CLI headless backend. `agy -p` subprocess + 드라이버 스키마 검증·재시도."""
import asyncio
import json
import re
import tempfile
from pydantic import BaseModel, ValidationError
from deep_research.backends.base import AgentBackend
_FENCE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL)
def extract_json_block(text: str) -> dict | None:
"""출력 텍스트에서 JSON 객체 추출: 펜스 우선, 없으면 최외곽 중괄호."""
m = _FENCE.search(text)
candidates = [m.group(1)] if m else []
if not candidates:
start, depth = -1, 0
for i, ch in enumerate(text):
if ch == "{":
if depth == 0:
start = i
depth += 1
elif ch == "}" and depth > 0:
depth -= 1
if depth == 0 and start >= 0:
candidates.append(text[start:i + 1])
break
for c in candidates:
try:
return json.loads(c)
except json.JSONDecodeError:
continue
return None
class AntigravityBackend(AgentBackend):
def __init__(self, *, retries: int = 2, neutral_cwd: str | None = None, model: str | None = None):
self.retries = retries
self.neutral_cwd = neutral_cwd or tempfile.gettempdir()
self.model = model
async def _invoke(self, prompt: str) -> str | None:
# 실 CLI 스모크 (2026-06-10): agy 에 `--cd`/`-m` 플래그 없음 ("flags provided but
# not defined" 즉사) — cwd 는 subprocess 인자로, 모델은 `--model`. stdin 차단(비-tty 안전).
cmd = ["agy", "-p", prompt]
if self.model:
cmd += ["--model", self.model]
proc = await asyncio.create_subprocess_exec(
*cmd, cwd=self.neutral_cwd, stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
out, _ = await proc.communicate()
if proc.returncode != 0:
return None
return out.decode("utf-8", "replace")
async def run_agent(self, prompt: str, schema: type[BaseModel], *, label: str):
schema_json = json.dumps(schema.model_json_schema())
full = (prompt + "\n\n## Output format\nReturn ONLY a JSON object conforming to this "
"JSON Schema (no prose, no markdown fences):\n" + schema_json)
for attempt in range(self.retries + 1):
text = await self._invoke(full if attempt == 0 else
full + "\n\nYour previous output was invalid. Return ONLY the JSON object.")
if text is None:
continue
obj = extract_json_block(text)
if obj is None:
continue
try:
return schema.model_validate(obj)
except ValidationError:
continue
return None
@@ -0,0 +1,11 @@
from abc import ABC, abstractmethod
from pydantic import BaseModel
class AgentBackend(ABC):
"""run_agent: 프롬프트를 1개 CLI headless 에이전트로 실행, schema 로 검증된 객체 반환.
실패/사용자-skip -> None (원본 .filter(Boolean) 의미)."""
@abstractmethod
async def run_agent(self, prompt: str, schema: type[BaseModel], *, label: str) -> BaseModel | None:
...
@@ -0,0 +1,89 @@
"""Codex CLI headless backend. `codex exec --json --output-schema` subprocess."""
import asyncio
import json
import os
import tempfile
from pydantic import BaseModel, ValidationError
from deep_research.backends.base import AgentBackend
def extract_final_json(jsonl_stdout: str) -> dict | None:
"""JSONL 이벤트 스트림에서 마지막 agent_message 의 text(JSON) 파싱."""
final = None
for line in jsonl_stdout.splitlines():
line = line.strip()
if not line:
continue
try:
evt = json.loads(line)
except json.JSONDecodeError:
continue
if evt.get("type") in ("agent_message", "message") and "text" in evt:
final = evt["text"]
# 실 CLI 포맷 (2026-06-10 스모크 확인): item.completed 에 중첩된 agent_message
item = evt.get("item")
if (evt.get("type") == "item.completed" and isinstance(item, dict)
and item.get("type") == "agent_message" and "text" in item):
final = item["text"]
if final is None:
return None
try:
return json.loads(final)
except json.JSONDecodeError:
return None
class CodexBackend(AgentBackend):
def __init__(self, *, neutral_cwd: str | None = None, model: str | None = None):
self.neutral_cwd = neutral_cwd or tempfile.gettempdir()
self.model = model
@staticmethod
def _strict_schema(schema: dict) -> dict:
"""OpenAI structured-output strict 요구사항 적용 (실 스모크 2026-06-10 발견):
모든 object 에 additionalProperties:false + 전 property required."""
def walk(node):
if isinstance(node, dict):
if node.get("type") == "object" or "properties" in node:
node["additionalProperties"] = False
if "properties" in node:
node["required"] = list(node["properties"].keys())
for v in node.values():
walk(v)
elif isinstance(node, list):
for v in node:
walk(v)
walk(schema)
return schema
async def run_agent(self, prompt: str, schema: type[BaseModel], *, label: str):
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fh:
json.dump(self._strict_schema(schema.model_json_schema()), fh)
schema_path = fh.name
# --skip-git-repo-check: neutral_cwd(/tmp)는 비신뢰 디렉터리 — 플래그 없으면 즉시 종료.
# stdin=DEVNULL: 비-tty 환경에서 codex 가 "Reading additional input from stdin" 으로
# 블록되는 것 방지 (실 CLI 스모크 2026-06-10 에서 발견).
cmd = ["codex", "exec", "--json", "--output-schema", schema_path,
"--cd", self.neutral_cwd, "-s", "read-only", "--skip-git-repo-check"]
if self.model:
cmd += ["-m", self.model]
cmd.append(prompt)
try:
proc = await asyncio.create_subprocess_exec(
*cmd, stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
out, _ = await proc.communicate()
finally:
try:
os.unlink(schema_path)
except OSError:
pass
if proc.returncode != 0:
return None
obj = extract_final_json(out.decode("utf-8", "replace"))
if obj is None:
return None
try:
return schema.model_validate(obj)
except ValidationError:
return None
@@ -0,0 +1,21 @@
from pydantic import BaseModel
from deep_research.backends.base import AgentBackend
class MockBackend(AgentBackend):
"""테스트 더블. label prefix -> 응답객체(또는 None) 매핑. 네트워크·subprocess 0.
responses 의 키는 label prefix. 가장 먼저 매칭되는 prefix 의 값을 반환.
값이 콜러블이면 (prompt, label) 로 호출해 동적 응답 가능."""
def __init__(self, responses: dict, default=None):
self.responses = responses
self.default = default
self.calls: list[str] = []
async def run_agent(self, prompt: str, schema: type[BaseModel], *, label: str):
self.calls.append(label)
for prefix, resp in self.responses.items():
if label.startswith(prefix):
return resp(prompt, label) if callable(resp) else resp
return self.default
@@ -0,0 +1,15 @@
"""원본 deep-research-wf JS 와 동일한 상수. 변경 금지(충실도)."""
from dataclasses import dataclass
@dataclass(frozen=True)
class Config:
VOTES_PER_CLAIM: int = 3
REFUTATIONS_REQUIRED: int = 2
MAX_FETCH: int = 15
MAX_VERIFY_CLAIMS: int = 25
CONCURRENCY: int = 10 # asyncio.Semaphore (원본 Workflow cap 대응)
ANTIGRAVITY_RETRIES: int = 2 # 스키마 검증 실패 시 재프롬프트 횟수
DEFAULT = Config()
+122
View File
@@ -0,0 +1,122 @@
"""결정론 제어 로직 — 원본 JS 와 1:1. 네트워크·LLM 의존 0."""
from urllib.parse import urlparse
from deep_research.config import Config
REL_RANK = {"high": 0, "medium": 1, "low": 2}
IMP_RANK = {"central": 0, "supporting": 1, "tangential": 2}
QUAL_RANK = {"primary": 0, "secondary": 1, "blog": 2, "forum": 3, "unreliable": 4}
def norm_url(u: str) -> str:
try:
p = urlparse(u)
host = (p.hostname or "")
if not host:
return u.lower()
if host.startswith("www."):
host = host[4:]
path = (p.path or "").rstrip("/")
return (host + path).lower()
except Exception:
return u.lower()
def host_of(u: str) -> str:
try:
h = urlparse(u).hostname or "unknown"
return h[4:] if h.startswith("www.") else h
except Exception:
return "unknown"
class Deduper:
"""원본 pipeline stage-2 의 dedup+budget 로직. 호출 순서에 결정론적."""
def __init__(self, config: Config):
self.seen: dict[str, dict] = {}
self.dupes: list[dict] = []
self.budget_dropped: list[dict] = []
self.fetch_slots = config.MAX_FETCH
def filter_novel(self, angle: str, results: list[dict]) -> list[dict]:
ordered = sorted(results, key=lambda r: REL_RANK[r["relevance"]])
novel: list[dict] = []
for r in ordered:
key = norm_url(r["url"])
if key in self.seen:
self.dupes.append({**r, "angle": angle, "dupOf": self.seen[key]})
continue
if self.fetch_slots <= 0 and REL_RANK[r["relevance"]] >= 1:
self.budget_dropped.append({**r, "angle": angle})
continue
self.seen[key] = {"angle": angle, "title": r["title"]}
self.fetch_slots -= 1
novel.append(r)
return novel
def rank_claims(claims: list[dict], max_verify: int) -> list[dict]:
return sorted(
claims,
key=lambda c: (IMP_RANK[c["importance"]], QUAL_RANK[c["sourceQuality"]]),
)[:max_verify]
def tally(verdicts: list, votes_per_claim: int, refutations_required: int) -> dict:
"""원본 verify 정족수 산식 1:1. None=기권."""
valid = [v for v in verdicts if v is not None]
refuted = sum(1 for v in valid if v["refuted"])
abstained = votes_per_claim - len(valid)
survives = len(valid) >= refutations_required and refuted < refutations_required
return {
"valid": valid,
"refutedVotes": refuted,
"abstained": abstained,
"survives": survives,
}
CONF_RANK = {"high": 0, "medium": 1, "low": 2}
def _vote_str(c: dict) -> str:
return str(len(c["valid"]) - c["refutedVotes"]) + "-" + str(c["refutedVotes"])
def build_synth_blocks(confirmed: list[dict], killed: list[dict]) -> tuple[str, str]:
"""원본 synthesize 의 block / killedBlock 문자열 조립."""
parts = []
for i, c in enumerate(confirmed):
non_refuted = [v for v in c["valid"] if not v["refuted"]]
best = sorted(non_refuted, key=lambda v: CONF_RANK[v["confidence"]])[0]
parts.append(
"### [" + str(i) + "] " + c["claim"] + "\n"
+ "Vote: " + _vote_str(c) + " · Source: " + c["sourceUrl"] + " (" + c["sourceQuality"] + ")\n"
+ 'Quote: "' + c["quote"] + '"\nVerifier evidence (' + best["confidence"] + "): " + best["evidence"] + "\n"
)
block = "\n".join(parts)
if killed:
killed_block = "\n## Refuted claims (for transparency)\n" + "\n".join(
'- "' + c["claim"] + '" (' + c["sourceUrl"] + ", vote " + _vote_str(c) + ")"
for c in killed
)
else:
killed_block = ""
return block, killed_block
def build_stats(*, angles, sources, claims, voted, confirmed, killed,
after_synth, dupes, budget_dropped, votes_per_claim) -> dict:
return {
"angles": angles,
"sourcesFetched": sources,
"claimsExtracted": claims,
"claimsVerified": voted,
"confirmed": confirmed,
"killed": killed,
"afterSynthesis": after_synth,
"urlDupes": dupes,
"budgetDropped": budget_dropped,
"agentCalls": 1 + angles + sources + (voted * votes_per_claim) + 1,
}
@@ -0,0 +1,217 @@
"""extract.py — Tiered Extraction 브로커: bulk 발췌를 외부 구독 CLI 에 위임 + 결정론 검증.
설계 (rules/extraction-tiering.md):
- 드라이버가 파일을 *직접 읽어* 줄번호 매긴 본문을 프롬프트에 내장 — 외부 엔진은
repo 접근이 불필요하다 (쓰기 위험 0 / 입력 결정론 / codex·agy·mock 동일 동작).
- 외부 발췌는 신뢰하지 않는다: 모든 verbatim 인용을 원문에 re-match 해 검증/정정/폐기
(quote-verifier 내장). 통과분만 digest 로 방출 — 상위 티어(opus)는 digest 만 소비.
- engine funnel: 어떤 엔진이 몇 파일을 처리/실패했는지 항상 기록 (no silent engine swap).
CLI:
python3 -m deep_research.extract --backend codex|antigravity|auto \
--question "<연구 질문>" --files f1.md f2.md ... [--out digest.md] [--max-quotes 8]
exit: 0 (digest 생성, 부분 실패는 funnel 에 기록) / 1 (전 파일 실패) / 2 (인자 오류)
"""
from __future__ import annotations
import argparse
import asyncio
import sys
from pathlib import Path
from pydantic import BaseModel, Field
MAX_FILE_CHARS = 60_000 # 파일당 프롬프트 상한 (초과분은 절단 + funnel 기록)
CONCURRENCY = 3
class Quote(BaseModel):
line: int = Field(description="1-based line number in the file")
quote: str = Field(description="byte-for-byte verbatim quote from that line (no paraphrase)")
class FileExtraction(BaseModel):
relevant: bool = Field(description="whether this file contains material relevant to the question")
summary: str = Field(description="2-4 sentence summary of what this file says about the question")
facts: list[str] = Field(default_factory=list,
description="key facts relevant to the question, each one sentence")
quotes: list[Quote] = Field(default_factory=list,
description="supporting verbatim quotes with line numbers")
PROMPT = """You are a read-only extraction worker. Below is ONE file with line numbers, \
and a research question. Extract ONLY what the file actually says — no inference, no outside \
knowledge, no paraphrase inside quotes.
Research question: {question}
Rules:
- quotes must be byte-for-byte substrings of a single line (copy exactly, without the line-number prefix).
- give the 1-based line number for each quote.
- at most {max_quotes} quotes; prefer the most decision-relevant lines.
- if the file is irrelevant to the question, set relevant=false with a one-line summary.
- respond ONLY with the JSON object matching the schema.
FILE: {path}
----------------------------------------
{numbered}
----------------------------------------"""
def numbered_content(text: str, limit: int = MAX_FILE_CHARS) -> tuple[str, bool]:
lines = text.splitlines()
out, total, truncated = [], 0, False
for i, line in enumerate(lines, start=1):
s = f"{i}\t{line}"
total += len(s) + 1
if total > limit:
truncated = True
break
out.append(s)
return "\n".join(out), truncated
def verify_quotes(path: Path, ext: FileExtraction) -> dict:
"""결정론 quote-verifier: verbatim 재대조. 반환: 검증 통계 + 정정된 quotes.
판정: 인용이 명시 라인에 있으면 PASS / 다른 라인에 있으면 CORRECTED(라인 정정) /
어디에도 없으면 DROPPED (위조·의역 — 폐기)."""
try:
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
except Exception:
return {"pass": 0, "corrected": 0, "dropped": len(ext.quotes), "kept": []}
n_pass = n_corr = n_drop = 0
kept: list[Quote] = []
for q in ext.quotes:
quote = q.quote.strip()
if not quote:
n_drop += 1
continue
if 1 <= q.line <= len(lines) and quote in lines[q.line - 1]:
n_pass += 1
kept.append(q)
continue
hit = next((i for i, l in enumerate(lines, start=1) if quote in l), None)
if hit is not None:
n_corr += 1
kept.append(Quote(line=hit, quote=quote))
else:
n_drop += 1
return {"pass": n_pass, "corrected": n_corr, "dropped": n_drop, "kept": kept}
def make_backend(name: str):
if name == "codex":
from deep_research.backends.codex import CodexBackend
return CodexBackend()
if name == "antigravity":
from deep_research.backends.antigravity import AntigravityBackend
return AntigravityBackend()
raise ValueError(name)
async def extract_file(backends: list[tuple[str, object]], question: str, path: Path,
max_quotes: int, sem: asyncio.Semaphore) -> dict:
"""파일 1개 발췌 — fallback 사다리 순서대로 시도. 반환 dict 는 digest 렌더 입력."""
try:
text = path.read_text(encoding="utf-8", errors="replace")
except Exception as e:
return {"path": str(path), "engine": None, "error": f"read 실패: {e}"}
numbered, truncated = numbered_content(text)
prompt = PROMPT.format(question=question, max_quotes=max_quotes,
path=path, numbered=numbered)
async with sem:
for engine_name, backend in backends:
result = await backend.run_agent(prompt, FileExtraction, label=f"extract:{path.name}")
if result is not None:
v = verify_quotes(path, result)
return {"path": str(path), "engine": engine_name, "truncated": truncated,
"relevant": result.relevant, "summary": result.summary,
"facts": result.facts, "verify": v}
return {"path": str(path), "engine": None, "error": "전 엔진 실패 (가용성/스키마)"}
def render_digest(question: str, results: list[dict]) -> str:
ok = [r for r in results if "error" not in r]
failed = [r for r in results if "error" in r]
engines: dict[str, int] = {}
for r in ok:
engines[r["engine"]] = engines.get(r["engine"], 0) + 1
out = [f"# Extraction Digest", f"**Question:** {question}",
f"**Files:** {len(results)} (성공 {len(ok)} / 실패 {len(failed)})",
f"**Engines:** " + (", ".join(f"{k}×{v}" for k, v in engines.items()) or "없음"), ""]
for r in ok:
v = r["verify"]
out.append(f"## {r['path']}" + (" (TRUNCATED)" if r.get("truncated") else ""))
if not r["relevant"]:
out.append(f"- 무관: {r['summary']}")
out.append("")
continue
out.append(f"- 요약: {r['summary']}")
for f in r["facts"]:
out.append(f"- {f}")
for q in v["kept"]:
out.append(f" > \"{q.quote}\"{r['path']}:{q.line}")
out.append(f"- 인용 검증: PASS {v['pass']} / 정정 {v['corrected']} / **폐기 {v['dropped']}**")
if r["facts"] and not v["kept"]:
# 검증 통과 인용이 0건이면 위 facts 는 근거 없는 주장 (계명 2 — 무검증 발췌 소비 금지)
out.append("- ⚠ 검증 인용 0건 — 위 facts 는 미검증 주장 (haiku/sonnet 재발췌 후보)")
out.append("")
if failed:
out.append("## 실패 (haiku/sonnet 재발췌 대상)")
for r in failed:
out.append(f"- {r['path']}: {r['error']}")
out.append("")
total_q = sum(r["verify"]["pass"] + r["verify"]["corrected"] + r["verify"]["dropped"] for r in ok)
kept_q = sum(len(r["verify"]["kept"]) for r in ok)
out.append("```wiki-stats")
out.append("agent: extract-broker")
out.append(f"found: {len(results)}")
out.append(f"processed: {len(ok)}")
out.append(f"dropped: {len(failed)}")
if failed:
out.append(f"dropped_reason: 엔진 실패 {len(failed)}건 (위 실패 목록 — 상위 티어 재발췌)")
out.append("```")
out.append(f"<!-- quotes: total {total_q}, verified-kept {kept_q} -->")
return "\n".join(out)
async def run(args) -> int:
order = {"codex": ["codex", "antigravity"],
"antigravity": ["antigravity", "codex"],
"auto": ["codex", "antigravity"]}[args.backend]
backends = []
for name in order:
try:
backends.append((name, make_backend(name)))
except Exception:
continue
if not backends:
print("error: 사용 가능한 backend 없음", file=sys.stderr)
return 1
sem = asyncio.Semaphore(CONCURRENCY)
results = await asyncio.gather(*[
extract_file(backends, args.question, Path(f), args.max_quotes, sem)
for f in args.files])
digest = render_digest(args.question, list(results))
if args.out:
Path(args.out).write_text(digest, encoding="utf-8")
print(f"digest → {args.out} ({len(digest)} chars)")
else:
print(digest)
return 0 if any("error" not in r for r in results) else 1
def main() -> None:
ap = argparse.ArgumentParser(description="Tiered extraction 브로커 (외부 구독 CLI + 결정론 검증)")
ap.add_argument("--backend", default="auto", choices=["codex", "antigravity", "auto"])
ap.add_argument("--question", required=True)
ap.add_argument("--files", nargs="+", required=True)
ap.add_argument("--out", help="digest 출력 파일 (생략 시 stdout)")
ap.add_argument("--max-quotes", type=int, default=8)
args = ap.parse_args()
sys.exit(asyncio.run(run(args)))
if __name__ == "__main__":
main()
@@ -0,0 +1,130 @@
"""asyncio 오케스트레이션. 원본 pipeline(무배리어)/parallel(배리어) 의미 재현."""
import asyncio
from deep_research.config import Config
from deep_research.core import Deduper, host_of, rank_claims, tally, build_synth_blocks, build_stats
from deep_research.schemas import Scope, Search, Extract, Verdict, Report
from deep_research import prompts
async def run_research(question: str, backend, *, config: Config) -> dict:
q = (question or "").strip()
if not q:
return {"error": "No research question provided."}
scope = await backend.run_agent(prompts.scope_prompt(q), Scope, label="scope")
if scope is None:
return {"error": "Scope agent returned no result — cannot decompose the question."}
deduper = Deduper(config)
sem = asyncio.Semaphore(config.CONCURRENCY)
dedup_lock = asyncio.Lock()
async def search_and_fetch(angle) -> list[dict]:
async with sem:
sr = await backend.run_agent(
prompts.search_prompt(q, angle), Search, label="search:" + angle.label)
if sr is None:
return []
async with dedup_lock: # 공유 상태(seen/fetch_slots) 임계구역 직렬화
novel = deduper.filter_novel(angle.label, [r.model_dump() for r in sr.results])
async def fetch_one(source: dict):
async with sem:
ext = await backend.run_agent(
prompts.fetch_prompt(q, source, angle.label),
Extract, label="fetch:" + host_of(source["url"]))
if ext is None:
return None
return {
"url": source["url"], "title": source["title"], "angle": angle.label,
"sourceQuality": ext.sourceQuality, "publishDate": ext.publishDate,
"claims": [{**c.model_dump(), "sourceUrl": source["url"], "sourceQuality": ext.sourceQuality}
for c in ext.claims],
}
fetched = await asyncio.gather(*[fetch_one(s) for s in novel])
return [f for f in fetched if f is not None]
per_angle = await asyncio.gather(*[search_and_fetch(a) for a in scope.angles])
all_sources = [s for sub in per_angle for s in sub]
all_claims = [c for s in all_sources for c in s["claims"]]
ranked = rank_claims(all_claims, config.MAX_VERIFY_CLAIMS)
def _sources_out():
return [{"url": s["url"], "quality": s["sourceQuality"], "angle": s["angle"],
"claimCount": len(s["claims"])} for s in all_sources]
if not ranked:
return {
"question": q,
"summary": f"No claims extracted. {len(all_sources)} sources fetched, all empty/failed.",
"findings": [], "refuted": [], "sources": _sources_out(),
"stats": {"angles": len(scope.angles), "sources": len(all_sources),
"claims": 0, "dupes": len(deduper.dupes)},
}
# ── Verify (배리어) ──
async def verify_claim(claim: dict) -> dict:
async def one_vote(v: int):
async with sem:
return await backend.run_agent(
prompts.verify_prompt(q, claim, v, config.VOTES_PER_CLAIM, config.REFUTATIONS_REQUIRED),
Verdict, label="v" + str(v) + ":" + claim["claim"][:40])
verdicts = await asyncio.gather(*[one_vote(v) for v in range(config.VOTES_PER_CLAIM)])
t = tally([vd.model_dump() if vd is not None else None for vd in verdicts],
config.VOTES_PER_CLAIM, config.REFUTATIONS_REQUIRED)
return {**claim, **t}
voted = await asyncio.gather(*[verify_claim(c) for c in ranked])
confirmed = [c for c in voted if c["survives"]]
killed = [c for c in voted if not c["survives"]]
def _refuted_out():
return [{"claim": c["claim"],
"vote": str(len(c["valid"]) - c["refutedVotes"]) + "-" + str(c["refutedVotes"]),
"source": c["sourceUrl"]} for c in killed]
if not confirmed:
return {
"question": q,
"summary": f"All {len(voted)} claims refuted by adversarial verification. Research inconclusive.",
"findings": [], "refuted": _refuted_out(), "sources": _sources_out(),
"stats": build_stats(angles=len(scope.angles), sources=len(all_sources),
claims=len(all_claims), voted=len(voted), confirmed=0,
killed=len(killed), after_synth=0, dupes=len(deduper.dupes),
budget_dropped=len(deduper.budget_dropped),
votes_per_claim=config.VOTES_PER_CLAIM),
}
# ── Synthesize ──
block, killed_block = build_synth_blocks(confirmed, killed)
report = await backend.run_agent(
prompts.synth_prompt(q, block, killed_block, len(confirmed), config.VOTES_PER_CLAIM),
Report, label="synthesize")
if report is None:
return {
"question": q,
"summary": f"Synthesis step was skipped or failed — returning {len(confirmed)} verified claims unmerged.",
"findings": [],
"confirmed": [{"claim": c["claim"], "source": c["sourceUrl"], "quote": c["quote"],
"vote": str(len(c["valid"]) - c["refutedVotes"]) + "-" + str(c["refutedVotes"])}
for c in confirmed],
"refuted": _refuted_out(), "sources": _sources_out(),
"stats": build_stats(angles=len(scope.angles), sources=len(all_sources),
claims=len(all_claims), voted=len(voted), confirmed=len(confirmed),
killed=len(killed), after_synth=0, dupes=len(deduper.dupes),
budget_dropped=len(deduper.budget_dropped),
votes_per_claim=config.VOTES_PER_CLAIM),
}
return {
"question": q,
**report.model_dump(),
"refuted": _refuted_out(), "sources": _sources_out(),
"stats": build_stats(angles=len(scope.angles), sources=len(all_sources),
claims=len(all_claims), voted=len(voted), confirmed=len(confirmed),
killed=len(killed), after_synth=len(report.findings),
dupes=len(deduper.dupes), budget_dropped=len(deduper.budget_dropped),
votes_per_claim=config.VOTES_PER_CLAIM),
}
@@ -0,0 +1,86 @@
"""원본 deep-research-wf JS 의 프롬프트 문자열 이식. 표현 변경 금지(충실도)."""
def scope_prompt(question: str) -> str:
return (
"Decompose this research question into complementary search angles.\n\n"
"## Question\n" + question + "\n\n"
"## Task\n"
"Generate 5 distinct web search queries that together cover the question from "
"different angles. Pick angles that suit the question's domain. Examples:\n"
"- broad/primary · academic/technical · recent news · contrarian/skeptical · practitioner/implementation\n"
"- For medical: anatomy · common causes · serious differentials · authoritative refs · red flags\n"
"- For tech: state-of-art · benchmarks · limitations · industry adoption · cost/tradeoffs\n\n"
"Make queries specific enough to surface high-signal results. Avoid redundancy.\n"
"Return: the question (verbatim or lightly normalized), a 1-2 sentence decomposition "
"strategy, and the angles.\n\nStructured output only."
)
def search_prompt(question: str, angle) -> str:
return (
"## Web Searcher: " + angle.label + "\n\n"
'Research question: "' + question + '"\n\n'
"Your angle: **" + angle.label + "** — " + (angle.rationale or "") + "\n"
"Search query: `" + angle.query + "`\n\n"
"## Task\nUse web search with the query above (or a refined version). Return the top "
"4-6 most relevant results.\n"
"Rank by relevance to the ORIGINAL question, not just the search query. Skip obvious "
"SEO spam/content farms.\n"
"Include a short snippet capturing why each result is relevant.\n\nStructured output only."
)
def fetch_prompt(question: str, source: dict, angle: str) -> str:
return (
"## Source Extractor\n\n"
'Research question: "' + question + '"\n\n'
"Fetch and extract key claims from this source:\n"
"**URL:** " + source["url"] + "\n**Title:** " + source["title"] + "\n**Found via:** " + angle + " search\n\n"
"## Task\n1. Fetch the page content.\n"
"2. Assess source quality: primary research/institution? secondary reporting? blog/opinion? forum? unreliable?\n"
"3. Extract 2-5 FALSIFIABLE claims that bear on the research question. Each claim must:\n"
" - be a concrete, checkable statement (not vague generalities)\n"
" - include a direct quote from the source as support\n"
" - be rated central/supporting/tangential to the research question\n"
"4. Note publish date if available.\n\n"
'If the fetch fails or the page is irrelevant/paywalled, return claims: [] and '
'sourceQuality: "unreliable".\n\nStructured output only.'
)
def verify_prompt(question: str, claim: dict, v: int, votes_per_claim: int, refutations_required: int) -> str:
return (
"## Adversarial Claim Verifier (voter " + str(v + 1) + "/" + str(votes_per_claim) + ")\n\n"
"Be SKEPTICAL. Try to REFUTE this claim. ≥" + str(refutations_required) + "/" + str(votes_per_claim) + " refutations kill it.\n\n"
"## Research question\n" + question + "\n\n"
'## Claim under review\n"' + claim["claim"] + '"\n\n'
"**Source:** " + claim["sourceUrl"] + " (" + claim["sourceQuality"] + ")\n"
'**Supporting quote:** "' + claim["quote"] + '"\n\n'
"## Checklist\n"
"1. Is the claim actually supported by the quote, or is it an overreach/misread?\n"
"2. Search for contradicting evidence — does any credible source dispute or heavily qualify this?\n"
"3. Is the source quality sufficient for the claim's strength? (extraordinary claims need primary sources)\n"
"4. Is the claim outdated? (check dates — old claims about fast-moving fields are suspect)\n"
"5. Is this a marketing claim / press release / cherry-picked benchmark / forum speculation?\n\n"
"**refuted=true** if: unsupported by quote / contradicted / low-quality source for strong claim / outdated / marketing fluff.\n"
"**refuted=false** ONLY if: claim is well-supported, current, and source quality matches claim strength.\n"
"Default to refuted=true if uncertain.\n\nStructured output only. Evidence MUST be specific."
)
def synth_prompt(question: str, block: str, killed_block: str, confirmed_count: int, votes_per_claim: int) -> str:
return (
"## Synthesis: research report\n\n"
"**Question:** " + question + "\n\n"
+ str(confirmed_count) + " claims survived " + str(votes_per_claim) + "-vote adversarial verification. "
"Merge semantic duplicates and synthesize.\n\n"
"## Confirmed claims\n" + block + "\n" + killed_block + "\n\n"
"## Instructions\n"
"1. Identify claims that say the same thing — merge them, combine their sources.\n"
"2. Group related claims into coherent findings. Each finding should directly address the research question.\n"
"3. Assign confidence per finding: high (multiple primary sources, unanimous votes), medium (secondary sources or split votes), low (single source or blog-quality).\n"
"4. Write a 3-5 sentence executive summary answering the research question.\n"
"5. Note caveats: what's uncertain, what sources were weak, what time-sensitivity applies.\n"
"6. List 2-4 open questions that emerged but weren't answered.\n\nStructured output only."
)
@@ -0,0 +1,43 @@
import json
def to_markdown(result: dict) -> str:
if "error" in result:
return "# Deep Research\n\n**Error:** " + result["error"]
lines = ["# Deep Research", "", "**Question:** " + result.get("question", ""), ""]
lines += ["## Summary", result.get("summary", ""), ""]
findings = result.get("findings", [])
if findings:
lines.append("## Findings")
for i, f in enumerate(findings, 1):
srcs = ", ".join(f.get("sources", []))
lines += [
f"### {i}. {f['claim']} _({f['confidence']})_",
f"{f.get('evidence', '')}",
f"Sources: {srcs}",
"",
]
caveats = result.get("caveats")
if caveats:
lines += ["## Caveats", caveats, ""]
oq = result.get("openQuestions")
if oq:
lines += ["## Open Questions"] + [f"- {q}" for q in oq] + [""]
refuted = result.get("refuted", [])
if refuted:
lines.append("## Refuted (transparency)")
for r in refuted:
lines.append(f"- \"{r['claim']}\" (vote {r.get('vote', '')}, {r.get('source', '')})")
lines.append("")
lines += ["## Stats", "```json", json.dumps(result.get("stats", {}), ensure_ascii=False, indent=2), "```"]
return "\n".join(lines)
def to_json(result: dict) -> str:
return json.dumps(result, ensure_ascii=False, indent=2)
@@ -0,0 +1,60 @@
"""원본 5 SCHEMA 의 Pydantic v2 포팅. LLM-facing 필드는 원본 JSON 키(camelCase) 유지."""
from typing import Literal, Optional
from pydantic import BaseModel, Field
class Angle(BaseModel):
label: str
query: str
rationale: Optional[str] = None
class Scope(BaseModel):
question: str
summary: str
angles: list[Angle] = Field(min_length=3, max_length=6)
class SearchResult(BaseModel):
url: str
title: str
snippet: Optional[str] = None
relevance: Literal["high", "medium", "low"]
class Search(BaseModel):
results: list[SearchResult] = Field(max_length=6)
class Claim(BaseModel):
claim: str
quote: str
importance: Literal["central", "supporting", "tangential"]
class Extract(BaseModel):
sourceQuality: Literal["primary", "secondary", "blog", "forum", "unreliable"]
publishDate: Optional[str] = None
claims: list[Claim] = Field(max_length=5)
class Verdict(BaseModel):
refuted: bool
evidence: str
confidence: Literal["high", "medium", "low"]
counterSource: Optional[str] = None
class Finding(BaseModel):
claim: str
confidence: Literal["high", "medium", "low"]
sources: list[str]
evidence: str
vote: Optional[str] = None
class Report(BaseModel):
summary: str
findings: list[Finding]
caveats: str
openQuestions: Optional[list[str]] = None
+125
View File
@@ -0,0 +1,125 @@
"""vote.py — cross-vendor 적대 검증 표 생성기 (wiki_quorum.py 워커).
N=3 quorum 의 표 2/3 을 외부 구독 CLI(codex/agy)로 — Claude 토큰 절감이면서
독립 실패 모드 덕에 falsification 은 강화된다 (rules/extraction-tiering.md).
출력은 ```wiki-verdict``` 블록 파일 — `wiki_quorum.py` 가 엔진 불가지로 그대로 집계.
CLI:
python3 -m deep_research.vote --backend codex|antigravity \
--findings findings.md [--context-files f1 f2 ...] --out /tmp/vote-codex.md
exit: 0 (표 파일 생성) / 1 (엔진 실패).
"""
from __future__ import annotations
import argparse
import asyncio
import re
import sys
from pathlib import Path
from typing import Literal
from pydantic import BaseModel, Field
from deep_research.extract import make_backend, numbered_content
MAX_CONTEXT_CHARS = 40_000
class FindingVote(BaseModel):
id: str = Field(description="the finding's ID exactly as given (no whitespace)")
action: Literal["KEEP", "DOWNGRADE", "REJECT"]
reason: str = Field(description="one-sentence justification")
class VoteResult(BaseModel):
votes: list[FindingVote]
PROMPT = """You are an independent adversarial reviewer. Your KPI is finding weaknesses, \
not approving. For EACH finding below, try to FALSIFY it via three checks: \
(1) PRACTICALITY — is it actionable/real in this context? \
(2) OVERCLAIM — does the evidence actually support the stated severity? \
(3) ASSUMPTION — does it rest on an unstated assumption that may not hold?
Default-refute: if any check leaves you uncertain, do NOT answer KEEP — answer at least \
DOWNGRADE. KEEP only when all three checks actively pass. REJECT when the finding is wrong \
or unsupported. Vote on EVERY finding (same IDs, no whitespace in IDs). \
Respond ONLY with the JSON object matching the schema.
FINDINGS:
----------------------------------------
{findings}
----------------------------------------
{context}"""
def build_context(files: list[str]) -> str:
if not files:
return ""
parts = ["REFERENCE FILES (read-only evidence):"]
budget = MAX_CONTEXT_CHARS
for f in files:
p = Path(f)
try:
text = p.read_text(encoding="utf-8", errors="replace")
except Exception:
continue
numbered, _ = numbered_content(text, limit=min(budget, 20_000))
parts.append(f"--- {f} ---\n{numbered}")
budget -= len(numbered)
if budget <= 0:
parts.append("--- (context budget exhausted — remaining files omitted) ---")
break
return "\n".join(parts)
def render_vote_block(backend_name: str, result: VoteResult, truncated_chars: int = 0) -> str:
lines = [f"# Adversarial vote ({backend_name})", ""]
if truncated_chars:
# no-silent-caps (rules/extraction-tiering.md 계명 3): 절단 사실을 표 자체에 기록.
lines += [f"> ⚠ findings input {truncated_chars} chars 중 {MAX_CONTEXT_CHARS} 까지만 표결 — "
f"잘린 finding 은 이 표에서 누락 (Claude 표로 보완 필요)", ""]
lines += ["```wiki-verdict", f"agent: {backend_name}-adversarial-vote"]
for v in result.votes:
fid = re.sub(r"\s+", "-", v.id.strip())
lines.append(f"finding: {fid} action: {v.action}")
lines.append("```")
lines.append("")
for v in result.votes:
lines.append(f"- {v.id}{v.action}: {v.reason}")
return "\n".join(lines)
async def run(args) -> int:
findings = Path(args.findings).read_text(encoding="utf-8", errors="replace")
truncated_chars = len(findings) if len(findings) > MAX_CONTEXT_CHARS else 0
if truncated_chars:
print(f"warning: findings {truncated_chars} chars > {MAX_CONTEXT_CHARS} — 절단됨. "
f"잘린 finding 은 이 표에서 누락되므로 분할 표결 또는 Claude 표 보완 필요 (no-silent-caps)",
file=sys.stderr)
prompt = PROMPT.format(findings=findings[:MAX_CONTEXT_CHARS],
context=build_context(args.context_files or []))
backend = make_backend(args.backend)
result = await backend.run_agent(prompt, VoteResult, label=f"vote:{args.backend}")
if result is None or not result.votes:
print(f"error: {args.backend} 표 생성 실패 — Claude 표로 대체하세요 (fallback 사다리)",
file=sys.stderr)
return 1
block = render_vote_block(args.backend, result, truncated_chars)
Path(args.out).write_text(block, encoding="utf-8")
print(f"vote → {args.out} ({len(result.votes)} findings)")
return 0
def main() -> None:
ap = argparse.ArgumentParser(description="cross-vendor 적대 검증 표 생성")
ap.add_argument("--backend", required=True, choices=["codex", "antigravity"])
ap.add_argument("--findings", required=True, help="findings 목록 파일 (ID 포함)")
ap.add_argument("--context-files", nargs="*", default=[])
ap.add_argument("--out", required=True)
args = ap.parse_args()
sys.exit(asyncio.run(run(args)))
if __name__ == "__main__":
main()
+20
View File
@@ -0,0 +1,20 @@
[project]
name = "deep-research-driver"
version = "0.1.0"
description = "Port of Claude Code deep-research harness to Codex/Antigravity CLIs"
requires-python = ">=3.11"
dependencies = ["pydantic>=2.6"]
[project.scripts]
deep-research = "deep_research.__main__:main"
[project.optional-dependencies]
dev = ["pytest>=8", "pytest-asyncio>=0.23"]
[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["tests"]
[build-system]
requires = ["setuptools>=68"]
build-backend = "setuptools.build_meta"
+153
View File
@@ -0,0 +1,153 @@
import pytest
from pydantic import ValidationError
from deep_research.schemas import Scope, Angle, Search, SearchResult, Extract, Claim, Verdict, Report
from deep_research.core import norm_url, Deduper, rank_claims, tally, build_synth_blocks, build_stats
from deep_research.config import Config
def test_scope_requires_min_3_angles():
with pytest.raises(ValidationError):
Scope(question="q", summary="s", angles=[Angle(label="a", query="x")])
def test_scope_accepts_5_angles():
angles = [Angle(label=f"a{i}", query="x") for i in range(5)]
s = Scope(question="q", summary="s", angles=angles)
assert len(s.angles) == 5
def test_search_relevance_enum_enforced():
with pytest.raises(ValidationError):
SearchResult(url="http://a", title="t", relevance="bogus")
def test_json_schema_export_works():
schema = Scope.model_json_schema()
assert "angles" in schema["properties"]
assert schema["properties"]["angles"]["minItems"] == 3
assert schema["properties"]["angles"]["maxItems"] == 6
def test_norm_url_strips_www_and_trailing_slash():
assert norm_url("http://www.Example.com/Path/") == "example.com/path"
def test_norm_url_bare_host():
assert norm_url("https://example.com") == "example.com"
def test_norm_url_fallback_on_garbage():
assert norm_url("not a url") == "not a url"
def test_deduper_filters_exact_dupes():
d = Deduper(Config(MAX_FETCH=15))
r = [{"url": "http://a.com/x", "title": "t", "relevance": "high"}]
assert len(d.filter_novel("angle1", r)) == 1
# same normalized url from another angle -> dup
r2 = [{"url": "http://www.a.com/x/", "title": "t2", "relevance": "high"}]
assert d.filter_novel("angle2", r2) == []
assert len(d.dupes) == 1
def test_deduper_budget_drops_medium_low_when_slots_exhausted():
d = Deduper(Config(MAX_FETCH=1))
first = [{"url": "http://a.com/1", "title": "t", "relevance": "high"}]
d.filter_novel("a1", first) # consumes the only slot
more = [
{"url": "http://b.com/2", "title": "t", "relevance": "medium"},
{"url": "http://c.com/3", "title": "t", "relevance": "low"},
]
assert d.filter_novel("a2", more) == []
assert len(d.budget_dropped) == 2
def test_deduper_high_passes_even_when_slots_exhausted():
# 원본: high(rank 0)는 budget 조건(rank>=1)에 안 걸려 slot<=0 이어도 통과
d = Deduper(Config(MAX_FETCH=1))
d.filter_novel("a1", [{"url": "http://a.com/1", "title": "t", "relevance": "high"}])
high = [{"url": "http://d.com/4", "title": "t", "relevance": "high"}]
assert len(d.filter_novel("a2", high)) == 1
assert d.fetch_slots == -1
def _claim(imp, qual, name="c"):
return {"claim": name, "importance": imp, "sourceQuality": qual}
def test_rank_claims_orders_by_importance_then_quality():
claims = [
_claim("tangential", "primary", "t-prim"),
_claim("central", "blog", "c-blog"),
_claim("central", "primary", "c-prim"),
]
ranked = rank_claims(claims, max_verify=25)
assert [c["claim"] for c in ranked] == ["c-prim", "c-blog", "t-prim"]
def test_rank_claims_truncates_to_max():
claims = [_claim("central", "primary", f"c{i}") for i in range(30)]
assert len(rank_claims(claims, max_verify=25)) == 25
def test_tally_survives_2_valid_0_refute():
v = [{"refuted": False}, {"refuted": False}, {"refuted": False}]
t = tally(v, votes_per_claim=3, refutations_required=2)
assert t["survives"] is True and t["refutedVotes"] == 0
def test_tally_killed_2_refute():
v = [{"refuted": True}, {"refuted": True}, {"refuted": False}]
t = tally(v, votes_per_claim=3, refutations_required=2)
assert t["survives"] is False and t["refutedVotes"] == 2
def test_tally_all_abstain_does_not_survive():
# ⚠️ 거짓 생존 차단: all-None -> refuted=0 이지만 valid<2 라 미생존
t = tally([None, None, None], votes_per_claim=3, refutations_required=2)
assert t["survives"] is False and t["abstained"] == 3
def test_tally_one_valid_two_abstain_does_not_survive():
t = tally([{"refuted": False}, None, None], votes_per_claim=3, refutations_required=2)
assert t["survives"] is False # valid(1) < 2
def test_tally_boundary_one_refute_two_valid_survives():
t = tally([{"refuted": True}, {"refuted": False}], votes_per_claim=3, refutations_required=2)
assert t["survives"] is True # 1 refute < 2 required, quorum met
def test_tally_boundary_two_refute_two_valid_killed():
t = tally([{"refuted": True}, {"refuted": True}], votes_per_claim=3, refutations_required=2)
assert t["survives"] is False # 2 refute == 2 required, strict < fails
def test_build_synth_blocks_includes_vote_and_source():
confirmed = [{
"claim": "X causes Y", "quote": "q", "sourceUrl": "http://a", "sourceQuality": "primary",
"valid": [{"refuted": False, "confidence": "high", "evidence": "e"}],
"refutedVotes": 0,
}]
killed = [{
"claim": "Z", "sourceUrl": "http://b", "valid": [{"refuted": True}], "refutedVotes": 1,
}]
block, killed_block = build_synth_blocks(confirmed, killed)
assert "X causes Y" in block and "1-0" in block
assert "Refuted claims" in killed_block and "Z" in killed_block
def test_build_synth_blocks_no_killed():
block, killed_block = build_synth_blocks([{
"claim": "X", "quote": "q", "sourceUrl": "http://a", "sourceQuality": "primary",
"valid": [{"refuted": False, "confidence": "high", "evidence": "e"}], "refutedVotes": 0,
}], [])
assert killed_block == ""
def test_build_stats_agent_calls_formula():
s = build_stats(angles=5, sources=12, claims=20, voted=18, confirmed=10, killed=8,
after_synth=6, dupes=3, budget_dropped=2, votes_per_claim=3)
# 1 + angles + sources + voted*votes_per_claim + 1
assert s["agentCalls"] == 1 + 5 + 12 + 18 * 3 + 1
assert s["confirmed"] == 10 and s["afterSynthesis"] == 6
@@ -0,0 +1,131 @@
"""extract.py(quote-verifier·funnel·fallback) + vote.py(블록 렌더) 테스트 — MockBackend."""
import asyncio
from pathlib import Path
import pytest
from deep_research.backends.mock import MockBackend
from deep_research.extract import (FileExtraction, Quote, extract_file,
numbered_content, render_digest, verify_quotes)
from deep_research.vote import FindingVote, VoteResult, render_vote_block
@pytest.fixture
def sample(tmp_path: Path) -> Path:
p = tmp_path / "note.md"
p.write_text("alpha line\nbeta line with policy X\ngamma line\n", encoding="utf-8")
return p
# ---------- quote-verifier (결정론 trust boundary) ----------
def test_verify_pass_corrected_dropped(sample):
ext = FileExtraction(relevant=True, summary="s", facts=[], quotes=[
Quote(line=2, quote="beta line with policy X"), # PASS
Quote(line=1, quote="gamma line"), # 다른 라인 → CORRECTED(3)
Quote(line=2, quote="완전 위조된 인용"), # DROPPED
])
v = verify_quotes(sample, ext)
assert (v["pass"], v["corrected"], v["dropped"]) == (1, 1, 1)
assert any(q.line == 3 and q.quote == "gamma line" for q in v["kept"])
assert all("위조" not in q.quote for q in v["kept"]) # 위조 인용은 digest 에 못 들어감
def test_numbered_content_truncation():
text = "\n".join(f"line-{i}" for i in range(1000))
numbered, truncated = numbered_content(text, limit=200)
assert truncated and numbered.startswith("1\tline-0")
# ---------- extract_file: fallback 사다리 + engine funnel ----------
def _ok(relevant=True):
return FileExtraction(relevant=relevant, summary="요약", facts=["사실1"],
quotes=[Quote(line=2, quote="beta line with policy X")])
def test_first_backend_wins(sample):
b1, b2 = MockBackend({"extract": _ok()}), MockBackend({"extract": _ok()})
r = asyncio.run(extract_file([("codex", b1), ("antigravity", b2)], "q", sample, 8,
asyncio.Semaphore(1)))
assert r["engine"] == "codex" and b2.calls == []
assert r["verify"]["pass"] == 1
def test_fallback_ladder(sample):
b1 = MockBackend({}, default=None) # codex 실패
b2 = MockBackend({"extract": _ok()}) # agy 성공
r = asyncio.run(extract_file([("codex", b1), ("antigravity", b2)], "q", sample, 8,
asyncio.Semaphore(1)))
assert r["engine"] == "antigravity" # silent engine swap 아님 — engine 기록
def test_all_fail_recorded(sample):
b = MockBackend({}, default=None)
r = asyncio.run(extract_file([("codex", b)], "q", sample, 8, asyncio.Semaphore(1)))
assert "error" in r
# ---------- digest 렌더: funnel 균형 ----------
def test_digest_funnel(sample):
ok = {"path": str(sample), "engine": "codex", "truncated": False, "relevant": True,
"summary": "s", "facts": ["f"],
"verify": {"pass": 1, "corrected": 0, "dropped": 1,
"kept": [Quote(line=2, quote="beta line with policy X")]}}
bad = {"path": "x.md", "engine": None, "error": "전 엔진 실패"}
d = render_digest("q", [ok, bad])
assert "found: 2" in d and "processed: 1" in d and "dropped: 1" in d
assert "dropped_reason" in d
assert f"{sample}:2" in d # file:line 포인터
assert "폐기 1" in d # 위조 인용 수 가시화
def test_digest_zero_quote_marker(sample):
"""인용 전멸(전부 DROPPED) 파일의 facts 는 미검증 주장 — 마커 필수 (계명 2)."""
ok = {"path": str(sample), "engine": "codex", "truncated": False, "relevant": True,
"summary": "s", "facts": ["근거 없는 주장"],
"verify": {"pass": 0, "corrected": 0, "dropped": 2, "kept": []}}
d = render_digest("q", [ok])
assert "검증 인용 0건" in d and "미검증 주장" in d
# 인용이 1건이라도 살아 있으면 마커 없음
ok["verify"] = {"pass": 1, "corrected": 0, "dropped": 1,
"kept": [Quote(line=2, quote="beta line with policy X")]}
assert "미검증 주장" not in render_digest("q", [ok])
# ---------- vote: wiki-verdict 블록 렌더 (wiki_quorum 호환) ----------
def _parse_with_wiki_rules(block: str):
"""wiki_quorum.py 의 실제 파서로 파싱 (엔진 불가지 계약 검증용)."""
import importlib.util
import sys
hooks = Path(__file__).resolve().parents[3] / ".claude" / "hooks"
sys.path.insert(0, str(hooks))
spec = importlib.util.spec_from_file_location("wr", hooks / "wiki_rules.py")
wr = importlib.util.module_from_spec(spec)
spec.loader.exec_module(wr)
return wr.parse_verdict_block(block)
def test_vote_block_quorum_compatible():
r = VoteResult(votes=[
FindingVote(id="4.1.1", action="KEEP", reason="ok"),
FindingVote(id="finding with space", action="REJECT", reason="bad"),
])
block = render_vote_block("codex", r)
assert "agent: codex-adversarial-vote" in block
assert "finding: 4.1.1 action: KEEP" in block
assert "finding: finding-with-space action: REJECT" in block # 공백 정규화
parsed = _parse_with_wiki_rules(block)
assert parsed and len(parsed["findings"]) == 2
def test_vote_block_truncation_note():
"""findings 절단 시 표 자체에 누락 사실 기록 (no-silent-caps) + quorum 파서 비파괴."""
r = VoteResult(votes=[FindingVote(id="1", action="KEEP", reason="ok")])
block = render_vote_block("codex", r, truncated_chars=99_999)
assert "잘린 finding 은 이 표에서 누락" in block
parsed = _parse_with_wiki_rules(block) # 노트는 fence 밖 — 파싱 영향 없음
assert parsed and len(parsed["findings"]) == 1
assert "누락" not in render_vote_block("codex", r) # 미절단 시 노트 없음
@@ -0,0 +1,136 @@
import pytest
from deep_research.pipeline import run_research
from deep_research.backends.mock import MockBackend
from deep_research.config import Config
from deep_research.schemas import Scope, Angle, Search, SearchResult, Extract, Claim, Verdict, Report
CFG = Config(MAX_FETCH=15, MAX_VERIFY_CLAIMS=25)
def _scope():
return Scope(question="Q", summary="s",
angles=[Angle(label=f"a{i}", query=f"q{i}") for i in range(3)])
def _search(prompt, label):
# 각 각도마다 고유 URL 1개
n = label.split(":")[1]
return Search(results=[SearchResult(url=f"http://{n}.com/x", title="t", relevance="high")])
def _extract(prompt, label):
return Extract(sourceQuality="primary",
claims=[Claim(claim="C-" + label, quote="q", importance="central")])
def _verdict_pass(prompt, label):
return Verdict(refuted=False, evidence="e", confidence="high")
def _report():
return Report(summary="done", findings=[], caveats="none")
async def test_happy_path_returns_report_and_stats():
backend = MockBackend({
"scope": _scope(),
"search:": _search,
"fetch:": _extract,
"v": _verdict_pass,
"synthesize": _report(),
})
out = await run_research("Q", backend, config=CFG)
assert out["summary"] == "done"
assert out["stats"]["confirmed"] == 3 # 3 각도 × 1 claim, 전부 생존
assert out["stats"]["angles"] == 3
async def test_empty_question_returns_error():
out = await run_research(" ", MockBackend({}), config=CFG)
assert "error" in out
async def test_no_claims_degenerate():
backend = MockBackend({
"scope": _scope(),
"search:": _search,
"fetch:": lambda p, l: Extract(sourceQuality="unreliable", claims=[]),
})
out = await run_research("Q", backend, config=CFG)
assert out["findings"] == [] and out["stats"]["claims"] == 0
async def test_all_refuted_degenerate():
backend = MockBackend({
"scope": _scope(),
"search:": _search,
"fetch:": _extract,
"v": lambda p, l: Verdict(refuted=True, evidence="e", confidence="high"),
})
out = await run_research("Q", backend, config=CFG)
assert out["findings"] == [] and out["stats"]["confirmed"] == 0
assert len(out["refuted"]) == 3
async def test_synth_failure_salvages_confirmed():
backend = MockBackend({
"scope": _scope(),
"search:": _search,
"fetch:": _extract,
"v": _verdict_pass,
"synthesize": None, # 합성 실패
})
out = await run_research("Q", backend, config=CFG)
assert out["findings"] == [] and len(out["confirmed"]) == 3
# ── Task 9: report.py ──────────────────────────────────────────────────────
from deep_research.report import to_markdown
def test_to_markdown_renders_summary_and_stats():
result = {
"question": "Q", "summary": "ans",
"findings": [{"claim": "F1", "confidence": "high", "sources": ["http://a"], "evidence": "e"}],
"caveats": "c", "refuted": [], "sources": [],
"stats": {"angles": 5, "confirmed": 1},
}
md = to_markdown(result)
assert "# Deep Research" in md and "ans" in md and "F1" in md
# ── Task 10: backends/codex.py ────────────────────────────────────────────
from deep_research.backends.codex import extract_final_json
def test_extract_final_json_from_jsonl():
jsonl = "\n".join([
'{"type":"reasoning","text":"thinking"}',
'{"type":"web_search","query":"x"}',
'{"type":"agent_message","text":"{\\"question\\":\\"Q\\",\\"summary\\":\\"s\\",\\"angles\\":[]}"}',
])
obj = extract_final_json(jsonl)
assert obj["summary"] == "s"
def test_extract_final_json_returns_none_when_absent():
assert extract_final_json('{"type":"reasoning","text":"only"}') is None
# ── Task 11: backends/antigravity.py ─────────────────────────────────────
from deep_research.backends.antigravity import extract_json_block
def test_extract_json_block_from_fenced():
text = 'prelude\n```json\n{"refuted": false, "evidence": "e", "confidence": "high"}\n```\ntrailing'
obj = extract_json_block(text)
assert obj["confidence"] == "high"
def test_extract_json_block_bare_object():
obj = extract_json_block('noise {"refuted": true, "evidence": "e", "confidence": "low"} more')
assert obj["refuted"] is True
def test_extract_json_block_none_when_absent():
assert extract_json_block("no json here") is None