init: llm-wiki-haness 하네스 설계
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.egg-info/
|
||||
.pytest_cache/
|
||||
@@ -0,0 +1,42 @@
|
||||
# deep-research driver
|
||||
|
||||
Claude Code 의 deep-research 하네스를 **codex-cli / antigravity-cli** 로 이식한 외부 Python 드라이버.
|
||||
|
||||
## 전제
|
||||
- `codex` 또는 `agy` CLI 설치 + 로그인(구독). 별도 검색 API 키 불필요 — 웹 조사는 각 CLI 네이티브 도구가 수행.
|
||||
|
||||
## 동작 방식 (중요)
|
||||
|
||||
이 드라이버는 codex/antigravity **안에서 도는 게 아니라 그 CLI 들을 워커로 호출**한다.
|
||||
당신이 `deep-research` 를 실행하면 → 결정론 제어(dedup·득표 산식)는 이 Python 이 맡고
|
||||
→ 각 단계의 LLM 작업은 `codex exec` / `agy -p` 를 subprocess 로 ~수십 회 호출해 처리한다.
|
||||
(Claude Code 의 `/deep-research` 스킬처럼 챗 안에서 트리거하는 구조가 아님.)
|
||||
|
||||
## 설치 / 사용
|
||||
|
||||
```bash
|
||||
# 1) 전역 명령으로 설치 (어느 디렉터리에서나 `deep-research`)
|
||||
pipx install /home/donghyeon/dev/llm-wiki-private/scripts/deep-research
|
||||
deep-research --backend codex "2026년 한국 전기차 보조금 정책 변화"
|
||||
deep-research --backend antigravity "..." --json
|
||||
|
||||
# 2) pipx 없이: 프로젝트 venv 의 console-script 사용
|
||||
cd scripts/deep-research && .venv/bin/pip install -e .
|
||||
.venv/bin/deep-research --backend codex "..."
|
||||
|
||||
# 3) 모듈로 직접 실행 (설치 없이)
|
||||
.venv/bin/python -m deep_research --backend codex "..."
|
||||
```
|
||||
|
||||
## 구조
|
||||
- `core.py` — 결정론 제어(dedup·예산·3-vote 정족수·랭킹). 원본 JS 와 1:1, 순수함수.
|
||||
- `backends/` — 플랫폼 어댑터(codex/antigravity/mock).
|
||||
- 충실도 기준 원본: `deep-research-wf_aecef33f-4cc.js` (spec 참조).
|
||||
|
||||
## 테스트
|
||||
|
||||
```bash
|
||||
pip install -e ".[dev]" && pytest
|
||||
```
|
||||
|
||||
MockBackend 로 네트워크 0 상태에서 6페이즈 + 퇴화 경로 3종 검증.
|
||||
@@ -0,0 +1,43 @@
|
||||
"""CLI: python -m deep_research --backend {codex|antigravity} "<질문>" [--json]"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import shutil
|
||||
import sys
|
||||
from deep_research.config import DEFAULT
|
||||
from deep_research.pipeline import run_research
|
||||
from deep_research.report import to_markdown, to_json
|
||||
|
||||
|
||||
def _make_backend(name: str):
|
||||
if name == "codex":
|
||||
if shutil.which("codex") is None:
|
||||
sys.exit("error: `codex` CLI not found on PATH. Install + login (subscription).")
|
||||
from deep_research.backends.codex import CodexBackend
|
||||
return CodexBackend()
|
||||
if name == "antigravity":
|
||||
if shutil.which("agy") is None:
|
||||
sys.exit("error: `agy` CLI not found on PATH. Install + login (subscription).")
|
||||
from deep_research.backends.antigravity import AntigravityBackend
|
||||
return AntigravityBackend(retries=DEFAULT.ANTIGRAVITY_RETRIES)
|
||||
sys.exit(f"error: unknown backend '{name}'")
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(prog="deep-research")
|
||||
ap.add_argument("question")
|
||||
ap.add_argument("--backend", required=True, choices=["codex", "antigravity"])
|
||||
ap.add_argument("--json", action="store_true", help="emit raw JSON instead of markdown")
|
||||
ap.add_argument("--concurrency", type=int, default=DEFAULT.CONCURRENCY)
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
if args.concurrency < 1:
|
||||
sys.exit("error: --concurrency must be >= 1")
|
||||
|
||||
backend = _make_backend(args.backend)
|
||||
cfg = DEFAULT.__class__(CONCURRENCY=args.concurrency)
|
||||
result = asyncio.run(run_research(args.question, backend, config=cfg))
|
||||
print(to_json(result) if args.json else to_markdown(result))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Antigravity CLI headless backend. `agy -p` subprocess + 드라이버 스키마 검증·재시도."""
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import tempfile
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from deep_research.backends.base import AgentBackend
|
||||
|
||||
_FENCE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL)
|
||||
|
||||
|
||||
def extract_json_block(text: str) -> dict | None:
|
||||
"""출력 텍스트에서 JSON 객체 추출: 펜스 우선, 없으면 최외곽 중괄호."""
|
||||
m = _FENCE.search(text)
|
||||
candidates = [m.group(1)] if m else []
|
||||
if not candidates:
|
||||
start, depth = -1, 0
|
||||
for i, ch in enumerate(text):
|
||||
if ch == "{":
|
||||
if depth == 0:
|
||||
start = i
|
||||
depth += 1
|
||||
elif ch == "}" and depth > 0:
|
||||
depth -= 1
|
||||
if depth == 0 and start >= 0:
|
||||
candidates.append(text[start:i + 1])
|
||||
break
|
||||
for c in candidates:
|
||||
try:
|
||||
return json.loads(c)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
class AntigravityBackend(AgentBackend):
|
||||
def __init__(self, *, retries: int = 2, neutral_cwd: str | None = None, model: str | None = None):
|
||||
self.retries = retries
|
||||
self.neutral_cwd = neutral_cwd or tempfile.gettempdir()
|
||||
self.model = model
|
||||
|
||||
async def _invoke(self, prompt: str) -> str | None:
|
||||
# 실 CLI 스모크 (2026-06-10): agy 에 `--cd`/`-m` 플래그 없음 ("flags provided but
|
||||
# not defined" 즉사) — cwd 는 subprocess 인자로, 모델은 `--model`. stdin 차단(비-tty 안전).
|
||||
cmd = ["agy", "-p", prompt]
|
||||
if self.model:
|
||||
cmd += ["--model", self.model]
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd, cwd=self.neutral_cwd, stdin=asyncio.subprocess.DEVNULL,
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
||||
out, _ = await proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
return out.decode("utf-8", "replace")
|
||||
|
||||
async def run_agent(self, prompt: str, schema: type[BaseModel], *, label: str):
|
||||
schema_json = json.dumps(schema.model_json_schema())
|
||||
full = (prompt + "\n\n## Output format\nReturn ONLY a JSON object conforming to this "
|
||||
"JSON Schema (no prose, no markdown fences):\n" + schema_json)
|
||||
for attempt in range(self.retries + 1):
|
||||
text = await self._invoke(full if attempt == 0 else
|
||||
full + "\n\nYour previous output was invalid. Return ONLY the JSON object.")
|
||||
if text is None:
|
||||
continue
|
||||
obj = extract_json_block(text)
|
||||
if obj is None:
|
||||
continue
|
||||
try:
|
||||
return schema.model_validate(obj)
|
||||
except ValidationError:
|
||||
continue
|
||||
return None
|
||||
@@ -0,0 +1,11 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class AgentBackend(ABC):
|
||||
"""run_agent: 프롬프트를 1개 CLI headless 에이전트로 실행, schema 로 검증된 객체 반환.
|
||||
실패/사용자-skip -> None (원본 .filter(Boolean) 의미)."""
|
||||
|
||||
@abstractmethod
|
||||
async def run_agent(self, prompt: str, schema: type[BaseModel], *, label: str) -> BaseModel | None:
|
||||
...
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Codex CLI headless backend. `codex exec --json --output-schema` subprocess."""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from deep_research.backends.base import AgentBackend
|
||||
|
||||
|
||||
def extract_final_json(jsonl_stdout: str) -> dict | None:
|
||||
"""JSONL 이벤트 스트림에서 마지막 agent_message 의 text(JSON) 파싱."""
|
||||
final = None
|
||||
for line in jsonl_stdout.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
evt = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if evt.get("type") in ("agent_message", "message") and "text" in evt:
|
||||
final = evt["text"]
|
||||
# 실 CLI 포맷 (2026-06-10 스모크 확인): item.completed 에 중첩된 agent_message
|
||||
item = evt.get("item")
|
||||
if (evt.get("type") == "item.completed" and isinstance(item, dict)
|
||||
and item.get("type") == "agent_message" and "text" in item):
|
||||
final = item["text"]
|
||||
if final is None:
|
||||
return None
|
||||
try:
|
||||
return json.loads(final)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
class CodexBackend(AgentBackend):
|
||||
def __init__(self, *, neutral_cwd: str | None = None, model: str | None = None):
|
||||
self.neutral_cwd = neutral_cwd or tempfile.gettempdir()
|
||||
self.model = model
|
||||
|
||||
@staticmethod
|
||||
def _strict_schema(schema: dict) -> dict:
|
||||
"""OpenAI structured-output strict 요구사항 적용 (실 스모크 2026-06-10 발견):
|
||||
모든 object 에 additionalProperties:false + 전 property required."""
|
||||
def walk(node):
|
||||
if isinstance(node, dict):
|
||||
if node.get("type") == "object" or "properties" in node:
|
||||
node["additionalProperties"] = False
|
||||
if "properties" in node:
|
||||
node["required"] = list(node["properties"].keys())
|
||||
for v in node.values():
|
||||
walk(v)
|
||||
elif isinstance(node, list):
|
||||
for v in node:
|
||||
walk(v)
|
||||
walk(schema)
|
||||
return schema
|
||||
|
||||
async def run_agent(self, prompt: str, schema: type[BaseModel], *, label: str):
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fh:
|
||||
json.dump(self._strict_schema(schema.model_json_schema()), fh)
|
||||
schema_path = fh.name
|
||||
# --skip-git-repo-check: neutral_cwd(/tmp)는 비신뢰 디렉터리 — 플래그 없으면 즉시 종료.
|
||||
# stdin=DEVNULL: 비-tty 환경에서 codex 가 "Reading additional input from stdin" 으로
|
||||
# 블록되는 것 방지 (실 CLI 스모크 2026-06-10 에서 발견).
|
||||
cmd = ["codex", "exec", "--json", "--output-schema", schema_path,
|
||||
"--cd", self.neutral_cwd, "-s", "read-only", "--skip-git-repo-check"]
|
||||
if self.model:
|
||||
cmd += ["-m", self.model]
|
||||
cmd.append(prompt)
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd, stdin=asyncio.subprocess.DEVNULL,
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
||||
out, _ = await proc.communicate()
|
||||
finally:
|
||||
try:
|
||||
os.unlink(schema_path)
|
||||
except OSError:
|
||||
pass
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
obj = extract_final_json(out.decode("utf-8", "replace"))
|
||||
if obj is None:
|
||||
return None
|
||||
try:
|
||||
return schema.model_validate(obj)
|
||||
except ValidationError:
|
||||
return None
|
||||
@@ -0,0 +1,21 @@
|
||||
from pydantic import BaseModel
|
||||
from deep_research.backends.base import AgentBackend
|
||||
|
||||
|
||||
class MockBackend(AgentBackend):
|
||||
"""테스트 더블. label prefix -> 응답객체(또는 None) 매핑. 네트워크·subprocess 0.
|
||||
|
||||
responses 의 키는 label prefix. 가장 먼저 매칭되는 prefix 의 값을 반환.
|
||||
값이 콜러블이면 (prompt, label) 로 호출해 동적 응답 가능."""
|
||||
|
||||
def __init__(self, responses: dict, default=None):
|
||||
self.responses = responses
|
||||
self.default = default
|
||||
self.calls: list[str] = []
|
||||
|
||||
async def run_agent(self, prompt: str, schema: type[BaseModel], *, label: str):
|
||||
self.calls.append(label)
|
||||
for prefix, resp in self.responses.items():
|
||||
if label.startswith(prefix):
|
||||
return resp(prompt, label) if callable(resp) else resp
|
||||
return self.default
|
||||
@@ -0,0 +1,15 @@
|
||||
"""원본 deep-research-wf JS 와 동일한 상수. 변경 금지(충실도)."""
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
VOTES_PER_CLAIM: int = 3
|
||||
REFUTATIONS_REQUIRED: int = 2
|
||||
MAX_FETCH: int = 15
|
||||
MAX_VERIFY_CLAIMS: int = 25
|
||||
CONCURRENCY: int = 10 # asyncio.Semaphore (원본 Workflow cap 대응)
|
||||
ANTIGRAVITY_RETRIES: int = 2 # 스키마 검증 실패 시 재프롬프트 횟수
|
||||
|
||||
|
||||
DEFAULT = Config()
|
||||
@@ -0,0 +1,122 @@
|
||||
"""결정론 제어 로직 — 원본 JS 와 1:1. 네트워크·LLM 의존 0."""
|
||||
from urllib.parse import urlparse
|
||||
from deep_research.config import Config
|
||||
|
||||
REL_RANK = {"high": 0, "medium": 1, "low": 2}
|
||||
IMP_RANK = {"central": 0, "supporting": 1, "tangential": 2}
|
||||
QUAL_RANK = {"primary": 0, "secondary": 1, "blog": 2, "forum": 3, "unreliable": 4}
|
||||
|
||||
|
||||
def norm_url(u: str) -> str:
|
||||
try:
|
||||
p = urlparse(u)
|
||||
host = (p.hostname or "")
|
||||
if not host:
|
||||
return u.lower()
|
||||
if host.startswith("www."):
|
||||
host = host[4:]
|
||||
path = (p.path or "").rstrip("/")
|
||||
return (host + path).lower()
|
||||
except Exception:
|
||||
return u.lower()
|
||||
|
||||
|
||||
def host_of(u: str) -> str:
|
||||
try:
|
||||
h = urlparse(u).hostname or "unknown"
|
||||
return h[4:] if h.startswith("www.") else h
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
class Deduper:
|
||||
"""원본 pipeline stage-2 의 dedup+budget 로직. 호출 순서에 결정론적."""
|
||||
|
||||
def __init__(self, config: Config):
|
||||
self.seen: dict[str, dict] = {}
|
||||
self.dupes: list[dict] = []
|
||||
self.budget_dropped: list[dict] = []
|
||||
self.fetch_slots = config.MAX_FETCH
|
||||
|
||||
def filter_novel(self, angle: str, results: list[dict]) -> list[dict]:
|
||||
ordered = sorted(results, key=lambda r: REL_RANK[r["relevance"]])
|
||||
novel: list[dict] = []
|
||||
for r in ordered:
|
||||
key = norm_url(r["url"])
|
||||
if key in self.seen:
|
||||
self.dupes.append({**r, "angle": angle, "dupOf": self.seen[key]})
|
||||
continue
|
||||
if self.fetch_slots <= 0 and REL_RANK[r["relevance"]] >= 1:
|
||||
self.budget_dropped.append({**r, "angle": angle})
|
||||
continue
|
||||
self.seen[key] = {"angle": angle, "title": r["title"]}
|
||||
self.fetch_slots -= 1
|
||||
novel.append(r)
|
||||
return novel
|
||||
|
||||
|
||||
def rank_claims(claims: list[dict], max_verify: int) -> list[dict]:
|
||||
return sorted(
|
||||
claims,
|
||||
key=lambda c: (IMP_RANK[c["importance"]], QUAL_RANK[c["sourceQuality"]]),
|
||||
)[:max_verify]
|
||||
|
||||
|
||||
def tally(verdicts: list, votes_per_claim: int, refutations_required: int) -> dict:
|
||||
"""원본 verify 정족수 산식 1:1. None=기권."""
|
||||
valid = [v for v in verdicts if v is not None]
|
||||
refuted = sum(1 for v in valid if v["refuted"])
|
||||
abstained = votes_per_claim - len(valid)
|
||||
survives = len(valid) >= refutations_required and refuted < refutations_required
|
||||
return {
|
||||
"valid": valid,
|
||||
"refutedVotes": refuted,
|
||||
"abstained": abstained,
|
||||
"survives": survives,
|
||||
}
|
||||
|
||||
|
||||
CONF_RANK = {"high": 0, "medium": 1, "low": 2}
|
||||
|
||||
|
||||
def _vote_str(c: dict) -> str:
|
||||
return str(len(c["valid"]) - c["refutedVotes"]) + "-" + str(c["refutedVotes"])
|
||||
|
||||
|
||||
def build_synth_blocks(confirmed: list[dict], killed: list[dict]) -> tuple[str, str]:
|
||||
"""원본 synthesize 의 block / killedBlock 문자열 조립."""
|
||||
parts = []
|
||||
for i, c in enumerate(confirmed):
|
||||
non_refuted = [v for v in c["valid"] if not v["refuted"]]
|
||||
best = sorted(non_refuted, key=lambda v: CONF_RANK[v["confidence"]])[0]
|
||||
parts.append(
|
||||
"### [" + str(i) + "] " + c["claim"] + "\n"
|
||||
+ "Vote: " + _vote_str(c) + " · Source: " + c["sourceUrl"] + " (" + c["sourceQuality"] + ")\n"
|
||||
+ 'Quote: "' + c["quote"] + '"\nVerifier evidence (' + best["confidence"] + "): " + best["evidence"] + "\n"
|
||||
)
|
||||
block = "\n".join(parts)
|
||||
|
||||
if killed:
|
||||
killed_block = "\n## Refuted claims (for transparency)\n" + "\n".join(
|
||||
'- "' + c["claim"] + '" (' + c["sourceUrl"] + ", vote " + _vote_str(c) + ")"
|
||||
for c in killed
|
||||
)
|
||||
else:
|
||||
killed_block = ""
|
||||
return block, killed_block
|
||||
|
||||
|
||||
def build_stats(*, angles, sources, claims, voted, confirmed, killed,
|
||||
after_synth, dupes, budget_dropped, votes_per_claim) -> dict:
|
||||
return {
|
||||
"angles": angles,
|
||||
"sourcesFetched": sources,
|
||||
"claimsExtracted": claims,
|
||||
"claimsVerified": voted,
|
||||
"confirmed": confirmed,
|
||||
"killed": killed,
|
||||
"afterSynthesis": after_synth,
|
||||
"urlDupes": dupes,
|
||||
"budgetDropped": budget_dropped,
|
||||
"agentCalls": 1 + angles + sources + (voted * votes_per_claim) + 1,
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
"""extract.py — Tiered Extraction 브로커: bulk 발췌를 외부 구독 CLI 에 위임 + 결정론 검증.
|
||||
|
||||
설계 (rules/extraction-tiering.md):
|
||||
- 드라이버가 파일을 *직접 읽어* 줄번호 매긴 본문을 프롬프트에 내장 — 외부 엔진은
|
||||
repo 접근이 불필요하다 (쓰기 위험 0 / 입력 결정론 / codex·agy·mock 동일 동작).
|
||||
- 외부 발췌는 신뢰하지 않는다: 모든 verbatim 인용을 원문에 re-match 해 검증/정정/폐기
|
||||
(quote-verifier 내장). 통과분만 digest 로 방출 — 상위 티어(opus)는 digest 만 소비.
|
||||
- engine funnel: 어떤 엔진이 몇 파일을 처리/실패했는지 항상 기록 (no silent engine swap).
|
||||
|
||||
CLI:
|
||||
python3 -m deep_research.extract --backend codex|antigravity|auto \
|
||||
--question "<연구 질문>" --files f1.md f2.md ... [--out digest.md] [--max-quotes 8]
|
||||
exit: 0 (digest 생성, 부분 실패는 funnel 에 기록) / 1 (전 파일 실패) / 2 (인자 오류)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
MAX_FILE_CHARS = 60_000 # 파일당 프롬프트 상한 (초과분은 절단 + funnel 기록)
|
||||
CONCURRENCY = 3
|
||||
|
||||
|
||||
class Quote(BaseModel):
|
||||
line: int = Field(description="1-based line number in the file")
|
||||
quote: str = Field(description="byte-for-byte verbatim quote from that line (no paraphrase)")
|
||||
|
||||
|
||||
class FileExtraction(BaseModel):
|
||||
relevant: bool = Field(description="whether this file contains material relevant to the question")
|
||||
summary: str = Field(description="2-4 sentence summary of what this file says about the question")
|
||||
facts: list[str] = Field(default_factory=list,
|
||||
description="key facts relevant to the question, each one sentence")
|
||||
quotes: list[Quote] = Field(default_factory=list,
|
||||
description="supporting verbatim quotes with line numbers")
|
||||
|
||||
|
||||
PROMPT = """You are a read-only extraction worker. Below is ONE file with line numbers, \
|
||||
and a research question. Extract ONLY what the file actually says — no inference, no outside \
|
||||
knowledge, no paraphrase inside quotes.
|
||||
|
||||
Research question: {question}
|
||||
|
||||
Rules:
|
||||
- quotes must be byte-for-byte substrings of a single line (copy exactly, without the line-number prefix).
|
||||
- give the 1-based line number for each quote.
|
||||
- at most {max_quotes} quotes; prefer the most decision-relevant lines.
|
||||
- if the file is irrelevant to the question, set relevant=false with a one-line summary.
|
||||
- respond ONLY with the JSON object matching the schema.
|
||||
|
||||
FILE: {path}
|
||||
----------------------------------------
|
||||
{numbered}
|
||||
----------------------------------------"""
|
||||
|
||||
|
||||
def numbered_content(text: str, limit: int = MAX_FILE_CHARS) -> tuple[str, bool]:
|
||||
lines = text.splitlines()
|
||||
out, total, truncated = [], 0, False
|
||||
for i, line in enumerate(lines, start=1):
|
||||
s = f"{i}\t{line}"
|
||||
total += len(s) + 1
|
||||
if total > limit:
|
||||
truncated = True
|
||||
break
|
||||
out.append(s)
|
||||
return "\n".join(out), truncated
|
||||
|
||||
|
||||
def verify_quotes(path: Path, ext: FileExtraction) -> dict:
|
||||
"""결정론 quote-verifier: verbatim 재대조. 반환: 검증 통계 + 정정된 quotes.
|
||||
|
||||
판정: 인용이 명시 라인에 있으면 PASS / 다른 라인에 있으면 CORRECTED(라인 정정) /
|
||||
어디에도 없으면 DROPPED (위조·의역 — 폐기)."""
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except Exception:
|
||||
return {"pass": 0, "corrected": 0, "dropped": len(ext.quotes), "kept": []}
|
||||
n_pass = n_corr = n_drop = 0
|
||||
kept: list[Quote] = []
|
||||
for q in ext.quotes:
|
||||
quote = q.quote.strip()
|
||||
if not quote:
|
||||
n_drop += 1
|
||||
continue
|
||||
if 1 <= q.line <= len(lines) and quote in lines[q.line - 1]:
|
||||
n_pass += 1
|
||||
kept.append(q)
|
||||
continue
|
||||
hit = next((i for i, l in enumerate(lines, start=1) if quote in l), None)
|
||||
if hit is not None:
|
||||
n_corr += 1
|
||||
kept.append(Quote(line=hit, quote=quote))
|
||||
else:
|
||||
n_drop += 1
|
||||
return {"pass": n_pass, "corrected": n_corr, "dropped": n_drop, "kept": kept}
|
||||
|
||||
|
||||
def make_backend(name: str):
|
||||
if name == "codex":
|
||||
from deep_research.backends.codex import CodexBackend
|
||||
return CodexBackend()
|
||||
if name == "antigravity":
|
||||
from deep_research.backends.antigravity import AntigravityBackend
|
||||
return AntigravityBackend()
|
||||
raise ValueError(name)
|
||||
|
||||
|
||||
async def extract_file(backends: list[tuple[str, object]], question: str, path: Path,
|
||||
max_quotes: int, sem: asyncio.Semaphore) -> dict:
|
||||
"""파일 1개 발췌 — fallback 사다리 순서대로 시도. 반환 dict 는 digest 렌더 입력."""
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
return {"path": str(path), "engine": None, "error": f"read 실패: {e}"}
|
||||
numbered, truncated = numbered_content(text)
|
||||
prompt = PROMPT.format(question=question, max_quotes=max_quotes,
|
||||
path=path, numbered=numbered)
|
||||
async with sem:
|
||||
for engine_name, backend in backends:
|
||||
result = await backend.run_agent(prompt, FileExtraction, label=f"extract:{path.name}")
|
||||
if result is not None:
|
||||
v = verify_quotes(path, result)
|
||||
return {"path": str(path), "engine": engine_name, "truncated": truncated,
|
||||
"relevant": result.relevant, "summary": result.summary,
|
||||
"facts": result.facts, "verify": v}
|
||||
return {"path": str(path), "engine": None, "error": "전 엔진 실패 (가용성/스키마)"}
|
||||
|
||||
|
||||
def render_digest(question: str, results: list[dict]) -> str:
|
||||
ok = [r for r in results if "error" not in r]
|
||||
failed = [r for r in results if "error" in r]
|
||||
engines: dict[str, int] = {}
|
||||
for r in ok:
|
||||
engines[r["engine"]] = engines.get(r["engine"], 0) + 1
|
||||
out = [f"# Extraction Digest", f"**Question:** {question}",
|
||||
f"**Files:** {len(results)} (성공 {len(ok)} / 실패 {len(failed)})",
|
||||
f"**Engines:** " + (", ".join(f"{k}×{v}" for k, v in engines.items()) or "없음"), ""]
|
||||
for r in ok:
|
||||
v = r["verify"]
|
||||
out.append(f"## {r['path']}" + (" (TRUNCATED)" if r.get("truncated") else ""))
|
||||
if not r["relevant"]:
|
||||
out.append(f"- 무관: {r['summary']}")
|
||||
out.append("")
|
||||
continue
|
||||
out.append(f"- 요약: {r['summary']}")
|
||||
for f in r["facts"]:
|
||||
out.append(f"- {f}")
|
||||
for q in v["kept"]:
|
||||
out.append(f" > \"{q.quote}\" — {r['path']}:{q.line}")
|
||||
out.append(f"- 인용 검증: PASS {v['pass']} / 정정 {v['corrected']} / **폐기 {v['dropped']}**")
|
||||
if r["facts"] and not v["kept"]:
|
||||
# 검증 통과 인용이 0건이면 위 facts 는 근거 없는 주장 (계명 2 — 무검증 발췌 소비 금지)
|
||||
out.append("- ⚠ 검증 인용 0건 — 위 facts 는 미검증 주장 (haiku/sonnet 재발췌 후보)")
|
||||
out.append("")
|
||||
if failed:
|
||||
out.append("## 실패 (haiku/sonnet 재발췌 대상)")
|
||||
for r in failed:
|
||||
out.append(f"- {r['path']}: {r['error']}")
|
||||
out.append("")
|
||||
total_q = sum(r["verify"]["pass"] + r["verify"]["corrected"] + r["verify"]["dropped"] for r in ok)
|
||||
kept_q = sum(len(r["verify"]["kept"]) for r in ok)
|
||||
out.append("```wiki-stats")
|
||||
out.append("agent: extract-broker")
|
||||
out.append(f"found: {len(results)}")
|
||||
out.append(f"processed: {len(ok)}")
|
||||
out.append(f"dropped: {len(failed)}")
|
||||
if failed:
|
||||
out.append(f"dropped_reason: 엔진 실패 {len(failed)}건 (위 실패 목록 — 상위 티어 재발췌)")
|
||||
out.append("```")
|
||||
out.append(f"<!-- quotes: total {total_q}, verified-kept {kept_q} -->")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
async def run(args) -> int:
|
||||
order = {"codex": ["codex", "antigravity"],
|
||||
"antigravity": ["antigravity", "codex"],
|
||||
"auto": ["codex", "antigravity"]}[args.backend]
|
||||
backends = []
|
||||
for name in order:
|
||||
try:
|
||||
backends.append((name, make_backend(name)))
|
||||
except Exception:
|
||||
continue
|
||||
if not backends:
|
||||
print("error: 사용 가능한 backend 없음", file=sys.stderr)
|
||||
return 1
|
||||
sem = asyncio.Semaphore(CONCURRENCY)
|
||||
results = await asyncio.gather(*[
|
||||
extract_file(backends, args.question, Path(f), args.max_quotes, sem)
|
||||
for f in args.files])
|
||||
digest = render_digest(args.question, list(results))
|
||||
if args.out:
|
||||
Path(args.out).write_text(digest, encoding="utf-8")
|
||||
print(f"digest → {args.out} ({len(digest)} chars)")
|
||||
else:
|
||||
print(digest)
|
||||
return 0 if any("error" not in r for r in results) else 1
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="Tiered extraction 브로커 (외부 구독 CLI + 결정론 검증)")
|
||||
ap.add_argument("--backend", default="auto", choices=["codex", "antigravity", "auto"])
|
||||
ap.add_argument("--question", required=True)
|
||||
ap.add_argument("--files", nargs="+", required=True)
|
||||
ap.add_argument("--out", help="digest 출력 파일 (생략 시 stdout)")
|
||||
ap.add_argument("--max-quotes", type=int, default=8)
|
||||
args = ap.parse_args()
|
||||
sys.exit(asyncio.run(run(args)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,130 @@
|
||||
"""asyncio 오케스트레이션. 원본 pipeline(무배리어)/parallel(배리어) 의미 재현."""
|
||||
import asyncio
|
||||
from deep_research.config import Config
|
||||
from deep_research.core import Deduper, host_of, rank_claims, tally, build_synth_blocks, build_stats
|
||||
from deep_research.schemas import Scope, Search, Extract, Verdict, Report
|
||||
from deep_research import prompts
|
||||
|
||||
|
||||
async def run_research(question: str, backend, *, config: Config) -> dict:
|
||||
q = (question or "").strip()
|
||||
if not q:
|
||||
return {"error": "No research question provided."}
|
||||
|
||||
scope = await backend.run_agent(prompts.scope_prompt(q), Scope, label="scope")
|
||||
if scope is None:
|
||||
return {"error": "Scope agent returned no result — cannot decompose the question."}
|
||||
|
||||
deduper = Deduper(config)
|
||||
sem = asyncio.Semaphore(config.CONCURRENCY)
|
||||
dedup_lock = asyncio.Lock()
|
||||
|
||||
async def search_and_fetch(angle) -> list[dict]:
|
||||
async with sem:
|
||||
sr = await backend.run_agent(
|
||||
prompts.search_prompt(q, angle), Search, label="search:" + angle.label)
|
||||
if sr is None:
|
||||
return []
|
||||
async with dedup_lock: # 공유 상태(seen/fetch_slots) 임계구역 직렬화
|
||||
novel = deduper.filter_novel(angle.label, [r.model_dump() for r in sr.results])
|
||||
|
||||
async def fetch_one(source: dict):
|
||||
async with sem:
|
||||
ext = await backend.run_agent(
|
||||
prompts.fetch_prompt(q, source, angle.label),
|
||||
Extract, label="fetch:" + host_of(source["url"]))
|
||||
if ext is None:
|
||||
return None
|
||||
return {
|
||||
"url": source["url"], "title": source["title"], "angle": angle.label,
|
||||
"sourceQuality": ext.sourceQuality, "publishDate": ext.publishDate,
|
||||
"claims": [{**c.model_dump(), "sourceUrl": source["url"], "sourceQuality": ext.sourceQuality}
|
||||
for c in ext.claims],
|
||||
}
|
||||
|
||||
fetched = await asyncio.gather(*[fetch_one(s) for s in novel])
|
||||
return [f for f in fetched if f is not None]
|
||||
|
||||
per_angle = await asyncio.gather(*[search_and_fetch(a) for a in scope.angles])
|
||||
all_sources = [s for sub in per_angle for s in sub]
|
||||
all_claims = [c for s in all_sources for c in s["claims"]]
|
||||
ranked = rank_claims(all_claims, config.MAX_VERIFY_CLAIMS)
|
||||
|
||||
def _sources_out():
|
||||
return [{"url": s["url"], "quality": s["sourceQuality"], "angle": s["angle"],
|
||||
"claimCount": len(s["claims"])} for s in all_sources]
|
||||
|
||||
if not ranked:
|
||||
return {
|
||||
"question": q,
|
||||
"summary": f"No claims extracted. {len(all_sources)} sources fetched, all empty/failed.",
|
||||
"findings": [], "refuted": [], "sources": _sources_out(),
|
||||
"stats": {"angles": len(scope.angles), "sources": len(all_sources),
|
||||
"claims": 0, "dupes": len(deduper.dupes)},
|
||||
}
|
||||
|
||||
# ── Verify (배리어) ──
|
||||
async def verify_claim(claim: dict) -> dict:
|
||||
async def one_vote(v: int):
|
||||
async with sem:
|
||||
return await backend.run_agent(
|
||||
prompts.verify_prompt(q, claim, v, config.VOTES_PER_CLAIM, config.REFUTATIONS_REQUIRED),
|
||||
Verdict, label="v" + str(v) + ":" + claim["claim"][:40])
|
||||
verdicts = await asyncio.gather(*[one_vote(v) for v in range(config.VOTES_PER_CLAIM)])
|
||||
t = tally([vd.model_dump() if vd is not None else None for vd in verdicts],
|
||||
config.VOTES_PER_CLAIM, config.REFUTATIONS_REQUIRED)
|
||||
return {**claim, **t}
|
||||
|
||||
voted = await asyncio.gather(*[verify_claim(c) for c in ranked])
|
||||
confirmed = [c for c in voted if c["survives"]]
|
||||
killed = [c for c in voted if not c["survives"]]
|
||||
|
||||
def _refuted_out():
|
||||
return [{"claim": c["claim"],
|
||||
"vote": str(len(c["valid"]) - c["refutedVotes"]) + "-" + str(c["refutedVotes"]),
|
||||
"source": c["sourceUrl"]} for c in killed]
|
||||
|
||||
if not confirmed:
|
||||
return {
|
||||
"question": q,
|
||||
"summary": f"All {len(voted)} claims refuted by adversarial verification. Research inconclusive.",
|
||||
"findings": [], "refuted": _refuted_out(), "sources": _sources_out(),
|
||||
"stats": build_stats(angles=len(scope.angles), sources=len(all_sources),
|
||||
claims=len(all_claims), voted=len(voted), confirmed=0,
|
||||
killed=len(killed), after_synth=0, dupes=len(deduper.dupes),
|
||||
budget_dropped=len(deduper.budget_dropped),
|
||||
votes_per_claim=config.VOTES_PER_CLAIM),
|
||||
}
|
||||
|
||||
# ── Synthesize ──
|
||||
block, killed_block = build_synth_blocks(confirmed, killed)
|
||||
report = await backend.run_agent(
|
||||
prompts.synth_prompt(q, block, killed_block, len(confirmed), config.VOTES_PER_CLAIM),
|
||||
Report, label="synthesize")
|
||||
|
||||
if report is None:
|
||||
return {
|
||||
"question": q,
|
||||
"summary": f"Synthesis step was skipped or failed — returning {len(confirmed)} verified claims unmerged.",
|
||||
"findings": [],
|
||||
"confirmed": [{"claim": c["claim"], "source": c["sourceUrl"], "quote": c["quote"],
|
||||
"vote": str(len(c["valid"]) - c["refutedVotes"]) + "-" + str(c["refutedVotes"])}
|
||||
for c in confirmed],
|
||||
"refuted": _refuted_out(), "sources": _sources_out(),
|
||||
"stats": build_stats(angles=len(scope.angles), sources=len(all_sources),
|
||||
claims=len(all_claims), voted=len(voted), confirmed=len(confirmed),
|
||||
killed=len(killed), after_synth=0, dupes=len(deduper.dupes),
|
||||
budget_dropped=len(deduper.budget_dropped),
|
||||
votes_per_claim=config.VOTES_PER_CLAIM),
|
||||
}
|
||||
|
||||
return {
|
||||
"question": q,
|
||||
**report.model_dump(),
|
||||
"refuted": _refuted_out(), "sources": _sources_out(),
|
||||
"stats": build_stats(angles=len(scope.angles), sources=len(all_sources),
|
||||
claims=len(all_claims), voted=len(voted), confirmed=len(confirmed),
|
||||
killed=len(killed), after_synth=len(report.findings),
|
||||
dupes=len(deduper.dupes), budget_dropped=len(deduper.budget_dropped),
|
||||
votes_per_claim=config.VOTES_PER_CLAIM),
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
"""원본 deep-research-wf JS 의 프롬프트 문자열 이식. 표현 변경 금지(충실도)."""
|
||||
|
||||
|
||||
def scope_prompt(question: str) -> str:
|
||||
return (
|
||||
"Decompose this research question into complementary search angles.\n\n"
|
||||
"## Question\n" + question + "\n\n"
|
||||
"## Task\n"
|
||||
"Generate 5 distinct web search queries that together cover the question from "
|
||||
"different angles. Pick angles that suit the question's domain. Examples:\n"
|
||||
"- broad/primary · academic/technical · recent news · contrarian/skeptical · practitioner/implementation\n"
|
||||
"- For medical: anatomy · common causes · serious differentials · authoritative refs · red flags\n"
|
||||
"- For tech: state-of-art · benchmarks · limitations · industry adoption · cost/tradeoffs\n\n"
|
||||
"Make queries specific enough to surface high-signal results. Avoid redundancy.\n"
|
||||
"Return: the question (verbatim or lightly normalized), a 1-2 sentence decomposition "
|
||||
"strategy, and the angles.\n\nStructured output only."
|
||||
)
|
||||
|
||||
|
||||
def search_prompt(question: str, angle) -> str:
|
||||
return (
|
||||
"## Web Searcher: " + angle.label + "\n\n"
|
||||
'Research question: "' + question + '"\n\n'
|
||||
"Your angle: **" + angle.label + "** — " + (angle.rationale or "") + "\n"
|
||||
"Search query: `" + angle.query + "`\n\n"
|
||||
"## Task\nUse web search with the query above (or a refined version). Return the top "
|
||||
"4-6 most relevant results.\n"
|
||||
"Rank by relevance to the ORIGINAL question, not just the search query. Skip obvious "
|
||||
"SEO spam/content farms.\n"
|
||||
"Include a short snippet capturing why each result is relevant.\n\nStructured output only."
|
||||
)
|
||||
|
||||
|
||||
def fetch_prompt(question: str, source: dict, angle: str) -> str:
|
||||
return (
|
||||
"## Source Extractor\n\n"
|
||||
'Research question: "' + question + '"\n\n'
|
||||
"Fetch and extract key claims from this source:\n"
|
||||
"**URL:** " + source["url"] + "\n**Title:** " + source["title"] + "\n**Found via:** " + angle + " search\n\n"
|
||||
"## Task\n1. Fetch the page content.\n"
|
||||
"2. Assess source quality: primary research/institution? secondary reporting? blog/opinion? forum? unreliable?\n"
|
||||
"3. Extract 2-5 FALSIFIABLE claims that bear on the research question. Each claim must:\n"
|
||||
" - be a concrete, checkable statement (not vague generalities)\n"
|
||||
" - include a direct quote from the source as support\n"
|
||||
" - be rated central/supporting/tangential to the research question\n"
|
||||
"4. Note publish date if available.\n\n"
|
||||
'If the fetch fails or the page is irrelevant/paywalled, return claims: [] and '
|
||||
'sourceQuality: "unreliable".\n\nStructured output only.'
|
||||
)
|
||||
|
||||
|
||||
def verify_prompt(question: str, claim: dict, v: int, votes_per_claim: int, refutations_required: int) -> str:
|
||||
return (
|
||||
"## Adversarial Claim Verifier (voter " + str(v + 1) + "/" + str(votes_per_claim) + ")\n\n"
|
||||
"Be SKEPTICAL. Try to REFUTE this claim. ≥" + str(refutations_required) + "/" + str(votes_per_claim) + " refutations kill it.\n\n"
|
||||
"## Research question\n" + question + "\n\n"
|
||||
'## Claim under review\n"' + claim["claim"] + '"\n\n'
|
||||
"**Source:** " + claim["sourceUrl"] + " (" + claim["sourceQuality"] + ")\n"
|
||||
'**Supporting quote:** "' + claim["quote"] + '"\n\n'
|
||||
"## Checklist\n"
|
||||
"1. Is the claim actually supported by the quote, or is it an overreach/misread?\n"
|
||||
"2. Search for contradicting evidence — does any credible source dispute or heavily qualify this?\n"
|
||||
"3. Is the source quality sufficient for the claim's strength? (extraordinary claims need primary sources)\n"
|
||||
"4. Is the claim outdated? (check dates — old claims about fast-moving fields are suspect)\n"
|
||||
"5. Is this a marketing claim / press release / cherry-picked benchmark / forum speculation?\n\n"
|
||||
"**refuted=true** if: unsupported by quote / contradicted / low-quality source for strong claim / outdated / marketing fluff.\n"
|
||||
"**refuted=false** ONLY if: claim is well-supported, current, and source quality matches claim strength.\n"
|
||||
"Default to refuted=true if uncertain.\n\nStructured output only. Evidence MUST be specific."
|
||||
)
|
||||
|
||||
|
||||
def synth_prompt(question: str, block: str, killed_block: str, confirmed_count: int, votes_per_claim: int) -> str:
|
||||
return (
|
||||
"## Synthesis: research report\n\n"
|
||||
"**Question:** " + question + "\n\n"
|
||||
+ str(confirmed_count) + " claims survived " + str(votes_per_claim) + "-vote adversarial verification. "
|
||||
"Merge semantic duplicates and synthesize.\n\n"
|
||||
"## Confirmed claims\n" + block + "\n" + killed_block + "\n\n"
|
||||
"## Instructions\n"
|
||||
"1. Identify claims that say the same thing — merge them, combine their sources.\n"
|
||||
"2. Group related claims into coherent findings. Each finding should directly address the research question.\n"
|
||||
"3. Assign confidence per finding: high (multiple primary sources, unanimous votes), medium (secondary sources or split votes), low (single source or blog-quality).\n"
|
||||
"4. Write a 3-5 sentence executive summary answering the research question.\n"
|
||||
"5. Note caveats: what's uncertain, what sources were weak, what time-sensitivity applies.\n"
|
||||
"6. List 2-4 open questions that emerged but weren't answered.\n\nStructured output only."
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
import json
|
||||
|
||||
|
||||
def to_markdown(result: dict) -> str:
|
||||
if "error" in result:
|
||||
return "# Deep Research\n\n**Error:** " + result["error"]
|
||||
|
||||
lines = ["# Deep Research", "", "**Question:** " + result.get("question", ""), ""]
|
||||
lines += ["## Summary", result.get("summary", ""), ""]
|
||||
|
||||
findings = result.get("findings", [])
|
||||
if findings:
|
||||
lines.append("## Findings")
|
||||
for i, f in enumerate(findings, 1):
|
||||
srcs = ", ".join(f.get("sources", []))
|
||||
lines += [
|
||||
f"### {i}. {f['claim']} _({f['confidence']})_",
|
||||
f"{f.get('evidence', '')}",
|
||||
f"Sources: {srcs}",
|
||||
"",
|
||||
]
|
||||
|
||||
caveats = result.get("caveats")
|
||||
if caveats:
|
||||
lines += ["## Caveats", caveats, ""]
|
||||
|
||||
oq = result.get("openQuestions")
|
||||
if oq:
|
||||
lines += ["## Open Questions"] + [f"- {q}" for q in oq] + [""]
|
||||
|
||||
refuted = result.get("refuted", [])
|
||||
if refuted:
|
||||
lines.append("## Refuted (transparency)")
|
||||
for r in refuted:
|
||||
lines.append(f"- \"{r['claim']}\" (vote {r.get('vote', '')}, {r.get('source', '')})")
|
||||
lines.append("")
|
||||
|
||||
lines += ["## Stats", "```json", json.dumps(result.get("stats", {}), ensure_ascii=False, indent=2), "```"]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def to_json(result: dict) -> str:
|
||||
return json.dumps(result, ensure_ascii=False, indent=2)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""원본 5 SCHEMA 의 Pydantic v2 포팅. LLM-facing 필드는 원본 JSON 키(camelCase) 유지."""
|
||||
from typing import Literal, Optional
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class Angle(BaseModel):
|
||||
label: str
|
||||
query: str
|
||||
rationale: Optional[str] = None
|
||||
|
||||
|
||||
class Scope(BaseModel):
|
||||
question: str
|
||||
summary: str
|
||||
angles: list[Angle] = Field(min_length=3, max_length=6)
|
||||
|
||||
|
||||
class SearchResult(BaseModel):
|
||||
url: str
|
||||
title: str
|
||||
snippet: Optional[str] = None
|
||||
relevance: Literal["high", "medium", "low"]
|
||||
|
||||
|
||||
class Search(BaseModel):
|
||||
results: list[SearchResult] = Field(max_length=6)
|
||||
|
||||
|
||||
class Claim(BaseModel):
|
||||
claim: str
|
||||
quote: str
|
||||
importance: Literal["central", "supporting", "tangential"]
|
||||
|
||||
|
||||
class Extract(BaseModel):
|
||||
sourceQuality: Literal["primary", "secondary", "blog", "forum", "unreliable"]
|
||||
publishDate: Optional[str] = None
|
||||
claims: list[Claim] = Field(max_length=5)
|
||||
|
||||
|
||||
class Verdict(BaseModel):
|
||||
refuted: bool
|
||||
evidence: str
|
||||
confidence: Literal["high", "medium", "low"]
|
||||
counterSource: Optional[str] = None
|
||||
|
||||
|
||||
class Finding(BaseModel):
|
||||
claim: str
|
||||
confidence: Literal["high", "medium", "low"]
|
||||
sources: list[str]
|
||||
evidence: str
|
||||
vote: Optional[str] = None
|
||||
|
||||
|
||||
class Report(BaseModel):
|
||||
summary: str
|
||||
findings: list[Finding]
|
||||
caveats: str
|
||||
openQuestions: Optional[list[str]] = None
|
||||
@@ -0,0 +1,125 @@
|
||||
"""vote.py — cross-vendor 적대 검증 표 생성기 (wiki_quorum.py 워커).
|
||||
|
||||
N=3 quorum 의 표 2/3 을 외부 구독 CLI(codex/agy)로 — Claude 토큰 절감이면서
|
||||
독립 실패 모드 덕에 falsification 은 강화된다 (rules/extraction-tiering.md).
|
||||
출력은 ```wiki-verdict``` 블록 파일 — `wiki_quorum.py` 가 엔진 불가지로 그대로 집계.
|
||||
|
||||
CLI:
|
||||
python3 -m deep_research.vote --backend codex|antigravity \
|
||||
--findings findings.md [--context-files f1 f2 ...] --out /tmp/vote-codex.md
|
||||
exit: 0 (표 파일 생성) / 1 (엔진 실패).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from deep_research.extract import make_backend, numbered_content
|
||||
|
||||
MAX_CONTEXT_CHARS = 40_000
|
||||
|
||||
|
||||
class FindingVote(BaseModel):
|
||||
id: str = Field(description="the finding's ID exactly as given (no whitespace)")
|
||||
action: Literal["KEEP", "DOWNGRADE", "REJECT"]
|
||||
reason: str = Field(description="one-sentence justification")
|
||||
|
||||
|
||||
class VoteResult(BaseModel):
|
||||
votes: list[FindingVote]
|
||||
|
||||
|
||||
PROMPT = """You are an independent adversarial reviewer. Your KPI is finding weaknesses, \
|
||||
not approving. For EACH finding below, try to FALSIFY it via three checks: \
|
||||
(1) PRACTICALITY — is it actionable/real in this context? \
|
||||
(2) OVERCLAIM — does the evidence actually support the stated severity? \
|
||||
(3) ASSUMPTION — does it rest on an unstated assumption that may not hold?
|
||||
|
||||
Default-refute: if any check leaves you uncertain, do NOT answer KEEP — answer at least \
|
||||
DOWNGRADE. KEEP only when all three checks actively pass. REJECT when the finding is wrong \
|
||||
or unsupported. Vote on EVERY finding (same IDs, no whitespace in IDs). \
|
||||
Respond ONLY with the JSON object matching the schema.
|
||||
|
||||
FINDINGS:
|
||||
----------------------------------------
|
||||
{findings}
|
||||
----------------------------------------
|
||||
{context}"""
|
||||
|
||||
|
||||
def build_context(files: list[str]) -> str:
|
||||
if not files:
|
||||
return ""
|
||||
parts = ["REFERENCE FILES (read-only evidence):"]
|
||||
budget = MAX_CONTEXT_CHARS
|
||||
for f in files:
|
||||
p = Path(f)
|
||||
try:
|
||||
text = p.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
continue
|
||||
numbered, _ = numbered_content(text, limit=min(budget, 20_000))
|
||||
parts.append(f"--- {f} ---\n{numbered}")
|
||||
budget -= len(numbered)
|
||||
if budget <= 0:
|
||||
parts.append("--- (context budget exhausted — remaining files omitted) ---")
|
||||
break
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def render_vote_block(backend_name: str, result: VoteResult, truncated_chars: int = 0) -> str:
|
||||
lines = [f"# Adversarial vote ({backend_name})", ""]
|
||||
if truncated_chars:
|
||||
# no-silent-caps (rules/extraction-tiering.md 계명 3): 절단 사실을 표 자체에 기록.
|
||||
lines += [f"> ⚠ findings input {truncated_chars} chars 중 {MAX_CONTEXT_CHARS} 까지만 표결 — "
|
||||
f"잘린 finding 은 이 표에서 누락 (Claude 표로 보완 필요)", ""]
|
||||
lines += ["```wiki-verdict", f"agent: {backend_name}-adversarial-vote"]
|
||||
for v in result.votes:
|
||||
fid = re.sub(r"\s+", "-", v.id.strip())
|
||||
lines.append(f"finding: {fid} action: {v.action}")
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
for v in result.votes:
|
||||
lines.append(f"- {v.id} → {v.action}: {v.reason}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def run(args) -> int:
|
||||
findings = Path(args.findings).read_text(encoding="utf-8", errors="replace")
|
||||
truncated_chars = len(findings) if len(findings) > MAX_CONTEXT_CHARS else 0
|
||||
if truncated_chars:
|
||||
print(f"warning: findings {truncated_chars} chars > {MAX_CONTEXT_CHARS} — 절단됨. "
|
||||
f"잘린 finding 은 이 표에서 누락되므로 분할 표결 또는 Claude 표 보완 필요 (no-silent-caps)",
|
||||
file=sys.stderr)
|
||||
prompt = PROMPT.format(findings=findings[:MAX_CONTEXT_CHARS],
|
||||
context=build_context(args.context_files or []))
|
||||
backend = make_backend(args.backend)
|
||||
result = await backend.run_agent(prompt, VoteResult, label=f"vote:{args.backend}")
|
||||
if result is None or not result.votes:
|
||||
print(f"error: {args.backend} 표 생성 실패 — Claude 표로 대체하세요 (fallback 사다리)",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
block = render_vote_block(args.backend, result, truncated_chars)
|
||||
Path(args.out).write_text(block, encoding="utf-8")
|
||||
print(f"vote → {args.out} ({len(result.votes)} findings)")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="cross-vendor 적대 검증 표 생성")
|
||||
ap.add_argument("--backend", required=True, choices=["codex", "antigravity"])
|
||||
ap.add_argument("--findings", required=True, help="findings 목록 파일 (ID 포함)")
|
||||
ap.add_argument("--context-files", nargs="*", default=[])
|
||||
ap.add_argument("--out", required=True)
|
||||
args = ap.parse_args()
|
||||
sys.exit(asyncio.run(run(args)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,20 @@
|
||||
[project]
|
||||
name = "deep-research-driver"
|
||||
version = "0.1.0"
|
||||
description = "Port of Claude Code deep-research harness to Codex/Antigravity CLIs"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = ["pydantic>=2.6"]
|
||||
|
||||
[project.scripts]
|
||||
deep-research = "deep_research.__main__:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = ["pytest>=8", "pytest-asyncio>=0.23"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
testpaths = ["tests"]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=68"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
@@ -0,0 +1,153 @@
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from deep_research.schemas import Scope, Angle, Search, SearchResult, Extract, Claim, Verdict, Report
|
||||
from deep_research.core import norm_url, Deduper, rank_claims, tally, build_synth_blocks, build_stats
|
||||
from deep_research.config import Config
|
||||
|
||||
|
||||
def test_scope_requires_min_3_angles():
|
||||
with pytest.raises(ValidationError):
|
||||
Scope(question="q", summary="s", angles=[Angle(label="a", query="x")])
|
||||
|
||||
|
||||
def test_scope_accepts_5_angles():
|
||||
angles = [Angle(label=f"a{i}", query="x") for i in range(5)]
|
||||
s = Scope(question="q", summary="s", angles=angles)
|
||||
assert len(s.angles) == 5
|
||||
|
||||
|
||||
def test_search_relevance_enum_enforced():
|
||||
with pytest.raises(ValidationError):
|
||||
SearchResult(url="http://a", title="t", relevance="bogus")
|
||||
|
||||
|
||||
def test_json_schema_export_works():
|
||||
schema = Scope.model_json_schema()
|
||||
assert "angles" in schema["properties"]
|
||||
assert schema["properties"]["angles"]["minItems"] == 3
|
||||
assert schema["properties"]["angles"]["maxItems"] == 6
|
||||
|
||||
|
||||
def test_norm_url_strips_www_and_trailing_slash():
|
||||
assert norm_url("http://www.Example.com/Path/") == "example.com/path"
|
||||
|
||||
|
||||
def test_norm_url_bare_host():
|
||||
assert norm_url("https://example.com") == "example.com"
|
||||
|
||||
|
||||
def test_norm_url_fallback_on_garbage():
|
||||
assert norm_url("not a url") == "not a url"
|
||||
|
||||
|
||||
def test_deduper_filters_exact_dupes():
|
||||
d = Deduper(Config(MAX_FETCH=15))
|
||||
r = [{"url": "http://a.com/x", "title": "t", "relevance": "high"}]
|
||||
assert len(d.filter_novel("angle1", r)) == 1
|
||||
# same normalized url from another angle -> dup
|
||||
r2 = [{"url": "http://www.a.com/x/", "title": "t2", "relevance": "high"}]
|
||||
assert d.filter_novel("angle2", r2) == []
|
||||
assert len(d.dupes) == 1
|
||||
|
||||
|
||||
def test_deduper_budget_drops_medium_low_when_slots_exhausted():
|
||||
d = Deduper(Config(MAX_FETCH=1))
|
||||
first = [{"url": "http://a.com/1", "title": "t", "relevance": "high"}]
|
||||
d.filter_novel("a1", first) # consumes the only slot
|
||||
more = [
|
||||
{"url": "http://b.com/2", "title": "t", "relevance": "medium"},
|
||||
{"url": "http://c.com/3", "title": "t", "relevance": "low"},
|
||||
]
|
||||
assert d.filter_novel("a2", more) == []
|
||||
assert len(d.budget_dropped) == 2
|
||||
|
||||
|
||||
def test_deduper_high_passes_even_when_slots_exhausted():
|
||||
# 원본: high(rank 0)는 budget 조건(rank>=1)에 안 걸려 slot<=0 이어도 통과
|
||||
d = Deduper(Config(MAX_FETCH=1))
|
||||
d.filter_novel("a1", [{"url": "http://a.com/1", "title": "t", "relevance": "high"}])
|
||||
high = [{"url": "http://d.com/4", "title": "t", "relevance": "high"}]
|
||||
assert len(d.filter_novel("a2", high)) == 1
|
||||
assert d.fetch_slots == -1
|
||||
|
||||
|
||||
def _claim(imp, qual, name="c"):
|
||||
return {"claim": name, "importance": imp, "sourceQuality": qual}
|
||||
|
||||
|
||||
def test_rank_claims_orders_by_importance_then_quality():
|
||||
claims = [
|
||||
_claim("tangential", "primary", "t-prim"),
|
||||
_claim("central", "blog", "c-blog"),
|
||||
_claim("central", "primary", "c-prim"),
|
||||
]
|
||||
ranked = rank_claims(claims, max_verify=25)
|
||||
assert [c["claim"] for c in ranked] == ["c-prim", "c-blog", "t-prim"]
|
||||
|
||||
|
||||
def test_rank_claims_truncates_to_max():
|
||||
claims = [_claim("central", "primary", f"c{i}") for i in range(30)]
|
||||
assert len(rank_claims(claims, max_verify=25)) == 25
|
||||
|
||||
|
||||
def test_tally_survives_2_valid_0_refute():
|
||||
v = [{"refuted": False}, {"refuted": False}, {"refuted": False}]
|
||||
t = tally(v, votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is True and t["refutedVotes"] == 0
|
||||
|
||||
|
||||
def test_tally_killed_2_refute():
|
||||
v = [{"refuted": True}, {"refuted": True}, {"refuted": False}]
|
||||
t = tally(v, votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is False and t["refutedVotes"] == 2
|
||||
|
||||
|
||||
def test_tally_all_abstain_does_not_survive():
|
||||
# ⚠️ 거짓 생존 차단: all-None -> refuted=0 이지만 valid<2 라 미생존
|
||||
t = tally([None, None, None], votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is False and t["abstained"] == 3
|
||||
|
||||
|
||||
def test_tally_one_valid_two_abstain_does_not_survive():
|
||||
t = tally([{"refuted": False}, None, None], votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is False # valid(1) < 2
|
||||
|
||||
|
||||
def test_tally_boundary_one_refute_two_valid_survives():
|
||||
t = tally([{"refuted": True}, {"refuted": False}], votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is True # 1 refute < 2 required, quorum met
|
||||
|
||||
|
||||
def test_tally_boundary_two_refute_two_valid_killed():
|
||||
t = tally([{"refuted": True}, {"refuted": True}], votes_per_claim=3, refutations_required=2)
|
||||
assert t["survives"] is False # 2 refute == 2 required, strict < fails
|
||||
|
||||
|
||||
def test_build_synth_blocks_includes_vote_and_source():
|
||||
confirmed = [{
|
||||
"claim": "X causes Y", "quote": "q", "sourceUrl": "http://a", "sourceQuality": "primary",
|
||||
"valid": [{"refuted": False, "confidence": "high", "evidence": "e"}],
|
||||
"refutedVotes": 0,
|
||||
}]
|
||||
killed = [{
|
||||
"claim": "Z", "sourceUrl": "http://b", "valid": [{"refuted": True}], "refutedVotes": 1,
|
||||
}]
|
||||
block, killed_block = build_synth_blocks(confirmed, killed)
|
||||
assert "X causes Y" in block and "1-0" in block
|
||||
assert "Refuted claims" in killed_block and "Z" in killed_block
|
||||
|
||||
|
||||
def test_build_synth_blocks_no_killed():
|
||||
block, killed_block = build_synth_blocks([{
|
||||
"claim": "X", "quote": "q", "sourceUrl": "http://a", "sourceQuality": "primary",
|
||||
"valid": [{"refuted": False, "confidence": "high", "evidence": "e"}], "refutedVotes": 0,
|
||||
}], [])
|
||||
assert killed_block == ""
|
||||
|
||||
|
||||
def test_build_stats_agent_calls_formula():
|
||||
s = build_stats(angles=5, sources=12, claims=20, voted=18, confirmed=10, killed=8,
|
||||
after_synth=6, dupes=3, budget_dropped=2, votes_per_claim=3)
|
||||
# 1 + angles + sources + voted*votes_per_claim + 1
|
||||
assert s["agentCalls"] == 1 + 5 + 12 + 18 * 3 + 1
|
||||
assert s["confirmed"] == 10 and s["afterSynthesis"] == 6
|
||||
@@ -0,0 +1,131 @@
|
||||
"""extract.py(quote-verifier·funnel·fallback) + vote.py(블록 렌더) 테스트 — MockBackend."""
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deep_research.backends.mock import MockBackend
|
||||
from deep_research.extract import (FileExtraction, Quote, extract_file,
|
||||
numbered_content, render_digest, verify_quotes)
|
||||
from deep_research.vote import FindingVote, VoteResult, render_vote_block
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample(tmp_path: Path) -> Path:
|
||||
p = tmp_path / "note.md"
|
||||
p.write_text("alpha line\nbeta line with policy X\ngamma line\n", encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
# ---------- quote-verifier (결정론 trust boundary) ----------
|
||||
|
||||
def test_verify_pass_corrected_dropped(sample):
|
||||
ext = FileExtraction(relevant=True, summary="s", facts=[], quotes=[
|
||||
Quote(line=2, quote="beta line with policy X"), # PASS
|
||||
Quote(line=1, quote="gamma line"), # 다른 라인 → CORRECTED(3)
|
||||
Quote(line=2, quote="완전 위조된 인용"), # DROPPED
|
||||
])
|
||||
v = verify_quotes(sample, ext)
|
||||
assert (v["pass"], v["corrected"], v["dropped"]) == (1, 1, 1)
|
||||
assert any(q.line == 3 and q.quote == "gamma line" for q in v["kept"])
|
||||
assert all("위조" not in q.quote for q in v["kept"]) # 위조 인용은 digest 에 못 들어감
|
||||
|
||||
|
||||
def test_numbered_content_truncation():
|
||||
text = "\n".join(f"line-{i}" for i in range(1000))
|
||||
numbered, truncated = numbered_content(text, limit=200)
|
||||
assert truncated and numbered.startswith("1\tline-0")
|
||||
|
||||
|
||||
# ---------- extract_file: fallback 사다리 + engine funnel ----------
|
||||
|
||||
def _ok(relevant=True):
|
||||
return FileExtraction(relevant=relevant, summary="요약", facts=["사실1"],
|
||||
quotes=[Quote(line=2, quote="beta line with policy X")])
|
||||
|
||||
|
||||
def test_first_backend_wins(sample):
|
||||
b1, b2 = MockBackend({"extract": _ok()}), MockBackend({"extract": _ok()})
|
||||
r = asyncio.run(extract_file([("codex", b1), ("antigravity", b2)], "q", sample, 8,
|
||||
asyncio.Semaphore(1)))
|
||||
assert r["engine"] == "codex" and b2.calls == []
|
||||
assert r["verify"]["pass"] == 1
|
||||
|
||||
|
||||
def test_fallback_ladder(sample):
|
||||
b1 = MockBackend({}, default=None) # codex 실패
|
||||
b2 = MockBackend({"extract": _ok()}) # agy 성공
|
||||
r = asyncio.run(extract_file([("codex", b1), ("antigravity", b2)], "q", sample, 8,
|
||||
asyncio.Semaphore(1)))
|
||||
assert r["engine"] == "antigravity" # silent engine swap 아님 — engine 기록
|
||||
|
||||
|
||||
def test_all_fail_recorded(sample):
|
||||
b = MockBackend({}, default=None)
|
||||
r = asyncio.run(extract_file([("codex", b)], "q", sample, 8, asyncio.Semaphore(1)))
|
||||
assert "error" in r
|
||||
|
||||
|
||||
# ---------- digest 렌더: funnel 균형 ----------
|
||||
|
||||
def test_digest_funnel(sample):
|
||||
ok = {"path": str(sample), "engine": "codex", "truncated": False, "relevant": True,
|
||||
"summary": "s", "facts": ["f"],
|
||||
"verify": {"pass": 1, "corrected": 0, "dropped": 1,
|
||||
"kept": [Quote(line=2, quote="beta line with policy X")]}}
|
||||
bad = {"path": "x.md", "engine": None, "error": "전 엔진 실패"}
|
||||
d = render_digest("q", [ok, bad])
|
||||
assert "found: 2" in d and "processed: 1" in d and "dropped: 1" in d
|
||||
assert "dropped_reason" in d
|
||||
assert f"{sample}:2" in d # file:line 포인터
|
||||
assert "폐기 1" in d # 위조 인용 수 가시화
|
||||
|
||||
|
||||
def test_digest_zero_quote_marker(sample):
|
||||
"""인용 전멸(전부 DROPPED) 파일의 facts 는 미검증 주장 — 마커 필수 (계명 2)."""
|
||||
ok = {"path": str(sample), "engine": "codex", "truncated": False, "relevant": True,
|
||||
"summary": "s", "facts": ["근거 없는 주장"],
|
||||
"verify": {"pass": 0, "corrected": 0, "dropped": 2, "kept": []}}
|
||||
d = render_digest("q", [ok])
|
||||
assert "검증 인용 0건" in d and "미검증 주장" in d
|
||||
# 인용이 1건이라도 살아 있으면 마커 없음
|
||||
ok["verify"] = {"pass": 1, "corrected": 0, "dropped": 1,
|
||||
"kept": [Quote(line=2, quote="beta line with policy X")]}
|
||||
assert "미검증 주장" not in render_digest("q", [ok])
|
||||
|
||||
|
||||
# ---------- vote: wiki-verdict 블록 렌더 (wiki_quorum 호환) ----------
|
||||
|
||||
def _parse_with_wiki_rules(block: str):
|
||||
"""wiki_quorum.py 의 실제 파서로 파싱 (엔진 불가지 계약 검증용)."""
|
||||
import importlib.util
|
||||
import sys
|
||||
hooks = Path(__file__).resolve().parents[3] / ".claude" / "hooks"
|
||||
sys.path.insert(0, str(hooks))
|
||||
spec = importlib.util.spec_from_file_location("wr", hooks / "wiki_rules.py")
|
||||
wr = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(wr)
|
||||
return wr.parse_verdict_block(block)
|
||||
|
||||
|
||||
def test_vote_block_quorum_compatible():
|
||||
r = VoteResult(votes=[
|
||||
FindingVote(id="4.1.1", action="KEEP", reason="ok"),
|
||||
FindingVote(id="finding with space", action="REJECT", reason="bad"),
|
||||
])
|
||||
block = render_vote_block("codex", r)
|
||||
assert "agent: codex-adversarial-vote" in block
|
||||
assert "finding: 4.1.1 action: KEEP" in block
|
||||
assert "finding: finding-with-space action: REJECT" in block # 공백 정규화
|
||||
parsed = _parse_with_wiki_rules(block)
|
||||
assert parsed and len(parsed["findings"]) == 2
|
||||
|
||||
|
||||
def test_vote_block_truncation_note():
|
||||
"""findings 절단 시 표 자체에 누락 사실 기록 (no-silent-caps) + quorum 파서 비파괴."""
|
||||
r = VoteResult(votes=[FindingVote(id="1", action="KEEP", reason="ok")])
|
||||
block = render_vote_block("codex", r, truncated_chars=99_999)
|
||||
assert "잘린 finding 은 이 표에서 누락" in block
|
||||
parsed = _parse_with_wiki_rules(block) # 노트는 fence 밖 — 파싱 영향 없음
|
||||
assert parsed and len(parsed["findings"]) == 1
|
||||
assert "누락" not in render_vote_block("codex", r) # 미절단 시 노트 없음
|
||||
@@ -0,0 +1,136 @@
|
||||
import pytest
|
||||
from deep_research.pipeline import run_research
|
||||
from deep_research.backends.mock import MockBackend
|
||||
from deep_research.config import Config
|
||||
from deep_research.schemas import Scope, Angle, Search, SearchResult, Extract, Claim, Verdict, Report
|
||||
|
||||
CFG = Config(MAX_FETCH=15, MAX_VERIFY_CLAIMS=25)
|
||||
|
||||
|
||||
def _scope():
|
||||
return Scope(question="Q", summary="s",
|
||||
angles=[Angle(label=f"a{i}", query=f"q{i}") for i in range(3)])
|
||||
|
||||
|
||||
def _search(prompt, label):
|
||||
# 각 각도마다 고유 URL 1개
|
||||
n = label.split(":")[1]
|
||||
return Search(results=[SearchResult(url=f"http://{n}.com/x", title="t", relevance="high")])
|
||||
|
||||
|
||||
def _extract(prompt, label):
|
||||
return Extract(sourceQuality="primary",
|
||||
claims=[Claim(claim="C-" + label, quote="q", importance="central")])
|
||||
|
||||
|
||||
def _verdict_pass(prompt, label):
|
||||
return Verdict(refuted=False, evidence="e", confidence="high")
|
||||
|
||||
|
||||
def _report():
|
||||
return Report(summary="done", findings=[], caveats="none")
|
||||
|
||||
|
||||
async def test_happy_path_returns_report_and_stats():
|
||||
backend = MockBackend({
|
||||
"scope": _scope(),
|
||||
"search:": _search,
|
||||
"fetch:": _extract,
|
||||
"v": _verdict_pass,
|
||||
"synthesize": _report(),
|
||||
})
|
||||
out = await run_research("Q", backend, config=CFG)
|
||||
assert out["summary"] == "done"
|
||||
assert out["stats"]["confirmed"] == 3 # 3 각도 × 1 claim, 전부 생존
|
||||
assert out["stats"]["angles"] == 3
|
||||
|
||||
|
||||
async def test_empty_question_returns_error():
|
||||
out = await run_research(" ", MockBackend({}), config=CFG)
|
||||
assert "error" in out
|
||||
|
||||
|
||||
async def test_no_claims_degenerate():
|
||||
backend = MockBackend({
|
||||
"scope": _scope(),
|
||||
"search:": _search,
|
||||
"fetch:": lambda p, l: Extract(sourceQuality="unreliable", claims=[]),
|
||||
})
|
||||
out = await run_research("Q", backend, config=CFG)
|
||||
assert out["findings"] == [] and out["stats"]["claims"] == 0
|
||||
|
||||
|
||||
async def test_all_refuted_degenerate():
|
||||
backend = MockBackend({
|
||||
"scope": _scope(),
|
||||
"search:": _search,
|
||||
"fetch:": _extract,
|
||||
"v": lambda p, l: Verdict(refuted=True, evidence="e", confidence="high"),
|
||||
})
|
||||
out = await run_research("Q", backend, config=CFG)
|
||||
assert out["findings"] == [] and out["stats"]["confirmed"] == 0
|
||||
assert len(out["refuted"]) == 3
|
||||
|
||||
|
||||
async def test_synth_failure_salvages_confirmed():
|
||||
backend = MockBackend({
|
||||
"scope": _scope(),
|
||||
"search:": _search,
|
||||
"fetch:": _extract,
|
||||
"v": _verdict_pass,
|
||||
"synthesize": None, # 합성 실패
|
||||
})
|
||||
out = await run_research("Q", backend, config=CFG)
|
||||
assert out["findings"] == [] and len(out["confirmed"]) == 3
|
||||
|
||||
|
||||
# ── Task 9: report.py ──────────────────────────────────────────────────────
|
||||
from deep_research.report import to_markdown
|
||||
|
||||
|
||||
def test_to_markdown_renders_summary_and_stats():
|
||||
result = {
|
||||
"question": "Q", "summary": "ans",
|
||||
"findings": [{"claim": "F1", "confidence": "high", "sources": ["http://a"], "evidence": "e"}],
|
||||
"caveats": "c", "refuted": [], "sources": [],
|
||||
"stats": {"angles": 5, "confirmed": 1},
|
||||
}
|
||||
md = to_markdown(result)
|
||||
assert "# Deep Research" in md and "ans" in md and "F1" in md
|
||||
|
||||
|
||||
# ── Task 10: backends/codex.py ────────────────────────────────────────────
|
||||
from deep_research.backends.codex import extract_final_json
|
||||
|
||||
|
||||
def test_extract_final_json_from_jsonl():
|
||||
jsonl = "\n".join([
|
||||
'{"type":"reasoning","text":"thinking"}',
|
||||
'{"type":"web_search","query":"x"}',
|
||||
'{"type":"agent_message","text":"{\\"question\\":\\"Q\\",\\"summary\\":\\"s\\",\\"angles\\":[]}"}',
|
||||
])
|
||||
obj = extract_final_json(jsonl)
|
||||
assert obj["summary"] == "s"
|
||||
|
||||
|
||||
def test_extract_final_json_returns_none_when_absent():
|
||||
assert extract_final_json('{"type":"reasoning","text":"only"}') is None
|
||||
|
||||
|
||||
# ── Task 11: backends/antigravity.py ─────────────────────────────────────
|
||||
from deep_research.backends.antigravity import extract_json_block
|
||||
|
||||
|
||||
def test_extract_json_block_from_fenced():
|
||||
text = 'prelude\n```json\n{"refuted": false, "evidence": "e", "confidence": "high"}\n```\ntrailing'
|
||||
obj = extract_json_block(text)
|
||||
assert obj["confidence"] == "high"
|
||||
|
||||
|
||||
def test_extract_json_block_bare_object():
|
||||
obj = extract_json_block('noise {"refuted": true, "evidence": "e", "confidence": "low"} more')
|
||||
assert obj["refuted"] is True
|
||||
|
||||
|
||||
def test_extract_json_block_none_when_absent():
|
||||
assert extract_json_block("no json here") is None
|
||||
Reference in New Issue
Block a user