123 lines
4.2 KiB
Python
123 lines
4.2 KiB
Python
"""결정론 제어 로직 — 원본 JS 와 1:1. 네트워크·LLM 의존 0."""
|
|
from urllib.parse import urlparse
|
|
from deep_research.config import Config
|
|
|
|
REL_RANK = {"high": 0, "medium": 1, "low": 2}
|
|
IMP_RANK = {"central": 0, "supporting": 1, "tangential": 2}
|
|
QUAL_RANK = {"primary": 0, "secondary": 1, "blog": 2, "forum": 3, "unreliable": 4}
|
|
|
|
|
|
def norm_url(u: str) -> str:
|
|
try:
|
|
p = urlparse(u)
|
|
host = (p.hostname or "")
|
|
if not host:
|
|
return u.lower()
|
|
if host.startswith("www."):
|
|
host = host[4:]
|
|
path = (p.path or "").rstrip("/")
|
|
return (host + path).lower()
|
|
except Exception:
|
|
return u.lower()
|
|
|
|
|
|
def host_of(u: str) -> str:
|
|
try:
|
|
h = urlparse(u).hostname or "unknown"
|
|
return h[4:] if h.startswith("www.") else h
|
|
except Exception:
|
|
return "unknown"
|
|
|
|
|
|
class Deduper:
|
|
"""원본 pipeline stage-2 의 dedup+budget 로직. 호출 순서에 결정론적."""
|
|
|
|
def __init__(self, config: Config):
|
|
self.seen: dict[str, dict] = {}
|
|
self.dupes: list[dict] = []
|
|
self.budget_dropped: list[dict] = []
|
|
self.fetch_slots = config.MAX_FETCH
|
|
|
|
def filter_novel(self, angle: str, results: list[dict]) -> list[dict]:
|
|
ordered = sorted(results, key=lambda r: REL_RANK[r["relevance"]])
|
|
novel: list[dict] = []
|
|
for r in ordered:
|
|
key = norm_url(r["url"])
|
|
if key in self.seen:
|
|
self.dupes.append({**r, "angle": angle, "dupOf": self.seen[key]})
|
|
continue
|
|
if self.fetch_slots <= 0 and REL_RANK[r["relevance"]] >= 1:
|
|
self.budget_dropped.append({**r, "angle": angle})
|
|
continue
|
|
self.seen[key] = {"angle": angle, "title": r["title"]}
|
|
self.fetch_slots -= 1
|
|
novel.append(r)
|
|
return novel
|
|
|
|
|
|
def rank_claims(claims: list[dict], max_verify: int) -> list[dict]:
|
|
return sorted(
|
|
claims,
|
|
key=lambda c: (IMP_RANK[c["importance"]], QUAL_RANK[c["sourceQuality"]]),
|
|
)[:max_verify]
|
|
|
|
|
|
def tally(verdicts: list, votes_per_claim: int, refutations_required: int) -> dict:
|
|
"""원본 verify 정족수 산식 1:1. None=기권."""
|
|
valid = [v for v in verdicts if v is not None]
|
|
refuted = sum(1 for v in valid if v["refuted"])
|
|
abstained = votes_per_claim - len(valid)
|
|
survives = len(valid) >= refutations_required and refuted < refutations_required
|
|
return {
|
|
"valid": valid,
|
|
"refutedVotes": refuted,
|
|
"abstained": abstained,
|
|
"survives": survives,
|
|
}
|
|
|
|
|
|
CONF_RANK = {"high": 0, "medium": 1, "low": 2}
|
|
|
|
|
|
def _vote_str(c: dict) -> str:
|
|
return str(len(c["valid"]) - c["refutedVotes"]) + "-" + str(c["refutedVotes"])
|
|
|
|
|
|
def build_synth_blocks(confirmed: list[dict], killed: list[dict]) -> tuple[str, str]:
|
|
"""원본 synthesize 의 block / killedBlock 문자열 조립."""
|
|
parts = []
|
|
for i, c in enumerate(confirmed):
|
|
non_refuted = [v for v in c["valid"] if not v["refuted"]]
|
|
best = sorted(non_refuted, key=lambda v: CONF_RANK[v["confidence"]])[0]
|
|
parts.append(
|
|
"### [" + str(i) + "] " + c["claim"] + "\n"
|
|
+ "Vote: " + _vote_str(c) + " · Source: " + c["sourceUrl"] + " (" + c["sourceQuality"] + ")\n"
|
|
+ 'Quote: "' + c["quote"] + '"\nVerifier evidence (' + best["confidence"] + "): " + best["evidence"] + "\n"
|
|
)
|
|
block = "\n".join(parts)
|
|
|
|
if killed:
|
|
killed_block = "\n## Refuted claims (for transparency)\n" + "\n".join(
|
|
'- "' + c["claim"] + '" (' + c["sourceUrl"] + ", vote " + _vote_str(c) + ")"
|
|
for c in killed
|
|
)
|
|
else:
|
|
killed_block = ""
|
|
return block, killed_block
|
|
|
|
|
|
def build_stats(*, angles, sources, claims, voted, confirmed, killed,
|
|
after_synth, dupes, budget_dropped, votes_per_claim) -> dict:
|
|
return {
|
|
"angles": angles,
|
|
"sourcesFetched": sources,
|
|
"claimsExtracted": claims,
|
|
"claimsVerified": voted,
|
|
"confirmed": confirmed,
|
|
"killed": killed,
|
|
"afterSynthesis": after_synth,
|
|
"urlDupes": dupes,
|
|
"budgetDropped": budget_dropped,
|
|
"agentCalls": 1 + angles + sources + (voted * votes_per_claim) + 1,
|
|
}
|