"""plan: 실행 전 검증 + 비용추정. judge 비용은 파일럿 18 만이 아니라 calibration + retry 를 포함해야 정직하다(단일결함 fixture 가 많으면 calibration 이 파일럿보다 클 수 있음).""" from . import judge, manifest, paths def estimate_judge_calls(n_calibration_fixtures, panel_size, arm_ids, retry_factor=1): """estimate total judge calls including pilot, calibration, and retry. pilot = pairwise combinations × 2 orientations × panel_size judges calibration = n_calibration_fixtures × panel_size × 2 orientations total = (pilot + calibration) × retry_factor """ pilot = len(judge.plan_calls(arm_ids, panel_size)) # 3-arm·3 → 18 # calibration: 각 fixture 를 gold 와 pairwise(panel×2 orientation) calib = n_calibration_fixtures * panel_size * 2 return (pilot + calib) * retry_factor def summary(n_calibration_fixtures=8, panel_size=3, retry_factor=2): """return dict with cost estimate and metadata for the benchmark plan. includes: - arms: per-arm commit and label - total-arm-runs: number of arms being evaluated - pilot-pairwise-calls: number of pilot pairwise judge calls (18 for 3-arm/panel-3) - estimated-judge-calls: total estimated judge calls including calibration and retry - preflight-violations: list of preflight check violations (empty if pass) - worktree-root: path to worktree root """ man = manifest.load() arm_ids = list(man["arms"]) pilot = len(judge.plan_calls(arm_ids, panel_size)) return { "arms": {a: {"commit": man["arms"][a]["commit"], "label": man["arms"][a]["label"]} for a in arm_ids}, "total-arm-runs": len(arm_ids), "pilot-pairwise-calls": pilot, "estimated-judge-calls": estimate_judge_calls(n_calibration_fixtures, panel_size, arm_ids, retry_factor), "preflight-violations": manifest.preflight(man), "worktree-root": paths.exec_root(""), }