41 lines
1.9 KiB
Python
41 lines
1.9 KiB
Python
"""plan: 실행 전 검증 + 비용추정. judge 비용은 파일럿 18 만이 아니라 calibration + retry 를 포함해야
|
||
정직하다(단일결함 fixture 가 많으면 calibration 이 파일럿보다 클 수 있음)."""
|
||
from . import judge, manifest, paths
|
||
|
||
|
||
def estimate_judge_calls(n_calibration_fixtures, panel_size, arm_ids, retry_factor=1):
|
||
"""estimate total judge calls including pilot, calibration, and retry.
|
||
|
||
pilot = pairwise combinations × 2 orientations × panel_size judges
|
||
calibration = n_calibration_fixtures × panel_size × 2 orientations
|
||
total = (pilot + calibration) × retry_factor
|
||
"""
|
||
pilot = len(judge.plan_calls(arm_ids, panel_size)) # 3-arm·3 → 18
|
||
# calibration: 각 fixture 를 gold 와 pairwise(panel×2 orientation)
|
||
calib = n_calibration_fixtures * panel_size * 2
|
||
return (pilot + calib) * retry_factor
|
||
|
||
|
||
def summary(n_calibration_fixtures=8, panel_size=3, retry_factor=2):
|
||
"""return dict with cost estimate and metadata for the benchmark plan.
|
||
|
||
includes:
|
||
- arms: per-arm commit and label
|
||
- total-arm-runs: number of arms being evaluated
|
||
- pilot-pairwise-calls: number of pilot pairwise judge calls (18 for 3-arm/panel-3)
|
||
- estimated-judge-calls: total estimated judge calls including calibration and retry
|
||
- preflight-violations: list of preflight check violations (empty if pass)
|
||
- worktree-root: path to worktree root
|
||
"""
|
||
man = manifest.load()
|
||
arm_ids = list(man["arms"])
|
||
pilot = len(judge.plan_calls(arm_ids, panel_size))
|
||
return {
|
||
"arms": {a: {"commit": man["arms"][a]["commit"], "label": man["arms"][a]["label"]} for a in arm_ids},
|
||
"total-arm-runs": len(arm_ids),
|
||
"pilot-pairwise-calls": pilot,
|
||
"estimated-judge-calls": estimate_judge_calls(n_calibration_fixtures, panel_size, arm_ids, retry_factor),
|
||
"preflight-violations": manifest.preflight(man),
|
||
"worktree-root": paths.exec_root("<run-id>"),
|
||
}
|