init: llm-wiki-haness 하네스 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,316 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Resolve and optionally execute neutral workflow execution contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any, Callable
|
||||
|
||||
import execution_profile
|
||||
|
||||
|
||||
DEFAULT_ROOT = Path(__file__).resolve().parents[2]
|
||||
MANIFEST = Path("harness/source/generation-manifest.json")
|
||||
RESULT_SCHEMA = "workflow-dispatch-result/v1"
|
||||
HEX_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
Runner = Callable[..., subprocess.CompletedProcess[str]]
|
||||
|
||||
|
||||
class DispatchError(ValueError):
|
||||
"""A workflow declaration or runtime result is not safe to dispatch."""
|
||||
|
||||
|
||||
def _safe_path(root: Path, value: Any, *, field: str, must_exist: bool = True) -> Path:
|
||||
if not isinstance(value, str) or not value or "\\" in value:
|
||||
raise DispatchError(f"{field} must be a non-empty repo-relative POSIX path")
|
||||
relative = Path(value)
|
||||
if relative.is_absolute() or ".." in relative.parts:
|
||||
raise DispatchError(f"{field} escapes repository: {value}")
|
||||
resolved = (root / relative).resolve()
|
||||
try:
|
||||
resolved.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise DispatchError(f"{field} escapes repository: {value}") from exc
|
||||
if must_exist and not resolved.is_file():
|
||||
raise DispatchError(f"{field} does not exist: {value}")
|
||||
return resolved
|
||||
|
||||
|
||||
def _workflow_inventory(root: Path) -> dict[str, Path]:
|
||||
manifest_path = root / MANIFEST
|
||||
document = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if document.get("schema_version") != 1 or not isinstance(document.get("sources"), list):
|
||||
raise DispatchError("generation manifest must use schema_version 1 with sources")
|
||||
workflows: dict[str, Path] = {}
|
||||
for index, item in enumerate(document["sources"]):
|
||||
if not isinstance(item, dict):
|
||||
raise DispatchError(f"sources[{index}] must be an object")
|
||||
metadata = _safe_path(root, item.get("metadata"), field=f"sources[{index}].metadata")
|
||||
data = json.loads(metadata.read_text(encoding="utf-8"))
|
||||
if data.get("source_kind") != "workflow":
|
||||
continue
|
||||
identifier = data.get("id")
|
||||
if not isinstance(identifier, str) or not identifier:
|
||||
raise DispatchError(f"{metadata.relative_to(root)}: workflow id is required")
|
||||
if identifier in workflows:
|
||||
raise DispatchError(f"duplicate workflow id: {identifier}")
|
||||
workflows[identifier] = metadata
|
||||
if not workflows:
|
||||
raise DispatchError("generation manifest declares no workflows")
|
||||
return workflows
|
||||
|
||||
|
||||
def _contract(root: Path, workflow: str) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
inventory = _workflow_inventory(root)
|
||||
metadata = inventory.get(workflow)
|
||||
if metadata is None:
|
||||
raise DispatchError(f"workflow is not declared by generation manifest: {workflow}")
|
||||
expected = (root / "harness/source/workflows" / f"{workflow}.json").resolve()
|
||||
if metadata != expected:
|
||||
raise DispatchError(f"workflow metadata must use neutral workflow directory: {metadata}")
|
||||
_identifier, contract = execution_profile.load_workflow_contract(
|
||||
workflow,
|
||||
root / "harness/source/workflows",
|
||||
)
|
||||
if contract["kind"] == "deterministic":
|
||||
entrypoint = _safe_path(root, contract.get("entrypoint"), field="execution_contract.entrypoint")
|
||||
if entrypoint.suffix != ".py":
|
||||
raise DispatchError("deterministic entrypoint must be a Python source file")
|
||||
return contract, json.loads(metadata.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def build_plan(
|
||||
root: Path,
|
||||
workflow: str,
|
||||
*,
|
||||
risk: str | None = None,
|
||||
finding_count: int = 0,
|
||||
public_claims_present: bool = False,
|
||||
claims_present: bool = False,
|
||||
phase: str = "baseline",
|
||||
) -> dict[str, Any]:
|
||||
if phase not in {"baseline", "final"}:
|
||||
raise DispatchError(f"unsupported resolution phase: {phase}")
|
||||
if phase == "baseline" and (finding_count or public_claims_present or claims_present):
|
||||
raise DispatchError("baseline resolution cannot declare findings or claims")
|
||||
root = root.resolve(strict=True)
|
||||
contract, _metadata = _contract(root, workflow)
|
||||
profiles = execution_profile.load_profiles(root / "harness/source/execution-profiles.json")
|
||||
policy = execution_profile.resolve_workflow_policy(
|
||||
profiles,
|
||||
workflow,
|
||||
root / "harness/source/workflows",
|
||||
risk,
|
||||
finding_count,
|
||||
public_claims_present,
|
||||
claims_present,
|
||||
)
|
||||
execution: dict[str, Any] = {
|
||||
"kind": contract["kind"],
|
||||
"explicit_execute_required": contract["kind"] == "deterministic",
|
||||
}
|
||||
if contract["kind"] == "deterministic":
|
||||
execution.update(
|
||||
{
|
||||
"entrypoint": contract["entrypoint"],
|
||||
"dry_run_first": contract["dry_run_first"],
|
||||
"result_schema": contract["result_schema"],
|
||||
}
|
||||
)
|
||||
return {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"status": "PLANNED",
|
||||
"workflow": workflow,
|
||||
"phase": phase,
|
||||
"profile": policy["profile"],
|
||||
"context": policy["context"],
|
||||
"always_checks": policy["always_checks"],
|
||||
"mandatory_gates": policy["mandatory_gates"],
|
||||
"review_intensity": policy["review_intensity"],
|
||||
"dispatch": policy["dispatch"],
|
||||
"semantic_review": policy["semantic_review"],
|
||||
"adversarial_review": policy["adversarial_review"],
|
||||
"output_contract": policy["output_contract"],
|
||||
"execution": execution,
|
||||
}
|
||||
|
||||
|
||||
def check_all(root: Path) -> dict[str, Any]:
|
||||
root = root.resolve(strict=True)
|
||||
findings: list[dict[str, str]] = []
|
||||
workflows: list[str] = []
|
||||
try:
|
||||
workflows = sorted(_workflow_inventory(root))
|
||||
except (DispatchError, OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
findings.append({"code": "WORKFLOW_INVENTORY_ERROR", "workflow": "", "message": str(exc)})
|
||||
for workflow in workflows:
|
||||
try:
|
||||
build_plan(root, workflow)
|
||||
except (DispatchError, execution_profile.ProfileError, OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
findings.append({"code": "WORKFLOW_CONTRACT_ERROR", "workflow": workflow, "message": str(exc)})
|
||||
return {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"status": "PASS" if not findings else "FAIL",
|
||||
"workflow_count": len(workflows),
|
||||
"findings": findings,
|
||||
}
|
||||
|
||||
|
||||
def _child_json(completed: subprocess.CompletedProcess[str], schema: str, phase: str) -> dict[str, Any]:
|
||||
try:
|
||||
document = json.loads(completed.stdout)
|
||||
except (json.JSONDecodeError, TypeError) as exc:
|
||||
raise DispatchError(f"{phase} returned non-JSON output") from exc
|
||||
if not isinstance(document, dict) or document.get("schema_version") != schema:
|
||||
raise DispatchError(f"{phase} result schema mismatch: expected {schema}")
|
||||
return document
|
||||
|
||||
|
||||
def _phase_outcome(
|
||||
plan: dict[str, Any],
|
||||
completed: subprocess.CompletedProcess[str],
|
||||
document: dict[str, Any],
|
||||
phase: str,
|
||||
success_status: str,
|
||||
) -> tuple[dict[str, Any] | None, int]:
|
||||
"""Preserve the shared 0/1/2 CLI envelope across child processes."""
|
||||
if completed.returncode not in {0, 1, 2}:
|
||||
raise DispatchError(f"{phase} returned unsupported exit code {completed.returncode}")
|
||||
if completed.returncode == 2:
|
||||
return {
|
||||
**plan,
|
||||
"status": "ERROR",
|
||||
"phase": phase,
|
||||
"runtime_result": document,
|
||||
}, 2
|
||||
if completed.returncode == 1:
|
||||
return {
|
||||
**plan,
|
||||
"status": "FAIL",
|
||||
"phase": phase,
|
||||
"runtime_result": document,
|
||||
}, 1
|
||||
if document.get("status") != success_status:
|
||||
raise DispatchError(f"{phase} exit 0 must return status {success_status}")
|
||||
return None, 0
|
||||
|
||||
|
||||
def execute_deterministic(
|
||||
root: Path,
|
||||
plan: dict[str, Any],
|
||||
arguments: list[str],
|
||||
*,
|
||||
runner: Runner = subprocess.run,
|
||||
) -> tuple[dict[str, Any], int]:
|
||||
if plan["execution"]["kind"] != "deterministic":
|
||||
return {**plan, "status": "FAIL", "error": {"code": "AGENTIC_EXECUTION_NOT_SUPPORTED"}}, 1
|
||||
required_reviews = [name for name, action in plan["dispatch"].items() if action == "dispatch"]
|
||||
if required_reviews:
|
||||
return {
|
||||
**plan,
|
||||
"status": "REVIEW_REQUIRED",
|
||||
"error": {"code": "REVIEW_DISPATCH_REQUIRED", "reviews": required_reviews},
|
||||
}, 1
|
||||
entrypoint = (root / plan["execution"]["entrypoint"]).resolve()
|
||||
schema = plan["execution"]["result_schema"]
|
||||
dry_command = [sys.executable, str(entrypoint), *arguments, "--dry-run"]
|
||||
dry = runner(dry_command, cwd=root, check=False, capture_output=True, text=True, timeout=120)
|
||||
dry_document = _child_json(dry, schema, "dry-run")
|
||||
outcome, exit_code = _phase_outcome(plan, dry, dry_document, "dry-run", "DRY_RUN")
|
||||
if outcome is not None:
|
||||
return outcome, exit_code
|
||||
plan_hash = dry_document.get("plan_sha256")
|
||||
if not isinstance(plan_hash, str) or not HEX_SHA256.fullmatch(plan_hash):
|
||||
raise DispatchError("dry-run did not return a valid plan_sha256")
|
||||
apply_command = [
|
||||
sys.executable,
|
||||
str(entrypoint),
|
||||
*arguments,
|
||||
"--apply",
|
||||
"--expected-plan-sha256",
|
||||
plan_hash,
|
||||
]
|
||||
applied = runner(apply_command, cwd=root, check=False, capture_output=True, text=True, timeout=120)
|
||||
applied_document = _child_json(applied, schema, "apply")
|
||||
outcome, exit_code = _phase_outcome(plan, applied, applied_document, "apply", "APPLIED")
|
||||
if outcome is not None:
|
||||
return outcome, exit_code
|
||||
if applied_document.get("plan_sha256") != plan_hash:
|
||||
raise DispatchError("apply result plan_sha256 does not match dry-run")
|
||||
return {
|
||||
**plan,
|
||||
"status": "APPLIED",
|
||||
"plan_sha256": plan_hash,
|
||||
"runtime_result": applied_document,
|
||||
}, 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("workflow", nargs="?")
|
||||
parser.add_argument("arguments", nargs="*")
|
||||
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
|
||||
parser.add_argument("--risk")
|
||||
parser.add_argument("--finding-count", type=int, default=0)
|
||||
parser.add_argument("--public-claims-present", action="store_true")
|
||||
parser.add_argument("--claims-present", action="store_true")
|
||||
parser.add_argument("--phase", choices=("baseline", "final"), default="baseline")
|
||||
parser.add_argument("--execute", action="store_true")
|
||||
parser.add_argument("--check-all", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
root = args.root.resolve(strict=True)
|
||||
if args.check_all:
|
||||
if args.workflow or args.arguments or args.execute:
|
||||
raise DispatchError("--check-all cannot be combined with a workflow or --execute")
|
||||
result = check_all(root)
|
||||
exit_code = 0 if result["status"] == "PASS" else 1
|
||||
else:
|
||||
if not args.workflow:
|
||||
raise DispatchError("workflow is required unless --check-all is used")
|
||||
plan = build_plan(
|
||||
root,
|
||||
args.workflow,
|
||||
risk=args.risk,
|
||||
finding_count=args.finding_count,
|
||||
public_claims_present=args.public_claims_present,
|
||||
claims_present=args.claims_present,
|
||||
phase=args.phase,
|
||||
)
|
||||
if args.execute:
|
||||
result, exit_code = execute_deterministic(root, plan, args.arguments)
|
||||
else:
|
||||
result, exit_code = plan, 0
|
||||
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
sys.stdout.write("\n")
|
||||
return exit_code
|
||||
except (
|
||||
DispatchError,
|
||||
execution_profile.ProfileError,
|
||||
OSError,
|
||||
UnicodeError,
|
||||
json.JSONDecodeError,
|
||||
subprocess.SubprocessError,
|
||||
) as exc:
|
||||
json.dump(
|
||||
{
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"status": "ERROR",
|
||||
"errors": [{"code": "WORKFLOW_DISPATCH_ERROR", "message": str(exc)}],
|
||||
},
|
||||
sys.stdout,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
sys.stdout.write("\n")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user