428 lines
25 KiB
Python
428 lines
25 KiB
Python
#!/usr/bin/env python3
|
|
"""Experience-foundation, organization design, and tool-neutral adapter regression."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
import yaml
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
HOOKS = os.path.join(ROOT, ".claude", "hooks")
|
|
WS = os.path.join(ROOT, ".claude", "tests", "fixtures", "experience-foundation-ws")
|
|
os.environ["CLAUDE_PROJECT_DIR"] = ROOT
|
|
os.environ["ORGOS_WORKSPACE"] = WS
|
|
sys.path.insert(0, HOOKS)
|
|
|
|
import state_engine as SE # noqa: E402
|
|
import lint_design_direction as LDD # noqa: E402
|
|
|
|
shutil.rmtree(WS, ignore_errors=True)
|
|
os.makedirs(WS, exist_ok=True)
|
|
passed = failed = 0
|
|
|
|
|
|
def check(name, condition):
|
|
global passed, failed
|
|
if condition:
|
|
passed += 1
|
|
print(f" PASS {name}")
|
|
else:
|
|
failed += 1
|
|
print(f" FAIL {name}")
|
|
|
|
|
|
def sha(path):
|
|
return hashlib.sha256(open(path, "rb").read()).hexdigest()
|
|
|
|
|
|
def report(wf, artifact_id, kind, stage, producer, body, tier="standard"):
|
|
directory = os.path.join(WS, "completion-records", wf)
|
|
os.makedirs(directory, exist_ok=True)
|
|
path = os.path.join(directory, artifact_id + ".report.yaml")
|
|
document = {
|
|
"report-type": "workflow-artifact", "artifact-kind": kind,
|
|
"artifact-version": 1, "tier": tier,
|
|
"identity": {"artifact-id": artifact_id, "workflow-id": wf,
|
|
"stage": stage, "producer-role-id": producer},
|
|
"report-header": {
|
|
"bottom-line": f"{kind} fixture", "decision-needed": {"needed": False},
|
|
"confidence": {"value": "Med", "derived-from": "evidence"},
|
|
"risks": [], "evidence": [{"source-uri": "README.md", "grade": "E3"}],
|
|
},
|
|
"payload": body,
|
|
}
|
|
with open(path, "w", encoding="utf-8") as handle:
|
|
yaml.safe_dump(document, handle, allow_unicode=True, sort_keys=False)
|
|
return path
|
|
|
|
|
|
def open_stage(wf, stage):
|
|
current = SE.read_ledger(wf).get("stage")
|
|
event = {"state-event-id": f"fixture-{wf}-{stage}", "event-type": "state-transition",
|
|
"workflow-id": wf, "from": current, "to": stage, "actor": "OPS-ORCH",
|
|
"effective-at": SE._now()}
|
|
return SE._atomic_event_transaction(wf, workflow_event=event)[0]
|
|
|
|
|
|
def submit_accept(wf, path, reviewer):
|
|
return SE.submit_artifact(wf, path, "OPS-ORCH")[0] and SE.review_artifact(
|
|
wf, path, "accepted", reviewer)[0]
|
|
|
|
|
|
print("== parent and typed applicability ==")
|
|
SE.init_ledger("parent", tier="light")
|
|
profile = report("parent", "profile", "workload-profile", "intake", "EXEC-CEO", {
|
|
"surfaces": {"ui": True, "public-api": False, "persistence": False, "infrastructure": False},
|
|
"surface-archetype": "interactive-learning", "experience-change": "incremental",
|
|
"risk": {"security-bearing": False, "data-migration": False, "external-side-effect": False,
|
|
"risk-level": "Low", "reversibility": "two-way-door", "blast-radius": "single-role",
|
|
"privacy": False, "regulatory": False, "slo-impact": False},
|
|
"required-capabilities": ["product", "design"], "product-feature": True,
|
|
}, tier="light")
|
|
check("interactive-learning workload submitted", SE.submit_artifact("parent", profile, "OPS-ORCH")[0])
|
|
check("fixture opens parent decide", open_stage("parent", "decide"))
|
|
packet = report("parent", "decision", "executive-decision-packet", "decide", "EXEC-CEO", {
|
|
"recommendation": "proceed", "selected-option-id": "a",
|
|
"evaluation-criteria": ["fit"],
|
|
"option-evaluations": [{"option-id": "a", "scores": {"fit": 1}, "evidence-refs": ["README.md"]}],
|
|
"tradeoffs": ["time"], "dissent": [], "kill-criteria": ["no demand"],
|
|
"revisit-conditions": ["market changes"], "evidence-refs": ["README.md"],
|
|
}, tier="light")
|
|
check("product decision accepted", submit_accept("parent", packet, "HUMAN-001"))
|
|
brief0 = os.path.join(WS, "brief-without-foundation.yaml")
|
|
with open(brief0, "w", encoding="utf-8") as handle:
|
|
yaml.safe_dump({"product-goal": "ship", "core-users": ["buyer"], "core-tasks": ["evaluate"],
|
|
"information-density": "medium", "required-accessibility": "WCAG AA",
|
|
"brand-constraints": ["legible"], "avoid-cliches": ["generic hero"],
|
|
"representative-screen-requirement": "core-task",
|
|
"tech-platform-constraints": ["web"]}, handle)
|
|
try:
|
|
SE.init_ledger("direction-too-early", plan="design-direction", parent_workflow="parent",
|
|
product_decision="decision", direction_input_brief=brief0)
|
|
early_blocked = False
|
|
except ValueError:
|
|
early_blocked = True
|
|
check("design-direction blocked before foundation", early_blocked)
|
|
try:
|
|
SE.init_ledger("foundation-light", plan="experience-foundation", tier="light",
|
|
parent_workflow="parent", product_decision="decision")
|
|
light_blocked = False
|
|
except ValueError:
|
|
light_blocked = True
|
|
check("required interactive-learning foundation cannot use light tier", light_blocked)
|
|
|
|
print("== benchmark -> strategy -> IA -> wireframes ==")
|
|
SE.init_ledger("foundation", plan="experience-foundation", tier="standard",
|
|
parent_workflow="parent", product_decision="decision")
|
|
check("foundation parent gate", SE.complete_stage("foundation", "OPS-ORCH", to="experience-benchmark")[0])
|
|
check("enter benchmark", SE.enter_stage("foundation", "experience-benchmark", actor="OPS-ORCH")[0])
|
|
shots = []
|
|
for index in range(10):
|
|
path = os.path.join(WS, f"shot-{index}.bin")
|
|
with open(path, "wb") as handle:
|
|
handle.write(f"capture-{index}".encode())
|
|
shots.append(path)
|
|
refs = []
|
|
for index in range(5):
|
|
refs.append({
|
|
"name": f"Reference {index}", "class": "direct" if index < 3 else "adjacent",
|
|
"source-url": f"https://example.com/{index}", "captured-at": "2026-07-20T00:00:00Z",
|
|
"screenshots": {
|
|
"desktop": [{"path": shots[index * 2], "sha256": sha(shots[index * 2])}],
|
|
"mobile": [{"path": shots[index * 2 + 1], "sha256": sha(shots[index * 2 + 1])}],
|
|
},
|
|
"flows": [{"name": "core", "steps": ["enter", "act"]}],
|
|
"information-architecture": {"root": ["entry", "detail"]},
|
|
"interaction-patterns": ["progressive disclosure"],
|
|
"content-strategy": {"message": "value before proof"},
|
|
"strengths": [{"claim": "clear task", "evidence-refs": [shots[index * 2]]}],
|
|
"weaknesses": [{"claim": "weak recovery", "evidence-refs": [shots[index * 2 + 1]]}],
|
|
})
|
|
benchmark = report("foundation", "benchmark", "competitive-experience-benchmark",
|
|
"experience-benchmark", "GTM-CI", {
|
|
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
|
"category": "learning product", "target-job": "understand a mechanism", "surface-archetype": "interactive-learning",
|
|
"references": refs,
|
|
"synthesis": {"table-stakes": ["clear value"], "adopt": ["proof"], "adapt": ["entry"],
|
|
"avoid": ["generic hero"], "differentiation-opportunities": ["guided evaluation"],
|
|
"unresolved-questions": []},
|
|
"no-copy-attestation": True,
|
|
})
|
|
check("benchmark accepted", submit_accept("foundation", benchmark, "EXEC-CPO"))
|
|
check("benchmark gate", SE.complete_stage("foundation", "OPS-ORCH", to="experience-strategy")[0])
|
|
check("enter strategy", SE.enter_stage("foundation", "experience-strategy", actor="OPS-ORCH")[0])
|
|
strategy = report("foundation", "strategy", "experience-strategy", "experience-strategy", "EXEC-CPO", {
|
|
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
|
"benchmark-ref": benchmark, "benchmark-sha256": sha(benchmark),
|
|
"experience-thesis": "guided confidence", "target-users": ["buyer"],
|
|
"jobs-to-be-done": ["evaluate"], "value-proposition": "decide with evidence",
|
|
"differentiation": ["recovery-aware proof"], "message-hierarchy": ["value", "proof", "action"],
|
|
"success-metrics": ["qualified action"], "decision": "proceed",
|
|
})
|
|
check("strategy accepted", submit_accept("foundation", strategy, "HUMAN-001"))
|
|
check("strategy alone cannot pass feasibility gate",
|
|
not SE.complete_stage("foundation", "OPS-ORCH", to="information-architecture")[0])
|
|
technical = report("foundation", "technical-feasibility", "experience-technical-feasibility",
|
|
"experience-strategy", "EXEC-CTO", {
|
|
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
|
"strategy-ref": strategy, "strategy-sha256": sha(strategy),
|
|
"architecture-fit": {"simulation": "deterministic browser model"},
|
|
"delivery-sustainability": {"content-unit": "versioned mechanism pack"},
|
|
"constraints": ["no production diagnosis claim"], "risks": ["content engine drift"],
|
|
"mitigations": ["versioned fixtures"], "verdict": "feasible",
|
|
})
|
|
check("technical feasibility accepted", submit_accept("foundation", technical, "HUMAN-001"))
|
|
check("technical-only cannot replace COO feasibility",
|
|
not SE.complete_stage("foundation", "OPS-ORCH", to="information-architecture")[0])
|
|
operational = report("foundation", "operational-feasibility", "experience-operational-feasibility",
|
|
"experience-strategy", "EXEC-COO", {
|
|
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
|
"strategy-ref": strategy, "strategy-sha256": sha(strategy),
|
|
"operating-model": {"owner": "learning content operations"},
|
|
"content-operations": {"release-unit": "mechanism deep dive"},
|
|
"support-and-recovery": {"boundary": "guided simulation support"},
|
|
"constraints": ["one validated mechanism at a time"], "risks": ["review queue"],
|
|
"mitigations": ["candidate release cap"], "verdict": "feasible",
|
|
})
|
|
check("operational feasibility accepted", submit_accept("foundation", operational, "HUMAN-001"))
|
|
check("strategy gate", SE.complete_stage("foundation", "OPS-ORCH", to="information-architecture")[0])
|
|
check("enter IA", SE.enter_stage("foundation", "information-architecture", actor="OPS-ORCH")[0])
|
|
blueprint = report("foundation", "blueprint", "experience-blueprint", "information-architecture", "DOC-IA", {
|
|
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
|
"benchmark-ref": benchmark, "benchmark-sha256": sha(benchmark),
|
|
"strategy-ref": strategy, "strategy-sha256": sha(strategy),
|
|
"experience-thesis": "guided confidence", "target-users": ["buyer"],
|
|
"jobs-to-be-done": ["evaluate"], "value-proposition": "decide with evidence",
|
|
"differentiation": ["recovery-aware proof"], "content-model": {"entities": ["claim", "proof"]},
|
|
"page-inventory": ["entry", "proof", "action"], "navigation-model": {"primary": ["proof"]},
|
|
"message-hierarchy": ["value", "proof", "action"],
|
|
"task-flows": [{"id": "evaluate", "steps": ["entry", "proof", "action"]}],
|
|
"state-matrix": {"default": {}, "loading": {}, "empty": {}, "error": {}, "partial": {}, "completed": {}},
|
|
"responsive-priorities": ["value", "action", "proof"],
|
|
"accessibility-intent": {"target": "WCAG AA"}, "success-metrics": ["qualified action"],
|
|
})
|
|
check("blueprint accepted", submit_accept("foundation", blueprint, "EXEC-CPO"))
|
|
check("blueprint gate", SE.complete_stage("foundation", "OPS-ORCH", to="wireframes")[0])
|
|
check("enter wireframes", SE.enter_stage("foundation", "wireframes", actor="OPS-ORCH")[0])
|
|
wireframes = report("foundation", "wireframes", "wireframe-set", "wireframes", "DES-PROD", {
|
|
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
|
"blueprint-ref": blueprint, "blueprint-sha256": sha(blueprint),
|
|
"screens": [{"id": "entry", "purpose": "explain value", "primary-action": "evaluate",
|
|
"content-priority": ["value", "proof"], "desktop": {"regions": 3},
|
|
"mobile": {"regions": 2}, "states": ["default", "loading", "error"]}],
|
|
"validation": {"information-scent": "passed", "task-completion": "passed",
|
|
"cognitive-load": "passed", "responsive-hierarchy": "passed"},
|
|
"art-direction-deferred": True,
|
|
})
|
|
check("wireframes accepted", submit_accept("foundation", wireframes, "EXEC-CPO"))
|
|
check("parent link registered", SE.register_experience_foundation("parent", "foundation"))
|
|
check("wireframe terminal gate", SE.complete_stage("foundation", "OPS-ORCH", to="foundation-approved")[0])
|
|
check("foundation enters terminal", SE.enter_stage("foundation", "foundation-approved", actor="OPS-ORCH")[0])
|
|
check("parent exact foundation valid", SE._has_experience_foundation("parent", SE.read_ledger("parent")))
|
|
approval = SE.read_ledger("parent")["experience-foundation-approval"]
|
|
check("parent link binds technical and operational exact revisions",
|
|
approval.get("technical-sha256") == sha(technical)
|
|
and approval.get("operational-sha256") == sha(operational))
|
|
|
|
print("== direction exact refs and organization design adapters ==")
|
|
brief = os.path.join(WS, "direction-input-brief.yaml")
|
|
brief_body = {"product-goal": "ship", "core-users": ["buyer"], "core-tasks": ["evaluate"],
|
|
"information-density": "medium", "required-accessibility": "WCAG AA",
|
|
"brand-constraints": ["legible"], "avoid-cliches": ["generic hero"],
|
|
"representative-screen-requirement": "core-task", "tech-platform-constraints": ["web"]}
|
|
for approval_key, brief_key in (("benchmark", "competitive-experience-benchmark"),
|
|
("blueprint", "experience-blueprint"), ("wireframe", "wireframe-set")):
|
|
brief_body[f"{brief_key}-ref"] = approval[f"{approval_key}-ref"]
|
|
brief_body[f"{brief_key}-sha256"] = approval[f"{approval_key}-sha256"]
|
|
with open(brief, "w", encoding="utf-8") as handle:
|
|
yaml.safe_dump(brief_body, handle, sort_keys=False)
|
|
try:
|
|
SE.init_ledger("direction", plan="design-direction", parent_workflow="parent",
|
|
product_decision="decision", direction_input_brief=brief)
|
|
direction_ok = True
|
|
except ValueError:
|
|
direction_ok = False
|
|
check("design-direction accepts exact foundation refs", direction_ok)
|
|
compiler = subprocess.run([sys.executable, os.path.join(HOOKS, "compile_design_system.py"), "--check"],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
check("organization design adapters are drift-free", compiler.returncode == 0)
|
|
query = subprocess.run([sys.executable, os.path.join(HOOKS, "design_registry.py"),
|
|
"--surface", "interactive-learning", "--state", "candidate"],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
query_body = json.loads(query.stdout) if query.returncode == 0 else {}
|
|
check("registry query returns interactive-learning patterns and page archetypes",
|
|
{"causal-learning-loop", "learning-feedback-recovery"}.issubset(
|
|
{item.get("id") for item in query_body.get("patterns", [])})
|
|
and {"interactive-learning-atlas", "interactive-learning-deep-dive"}.issubset(
|
|
{item.get("id") for item in query_body.get("page-archetypes", [])}))
|
|
|
|
print("== DESIGN.md diff and local duplicate adherence ==")
|
|
generated_design = os.path.join(ROOT, "org-os", "08-design", "generated", "DESIGN.md")
|
|
design_diff = subprocess.run([sys.executable, os.path.join(HOOKS, "compile_design_system.py"),
|
|
"--diff", generated_design],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
check("DESIGN.md exact canonical diff is clean", design_diff.returncode == 0)
|
|
drift_design = os.path.join(WS, "drift-DESIGN.md")
|
|
with open(generated_design, encoding="utf-8") as source, open(drift_design, "w", encoding="utf-8") as target:
|
|
target.write(source.read() + "\nmanual drift\n")
|
|
design_drift = subprocess.run([sys.executable, os.path.join(HOOKS, "compile_design_system.py"),
|
|
"--diff", drift_design],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
check("DESIGN.md drift emits unified diff", design_drift.returncode == 1
|
|
and "--- " in design_drift.stdout and "+++ " in design_drift.stdout)
|
|
release = os.path.join(ROOT, "org-os", "08-design", "releases", "0.1.0.yaml")
|
|
ui_report = os.path.join(WS, "ui-design.yaml")
|
|
target_dir = os.path.join(WS, "design-target")
|
|
os.makedirs(os.path.join(target_dir, "src", "components"), exist_ok=True)
|
|
button_path = os.path.join(target_dir, "src", "components", "Button.tsx")
|
|
with open(button_path, "w", encoding="utf-8") as handle:
|
|
handle.write("export function Button(){ return null }\n")
|
|
|
|
|
|
def write_ui_delta(components):
|
|
with open(ui_report, "w", encoding="utf-8") as handle:
|
|
yaml.safe_dump({"payload": {"design-system-bindings": [{
|
|
"release-id": "org-design-0.1.0", "release-ref": release,
|
|
"release-sha256": sha(release), "component-ids": ["button"],
|
|
"delta": {"tokens": [], "components": components},
|
|
}]}}, handle, sort_keys=False)
|
|
|
|
|
|
write_ui_delta([])
|
|
adherence = subprocess.run([sys.executable, os.path.join(HOOKS, "lint_design_system_adherence.py"),
|
|
"--ui-report", ui_report, "--target", target_dir],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
check("organization component local reimplementation is rejected", adherence.returncode != 0
|
|
and "organization component local duplicate" in adherence.stderr)
|
|
write_ui_delta([{"id": "button", "reason": "project adapter boundary"}])
|
|
adherence_delta = subprocess.run([sys.executable, os.path.join(HOOKS, "lint_design_system_adherence.py"),
|
|
"--ui-report", ui_report, "--target", target_dir],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
check("declared project component delta is accepted", adherence_delta.returncode == 0)
|
|
os.makedirs(os.path.join(target_dir, "src", "ui"), exist_ok=True)
|
|
with open(os.path.join(target_dir, "src", "ui", "Button.jsx"), "w", encoding="utf-8") as handle:
|
|
handle.write("export const Button = () => null\n")
|
|
adherence_duplicate = subprocess.run([
|
|
sys.executable, os.path.join(HOOKS, "lint_design_system_adherence.py"),
|
|
"--ui-report", ui_report, "--target", target_dir],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
check("duplicate local component ids remain rejected despite delta", adherence_duplicate.returncode != 0
|
|
and "local duplicate component" in adherence_duplicate.stderr)
|
|
|
|
print("== Hyeonworks controlled first-draft experiment contract ==")
|
|
experiment_hook = os.path.join(HOOKS, "first_draft_experiment.py")
|
|
planned_manifest = os.path.join(ROOT, "hyeonworks", "experiments", "experience-foundation-ab",
|
|
"experiment.yaml")
|
|
planned = subprocess.run([sys.executable, experiment_hook, "validate", planned_manifest],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
check("Hyeonworks planned experiment validates without claiming completion", planned.returncode == 0)
|
|
planned_complete = subprocess.run([sys.executable, experiment_hook, "validate", planned_manifest,
|
|
"--require-complete"],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
check("planned experiment cannot claim completed evidence", planned_complete.returncode != 0)
|
|
|
|
experiment_dir = os.path.join(WS, "completed-ab")
|
|
os.makedirs(experiment_dir, exist_ok=True)
|
|
request_path = os.path.join(experiment_dir, "request.md")
|
|
with open(request_path, "w", encoding="utf-8") as handle:
|
|
handle.write("same controlled request\n")
|
|
input_kinds = ["competitive-experience-benchmark", "experience-blueprint", "wireframe-set",
|
|
"design-md", "component-registry"]
|
|
input_bindings = []
|
|
for kind in input_kinds:
|
|
path = os.path.join(experiment_dir, kind + ".yaml")
|
|
with open(path, "w", encoding="utf-8") as handle:
|
|
handle.write(f"kind: {kind}\n")
|
|
input_bindings.append({"kind": kind, "ref": path, "sha256": sha(path)})
|
|
outputs = {}
|
|
evaluations = {}
|
|
for arm_id, score, revisions, tokens in (("A", 2, 4, 12000), ("B", 5, 1, 7000)):
|
|
output_path = os.path.join(experiment_dir, f"arm-{arm_id}.html")
|
|
desktop = os.path.join(experiment_dir, f"arm-{arm_id}-desktop.png")
|
|
mobile = os.path.join(experiment_dir, f"arm-{arm_id}-mobile.png")
|
|
with open(output_path, "w", encoding="utf-8") as handle:
|
|
handle.write(f"<main>arm {arm_id}</main>\n")
|
|
for screenshot in (desktop, mobile):
|
|
with open(screenshot, "wb") as handle:
|
|
handle.write(b"\x89PNG\r\n\x1a\n" + (f"render-{arm_id}-" * 300).encode())
|
|
output = {"ref": output_path, "sha256": sha(output_path)}
|
|
outputs[arm_id] = output
|
|
evaluation_path = os.path.join(experiment_dir, f"evaluation-{arm_id}.yaml")
|
|
metrics = {key: score for key in (
|
|
"product-purpose-understood-in-5s", "primary-action-found", "information-hierarchy",
|
|
"category-quality-parity", "visual-craft", "mobile-clarity",
|
|
"design-system-consistency", "human-preference")}
|
|
metrics.update({"revision-count-to-acceptance": revisions, "tokens-to-acceptance": tokens})
|
|
with open(evaluation_path, "w", encoding="utf-8") as handle:
|
|
yaml.safe_dump({
|
|
"experiment-id": "fixture-ab", "subject": "Hyeonworks", "arm-id": arm_id,
|
|
"model-id": "same-model", "request-sha256": sha(request_path),
|
|
"first-result-only": True, "revision-count-at-capture": 0,
|
|
"output-ref": output_path, "output-sha256": sha(output_path),
|
|
"screenshots": {
|
|
"desktop": {"path": desktop, "sha256": sha(desktop)},
|
|
"mobile": {"path": mobile, "sha256": sha(mobile)}},
|
|
"evaluator-blinded": True, "metrics": metrics,
|
|
}, handle, sort_keys=False)
|
|
evaluations[arm_id] = {"ref": evaluation_path, "sha256": sha(evaluation_path)}
|
|
completed_manifest = os.path.join(experiment_dir, "experiment.yaml")
|
|
control = {"same-model": True, "same-request-sha256": True, "first-result-only": True,
|
|
"generation-attempts-per-arm": 1, "revision-count-at-capture": 0,
|
|
"evaluator-blinded": True}
|
|
with open(completed_manifest, "w", encoding="utf-8") as handle:
|
|
yaml.safe_dump({"first-draft-experiment": {
|
|
"version": 1, "experiment-id": "fixture-ab", "subject": "Hyeonworks",
|
|
"status": "completed", "model-id": "same-model",
|
|
"request": {"ref": request_path, "sha256": sha(request_path)}, "control": control,
|
|
"arms": {
|
|
"A": {"treatment": "current-harness", "required-input-kinds": ["request"],
|
|
"forbidden-input-kinds": input_kinds, "inputs": [],
|
|
"output": outputs["A"], "evaluation": evaluations["A"]},
|
|
"B": {"treatment": "experience-foundation", "required-input-kinds": input_kinds,
|
|
"generation-scope": "representative-section-or-core-screen",
|
|
"inputs": input_bindings, "output": outputs["B"], "evaluation": evaluations["B"]},
|
|
}}}, handle, sort_keys=False)
|
|
completed = subprocess.run([sys.executable, experiment_hook, "validate", completed_manifest,
|
|
"--require-complete"], cwd=ROOT, capture_output=True, text=True)
|
|
check("completed A/B requires exact model/request/input/output/evaluation evidence",
|
|
completed.returncode == 0)
|
|
comparison = subprocess.run([sys.executable, experiment_hook, "compare", completed_manifest],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
comparison_body = json.loads(comparison.stdout) if comparison.returncode == 0 else {}
|
|
check("first-draft metrics comparison applies explicit decision rule",
|
|
comparison.returncode == 0 and comparison_body.get("treatment-supported") is True)
|
|
|
|
print("== HUMAN none-of-the-above path ==")
|
|
dset_path = os.path.join(WS, "direction-set.yaml")
|
|
with open(dset_path, "w", encoding="utf-8") as handle:
|
|
yaml.safe_dump({"directions": [{"id": "A"}, {"id": "B"}, {"id": "C"}]}, handle)
|
|
none_path = os.path.join(WS, "none-selection.yaml")
|
|
with open(none_path, "w", encoding="utf-8") as handle:
|
|
yaml.safe_dump({
|
|
"direction-set-ref": dset_path, "direction-set-sha256": sha(dset_path),
|
|
"selection-decision": "none-of-the-above",
|
|
"rejected-directions": [{"id": value, "reason": "benchmark-relative quality floor 미달"}
|
|
for value in ("A", "B", "C")],
|
|
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
|
"direction-input-brief-sha256": "0" * 64,
|
|
"selection-acceptance-receipt": "human-receipt",
|
|
}, handle)
|
|
hard, _warn = LDD.lint_selected_direction(none_path, dset_path)
|
|
check("none-of-the-above rejects all without averaging", not hard)
|
|
contracts = SE.load_contracts()
|
|
selected_contract = contracts["artifact-kinds"]["selected-direction"]
|
|
check("selected-direction decision remains HUMAN-only", selected_contract.get("required-reviewer-roles") == ["HUMAN-001"])
|
|
dd_transitions = contracts["workflows"]["design-direction"].get("additional-transitions", [])
|
|
check("none decision routes back to discovery", any(
|
|
item.get("from") == "design-direction-decision"
|
|
and item.get("to") == "design-direction-discovery"
|
|
and "none-of-the-above-recorded" in item.get("required-conditions", [])
|
|
for item in dd_transitions))
|
|
|
|
print(f"\n{passed} passed, {failed} failed")
|
|
shutil.rmtree(WS, ignore_errors=True)
|
|
raise SystemExit(1 if failed else 0)
|