init: company-haness 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,427 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Experience-foundation, organization design, and tool-neutral adapter regression."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
HOOKS = os.path.join(ROOT, ".claude", "hooks")
|
||||
WS = os.path.join(ROOT, ".claude", "tests", "fixtures", "experience-foundation-ws")
|
||||
os.environ["CLAUDE_PROJECT_DIR"] = ROOT
|
||||
os.environ["ORGOS_WORKSPACE"] = WS
|
||||
sys.path.insert(0, HOOKS)
|
||||
|
||||
import state_engine as SE # noqa: E402
|
||||
import lint_design_direction as LDD # noqa: E402
|
||||
|
||||
shutil.rmtree(WS, ignore_errors=True)
|
||||
os.makedirs(WS, exist_ok=True)
|
||||
passed = failed = 0
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
global passed, failed
|
||||
if condition:
|
||||
passed += 1
|
||||
print(f" PASS {name}")
|
||||
else:
|
||||
failed += 1
|
||||
print(f" FAIL {name}")
|
||||
|
||||
|
||||
def sha(path):
|
||||
return hashlib.sha256(open(path, "rb").read()).hexdigest()
|
||||
|
||||
|
||||
def report(wf, artifact_id, kind, stage, producer, body, tier="standard"):
|
||||
directory = os.path.join(WS, "completion-records", wf)
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
path = os.path.join(directory, artifact_id + ".report.yaml")
|
||||
document = {
|
||||
"report-type": "workflow-artifact", "artifact-kind": kind,
|
||||
"artifact-version": 1, "tier": tier,
|
||||
"identity": {"artifact-id": artifact_id, "workflow-id": wf,
|
||||
"stage": stage, "producer-role-id": producer},
|
||||
"report-header": {
|
||||
"bottom-line": f"{kind} fixture", "decision-needed": {"needed": False},
|
||||
"confidence": {"value": "Med", "derived-from": "evidence"},
|
||||
"risks": [], "evidence": [{"source-uri": "README.md", "grade": "E3"}],
|
||||
},
|
||||
"payload": body,
|
||||
}
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
yaml.safe_dump(document, handle, allow_unicode=True, sort_keys=False)
|
||||
return path
|
||||
|
||||
|
||||
def open_stage(wf, stage):
|
||||
current = SE.read_ledger(wf).get("stage")
|
||||
event = {"state-event-id": f"fixture-{wf}-{stage}", "event-type": "state-transition",
|
||||
"workflow-id": wf, "from": current, "to": stage, "actor": "OPS-ORCH",
|
||||
"effective-at": SE._now()}
|
||||
return SE._atomic_event_transaction(wf, workflow_event=event)[0]
|
||||
|
||||
|
||||
def submit_accept(wf, path, reviewer):
|
||||
return SE.submit_artifact(wf, path, "OPS-ORCH")[0] and SE.review_artifact(
|
||||
wf, path, "accepted", reviewer)[0]
|
||||
|
||||
|
||||
print("== parent and typed applicability ==")
|
||||
SE.init_ledger("parent", tier="light")
|
||||
profile = report("parent", "profile", "workload-profile", "intake", "EXEC-CEO", {
|
||||
"surfaces": {"ui": True, "public-api": False, "persistence": False, "infrastructure": False},
|
||||
"surface-archetype": "interactive-learning", "experience-change": "incremental",
|
||||
"risk": {"security-bearing": False, "data-migration": False, "external-side-effect": False,
|
||||
"risk-level": "Low", "reversibility": "two-way-door", "blast-radius": "single-role",
|
||||
"privacy": False, "regulatory": False, "slo-impact": False},
|
||||
"required-capabilities": ["product", "design"], "product-feature": True,
|
||||
}, tier="light")
|
||||
check("interactive-learning workload submitted", SE.submit_artifact("parent", profile, "OPS-ORCH")[0])
|
||||
check("fixture opens parent decide", open_stage("parent", "decide"))
|
||||
packet = report("parent", "decision", "executive-decision-packet", "decide", "EXEC-CEO", {
|
||||
"recommendation": "proceed", "selected-option-id": "a",
|
||||
"evaluation-criteria": ["fit"],
|
||||
"option-evaluations": [{"option-id": "a", "scores": {"fit": 1}, "evidence-refs": ["README.md"]}],
|
||||
"tradeoffs": ["time"], "dissent": [], "kill-criteria": ["no demand"],
|
||||
"revisit-conditions": ["market changes"], "evidence-refs": ["README.md"],
|
||||
}, tier="light")
|
||||
check("product decision accepted", submit_accept("parent", packet, "HUMAN-001"))
|
||||
brief0 = os.path.join(WS, "brief-without-foundation.yaml")
|
||||
with open(brief0, "w", encoding="utf-8") as handle:
|
||||
yaml.safe_dump({"product-goal": "ship", "core-users": ["buyer"], "core-tasks": ["evaluate"],
|
||||
"information-density": "medium", "required-accessibility": "WCAG AA",
|
||||
"brand-constraints": ["legible"], "avoid-cliches": ["generic hero"],
|
||||
"representative-screen-requirement": "core-task",
|
||||
"tech-platform-constraints": ["web"]}, handle)
|
||||
try:
|
||||
SE.init_ledger("direction-too-early", plan="design-direction", parent_workflow="parent",
|
||||
product_decision="decision", direction_input_brief=brief0)
|
||||
early_blocked = False
|
||||
except ValueError:
|
||||
early_blocked = True
|
||||
check("design-direction blocked before foundation", early_blocked)
|
||||
try:
|
||||
SE.init_ledger("foundation-light", plan="experience-foundation", tier="light",
|
||||
parent_workflow="parent", product_decision="decision")
|
||||
light_blocked = False
|
||||
except ValueError:
|
||||
light_blocked = True
|
||||
check("required interactive-learning foundation cannot use light tier", light_blocked)
|
||||
|
||||
print("== benchmark -> strategy -> IA -> wireframes ==")
|
||||
SE.init_ledger("foundation", plan="experience-foundation", tier="standard",
|
||||
parent_workflow="parent", product_decision="decision")
|
||||
check("foundation parent gate", SE.complete_stage("foundation", "OPS-ORCH", to="experience-benchmark")[0])
|
||||
check("enter benchmark", SE.enter_stage("foundation", "experience-benchmark", actor="OPS-ORCH")[0])
|
||||
shots = []
|
||||
for index in range(10):
|
||||
path = os.path.join(WS, f"shot-{index}.bin")
|
||||
with open(path, "wb") as handle:
|
||||
handle.write(f"capture-{index}".encode())
|
||||
shots.append(path)
|
||||
refs = []
|
||||
for index in range(5):
|
||||
refs.append({
|
||||
"name": f"Reference {index}", "class": "direct" if index < 3 else "adjacent",
|
||||
"source-url": f"https://example.com/{index}", "captured-at": "2026-07-20T00:00:00Z",
|
||||
"screenshots": {
|
||||
"desktop": [{"path": shots[index * 2], "sha256": sha(shots[index * 2])}],
|
||||
"mobile": [{"path": shots[index * 2 + 1], "sha256": sha(shots[index * 2 + 1])}],
|
||||
},
|
||||
"flows": [{"name": "core", "steps": ["enter", "act"]}],
|
||||
"information-architecture": {"root": ["entry", "detail"]},
|
||||
"interaction-patterns": ["progressive disclosure"],
|
||||
"content-strategy": {"message": "value before proof"},
|
||||
"strengths": [{"claim": "clear task", "evidence-refs": [shots[index * 2]]}],
|
||||
"weaknesses": [{"claim": "weak recovery", "evidence-refs": [shots[index * 2 + 1]]}],
|
||||
})
|
||||
benchmark = report("foundation", "benchmark", "competitive-experience-benchmark",
|
||||
"experience-benchmark", "GTM-CI", {
|
||||
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
||||
"category": "learning product", "target-job": "understand a mechanism", "surface-archetype": "interactive-learning",
|
||||
"references": refs,
|
||||
"synthesis": {"table-stakes": ["clear value"], "adopt": ["proof"], "adapt": ["entry"],
|
||||
"avoid": ["generic hero"], "differentiation-opportunities": ["guided evaluation"],
|
||||
"unresolved-questions": []},
|
||||
"no-copy-attestation": True,
|
||||
})
|
||||
check("benchmark accepted", submit_accept("foundation", benchmark, "EXEC-CPO"))
|
||||
check("benchmark gate", SE.complete_stage("foundation", "OPS-ORCH", to="experience-strategy")[0])
|
||||
check("enter strategy", SE.enter_stage("foundation", "experience-strategy", actor="OPS-ORCH")[0])
|
||||
strategy = report("foundation", "strategy", "experience-strategy", "experience-strategy", "EXEC-CPO", {
|
||||
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
||||
"benchmark-ref": benchmark, "benchmark-sha256": sha(benchmark),
|
||||
"experience-thesis": "guided confidence", "target-users": ["buyer"],
|
||||
"jobs-to-be-done": ["evaluate"], "value-proposition": "decide with evidence",
|
||||
"differentiation": ["recovery-aware proof"], "message-hierarchy": ["value", "proof", "action"],
|
||||
"success-metrics": ["qualified action"], "decision": "proceed",
|
||||
})
|
||||
check("strategy accepted", submit_accept("foundation", strategy, "HUMAN-001"))
|
||||
check("strategy alone cannot pass feasibility gate",
|
||||
not SE.complete_stage("foundation", "OPS-ORCH", to="information-architecture")[0])
|
||||
technical = report("foundation", "technical-feasibility", "experience-technical-feasibility",
|
||||
"experience-strategy", "EXEC-CTO", {
|
||||
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
||||
"strategy-ref": strategy, "strategy-sha256": sha(strategy),
|
||||
"architecture-fit": {"simulation": "deterministic browser model"},
|
||||
"delivery-sustainability": {"content-unit": "versioned mechanism pack"},
|
||||
"constraints": ["no production diagnosis claim"], "risks": ["content engine drift"],
|
||||
"mitigations": ["versioned fixtures"], "verdict": "feasible",
|
||||
})
|
||||
check("technical feasibility accepted", submit_accept("foundation", technical, "HUMAN-001"))
|
||||
check("technical-only cannot replace COO feasibility",
|
||||
not SE.complete_stage("foundation", "OPS-ORCH", to="information-architecture")[0])
|
||||
operational = report("foundation", "operational-feasibility", "experience-operational-feasibility",
|
||||
"experience-strategy", "EXEC-COO", {
|
||||
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
||||
"strategy-ref": strategy, "strategy-sha256": sha(strategy),
|
||||
"operating-model": {"owner": "learning content operations"},
|
||||
"content-operations": {"release-unit": "mechanism deep dive"},
|
||||
"support-and-recovery": {"boundary": "guided simulation support"},
|
||||
"constraints": ["one validated mechanism at a time"], "risks": ["review queue"],
|
||||
"mitigations": ["candidate release cap"], "verdict": "feasible",
|
||||
})
|
||||
check("operational feasibility accepted", submit_accept("foundation", operational, "HUMAN-001"))
|
||||
check("strategy gate", SE.complete_stage("foundation", "OPS-ORCH", to="information-architecture")[0])
|
||||
check("enter IA", SE.enter_stage("foundation", "information-architecture", actor="OPS-ORCH")[0])
|
||||
blueprint = report("foundation", "blueprint", "experience-blueprint", "information-architecture", "DOC-IA", {
|
||||
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
||||
"benchmark-ref": benchmark, "benchmark-sha256": sha(benchmark),
|
||||
"strategy-ref": strategy, "strategy-sha256": sha(strategy),
|
||||
"experience-thesis": "guided confidence", "target-users": ["buyer"],
|
||||
"jobs-to-be-done": ["evaluate"], "value-proposition": "decide with evidence",
|
||||
"differentiation": ["recovery-aware proof"], "content-model": {"entities": ["claim", "proof"]},
|
||||
"page-inventory": ["entry", "proof", "action"], "navigation-model": {"primary": ["proof"]},
|
||||
"message-hierarchy": ["value", "proof", "action"],
|
||||
"task-flows": [{"id": "evaluate", "steps": ["entry", "proof", "action"]}],
|
||||
"state-matrix": {"default": {}, "loading": {}, "empty": {}, "error": {}, "partial": {}, "completed": {}},
|
||||
"responsive-priorities": ["value", "action", "proof"],
|
||||
"accessibility-intent": {"target": "WCAG AA"}, "success-metrics": ["qualified action"],
|
||||
})
|
||||
check("blueprint accepted", submit_accept("foundation", blueprint, "EXEC-CPO"))
|
||||
check("blueprint gate", SE.complete_stage("foundation", "OPS-ORCH", to="wireframes")[0])
|
||||
check("enter wireframes", SE.enter_stage("foundation", "wireframes", actor="OPS-ORCH")[0])
|
||||
wireframes = report("foundation", "wireframes", "wireframe-set", "wireframes", "DES-PROD", {
|
||||
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
||||
"blueprint-ref": blueprint, "blueprint-sha256": sha(blueprint),
|
||||
"screens": [{"id": "entry", "purpose": "explain value", "primary-action": "evaluate",
|
||||
"content-priority": ["value", "proof"], "desktop": {"regions": 3},
|
||||
"mobile": {"regions": 2}, "states": ["default", "loading", "error"]}],
|
||||
"validation": {"information-scent": "passed", "task-completion": "passed",
|
||||
"cognitive-load": "passed", "responsive-hierarchy": "passed"},
|
||||
"art-direction-deferred": True,
|
||||
})
|
||||
check("wireframes accepted", submit_accept("foundation", wireframes, "EXEC-CPO"))
|
||||
check("parent link registered", SE.register_experience_foundation("parent", "foundation"))
|
||||
check("wireframe terminal gate", SE.complete_stage("foundation", "OPS-ORCH", to="foundation-approved")[0])
|
||||
check("foundation enters terminal", SE.enter_stage("foundation", "foundation-approved", actor="OPS-ORCH")[0])
|
||||
check("parent exact foundation valid", SE._has_experience_foundation("parent", SE.read_ledger("parent")))
|
||||
approval = SE.read_ledger("parent")["experience-foundation-approval"]
|
||||
check("parent link binds technical and operational exact revisions",
|
||||
approval.get("technical-sha256") == sha(technical)
|
||||
and approval.get("operational-sha256") == sha(operational))
|
||||
|
||||
print("== direction exact refs and organization design adapters ==")
|
||||
brief = os.path.join(WS, "direction-input-brief.yaml")
|
||||
brief_body = {"product-goal": "ship", "core-users": ["buyer"], "core-tasks": ["evaluate"],
|
||||
"information-density": "medium", "required-accessibility": "WCAG AA",
|
||||
"brand-constraints": ["legible"], "avoid-cliches": ["generic hero"],
|
||||
"representative-screen-requirement": "core-task", "tech-platform-constraints": ["web"]}
|
||||
for approval_key, brief_key in (("benchmark", "competitive-experience-benchmark"),
|
||||
("blueprint", "experience-blueprint"), ("wireframe", "wireframe-set")):
|
||||
brief_body[f"{brief_key}-ref"] = approval[f"{approval_key}-ref"]
|
||||
brief_body[f"{brief_key}-sha256"] = approval[f"{approval_key}-sha256"]
|
||||
with open(brief, "w", encoding="utf-8") as handle:
|
||||
yaml.safe_dump(brief_body, handle, sort_keys=False)
|
||||
try:
|
||||
SE.init_ledger("direction", plan="design-direction", parent_workflow="parent",
|
||||
product_decision="decision", direction_input_brief=brief)
|
||||
direction_ok = True
|
||||
except ValueError:
|
||||
direction_ok = False
|
||||
check("design-direction accepts exact foundation refs", direction_ok)
|
||||
compiler = subprocess.run([sys.executable, os.path.join(HOOKS, "compile_design_system.py"), "--check"],
|
||||
cwd=ROOT, capture_output=True, text=True)
|
||||
check("organization design adapters are drift-free", compiler.returncode == 0)
|
||||
query = subprocess.run([sys.executable, os.path.join(HOOKS, "design_registry.py"),
|
||||
"--surface", "interactive-learning", "--state", "candidate"],
|
||||
cwd=ROOT, capture_output=True, text=True)
|
||||
query_body = json.loads(query.stdout) if query.returncode == 0 else {}
|
||||
check("registry query returns interactive-learning patterns and page archetypes",
|
||||
{"causal-learning-loop", "learning-feedback-recovery"}.issubset(
|
||||
{item.get("id") for item in query_body.get("patterns", [])})
|
||||
and {"interactive-learning-atlas", "interactive-learning-deep-dive"}.issubset(
|
||||
{item.get("id") for item in query_body.get("page-archetypes", [])}))
|
||||
|
||||
print("== DESIGN.md diff and local duplicate adherence ==")
|
||||
generated_design = os.path.join(ROOT, "org-os", "08-design", "generated", "DESIGN.md")
|
||||
design_diff = subprocess.run([sys.executable, os.path.join(HOOKS, "compile_design_system.py"),
|
||||
"--diff", generated_design],
|
||||
cwd=ROOT, capture_output=True, text=True)
|
||||
check("DESIGN.md exact canonical diff is clean", design_diff.returncode == 0)
|
||||
drift_design = os.path.join(WS, "drift-DESIGN.md")
|
||||
with open(generated_design, encoding="utf-8") as source, open(drift_design, "w", encoding="utf-8") as target:
|
||||
target.write(source.read() + "\nmanual drift\n")
|
||||
design_drift = subprocess.run([sys.executable, os.path.join(HOOKS, "compile_design_system.py"),
|
||||
"--diff", drift_design],
|
||||
cwd=ROOT, capture_output=True, text=True)
|
||||
check("DESIGN.md drift emits unified diff", design_drift.returncode == 1
|
||||
and "--- " in design_drift.stdout and "+++ " in design_drift.stdout)
|
||||
release = os.path.join(ROOT, "org-os", "08-design", "releases", "0.1.0.yaml")
|
||||
ui_report = os.path.join(WS, "ui-design.yaml")
|
||||
target_dir = os.path.join(WS, "design-target")
|
||||
os.makedirs(os.path.join(target_dir, "src", "components"), exist_ok=True)
|
||||
button_path = os.path.join(target_dir, "src", "components", "Button.tsx")
|
||||
with open(button_path, "w", encoding="utf-8") as handle:
|
||||
handle.write("export function Button(){ return null }\n")
|
||||
|
||||
|
||||
def write_ui_delta(components):
|
||||
with open(ui_report, "w", encoding="utf-8") as handle:
|
||||
yaml.safe_dump({"payload": {"design-system-bindings": [{
|
||||
"release-id": "org-design-0.1.0", "release-ref": release,
|
||||
"release-sha256": sha(release), "component-ids": ["button"],
|
||||
"delta": {"tokens": [], "components": components},
|
||||
}]}}, handle, sort_keys=False)
|
||||
|
||||
|
||||
write_ui_delta([])
|
||||
adherence = subprocess.run([sys.executable, os.path.join(HOOKS, "lint_design_system_adherence.py"),
|
||||
"--ui-report", ui_report, "--target", target_dir],
|
||||
cwd=ROOT, capture_output=True, text=True)
|
||||
check("organization component local reimplementation is rejected", adherence.returncode != 0
|
||||
and "organization component local duplicate" in adherence.stderr)
|
||||
write_ui_delta([{"id": "button", "reason": "project adapter boundary"}])
|
||||
adherence_delta = subprocess.run([sys.executable, os.path.join(HOOKS, "lint_design_system_adherence.py"),
|
||||
"--ui-report", ui_report, "--target", target_dir],
|
||||
cwd=ROOT, capture_output=True, text=True)
|
||||
check("declared project component delta is accepted", adherence_delta.returncode == 0)
|
||||
os.makedirs(os.path.join(target_dir, "src", "ui"), exist_ok=True)
|
||||
with open(os.path.join(target_dir, "src", "ui", "Button.jsx"), "w", encoding="utf-8") as handle:
|
||||
handle.write("export const Button = () => null\n")
|
||||
adherence_duplicate = subprocess.run([
|
||||
sys.executable, os.path.join(HOOKS, "lint_design_system_adherence.py"),
|
||||
"--ui-report", ui_report, "--target", target_dir],
|
||||
cwd=ROOT, capture_output=True, text=True)
|
||||
check("duplicate local component ids remain rejected despite delta", adherence_duplicate.returncode != 0
|
||||
and "local duplicate component" in adherence_duplicate.stderr)
|
||||
|
||||
print("== Hyeonworks controlled first-draft experiment contract ==")
|
||||
experiment_hook = os.path.join(HOOKS, "first_draft_experiment.py")
|
||||
planned_manifest = os.path.join(ROOT, "hyeonworks", "experiments", "experience-foundation-ab",
|
||||
"experiment.yaml")
|
||||
planned = subprocess.run([sys.executable, experiment_hook, "validate", planned_manifest],
|
||||
cwd=ROOT, capture_output=True, text=True)
|
||||
check("Hyeonworks planned experiment validates without claiming completion", planned.returncode == 0)
|
||||
planned_complete = subprocess.run([sys.executable, experiment_hook, "validate", planned_manifest,
|
||||
"--require-complete"],
|
||||
cwd=ROOT, capture_output=True, text=True)
|
||||
check("planned experiment cannot claim completed evidence", planned_complete.returncode != 0)
|
||||
|
||||
experiment_dir = os.path.join(WS, "completed-ab")
|
||||
os.makedirs(experiment_dir, exist_ok=True)
|
||||
request_path = os.path.join(experiment_dir, "request.md")
|
||||
with open(request_path, "w", encoding="utf-8") as handle:
|
||||
handle.write("same controlled request\n")
|
||||
input_kinds = ["competitive-experience-benchmark", "experience-blueprint", "wireframe-set",
|
||||
"design-md", "component-registry"]
|
||||
input_bindings = []
|
||||
for kind in input_kinds:
|
||||
path = os.path.join(experiment_dir, kind + ".yaml")
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
handle.write(f"kind: {kind}\n")
|
||||
input_bindings.append({"kind": kind, "ref": path, "sha256": sha(path)})
|
||||
outputs = {}
|
||||
evaluations = {}
|
||||
for arm_id, score, revisions, tokens in (("A", 2, 4, 12000), ("B", 5, 1, 7000)):
|
||||
output_path = os.path.join(experiment_dir, f"arm-{arm_id}.html")
|
||||
desktop = os.path.join(experiment_dir, f"arm-{arm_id}-desktop.png")
|
||||
mobile = os.path.join(experiment_dir, f"arm-{arm_id}-mobile.png")
|
||||
with open(output_path, "w", encoding="utf-8") as handle:
|
||||
handle.write(f"<main>arm {arm_id}</main>\n")
|
||||
for screenshot in (desktop, mobile):
|
||||
with open(screenshot, "wb") as handle:
|
||||
handle.write(b"\x89PNG\r\n\x1a\n" + (f"render-{arm_id}-" * 300).encode())
|
||||
output = {"ref": output_path, "sha256": sha(output_path)}
|
||||
outputs[arm_id] = output
|
||||
evaluation_path = os.path.join(experiment_dir, f"evaluation-{arm_id}.yaml")
|
||||
metrics = {key: score for key in (
|
||||
"product-purpose-understood-in-5s", "primary-action-found", "information-hierarchy",
|
||||
"category-quality-parity", "visual-craft", "mobile-clarity",
|
||||
"design-system-consistency", "human-preference")}
|
||||
metrics.update({"revision-count-to-acceptance": revisions, "tokens-to-acceptance": tokens})
|
||||
with open(evaluation_path, "w", encoding="utf-8") as handle:
|
||||
yaml.safe_dump({
|
||||
"experiment-id": "fixture-ab", "subject": "Hyeonworks", "arm-id": arm_id,
|
||||
"model-id": "same-model", "request-sha256": sha(request_path),
|
||||
"first-result-only": True, "revision-count-at-capture": 0,
|
||||
"output-ref": output_path, "output-sha256": sha(output_path),
|
||||
"screenshots": {
|
||||
"desktop": {"path": desktop, "sha256": sha(desktop)},
|
||||
"mobile": {"path": mobile, "sha256": sha(mobile)}},
|
||||
"evaluator-blinded": True, "metrics": metrics,
|
||||
}, handle, sort_keys=False)
|
||||
evaluations[arm_id] = {"ref": evaluation_path, "sha256": sha(evaluation_path)}
|
||||
completed_manifest = os.path.join(experiment_dir, "experiment.yaml")
|
||||
control = {"same-model": True, "same-request-sha256": True, "first-result-only": True,
|
||||
"generation-attempts-per-arm": 1, "revision-count-at-capture": 0,
|
||||
"evaluator-blinded": True}
|
||||
with open(completed_manifest, "w", encoding="utf-8") as handle:
|
||||
yaml.safe_dump({"first-draft-experiment": {
|
||||
"version": 1, "experiment-id": "fixture-ab", "subject": "Hyeonworks",
|
||||
"status": "completed", "model-id": "same-model",
|
||||
"request": {"ref": request_path, "sha256": sha(request_path)}, "control": control,
|
||||
"arms": {
|
||||
"A": {"treatment": "current-harness", "required-input-kinds": ["request"],
|
||||
"forbidden-input-kinds": input_kinds, "inputs": [],
|
||||
"output": outputs["A"], "evaluation": evaluations["A"]},
|
||||
"B": {"treatment": "experience-foundation", "required-input-kinds": input_kinds,
|
||||
"generation-scope": "representative-section-or-core-screen",
|
||||
"inputs": input_bindings, "output": outputs["B"], "evaluation": evaluations["B"]},
|
||||
}}}, handle, sort_keys=False)
|
||||
completed = subprocess.run([sys.executable, experiment_hook, "validate", completed_manifest,
|
||||
"--require-complete"], cwd=ROOT, capture_output=True, text=True)
|
||||
check("completed A/B requires exact model/request/input/output/evaluation evidence",
|
||||
completed.returncode == 0)
|
||||
comparison = subprocess.run([sys.executable, experiment_hook, "compare", completed_manifest],
|
||||
cwd=ROOT, capture_output=True, text=True)
|
||||
comparison_body = json.loads(comparison.stdout) if comparison.returncode == 0 else {}
|
||||
check("first-draft metrics comparison applies explicit decision rule",
|
||||
comparison.returncode == 0 and comparison_body.get("treatment-supported") is True)
|
||||
|
||||
print("== HUMAN none-of-the-above path ==")
|
||||
dset_path = os.path.join(WS, "direction-set.yaml")
|
||||
with open(dset_path, "w", encoding="utf-8") as handle:
|
||||
yaml.safe_dump({"directions": [{"id": "A"}, {"id": "B"}, {"id": "C"}]}, handle)
|
||||
none_path = os.path.join(WS, "none-selection.yaml")
|
||||
with open(none_path, "w", encoding="utf-8") as handle:
|
||||
yaml.safe_dump({
|
||||
"direction-set-ref": dset_path, "direction-set-sha256": sha(dset_path),
|
||||
"selection-decision": "none-of-the-above",
|
||||
"rejected-directions": [{"id": value, "reason": "benchmark-relative quality floor 미달"}
|
||||
for value in ("A", "B", "C")],
|
||||
"parent-workflow-id": "parent", "product-decision-id": "decision",
|
||||
"direction-input-brief-sha256": "0" * 64,
|
||||
"selection-acceptance-receipt": "human-receipt",
|
||||
}, handle)
|
||||
hard, _warn = LDD.lint_selected_direction(none_path, dset_path)
|
||||
check("none-of-the-above rejects all without averaging", not hard)
|
||||
contracts = SE.load_contracts()
|
||||
selected_contract = contracts["artifact-kinds"]["selected-direction"]
|
||||
check("selected-direction decision remains HUMAN-only", selected_contract.get("required-reviewer-roles") == ["HUMAN-001"])
|
||||
dd_transitions = contracts["workflows"]["design-direction"].get("additional-transitions", [])
|
||||
check("none decision routes back to discovery", any(
|
||||
item.get("from") == "design-direction-decision"
|
||||
and item.get("to") == "design-direction-discovery"
|
||||
and "none-of-the-above-recorded" in item.get("required-conditions", [])
|
||||
for item in dd_transitions))
|
||||
|
||||
print(f"\n{passed} passed, {failed} failed")
|
||||
shutil.rmtree(WS, ignore_errors=True)
|
||||
raise SystemExit(1 if failed else 0)
|
||||
Reference in New Issue
Block a user