#!/usr/bin/env python3 """Subagent lifecycle enforcement tests (finding #2). Drives the real hooks end-to-end by piping representative SubagentStart -> SubagentStop JSON payloads at the scripts as subprocesses. Each case runs in its own fresh temp workspace (ORGOS_WORKSPACE) with seeded report/registry files, so cases never interfere. No pytest needed. Exit 0 = all pass. Cases (spec WP-2): - valid report -> pass(0) - registered agent, missing report -> block(2) - malformed YAML report -> block(2) - report path escaping workspace -> block(2) - two concurrent agents A & B, neither validated against the other's report * via per-agent expected_report_dir (priority 3) * via workflow/role filter on the recursive search (priority 4) - unregistered helper, no report -> allow(0) (don't over-block) - malformed hook JSON on stdin -> block(2) - --main: no report allow(0); invalid final report block(2) - subagent_register: malformed JSON never crashes; valid writes a registry line """ import json import os import shutil import subprocess import sys import tempfile import time ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) HOOKS = os.path.join(ROOT, ".claude", "hooks") PY = sys.executable passed, failed = 0, 0 _tmpdirs = [] def check(name, ok): global passed, failed if ok: passed += 1 print(f" PASS {name}") else: failed += 1 print(f" FAIL {name}") def new_ws(): d = tempfile.mkdtemp(prefix="orgos-lifecycle-") _tmpdirs.append(d) return d def run(script, args=None, stdin="", ws=None, extra_env=None): env = dict(os.environ) env["CLAUDE_PROJECT_DIR"] = ROOT if ws is not None: env["ORGOS_WORKSPACE"] = ws env.pop("CLAUDE_REPORT_PATH", None) if extra_env: env.update(extra_env) return subprocess.run( [PY, os.path.join(HOOKS, script)] + (args or []), input=stdin, capture_output=True, text=True, env=env, ) def start(ws, **payload): """Pipe a SubagentStart payload at subagent_register.py.""" return run("subagent_register.py", stdin=json.dumps(payload), ws=ws) def stop(ws, args=None, **payload): """Pipe a SubagentStop payload at stop_validate.py.""" return run("stop_validate.py", args=args, stdin=json.dumps(payload), ws=ws) def records_dir(ws): return os.path.join(ws, "completion-records") def seed_evidence(ws): """Create a real evidence artifact and return its absolute path.""" p = os.path.join(ws, "evidence.log") with open(p, "w", encoding="utf-8") as f: f.write("run: 12 passed\n") return p def valid_report(path, ws, role_id="PROD-PM"): """P0-3/P0-5: 유효한 report 는 이제 report-type + identity(report-id/workflow-id/role-id)를 반드시 담는다. workflow-id 는 상위 폴더명(records_dir//…)에서 유추한다. role-id 는 등록된 실역할(PROD-PM 기본). 소유 판정은 경로(폴더=워크플로, 파일명 prefix=rec.role)로 하므로 내부 concrete card에서 role-id가 추론된 경우 내부 producer 역할도 그 역할과 일치해야 한다.""" os.makedirs(os.path.dirname(path), exist_ok=True) ev = seed_evidence(ws) rid = os.path.basename(path)[: -len(".report.yaml")] wf = os.path.basename(os.path.dirname(path)) with open(path, "w", encoding="utf-8") as f: f.write( "report-type: work\n" f"report-id: {rid}\n" f"workflow-id: {wf}\n" # P3-B cutover: 이 lifecycle 테스트는 report 해소/바인딩/freshness 를 검증(계약 아님). # 기본 role(PROD-PM)이 활성화되면 standard-tier 는 method-execution 을 요구하므로, # 여기선 tier: light 로 두어 강제 면제(subprocess 가 tempdir 파일을 읽어 production # 컨텍스트가 되므로 TST-* 는 가드에 막힘 — light 면제가 올바른 경로). "tier: light\n" f"role-id: {role_id}\n" 'work-summary: "결과 요약."\n' "report-header:\n" ' bottom-line: "결론이 있다."\n' " decision-needed: { needed: false }\n" " confidence: { value: Med, derived-from: evidence }\n" " risks: []\n" " evidence:\n" f" - source-uri: {ev}\n" " grade: E3\n" ) return path def valid_report_with_created(path, ws, created_at, role_id="PROD-PM"): """A valid report that ALSO declares an explicit created-at header field. Simulates a PRE-MINTED report: the orchestrator mints the path/id + a created-at stamp BEFORE spawning, so created-at can predate the worker's own SubagentStart.""" valid_report(path, ws, role_id=role_id) with open(path, encoding="utf-8") as f: body = f.read() body = body.replace("report-type: work\n", f"report-type: work\ncreated-at: {created_at}\n", 1) with open(path, "w", encoding="utf-8") as f: f.write(body) return path def invalid_report(path): """Structurally valid YAML but violates the report contract (empty BLUF).""" os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf-8") as f: f.write( "report-header:\n" ' bottom-line: ""\n' " decision-needed: { needed: false }\n" " confidence: { value: Med }\n" " risks: []\n" " evidence: []\n" ) return path def malformed_report(path): """Unparseable YAML (unclosed flow mapping).""" os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf-8") as f: f.write("report-header: { bottom-line: 'x'\n") # missing closing brace return path def touch(path, ts): os.utime(path, (ts, ts)) # --------------------------------------------------------------------------- 1 print("== valid report -> pass(0) ==") ws = new_ws() edir = os.path.join(records_dir(ws), "wf-ok") valid_report(os.path.join(edir, "role-ok-20260101T000000Z.report.yaml"), ws) r = start(ws, agent_id="A-ok", agent_type="prod-pm", workflow_id="wf-ok", role="role-ok", expected_report_dir=edir) check("subagent_register accepts SubagentStart (exit 0)", r.returncode == 0) r = stop(ws, agent_id="A-ok", last_assistant_message="작업 완료.") check("valid report resolved via registry -> 0", r.returncode == 0) # --------------------------------------------------------------------------- 2 print("== registered agent, missing report -> block(2) ==") ws = new_ws() edir = os.path.join(records_dir(ws), "wf-missing") start(ws, agent_id="A-missing", agent_type="prod-pm", workflow_id="wf-missing", role="role-missing", expected_report_dir=edir) r = stop(ws, agent_id="A-missing", last_assistant_message="다 했어요(보고서는 안 씀).") check("report-producing agent w/ no report -> 2", r.returncode == 2) check("block reason mentions produced no report", "produced no report" in r.stderr) # --------------------------------------------------------------------------- 3 print("== malformed YAML report -> block(2) ==") ws = new_ws() edir = os.path.join(records_dir(ws), "wf-badyaml") malformed_report(os.path.join(edir, "role-y-20260101T000000Z.report.yaml")) start(ws, agent_id="A-yaml", agent_type="prod-pm", workflow_id="wf-badyaml", role="role-y", expected_report_dir=edir) r = stop(ws, agent_id="A-yaml", last_assistant_message="완료.") check("unparseable report -> 2", r.returncode == 2) check("block reason mentions YAML parse", "YAML" in r.stderr) # --------------------------------------------------------------------------- 4 # Out-of-workspace declared paths are SKIPPED (never bound, never escape-blocked). # Security is preserved by the missing-report fail-close; false positives from merely # quoting an existing external report path are eliminated (observed live: a Stop hook # blocked a session whose final message quoted a real report path under a scratch dir). print("== SECURITY: existing out-of-ws report + NO real in-ws report -> block(2) via missing report ==") ws = new_ws() outside_dir = new_ws() # a *different* temp dir, outside the workspace outside = os.path.join(outside_dir, "evil.report.yaml") valid_report(outside, outside_dir) # a real, VALID report — but OUTSIDE the workspace start(ws, agent_id="A-escape", agent_type="prod-pm", workflow_id="wf-esc", role="role-esc") r = stop(ws, agent_id="A-escape", last_assistant_message=f"보고서 경로: {outside}") # Agent cannot satisfy validation by pointing OUTSIDE the workspace: the outside path # is not bound, so a report-producing agent with no real in-ws report is fail-closed. check("existing out-of-ws report, no in-ws report -> 2 (cannot pass by pointing out)", r.returncode == 2) check("blocked for MISSING report, not escape", "produced no report" in r.stderr and "이탈" not in r.stderr) print("== NO FALSE POSITIVE: existing out-of-ws mention + real in-ws report -> 0 ==") ws = new_ws() edir = os.path.join(records_dir(ws), "wf-quote") valid_report(os.path.join(edir, "role-q-20260101T000000Z.report.yaml"), ws) outside_dir2 = new_ws() outside2 = os.path.join(outside_dir2, "peer.report.yaml") valid_report(outside2, outside_dir2) # a real report elsewhere the message quotes start(ws, agent_id="A-quote", agent_type="prod-pm", workflow_id="wf-quote", role="role-q", expected_report_dir=edir) r = stop(ws, agent_id="A-quote", last_assistant_message=f"다른 워크스페이스의 {outside2} 를 참고했고, 내 보고서는 작성 완료.") check("existing out-of-ws mention ignored, real in-ws report resolved -> 0", r.returncode == 0) check("resolved the real in-workspace report (not the quoted external one)", "role-q-2026" in r.stdout) print("== mere mention of NON-EXISTENT out-of-workspace path -> NOT blocked ==") # The core false-positive fix: an agent whose message/transcript merely mentions a # non-existent *.report.yaml string outside the workspace (e.g. it read a file # containing "lowrole.report.yaml") must NOT be blocked. Resolution must fall through # to the agent's REAL in-workspace report (Priority 3/4) and pass. ws = new_ws() edir = os.path.join(records_dir(ws), "wf-mention") valid_report(os.path.join(edir, "role-m-20260101T000000Z.report.yaml"), ws) ghost = os.path.join(tempfile.gettempdir(), "does-not-exist-lowrole.report.yaml") assert not os.path.exists(ghost) start(ws, agent_id="A-mention", agent_type="prod-pm", workflow_id="wf-mention", role="role-m", expected_report_dir=edir) r = stop(ws, agent_id="A-mention", last_assistant_message=f"참고로 {ghost} 라는 문자열을 파일에서 봤고, 작업은 끝냈습니다.") check("non-existent out-of-ws mention -> 0 (falls through to real report)", r.returncode == 0) check("resolved the real in-workspace report", "role-m-2026" in r.stdout) # --------------------------------------------------------------------------- 5 print("== concurrent A & B: neither validated against the other's report ==") # A valid (older), B invalid (NEWER). A naive 'newest report anywhere' resolver # would make A grab B's newer invalid report and wrongly block. Correct per-agent # binding: A -> A's valid (0), B -> B's invalid (2). ws = new_ws() a_dir = os.path.join(records_dir(ws), "wf-a") b_dir = os.path.join(records_dir(ws), "wf-b") a_rep = valid_report(os.path.join(a_dir, "role-a-20260101T000000Z.report.yaml"), ws) b_rep = invalid_report(os.path.join(b_dir, "role-b-20260101T000000Z.report.yaml")) now = time.time() touch(a_rep, now) # A older, but still fresh for this run touch(b_rep, now + 1) # B newer start(ws, agent_id="A", agent_type="prod-pm", workflow_id="wf-a", role="role-a", expected_report_dir=a_dir) start(ws, agent_id="B", agent_type="prod-pm", workflow_id="wf-b", role="role-b", expected_report_dir=b_dir) ra = stop(ws, agent_id="A", last_assistant_message="A done.") rb = stop(ws, agent_id="B", last_assistant_message="B done.") check("A stop validates A's VALID report -> 0 (not B's newer invalid)", ra.returncode == 0) check("B stop validates B's INVALID report -> 2", rb.returncode == 2) check("A did not resolve B's report path", b_rep not in ra.stdout and "role-b" not in ra.stdout) print("== concurrent isolation via priority-4 workflow/role filter ==") # Same race, but agents carry NO expected_report_dir -> resolution falls to the # recursive records_dir search, which MUST filter by workflow/role. We seed the # registry lines directly to force priority 4 (register would auto-fill a dir). ws = new_ws() c_dir = os.path.join(records_dir(ws), "wf-c") d_dir = os.path.join(records_dir(ws), "wf-d") c_rep = valid_report(os.path.join(c_dir, "role-c-20260101T000000Z.report.yaml"), ws) d_rep = invalid_report(os.path.join(d_dir, "role-d-20260101T000000Z.report.yaml")) touch(c_rep, time.time() - 100) # C older touch(d_rep, time.time()) # D newer distractor os.makedirs(os.path.join(ws, "state"), exist_ok=True) with open(os.path.join(ws, "state", "subagent-registry.jsonl"), "w", encoding="utf-8") as f: f.write(json.dumps({"agent_id": "C", "agent_type": "prod-pm", "workflow_id": "wf-c", "role": "role-c", "report_producing": True}) + "\n") f.write(json.dumps({"agent_id": "D", "agent_type": "prod-pm", "workflow_id": "wf-d", "role": "role-d", "report_producing": True}) + "\n") rc = stop(ws, agent_id="C", last_assistant_message="C done.") rd = stop(ws, agent_id="D", last_assistant_message="D done.") check("C -> its own VALID report via workflow filter -> 0 (not D's newer)", rc.returncode == 0) check("D -> its own INVALID report -> 2", rd.returncode == 2) # --------------------------------------------------------------------------- 6 print("== unregistered helper, no report -> allow(0) ==") ws = new_ws() r = stop(ws, agent_id="ghost-helper", last_assistant_message="읽기만 함.") check("unknown/never-registered agent, no report -> 0", r.returncode == 0) print("== registered NON-report-producing agent, no report -> allow(0) ==") ws = new_ws() start(ws, agent_id="helper-1", agent_type="explore", report_producing=False) r = stop(ws, agent_id="helper-1", last_assistant_message="탐색 결과만 반환.") check("registry says not report-producing -> 0", r.returncode == 0) print("== NON-report-producing helper in a POPULATED workspace -> allow(0) " "(regression: finding #2 fail-CLOSED false positive) ==") # A read-only helper (general-purpose/explore), registered report_producing=false, # stopping in a workspace that ALREADY holds an unrelated INVALID report from another # workflow MUST still exit 0. Before the fix, Priority-4 grabbed the newest report # anywhere (a rec with empty workflow+role match-alled every path) and fail-closed- # blocked the helper against a peer's report — the exact live block observed. ws = new_ws() invalid_report(os.path.join(records_dir(ws), "wf-other", "role-x-20260101T000000Z.report.yaml")) start(ws, agent_id="helper-pop", agent_type="general-purpose", report_producing=False) r = stop(ws, agent_id="helper-pop", last_assistant_message="감사 완료(읽기 전용).") check("non-report-producing helper in populated ws -> 0 (not bound to newest peer report)", r.returncode == 0) print("== helper/auditor that QUOTES an existing in-workspace report path -> allow(0) " "(regression: Priority-1 mention-binding must not fail-close a non-producer) ==") # The exact live block: an audit/review agent's final message MENTIONS an existing # *.report.yaml path. Priority 1 must NOT treat that mention as a self-declared report # and validate a peer's (invalid) report. A read-only helper (unregistered OR registered # report_producing=false) that quotes such a path must still stop cleanly. ws = new_ws() peer = invalid_report(os.path.join(records_dir(ws), "wf-peer", "role-p-20260101T000000Z.report.yaml")) quote = f"I reviewed {os.path.relpath(peer, ws)} and {peer} during the audit." # (a) unregistered auditor quoting the path r = stop(ws, agent_id="auditor-unreg", last_assistant_message=quote) check("unregistered auditor quoting an existing report path -> 0 (not bound to it)", r.returncode == 0) # (b) registered report_producing=false helper quoting the path start(ws, agent_id="auditor-reg", agent_type="general-purpose", report_producing=False) r = stop(ws, agent_id="auditor-reg", last_assistant_message=quote) check("registered non-producer quoting an existing report path -> 0 (not bound to it)", r.returncode == 0) # --------------------------------------------------------------------------- 7 print("== malformed hook JSON on stdin -> block(2) ==") ws = new_ws() r = run("stop_validate.py", stdin="{not json", ws=ws) check("malformed SubagentStop JSON -> 2", r.returncode == 2) check("block reason mentions malformed", "malformed" in r.stderr) # --------------------------------------------------------------------------- 8 print("== --main policy ==") ws = new_ws() # empty workspace, no reports r = run("stop_validate.py", args=["--main"], stdin="{}", ws=ws) check("--main with no report -> 0 (main may be read-only)", r.returncode == 0) ws = new_ws() invalid_report(os.path.join(records_dir(ws), "wf-final", "role-f-20260101T000000Z.report.yaml")) r = run("stop_validate.py", args=["--main"], stdin="{}", ws=ws) # --main is ADVISORY: an invalid final report must NOT block the main session # ("newest report anywhere" can't be bound to this session); it only warns. check("--main with INVALID final report -> 0 (advisory warn, not block)", r.returncode == 0) check("--main invalid final report emits advisory WARN", "WARN (advisory, --main)" in r.stderr) ws = new_ws() valid_report(os.path.join(records_dir(ws), "wf-final", "role-f-20260101T000000Z.report.yaml"), ws) r = run("stop_validate.py", args=["--main"], stdin="{}", ws=ws) check("--main with VALID final report -> 0", r.returncode == 0) # --------------------------------------------------------------------------- 9 print("== subagent_register robustness ==") ws = new_ws() r = run("subagent_register.py", stdin="{not json at all", ws=ws) check("register malformed JSON -> exit 0 (no crash)", r.returncode == 0) check("register malformed JSON logs to stderr", "malformed" in r.stderr) r = run("subagent_register.py", stdin=json.dumps({"agent_type": "prod-pm"}), ws=ws) check("register missing agent_id -> exit 0 (no crash)", r.returncode == 0) r = start(ws, agent_id="reg-1", agent_type="prod-pm", workflow_id="wf-reg", role="role-r") check("register valid -> exit 0", r.returncode == 0) reg_path = os.path.join(ws, "state", "subagent-registry.jsonl") check("registry file written (jsonl)", os.path.exists(reg_path)) line = [l for l in open(reg_path, encoding="utf-8").read().splitlines() if '"reg-1"' in l] rec = json.loads(line[0]) if line else {} check("registry record has C4 fields", rec.get("agent_id") == "reg-1" and rec.get("report_producing") is True and "started_at" in rec and rec.get("workflow_id") == "wf-reg") check("register auto-fills expected_report_dir from workflow", rec.get("expected_report_dir", "").endswith(os.path.join("completion-records", "wf-reg"))) start(ws, agent_id="reg-native-role", agent_type="prod-pm") native_lines = [json.loads(line) for line in open(reg_path, encoding="utf-8") if line.strip() and "reg-native-role" in line] check("native SubagentStart infers concrete role-id from agent card", native_lines and native_lines[-1].get("role") == "PROD-PM") # --------------------------------------------------------------------------- 10 # Native SubagentStart payload reality: Claude Code's native event provides only # agent_id + agent_type (NO workflow_id/role, and NO prompt), so the registry record # for a report-producing worker carries NO identity. Such an agent still declares its # OWN valid, fresh, in-workspace report in its final message — it MUST bind and pass. # Before the fix, owns_report()->matches_agent() returned False for a no-identity rec, # so the agent's own valid report was never bound and it fail-closed as "produced no # report" — an infinite Stop-block loop hitting EVERY Org OS worker. print("== no-identity registered producer binds its OWN declared report (native payload) ==") ws = new_ws() edir = os.path.join(records_dir(ws), "wf-native") rp = valid_report(os.path.join(edir, "prod-pm-customer.report.yaml"), ws) # native SubagentStart: agent_id + agent_type only (no workflow_id/role/prompt) start(ws, agent_id="A-native", agent_type="prod-pm") r = stop(ws, agent_id="A-native", last_assistant_message=f"report-path: {rp}") check("no-identity producer binds own declared valid report -> pass(0)", r.returncode == 0) # fail-closed preserved: same no-identity producer that declares NO report still blocks. ws2 = new_ws() start(ws2, agent_id="A-native-empty", agent_type="prod-pm") r = stop(ws2, agent_id="A-native-empty", last_assistant_message="done, no report path here") check("no-identity producer with NO report still blocks -> (2)", r.returncode == 2) # priority-4 stays closed: a no-identity producer that declares nothing must NOT be # bound to a PEER's report merely present under records_dir (leniency is priority-1 only). ws3 = new_ws() valid_report(os.path.join(records_dir(ws3), "wf-peer", "role-peer-20260101T000000Z.report.yaml"), ws3) start(ws3, agent_id="A-native-peer", agent_type="prod-pm") r = stop(ws3, agent_id="A-native-peer", last_assistant_message="finished") check("no-identity producer does NOT grab a peer report via priority-4 -> (2)", r.returncode == 2) # Same workflow, different concrete roles: a PROD-PM cannot declare or be assigned the # STR-ANALYST report merely because it is newer or in the shared workflow directory. print("== concrete-card identity blocks same-workflow cross-role substitution ==") ws4 = new_ws() shared = os.path.join(records_dir(ws4), "wf-shared") prod_report = valid_report(os.path.join(shared, "prod-pm-own.report.yaml"), ws4, role_id="PROD-PM") str_report = valid_report(os.path.join(shared, "str-analyst-peer.report.yaml"), ws4, role_id="STR-ANALYST") touch(prod_report, time.time() - 2) touch(str_report, time.time()) start(ws4, agent_id="A-prod-shared", agent_type="prod-pm", workflow_id="wf-shared", expected_report_dir=shared) r = stop(ws4, agent_id="A-prod-shared", last_assistant_message=f"peer report: {str_report}") check("priority-3 filters newer peer and resolves concrete role's own report", r.returncode == 0 and "prod-pm-own" in r.stdout and "str-analyst-peer" not in r.stdout) ws5 = new_ws() shared5 = os.path.join(records_dir(ws5), "wf-shared-only-peer") str_only = valid_report(os.path.join(shared5, "str-analyst-only.report.yaml"), ws5, role_id="STR-ANALYST") start(ws5, agent_id="A-prod-no-own", agent_type="prod-pm", workflow_id="wf-shared-only-peer", expected_report_dir=shared5) r = stop(ws5, agent_id="A-prod-no-own", last_assistant_message=f"report-path: {str_only}") check("concrete producer cannot satisfy stop with another role's report", r.returncode == 2 and "produced no report" in r.stderr) # --------------------------------------------------------------------------- 11 # P1 (pre-mint freshness skew): the orchestrator mints a report path + created-at stamp # BEFORE spawning the worker, then spends time compiling context packages, so the # worker's SubagentStart (started_at) lands AFTER the minted created-at. The worker # writes that pre-minted report THIS run (fresh mtime) but keeps the old created-at. # Freshness must key on the file's actual mtime (filesystem truth), NOT the self-declared # created-at field, or every pre-minted report fail-closes as "stale" and the worker must # re-emit it — the observed ~2x token blow-up across a 5-worker wave. print("== pre-minted created-at (old) + fresh mtime binds & passes; stale mtime still blocks ==") ws = new_ws() edir = os.path.join(records_dir(ws), "wf-premint") start(ws, agent_id="A-premint", agent_type="prod-pm") # native payload: no identity captured # worker writes its report AFTER start -> mtime is fresh; created-at is the old pre-mint stamp rp = valid_report_with_created( os.path.join(edir, "prod-pm-premint.report.yaml"), ws, created_at="20200101T000000Z") r = stop(ws, agent_id="A-premint", last_assistant_message=f"report-path: {rp}") check("pre-minted created-at (old) but fresh mtime -> pass(0) (no false 'stale')", r.returncode == 0) # security preserved: a GENUINELY stale report (old created-at AND old mtime = cross-run # reuse) MUST still be rejected, so a worker cannot escape fail-closed by re-declaring last # run's report. mtime-based freshness still catches this (the file was not written this run). ws2 = new_ws() edir2 = os.path.join(records_dir(ws2), "wf-stale") start(ws2, agent_id="A-stale", agent_type="prod-pm") rp2 = valid_report_with_created( os.path.join(edir2, "prod-pm-stale.report.yaml"), ws2, created_at="20200101T000000Z") touch(rp2, time.time() - 3600) # cross-run reuse: file last written an hour ago r = stop(ws2, agent_id="A-stale", last_assistant_message=f"report-path: {rp2}") check("genuinely stale report (old mtime) still blocked -> 2 (stale-reuse defence preserved)", r.returncode == 2) # --------------------------------------------------------------------------- cleanup for d in _tmpdirs: shutil.rmtree(d, ignore_errors=True) print(f"\n{passed} passed, {failed} failed") sys.exit(1 if failed else 0)