"""schemaVersion 4 런이 command-pedagogy artifact/hash 계약을 빠뜨리지 않는지 본다.""" from __future__ import annotations import copy import hashlib import json import os import subprocess import sys import tempfile import unittest ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) SCRIPT = os.path.join(ROOT, "scripts", "verify-pipeline-run.py") sys.path.insert(0, os.path.join(ROOT, "scripts")) from command_pedagogy import analyze_commands # noqa: E402 STAGE_SPECS = { "S1": ("analyzing-codebase-for-tech-log", "ssot-analyst", True), "S2": ("deriving-tech-log-root-tree", "tree-deriver", True), "S3": ("writing-tech-log-records", "record-writer", False), "S4": ("technical-visualizer", "diagram-maker", True), "S5": ("rewriting-technical-prose-naturally", "prose-rewriter", False), "S6": ("writing-as-the-person-who-did-it", "voice-writer", False), "S7": ("publishing-tech-log-to-studio", "studio-validator", True), } def verify(path: str): p = subprocess.run( [sys.executable, SCRIPT, path], cwd=ROOT, capture_output=True, text=True ) return p.returncode, p.stdout + p.stderr def skill_echo(skill: str) -> str: path = os.path.join(ROOT, ".agents", "skills", skill, "SKILL.md") with open(path, encoding="utf-8") as fh: for line in fh: value = line.strip() if len(value) >= 30 and not value.startswith(("#", "---", "name:", "description:")): return value raise AssertionError(f"usable skill echo not found: {skill}") def stage(stage_id: str) -> dict: skill, agent, skippable = STAGE_SPECS[stage_id] base = { "id": stage_id, "name": stage_id, "skill": skill, "runBy": agent, "status": "SKIPPED" if skippable else "DONE", "skipReason": "synthetic fixture에서 생략" if skippable else "", "skillEcho": "" if skippable else skill_echo(skill), "skillRevision": None, "inputs": [], "outputs": [], "gates": [], "notes": "", } if stage_id == "S3": base["gates"] = [ {"cmd": "node check_body.mjs", "exit": 0}, {"cmd": "node check_prose.mjs", "exit": 0}, {"cmd": "node check_evidence.mjs keycloak --repo", "exit": 0}, ] elif stage_id == "S5": base["gates"] = [ {"cmd": "node check_prose.mjs", "exit": 0}, {"cmd": "node style_profile.mjs", "exit": 0}, {"cmd": "node check_body.mjs", "exit": 0}, {"cmd": "node check_evidence.mjs keycloak --repo", "exit": 0}, ] elif stage_id == "S6": base["gates"] = [ {"cmd": "node check_voice.mjs", "exit": 0}, {"cmd": "node check_prose.mjs", "exit": 0}, {"cmd": "node check_body.mjs", "exit": 0}, {"cmd": "node check_evidence.mjs keycloak --repo", "exit": 0}, ] return base def done_role(agent: str, verdict: str | None = None) -> dict: role = { "runBy": agent, "status": "DONE", "skipReason": "", "notes": "", } if agent.startswith("command-pedagogy-"): role.update( skill="writing-practitioner-guides", skillEcho=skill_echo("writing-practitioner-guides"), ) if verdict is not None: role["verdict"] = verdict return role def skipped_role(agent: str, reason: str) -> dict: role = done_role(agent) role["status"] = "SKIPPED" role["skipReason"] = reason role.pop("verdict", None) return role def good_reviews(*, shell_blocks: int = 1, findings: int = 0) -> dict: needs_edit = findings > 0 return { "commandPedagogy": { "initialAnalysis": { "cmd": "python3 scripts/check-command-pedagogy.py docs/keycloak/final/document.md", "exit": 0, "shellBlocks": shell_blocks, "findings": findings, "majorFindings": 0, }, "finalAnalysis": { "cmd": "python3 scripts/check-command-pedagogy.py docs/keycloak/final/document.md", "exit": 0, "shellBlocks": shell_blocks, "findings": 0, "majorFindings": 0, }, "planner": ( done_role("command-pedagogy-planner") if needs_edit else skipped_role("command-pedagogy-planner", "deterministic finding 없음") ), "editor": ( done_role("command-pedagogy-editor") if needs_edit else skipped_role("command-pedagogy-editor", "deterministic finding 없음") ), "reviewer": ( done_role("command-pedagogy-reviewer", "PASS") if shell_blocks else skipped_role("command-pedagogy-reviewer", "shell/CLI block 없음") ), }, "technicalEvidence": done_role("fact-reviewer", "PASS"), } def synthetic_run() -> dict: return { "schemaVersion": 4, "runId": "synthetic-command-pedagogy", "project": "keycloak", "record": "docs/keycloak/final/document.md", "startedAt": "2026-09-17T20:00:00+09:00", "finishedAt": "2026-09-17T20:30:00+09:00", "stages": [stage(sid) for sid in STAGE_SPECS], "qualityReviews": good_reviews(), } class Schema3ReviewContractTest(unittest.TestCase): def setUp(self): runs_dir = os.path.join(ROOT, "runs") os.makedirs(runs_dir, exist_ok=True) self.tmp = tempfile.TemporaryDirectory(prefix="test-command-pedagogy-", dir=runs_dir) self.addCleanup(self.tmp.cleanup) self.record_path = os.path.join(self.tmp.name, "record.md") with open(self.record_path, "w", encoding="utf-8") as fh: fh.write("실행 확인:\n\n```bash\nkubectl get pods\n```\n") self.record_rel = os.path.relpath(self.record_path, ROOT) self.base = synthetic_run() self.base["record"] = self.record_rel self.base["qualityReviews"] = self.make_reviews() def artifact(self, name: str, value: dict) -> dict: path = os.path.join(self.tmp.name, name) with open(path, "w", encoding="utf-8") as fh: json.dump(value, fh, ensure_ascii=False, indent=2) fh.write("\n") with open(path, "rb") as fh: digest = hashlib.sha256(fh.read()).hexdigest() return {"path": os.path.relpath(path, ROOT), "sha256": digest} def record_sha256(self) -> str: with open(self.record_path, "rb") as fh: return hashlib.sha256(fh.read()).hexdigest() def make_reviews(self, *, findings: int = 0) -> dict: with open(self.record_path, encoding="utf-8") as fh: text = fh.read() final_analysis = analyze_commands("synthetic-command-pedagogy", text, mode="operator") initial_analysis = copy.deepcopy(final_analysis) block = initial_analysis["blocks"][0] if initial_analysis["blocks"] else None if findings: initial_analysis["result"] = "WARN" initial_analysis["requires_editor"] = True initial_analysis["findings"] = [ { "block_id": block["id"], "code": f"synthetic-{idx}", "severity": "minor", "evidence": "fixture", "instruction": "fixture", } for idx in range(findings) ] initial_receipt = self.artifact("command-initial.json", initial_analysis) final_receipt = self.artifact("command-final.json", final_analysis) source_sha = self.record_sha256() if findings: plan = { "schema_version": "1.0", "section_id": "synthetic-command-pedagogy", "source_sha256": initial_analysis["source_sha256"], "mode": "operator", "command_groups": [ { "id": "inspect-pods", "block_id": block["id"], "source_sha256": block["source_sha256"], "goal": "파드 상태를 확인한다.", "execution_context": {"host": "local", "cwd": "."}, "prerequisites": [], "steps": [ { "command": "kubectl get pods", "reason": "현재 파드 목록을 본다.", "expected_result": "파드 목록이 출력된다.", } ], "cleanup": [], } ], } patch = { "schema_version": "1.0", "source_sha256": initial_analysis["source_sha256"], "patches": [], } planner = done_role("command-pedagogy-planner") planner["artifact"] = self.artifact("command-plan.json", plan) editor = done_role("command-pedagogy-editor") editor["artifact"] = self.artifact("command-patch-set.json", patch) else: planner = skipped_role("command-pedagogy-planner", "deterministic finding 없음") planner["artifact"] = None editor = skipped_role("command-pedagogy-editor", "deterministic finding 없음") editor["artifact"] = None if final_analysis["blocks"]: review = { "scope": "command-pedagogy", "reviewer": "command-pedagogy-reviewer", "verdict": "PASS", "source_sha256": source_sha, "findings": [], "notes": "", } reviewer = done_role("command-pedagogy-reviewer", "PASS") reviewer["sourceSha256"] = source_sha reviewer["artifact"] = self.artifact("command-review.json", review) else: reviewer = skipped_role("command-pedagogy-reviewer", "shell/CLI block 없음") reviewer["sourceSha256"] = source_sha reviewer["artifact"] = None technical = done_role("fact-reviewer", "PASS") technical["sourceSha256"] = source_sha return { "commandPedagogy": { "initialAnalysis": { "cmd": f"python3 scripts/check-command-pedagogy.py {self.record_rel} --mode operator", "exit": 0, "shellBlocks": len(initial_analysis["blocks"]), "findings": len(initial_analysis["findings"]), "majorFindings": sum(1 for f in initial_analysis["findings"] if f["severity"] == "major"), "artifact": initial_receipt, }, "finalAnalysis": { "cmd": f"python3 scripts/check-command-pedagogy.py {self.record_rel} --mode operator", "exit": 0, "shellBlocks": len(final_analysis["blocks"]), "findings": len(final_analysis["findings"]), "majorFindings": sum(1 for f in final_analysis["findings"] if f["severity"] == "major"), "artifact": final_receipt, }, "planner": planner, "editor": editor, "reviewer": reviewer, }, "technicalEvidence": technical, } def write(self, mutate=None): data = copy.deepcopy(self.base) if mutate: mutate(data) path = os.path.join(self.tmp.name, "run.json") with open(path, "w", encoding="utf-8") as fh: json.dump(data, fh, ensure_ascii=False, indent=2) return path def test_schema3_with_complete_reviews_passes(self): code, out = verify(self.write()) self.assertEqual(0, code, out) def test_schema3_legacy_quality_receipts_do_not_require_new_artifact_hash_fields(self): def mutate(data): data["schemaVersion"] = 3 command = data["qualityReviews"]["commandPedagogy"] command["initialAnalysis"].pop("artifact", None) command["finalAnalysis"].pop("artifact", None) for name in ("planner", "editor", "reviewer"): command[name].pop("artifact", None) command[name].pop("sourceSha256", None) data["qualityReviews"]["technicalEvidence"].pop("sourceSha256", None) code, out = verify(self.write(mutate)) self.assertEqual(0, code, out) def test_schema3_requires_quality_review_receipts(self): code, out = verify(self.write(lambda d: d.pop("qualityReviews"))) self.assertEqual(1, code) self.assertIn("품질 검토 원장이 없다", out) def test_findings_require_planner_and_editor(self): def mutate(data): data["qualityReviews"] = self.make_reviews(findings=2) data["qualityReviews"]["commandPedagogy"]["planner"] = skipped_role( "command-pedagogy-planner", "임의 생략" ) code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("명령 finding이 있는데 planner가 끝나지 않았다", out) def test_shell_blocks_require_independent_reviewer(self): def mutate(data): data["qualityReviews"]["commandPedagogy"]["reviewer"] = skipped_role( "command-pedagogy-reviewer", "임의 생략" ) code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("shell/CLI가 있는데 command reviewer가 끝나지 않았다", out) def test_uncertain_command_review_blocks_acceptance(self): def mutate(data): data["qualityReviews"]["commandPedagogy"]["reviewer"]["verdict"] = "UNCERTAIN" code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("command-pedagogy review가 통과하지 못했다", out) def test_final_major_finding_blocks_acceptance(self): def mutate(data): data["qualityReviews"]["commandPedagogy"]["finalAnalysis"]["majorFindings"] = 1 code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("major command finding이 남았다", out) def test_command_repair_cannot_remove_all_shell_blocks(self): def mutate(data): data["qualityReviews"]["commandPedagogy"]["finalAnalysis"]["shellBlocks"] = 0 data["qualityReviews"]["commandPedagogy"]["reviewer"] = skipped_role( "command-pedagogy-reviewer", "최종 shell block 없음" ) code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("command repair가 모든 shell block을 없앴다", out) def test_fact_review_runs_after_repairs_and_must_pass(self): def mutate(data): data["qualityReviews"]["technicalEvidence"]["verdict"] = "FAIL" code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("technical-evidence review가 통과하지 못했다", out) def test_command_free_record_skips_command_roles_but_keeps_fact_review(self): with open(self.record_path, "w", encoding="utf-8") as fh: fh.write("명령어가 없는 설명 문단이다.\n") self.base["qualityReviews"] = self.make_reviews() code, out = verify(self.write()) self.assertEqual(0, code, out) def test_initial_analysis_requires_path_and_sha_evidence(self): def mutate(data): data["qualityReviews"]["commandPedagogy"]["initialAnalysis"].pop("artifact") code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("command artifact 영수증이 없다", out) def test_artifact_sha_mismatch_blocks_acceptance(self): def mutate(data): data["qualityReviews"]["commandPedagogy"]["finalAnalysis"]["artifact"]["sha256"] = "0" * 64 code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("command artifact sha256이 실제 파일과 다르다", out) def test_plan_and_patch_artifacts_are_required_when_findings_exist(self): self.base["qualityReviews"] = self.make_reviews(findings=1) def mutate(data): data["qualityReviews"]["commandPedagogy"]["planner"].pop("artifact") data["qualityReviews"]["commandPedagogy"]["editor"].pop("artifact") code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("command role artifact 영수증이 없다", out) def test_command_review_is_bound_to_final_publication_hash(self): def mutate(data): data["qualityReviews"]["commandPedagogy"]["reviewer"]["sourceSha256"] = "0" * 64 code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("최종 publication hash와 다르다", out) def test_technical_evidence_review_is_bound_to_final_publication_hash(self): def mutate(data): data["qualityReviews"]["technicalEvidence"]["sourceSha256"] = "0" * 64 code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("최종 publication hash와 다르다", out) def test_command_review_artifact_is_required_for_command_content(self): def mutate(data): data["qualityReviews"]["commandPedagogy"]["reviewer"].pop("artifact") code, out = verify(self.write(mutate)) self.assertEqual(1, code) self.assertIn("command role artifact 영수증이 없다", out) class Schema3InitTest(unittest.TestCase): def test_init_uses_current_schema_and_prepares_review_receipts(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, "run.json") p = subprocess.run( [ sys.executable, SCRIPT, "--init", path, "--project", "keycloak", "--record", "docs/keycloak/final/document.md", ], cwd=ROOT, capture_output=True, text=True, ) self.assertEqual(0, p.returncode, p.stdout + p.stderr) with open(path, encoding="utf-8") as fh: data = json.load(fh) self.assertEqual(5, data["schemaVersion"]) self.assertIn("commandPedagogy", data["qualityReviews"]) self.assertIn("technicalEvidence", data["qualityReviews"]) command = data["qualityReviews"]["commandPedagogy"] self.assertIn("artifact", command["initialAnalysis"]) self.assertIn("artifact", command["finalAnalysis"]) self.assertIn("artifact", command["planner"]) self.assertIn("artifact", command["editor"]) self.assertIn("artifact", command["reviewer"]) self.assertIn("sourceSha256", command["reviewer"]) technical = data["qualityReviews"]["technicalEvidence"] self.assertIn("sourceSha256", technical) self.assertIn("liveSourceReconciliation", technical) self.assertIn("liveSourceReason", technical) self.assertIn("acceptedByProjectReview", technical) if __name__ == "__main__": unittest.main()