471 lines
19 KiB
Python
471 lines
19 KiB
Python
"""schemaVersion 4 런이 command-pedagogy artifact/hash 계약을 빠뜨리지 않는지 본다."""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
SCRIPT = os.path.join(ROOT, "scripts", "verify-pipeline-run.py")
|
|
sys.path.insert(0, os.path.join(ROOT, "scripts"))
|
|
from command_pedagogy import analyze_commands # noqa: E402
|
|
|
|
STAGE_SPECS = {
|
|
"S1": ("analyzing-codebase-for-tech-log", "ssot-analyst", True),
|
|
"S2": ("deriving-tech-log-root-tree", "tree-deriver", True),
|
|
"S3": ("writing-tech-log-records", "record-writer", False),
|
|
"S4": ("technical-visualizer", "diagram-maker", True),
|
|
"S5": ("rewriting-technical-prose-naturally", "prose-rewriter", False),
|
|
"S6": ("writing-as-the-person-who-did-it", "voice-writer", False),
|
|
"S7": ("publishing-tech-log-to-studio", "studio-validator", True),
|
|
}
|
|
|
|
|
|
def verify(path: str):
|
|
p = subprocess.run(
|
|
[sys.executable, SCRIPT, path], cwd=ROOT, capture_output=True, text=True
|
|
)
|
|
return p.returncode, p.stdout + p.stderr
|
|
|
|
|
|
def skill_echo(skill: str) -> str:
|
|
path = os.path.join(ROOT, ".agents", "skills", skill, "SKILL.md")
|
|
with open(path, encoding="utf-8") as fh:
|
|
for line in fh:
|
|
value = line.strip()
|
|
if len(value) >= 30 and not value.startswith(("#", "---", "name:", "description:")):
|
|
return value
|
|
raise AssertionError(f"usable skill echo not found: {skill}")
|
|
|
|
|
|
def stage(stage_id: str) -> dict:
|
|
skill, agent, skippable = STAGE_SPECS[stage_id]
|
|
base = {
|
|
"id": stage_id,
|
|
"name": stage_id,
|
|
"skill": skill,
|
|
"runBy": agent,
|
|
"status": "SKIPPED" if skippable else "DONE",
|
|
"skipReason": "synthetic fixture에서 생략" if skippable else "",
|
|
"skillEcho": "" if skippable else skill_echo(skill),
|
|
"skillRevision": None,
|
|
"inputs": [],
|
|
"outputs": [],
|
|
"gates": [],
|
|
"notes": "",
|
|
}
|
|
if stage_id == "S3":
|
|
base["gates"] = [
|
|
{"cmd": "node check_body.mjs", "exit": 0},
|
|
{"cmd": "node check_prose.mjs", "exit": 0},
|
|
{"cmd": "node check_evidence.mjs keycloak --repo", "exit": 0},
|
|
]
|
|
elif stage_id == "S5":
|
|
base["gates"] = [
|
|
{"cmd": "node check_prose.mjs", "exit": 0},
|
|
{"cmd": "node style_profile.mjs", "exit": 0},
|
|
{"cmd": "node check_body.mjs", "exit": 0},
|
|
{"cmd": "node check_evidence.mjs keycloak --repo", "exit": 0},
|
|
]
|
|
elif stage_id == "S6":
|
|
base["gates"] = [
|
|
{"cmd": "node check_voice.mjs", "exit": 0},
|
|
{"cmd": "node check_prose.mjs", "exit": 0},
|
|
{"cmd": "node check_body.mjs", "exit": 0},
|
|
{"cmd": "node check_evidence.mjs keycloak --repo", "exit": 0},
|
|
]
|
|
return base
|
|
|
|
|
|
def done_role(agent: str, verdict: str | None = None) -> dict:
|
|
role = {
|
|
"runBy": agent,
|
|
"status": "DONE",
|
|
"skipReason": "",
|
|
"notes": "",
|
|
}
|
|
if agent.startswith("command-pedagogy-"):
|
|
role.update(
|
|
skill="writing-practitioner-guides",
|
|
skillEcho=skill_echo("writing-practitioner-guides"),
|
|
)
|
|
if verdict is not None:
|
|
role["verdict"] = verdict
|
|
return role
|
|
|
|
|
|
def skipped_role(agent: str, reason: str) -> dict:
|
|
role = done_role(agent)
|
|
role["status"] = "SKIPPED"
|
|
role["skipReason"] = reason
|
|
role.pop("verdict", None)
|
|
return role
|
|
|
|
|
|
def good_reviews(*, shell_blocks: int = 1, findings: int = 0) -> dict:
|
|
needs_edit = findings > 0
|
|
return {
|
|
"commandPedagogy": {
|
|
"initialAnalysis": {
|
|
"cmd": "python3 scripts/check-command-pedagogy.py docs/keycloak/final/document.md",
|
|
"exit": 0,
|
|
"shellBlocks": shell_blocks,
|
|
"findings": findings,
|
|
"majorFindings": 0,
|
|
},
|
|
"finalAnalysis": {
|
|
"cmd": "python3 scripts/check-command-pedagogy.py docs/keycloak/final/document.md",
|
|
"exit": 0,
|
|
"shellBlocks": shell_blocks,
|
|
"findings": 0,
|
|
"majorFindings": 0,
|
|
},
|
|
"planner": (
|
|
done_role("command-pedagogy-planner")
|
|
if needs_edit
|
|
else skipped_role("command-pedagogy-planner", "deterministic finding 없음")
|
|
),
|
|
"editor": (
|
|
done_role("command-pedagogy-editor")
|
|
if needs_edit
|
|
else skipped_role("command-pedagogy-editor", "deterministic finding 없음")
|
|
),
|
|
"reviewer": (
|
|
done_role("command-pedagogy-reviewer", "PASS")
|
|
if shell_blocks
|
|
else skipped_role("command-pedagogy-reviewer", "shell/CLI block 없음")
|
|
),
|
|
},
|
|
"technicalEvidence": done_role("fact-reviewer", "PASS"),
|
|
}
|
|
|
|
|
|
def synthetic_run() -> dict:
|
|
return {
|
|
"schemaVersion": 4,
|
|
"runId": "synthetic-command-pedagogy",
|
|
"project": "keycloak",
|
|
"record": "docs/keycloak/final/document.md",
|
|
"startedAt": "2026-09-17T20:00:00+09:00",
|
|
"finishedAt": "2026-09-17T20:30:00+09:00",
|
|
"stages": [stage(sid) for sid in STAGE_SPECS],
|
|
"qualityReviews": good_reviews(),
|
|
}
|
|
|
|
|
|
class Schema3ReviewContractTest(unittest.TestCase):
|
|
def setUp(self):
|
|
runs_dir = os.path.join(ROOT, "runs")
|
|
os.makedirs(runs_dir, exist_ok=True)
|
|
self.tmp = tempfile.TemporaryDirectory(prefix="test-command-pedagogy-", dir=runs_dir)
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.record_path = os.path.join(self.tmp.name, "record.md")
|
|
with open(self.record_path, "w", encoding="utf-8") as fh:
|
|
fh.write("실행 확인:\n\n```bash\nkubectl get pods\n```\n")
|
|
self.record_rel = os.path.relpath(self.record_path, ROOT)
|
|
self.base = synthetic_run()
|
|
self.base["record"] = self.record_rel
|
|
self.base["qualityReviews"] = self.make_reviews()
|
|
|
|
def artifact(self, name: str, value: dict) -> dict:
|
|
path = os.path.join(self.tmp.name, name)
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
json.dump(value, fh, ensure_ascii=False, indent=2)
|
|
fh.write("\n")
|
|
with open(path, "rb") as fh:
|
|
digest = hashlib.sha256(fh.read()).hexdigest()
|
|
return {"path": os.path.relpath(path, ROOT), "sha256": digest}
|
|
|
|
def record_sha256(self) -> str:
|
|
with open(self.record_path, "rb") as fh:
|
|
return hashlib.sha256(fh.read()).hexdigest()
|
|
|
|
def make_reviews(self, *, findings: int = 0) -> dict:
|
|
with open(self.record_path, encoding="utf-8") as fh:
|
|
text = fh.read()
|
|
final_analysis = analyze_commands("synthetic-command-pedagogy", text, mode="operator")
|
|
initial_analysis = copy.deepcopy(final_analysis)
|
|
block = initial_analysis["blocks"][0] if initial_analysis["blocks"] else None
|
|
if findings:
|
|
initial_analysis["result"] = "WARN"
|
|
initial_analysis["requires_editor"] = True
|
|
initial_analysis["findings"] = [
|
|
{
|
|
"block_id": block["id"],
|
|
"code": f"synthetic-{idx}",
|
|
"severity": "minor",
|
|
"evidence": "fixture",
|
|
"instruction": "fixture",
|
|
}
|
|
for idx in range(findings)
|
|
]
|
|
initial_receipt = self.artifact("command-initial.json", initial_analysis)
|
|
final_receipt = self.artifact("command-final.json", final_analysis)
|
|
source_sha = self.record_sha256()
|
|
|
|
if findings:
|
|
plan = {
|
|
"schema_version": "1.0",
|
|
"section_id": "synthetic-command-pedagogy",
|
|
"source_sha256": initial_analysis["source_sha256"],
|
|
"mode": "operator",
|
|
"command_groups": [
|
|
{
|
|
"id": "inspect-pods",
|
|
"block_id": block["id"],
|
|
"source_sha256": block["source_sha256"],
|
|
"goal": "파드 상태를 확인한다.",
|
|
"execution_context": {"host": "local", "cwd": "."},
|
|
"prerequisites": [],
|
|
"steps": [
|
|
{
|
|
"command": "kubectl get pods",
|
|
"reason": "현재 파드 목록을 본다.",
|
|
"expected_result": "파드 목록이 출력된다.",
|
|
}
|
|
],
|
|
"cleanup": [],
|
|
}
|
|
],
|
|
}
|
|
patch = {
|
|
"schema_version": "1.0",
|
|
"source_sha256": initial_analysis["source_sha256"],
|
|
"patches": [],
|
|
}
|
|
planner = done_role("command-pedagogy-planner")
|
|
planner["artifact"] = self.artifact("command-plan.json", plan)
|
|
editor = done_role("command-pedagogy-editor")
|
|
editor["artifact"] = self.artifact("command-patch-set.json", patch)
|
|
else:
|
|
planner = skipped_role("command-pedagogy-planner", "deterministic finding 없음")
|
|
planner["artifact"] = None
|
|
editor = skipped_role("command-pedagogy-editor", "deterministic finding 없음")
|
|
editor["artifact"] = None
|
|
|
|
if final_analysis["blocks"]:
|
|
review = {
|
|
"scope": "command-pedagogy",
|
|
"reviewer": "command-pedagogy-reviewer",
|
|
"verdict": "PASS",
|
|
"source_sha256": source_sha,
|
|
"findings": [],
|
|
"notes": "",
|
|
}
|
|
reviewer = done_role("command-pedagogy-reviewer", "PASS")
|
|
reviewer["sourceSha256"] = source_sha
|
|
reviewer["artifact"] = self.artifact("command-review.json", review)
|
|
else:
|
|
reviewer = skipped_role("command-pedagogy-reviewer", "shell/CLI block 없음")
|
|
reviewer["sourceSha256"] = source_sha
|
|
reviewer["artifact"] = None
|
|
|
|
technical = done_role("fact-reviewer", "PASS")
|
|
technical["sourceSha256"] = source_sha
|
|
return {
|
|
"commandPedagogy": {
|
|
"initialAnalysis": {
|
|
"cmd": f"python3 scripts/check-command-pedagogy.py {self.record_rel} --mode operator",
|
|
"exit": 0,
|
|
"shellBlocks": len(initial_analysis["blocks"]),
|
|
"findings": len(initial_analysis["findings"]),
|
|
"majorFindings": sum(1 for f in initial_analysis["findings"] if f["severity"] == "major"),
|
|
"artifact": initial_receipt,
|
|
},
|
|
"finalAnalysis": {
|
|
"cmd": f"python3 scripts/check-command-pedagogy.py {self.record_rel} --mode operator",
|
|
"exit": 0,
|
|
"shellBlocks": len(final_analysis["blocks"]),
|
|
"findings": len(final_analysis["findings"]),
|
|
"majorFindings": sum(1 for f in final_analysis["findings"] if f["severity"] == "major"),
|
|
"artifact": final_receipt,
|
|
},
|
|
"planner": planner,
|
|
"editor": editor,
|
|
"reviewer": reviewer,
|
|
},
|
|
"technicalEvidence": technical,
|
|
}
|
|
|
|
def write(self, mutate=None):
|
|
data = copy.deepcopy(self.base)
|
|
if mutate:
|
|
mutate(data)
|
|
path = os.path.join(self.tmp.name, "run.json")
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
json.dump(data, fh, ensure_ascii=False, indent=2)
|
|
return path
|
|
|
|
def test_schema3_with_complete_reviews_passes(self):
|
|
code, out = verify(self.write())
|
|
self.assertEqual(0, code, out)
|
|
|
|
|
|
def test_schema3_legacy_quality_receipts_do_not_require_new_artifact_hash_fields(self):
|
|
def mutate(data):
|
|
data["schemaVersion"] = 3
|
|
command = data["qualityReviews"]["commandPedagogy"]
|
|
command["initialAnalysis"].pop("artifact", None)
|
|
command["finalAnalysis"].pop("artifact", None)
|
|
for name in ("planner", "editor", "reviewer"):
|
|
command[name].pop("artifact", None)
|
|
command[name].pop("sourceSha256", None)
|
|
data["qualityReviews"]["technicalEvidence"].pop("sourceSha256", None)
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(0, code, out)
|
|
|
|
def test_schema3_requires_quality_review_receipts(self):
|
|
code, out = verify(self.write(lambda d: d.pop("qualityReviews")))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("품질 검토 원장이 없다", out)
|
|
|
|
def test_findings_require_planner_and_editor(self):
|
|
def mutate(data):
|
|
data["qualityReviews"] = self.make_reviews(findings=2)
|
|
data["qualityReviews"]["commandPedagogy"]["planner"] = skipped_role(
|
|
"command-pedagogy-planner", "임의 생략"
|
|
)
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("명령 finding이 있는데 planner가 끝나지 않았다", out)
|
|
|
|
def test_shell_blocks_require_independent_reviewer(self):
|
|
def mutate(data):
|
|
data["qualityReviews"]["commandPedagogy"]["reviewer"] = skipped_role(
|
|
"command-pedagogy-reviewer", "임의 생략"
|
|
)
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("shell/CLI가 있는데 command reviewer가 끝나지 않았다", out)
|
|
|
|
def test_uncertain_command_review_blocks_acceptance(self):
|
|
def mutate(data):
|
|
data["qualityReviews"]["commandPedagogy"]["reviewer"]["verdict"] = "UNCERTAIN"
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("command-pedagogy review가 통과하지 못했다", out)
|
|
|
|
def test_final_major_finding_blocks_acceptance(self):
|
|
def mutate(data):
|
|
data["qualityReviews"]["commandPedagogy"]["finalAnalysis"]["majorFindings"] = 1
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("major command finding이 남았다", out)
|
|
|
|
def test_command_repair_cannot_remove_all_shell_blocks(self):
|
|
def mutate(data):
|
|
data["qualityReviews"]["commandPedagogy"]["finalAnalysis"]["shellBlocks"] = 0
|
|
data["qualityReviews"]["commandPedagogy"]["reviewer"] = skipped_role(
|
|
"command-pedagogy-reviewer", "최종 shell block 없음"
|
|
)
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("command repair가 모든 shell block을 없앴다", out)
|
|
|
|
def test_fact_review_runs_after_repairs_and_must_pass(self):
|
|
def mutate(data):
|
|
data["qualityReviews"]["technicalEvidence"]["verdict"] = "FAIL"
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("technical-evidence review가 통과하지 못했다", out)
|
|
|
|
def test_command_free_record_skips_command_roles_but_keeps_fact_review(self):
|
|
with open(self.record_path, "w", encoding="utf-8") as fh:
|
|
fh.write("명령어가 없는 설명 문단이다.\n")
|
|
self.base["qualityReviews"] = self.make_reviews()
|
|
code, out = verify(self.write())
|
|
self.assertEqual(0, code, out)
|
|
|
|
|
|
def test_initial_analysis_requires_path_and_sha_evidence(self):
|
|
def mutate(data):
|
|
data["qualityReviews"]["commandPedagogy"]["initialAnalysis"].pop("artifact")
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("command artifact 영수증이 없다", out)
|
|
|
|
def test_artifact_sha_mismatch_blocks_acceptance(self):
|
|
def mutate(data):
|
|
data["qualityReviews"]["commandPedagogy"]["finalAnalysis"]["artifact"]["sha256"] = "0" * 64
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("command artifact sha256이 실제 파일과 다르다", out)
|
|
|
|
def test_plan_and_patch_artifacts_are_required_when_findings_exist(self):
|
|
self.base["qualityReviews"] = self.make_reviews(findings=1)
|
|
def mutate(data):
|
|
data["qualityReviews"]["commandPedagogy"]["planner"].pop("artifact")
|
|
data["qualityReviews"]["commandPedagogy"]["editor"].pop("artifact")
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("command role artifact 영수증이 없다", out)
|
|
|
|
def test_command_review_is_bound_to_final_publication_hash(self):
|
|
def mutate(data):
|
|
data["qualityReviews"]["commandPedagogy"]["reviewer"]["sourceSha256"] = "0" * 64
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("최종 publication hash와 다르다", out)
|
|
|
|
def test_technical_evidence_review_is_bound_to_final_publication_hash(self):
|
|
def mutate(data):
|
|
data["qualityReviews"]["technicalEvidence"]["sourceSha256"] = "0" * 64
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("최종 publication hash와 다르다", out)
|
|
|
|
def test_command_review_artifact_is_required_for_command_content(self):
|
|
def mutate(data):
|
|
data["qualityReviews"]["commandPedagogy"]["reviewer"].pop("artifact")
|
|
code, out = verify(self.write(mutate))
|
|
self.assertEqual(1, code)
|
|
self.assertIn("command role artifact 영수증이 없다", out)
|
|
|
|
|
|
class Schema3InitTest(unittest.TestCase):
|
|
def test_init_uses_current_schema_and_prepares_review_receipts(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
path = os.path.join(d, "run.json")
|
|
p = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SCRIPT,
|
|
"--init",
|
|
path,
|
|
"--project",
|
|
"keycloak",
|
|
"--record",
|
|
"docs/keycloak/final/document.md",
|
|
],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
self.assertEqual(0, p.returncode, p.stdout + p.stderr)
|
|
with open(path, encoding="utf-8") as fh:
|
|
data = json.load(fh)
|
|
self.assertEqual(5, data["schemaVersion"])
|
|
self.assertIn("commandPedagogy", data["qualityReviews"])
|
|
self.assertIn("technicalEvidence", data["qualityReviews"])
|
|
command = data["qualityReviews"]["commandPedagogy"]
|
|
self.assertIn("artifact", command["initialAnalysis"])
|
|
self.assertIn("artifact", command["finalAnalysis"])
|
|
self.assertIn("artifact", command["planner"])
|
|
self.assertIn("artifact", command["editor"])
|
|
self.assertIn("artifact", command["reviewer"])
|
|
self.assertIn("sourceSha256", command["reviewer"])
|
|
technical = data["qualityReviews"]["technicalEvidence"]
|
|
self.assertIn("sourceSha256", technical)
|
|
self.assertIn("liveSourceReconciliation", technical)
|
|
self.assertIn("liveSourceReason", technical)
|
|
self.assertIn("acceptedByProjectReview", technical)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|