224 lines
13 KiB
Python
224 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""P1-E (#11) permission-boundary tests — standalone (no pytest). exit 0 = all pass.
|
|
|
|
Covers:
|
|
(A) guard_tools.py hardening — the concrete bypasses finding #11 flagged are now BLOCKED
|
|
(git -C . push, .env read via python/node/redirection, Bash redirection overwrite of an
|
|
existing .report.yaml, NotebookEdit on an existing report path, malformed hook JSON).
|
|
(B) normal dev commands STILL pass (exit 0) — the guard must not trap ordinary development.
|
|
(C) previously-blocked cases stay blocked (regression guard for test_enforcement's 9 asserts).
|
|
(D) .claude/settings.json `permissions` block is valid and denies/asks the key side-effects,
|
|
with the existing `hooks` block preserved.
|
|
"""
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
HOOKS = os.path.join(ROOT, ".claude", "hooks")
|
|
PY = sys.executable
|
|
|
|
passed, failed = 0, 0
|
|
|
|
|
|
def check(name, ok):
|
|
global passed, failed
|
|
if ok:
|
|
passed += 1
|
|
print(f" PASS {name}")
|
|
else:
|
|
failed += 1
|
|
print(f" FAIL {name}")
|
|
|
|
|
|
def raw(stdin):
|
|
"""Run guard_tools with an arbitrary raw stdin string; return exit code."""
|
|
e = dict(os.environ)
|
|
e["CLAUDE_PROJECT_DIR"] = ROOT
|
|
return subprocess.run(
|
|
[PY, os.path.join(HOOKS, "guard_tools.py")],
|
|
input=stdin, capture_output=True, text=True, env=e,
|
|
).returncode
|
|
|
|
|
|
def gt(tool, ti):
|
|
return raw(json.dumps({"tool_name": tool, "tool_input": ti}))
|
|
|
|
|
|
# scratch workspace with a REAL existing completion-record report (for immutable checks)
|
|
WS = tempfile.mkdtemp(prefix="p1e_")
|
|
REPDIR = os.path.join(WS, "completion-records", "wf-p1e")
|
|
os.makedirs(REPDIR, exist_ok=True)
|
|
EXIST_REPORT = os.path.join(REPDIR, "role-x-20260101T000000Z.report.yaml")
|
|
with open(EXIST_REPORT, "w") as f:
|
|
f.write("report-header: { bottom-line: t }\n")
|
|
NEW_REPORT = os.path.join(REPDIR, "role-y-20990101T000000Z.report.yaml") # does NOT exist
|
|
|
|
print("== (A) newly-closed bypasses are BLOCKED ==")
|
|
# git push with -C / other flag variants (old regex \bgit\s+push\b slipped past these)
|
|
check("git -C . push -> 2", gt("Bash", {"command": "git -C . push origin main"}) == 2)
|
|
check("git -c user.name=x push -> 2", gt("Bash", {"command": "git -c user.name=x push"}) == 2)
|
|
check("git --git-dir=/r push -> 2", gt("Bash", {"command": "git --git-dir=/r push"}) == 2)
|
|
check("cd sub && git push -> 2", gt("Bash", {"command": "cd sub && git push"}) == 2)
|
|
# secret read via python/node/ruby/env/xargs/redirection (old code only matched cat/less/head/tail/grep/cp/scp)
|
|
check("python -c open('.env') -> 2", gt("Bash", {"command": "python -c \"print(open('.env').read())\""}) == 2)
|
|
check("node fs .env -> 2", gt("Bash", {"command": "node -e \"require('fs').readFileSync('.env')\""}) == 2)
|
|
check("env cat .env (env prefix) -> 2", gt("Bash", {"command": "env FOO=1 cat .env"}) == 2)
|
|
check("find | xargs cat .env -> 2", gt("Bash", {"command": "find . -name .env | xargs cat"}) == 2)
|
|
check("read .env via redirection (< .env) -> 2", gt("Bash", {"command": "read x < .env"}) == 2)
|
|
# Bash redirection / tee / dd overwrite of an EXISTING immutable report
|
|
check("redirect > existing report -> 2", gt("Bash", {"command": f"echo x > {EXIST_REPORT}"}) == 2)
|
|
check("append >> existing report -> 2", gt("Bash", {"command": f"echo x >> {EXIST_REPORT}"}) == 2)
|
|
check("tee existing report -> 2", gt("Bash", {"command": f"echo x | tee {EXIST_REPORT}"}) == 2)
|
|
check("dd of= existing report -> 2", gt("Bash", {"command": f"dd if=/dev/null of={EXIST_REPORT}"}) == 2)
|
|
# NotebookEdit uses notebook_path (old code only read file_path -> bypass)
|
|
check("NotebookEdit on existing report (notebook_path) -> 2",
|
|
gt("NotebookEdit", {"notebook_path": EXIST_REPORT}) == 2)
|
|
# malformed / unparseable hook JSON -> fail-closed (block)
|
|
check("malformed JSON -> 2 (fail-closed)", raw("{not json") == 2)
|
|
check("empty stdin -> 2 (fail-closed)", raw("") == 2)
|
|
check("non-object JSON (array) -> 2 (fail-closed)", raw("[1,2,3]") == 2)
|
|
|
|
print("== (A2) finding #11 completion: language-level report write + Read/Grep/Glob secret ==")
|
|
# language-level write (python/node/shutil/pathlib) onto an EXISTING immutable report -> BLOCK
|
|
check("python open('w') existing report -> 2",
|
|
gt("Bash", {"command": f"python3 -c \"open('{EXIST_REPORT}','w').write('x')\""}) == 2)
|
|
check("python open('a') existing report -> 2",
|
|
gt("Bash", {"command": f"python3 -c \"open('{EXIST_REPORT}','a').write('x')\""}) == 2)
|
|
check("node writeFileSync existing report -> 2",
|
|
gt("Bash", {"command": f"node -e \"require('fs').writeFileSync('{EXIST_REPORT}','x')\""}) == 2)
|
|
check("shutil.move onto existing report -> 2",
|
|
gt("Bash", {"command": f"python3 -c \"import shutil; shutil.move('a','{EXIST_REPORT}')\""}) == 2)
|
|
check("pathlib write_text onto existing report -> 2",
|
|
gt("Bash", {"command": f"python3 -c \"from pathlib import Path; Path('{EXIST_REPORT}').write_text('x')\""}) == 2)
|
|
# reading a report is fine (must not false-block)
|
|
check("python open('r') report -> 0 (read allowed)",
|
|
gt("Bash", {"command": f"python3 -c \"print(open('{EXIST_REPORT}').read())\""}) == 0)
|
|
check("python open('w') NEW report path -> 0 (creating, not overwriting)",
|
|
gt("Bash", {"command": f"python3 -c \"open('{NEW_REPORT}','w').write('x')\""}) == 0)
|
|
# Read/Grep/Glob explicitly targeting secret/credential paths -> BLOCK (2차; native Read deny = 1차)
|
|
check("Grep path=/etc/shadow -> 2", gt("Grep", {"pattern": "x", "path": "/etc/shadow"}) == 2)
|
|
check("Glob **/.ssh/id_rsa -> 2", gt("Glob", {"pattern": "**/.ssh/id_rsa"}) == 2)
|
|
check("Read .aws/credentials -> 2", gt("Read", {"file_path": "~/.aws/credentials"}) == 2)
|
|
check("Grep glob=.env -> 2", gt("Grep", {"pattern": "KEY", "glob": ".env"}) == 2)
|
|
# Read/Grep/Glob on ordinary paths -> allow (no over-block)
|
|
check("Grep normal path=. -> 0", gt("Grep", {"pattern": "foo", "path": "."}) == 0)
|
|
check("Glob **/*.py -> 0", gt("Glob", {"pattern": "**/*.py"}) == 0)
|
|
check("Read README.md -> 0", gt("Read", {"file_path": "README.md"}) == 0)
|
|
check("Grep .env.example (template) -> 0", gt("Grep", {"pattern": "x", "glob": ".env.example"}) == 0)
|
|
|
|
print("== (B) normal dev commands STILL pass (exit 0) — no session trap ==")
|
|
check("git status -> 0", gt("Bash", {"command": "git status"}) == 0)
|
|
check("git diff HEAD -> 0", gt("Bash", {"command": "git diff HEAD"}) == 0)
|
|
check("git add -A -> 0", gt("Bash", {"command": "git add -A"}) == 0)
|
|
check("git commit -m 'fix push handler' -> 0 (push in message, subcmd=commit)",
|
|
gt("Bash", {"command": "git commit -m 'fix push handler'"}) == 0)
|
|
check("git log --oneline -> 0", gt("Bash", {"command": "git log --oneline -5"}) == 0)
|
|
check("python3 -V -> 0", gt("Bash", {"command": "python3 -V"}) == 0)
|
|
check("python3 -c import os -> 0", gt("Bash", {"command": "python3 -c 'import os,sys'"}) == 0)
|
|
check("ls -la -> 0", gt("Bash", {"command": "ls -la"}) == 0)
|
|
check("grep -r foo src -> 0", gt("Bash", {"command": "grep -rn foo src/"}) == 0)
|
|
check("npm test -> 0", gt("Bash", {"command": "npm test"}) == 0)
|
|
check("pytest -q -> 0", gt("Bash", {"command": "pytest -q"}) == 0)
|
|
check("source .venv/bin/activate -> 0 (venv, not .env)",
|
|
gt("Bash", {"command": "source .venv/bin/activate"}) == 0)
|
|
check("source ./env/bin/activate -> 0 (env dir, not .env file)",
|
|
gt("Bash", {"command": "source ./env/bin/activate"}) == 0)
|
|
check("cat .env.example -> 0 (template, not secret)", gt("Bash", {"command": "cat .env.example"}) == 0)
|
|
check("pip install python-dotenv -> 0 (no .env token)",
|
|
gt("Bash", {"command": "pip install python-dotenv"}) == 0)
|
|
check("echo client.environment -> 0 (no .env filename)",
|
|
gt("Bash", {"command": "echo client.environment"}) == 0)
|
|
check("redirect > NEW report path -> 0 (creating, not overwriting)",
|
|
gt("Bash", {"command": f"echo x > {NEW_REPORT}"}) == 0)
|
|
check("Read tool -> 0", gt("Read", {"file_path": "x"}) == 0)
|
|
check("Write normal file -> 0", gt("Write", {"file_path": "org-os/x.yaml"}) == 0)
|
|
check("Write NEW report path -> 0", gt("Write", {"file_path": NEW_REPORT}) == 0)
|
|
check("NotebookEdit NEW notebook path -> 0", gt("NotebookEdit", {"notebook_path": NEW_REPORT}) == 0)
|
|
|
|
print("== (C) previously-blocked cases stay blocked (test_enforcement regression parity) ==")
|
|
check("plain git push origin main -> 2", gt("Bash", {"command": "git push origin main"}) == 2)
|
|
check("gh pr create -> 2", gt("Bash", {"command": "gh pr create --title x"}) == 2)
|
|
check("cat .env -> 2", gt("Bash", {"command": "cat .env"}) == 2)
|
|
check("rm -rf build/ -> 2", gt("Bash", {"command": "rm -rf build/"}) == 2)
|
|
check("kubectl apply -> 2", gt("Bash", {"command": "kubectl apply -f d.yaml"}) == 2)
|
|
check("write config/.env -> 2", gt("Write", {"file_path": "config/.env"}) == 2)
|
|
check("Write existing report (immutable) -> 2", gt("Write", {"file_path": EXIST_REPORT}) == 2)
|
|
check("Edit existing report (immutable) -> 2", gt("Edit", {"file_path": EXIST_REPORT}) == 2)
|
|
|
|
print("== (C2) company-context.yaml SoT — Write/Edit/Bash direct writes blocked, committer CLI allowed ==")
|
|
COMPANY_CTX = "org-os/01-company/company-context.yaml"
|
|
check("Write company-context.yaml -> 2 (blocked)",
|
|
gt("Write", {"file_path": COMPANY_CTX, "content": "x"}) == 2)
|
|
check("Edit company-context.yaml -> 2 (blocked)",
|
|
gt("Edit", {"file_path": COMPANY_CTX}) == 2)
|
|
check("Bash redirect > company-context.yaml -> 2 (blocked)",
|
|
gt("Bash", {"command": f"echo x > {COMPANY_CTX}"}) == 2)
|
|
check("Bash tee company-context.yaml -> 2 (blocked)",
|
|
gt("Bash", {"command": f"echo x | tee {COMPANY_CTX}"}) == 2)
|
|
check("Bash commit_company_context.py invocation -> 0 (sanctioned writer allowed, NOT blocked)",
|
|
gt("Bash", {"command":
|
|
"python3 .claude/hooks/commit_company_context.py "
|
|
"--workflow w --candidate /tmp/c.yaml"}) == 0)
|
|
check("Write DIFFERENT company-context.yaml path -> 0 (suffix requires org-os/01-company/ prefix)",
|
|
gt("Write", {"file_path": "/tmp/foo/company-context.yaml", "content": "x"}) == 0)
|
|
|
|
print("== (D) settings.json permissions valid + denies key side-effects + hooks preserved ==")
|
|
with open(os.path.join(ROOT, ".claude", "settings.json")) as f:
|
|
cfg = json.load(f) # raises if invalid JSON
|
|
perms = cfg.get("permissions", {})
|
|
deny = perms.get("deny", [])
|
|
ask = perms.get("ask", [])
|
|
check("settings.json is valid JSON and has permissions block", isinstance(perms, dict) and bool(perms))
|
|
check("hooks block preserved", "hooks" in cfg and "PreToolUse" in cfg["hooks"])
|
|
check("PreToolUse still wires guard_tools.py",
|
|
any("guard_tools.py" in h.get("command", "")
|
|
for grp in cfg["hooks"]["PreToolUse"] for h in grp.get("hooks", [])))
|
|
check("PreToolUse matcher covers Read|Grep|Glob (finding #11 secret 2차)",
|
|
any(all(t in grp.get("matcher", "") for t in ("Read", "Grep", "Glob"))
|
|
for grp in cfg["hooks"]["PreToolUse"]))
|
|
check("deny secret read Read(./.env)", "Read(./.env)" in deny)
|
|
check("deny secret read Read(**/.env)", "Read(**/.env)" in deny)
|
|
check("deny credentials Read(**/.aws/credentials)", "Read(**/.aws/credentials)" in deny)
|
|
check("deny destructive Bash(rm -rf:*)", "Bash(rm -rf:*)" in deny)
|
|
check("ask remote push Bash(git push:*)", "Bash(git push:*)" in ask)
|
|
check("ask PR create Bash(gh pr create:*)", "Bash(gh pr create:*)" in ask)
|
|
check("ask deploy Bash(kubectl:*)", "Bash(kubectl:*)" in ask)
|
|
check("ask deploy Bash(terraform apply:*)", "Bash(terraform apply:*)" in ask)
|
|
check("push/PR/deploy are ask (human-approvable), NOT hard-denied",
|
|
not any(x in deny for x in ("Bash(git push:*)", "Bash(gh pr create:*)", "Bash(kubectl:*)")))
|
|
|
|
print("== (E) #11 deep: guard 의 side-effect deny 는 tool-permission-matrix SoT에서 온다 ==")
|
|
import importlib as _il2 # noqa: E402
|
|
import yaml as _yl # noqa: E402
|
|
sys.path.insert(0, HOOKS)
|
|
_G = _il2.import_module("guard_tools")
|
|
_mx = _yl.safe_load(open(os.path.join(ROOT, "org-os/00-role-registry/tool-permission-matrix.yaml")))
|
|
_se = _mx["tool-permission-matrix"]["default-policy"]["external-side-effects"]
|
|
_denied_in_yaml = {k for k, v in _se.items() if str(v).strip() == "denied"}
|
|
check("guard.DENIED_CATEGORIES == matrix default-deny (하드코딩 아님)",
|
|
set(_G.DENIED_CATEGORIES) == _denied_in_yaml)
|
|
check("structural cat(destructive/git-push)은 매트릭스와 무관하게 항상 강제",
|
|
_G._category_active("destructive") and _G._category_active("git-push"))
|
|
# 매트릭스에서 slack을 approval-required로 바꾸면 guard가 slack을 하드블록하지 않는다(SoT 반영).
|
|
_tmp_mx = tempfile.mktemp(suffix=".yaml")
|
|
_mx2 = json.loads(json.dumps(_mx))
|
|
_mx2["tool-permission-matrix"]["default-policy"]["external-side-effects"]["slack"] = "approval-required"
|
|
_yl.safe_dump(_mx2, open(_tmp_mx, "w"))
|
|
_orig = _G._MATRIX
|
|
try:
|
|
_G._MATRIX = _tmp_mx
|
|
_denied2 = _G._denied_categories()
|
|
check("매트릭스가 slack=approval-required면 denied 집합에서 빠진다(SoT-driven)",
|
|
"slack" not in _denied2 and "deploy" in _denied2)
|
|
finally:
|
|
_G._MATRIX = _orig
|
|
os.remove(_tmp_mx)
|
|
|
|
shutil.rmtree(WS, ignore_errors=True)
|
|
print(f"\n{passed} passed, {failed} failed")
|
|
sys.exit(1 if failed else 0)
|