566 lines
22 KiB
Python
566 lines
22 KiB
Python
"""명령어 교육성 분석기 회귀 시험.
|
|
|
|
정확한 명령을 금지하는 검사가 아니라, 사람이 따라 하기 어려운 압축 표현을
|
|
후속 planner/editor/reviewer가 볼 수 있게 결정론적으로 표시하는지 본다.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
sys.path.insert(0, os.path.join(ROOT, "scripts"))
|
|
|
|
from command_pedagogy import ( # noqa: E402
|
|
analyze_commands,
|
|
apply_command_patch_set,
|
|
validate_command_patch_set,
|
|
validate_command_plan,
|
|
)
|
|
|
|
|
|
class CommandPedagogyAnalyzerTest(unittest.TestCase):
|
|
def codes(self, text: str) -> set[str]:
|
|
result = analyze_commands("record", text)
|
|
return {finding["code"] for finding in result["findings"]}
|
|
|
|
def test_sed_in_place_and_command_substitution_are_signals(self):
|
|
text = '''```bash
|
|
sed -i \\
|
|
-e "s|__LAB_HOST_KEY__|$(cat ~/.ssh/id_ed25519.pub)|" \\
|
|
lab.yaml
|
|
```
|
|
'''
|
|
self.assertEqual(
|
|
{"in-place-text-rewrite", "command-substitution"},
|
|
self.codes(text),
|
|
)
|
|
|
|
def test_sed_print_mode_is_not_in_place_rewrite(self):
|
|
text = r'''```bash
|
|
sed -n '/^COPY public.databasechangelog /,/^\\\.$/p' dump.sql | wc -l
|
|
```
|
|
'''
|
|
self.assertNotIn("in-place-text-rewrite", self.codes(text))
|
|
|
|
def test_remote_compound_flow_exposes_ip_and_cleanup_chain(self):
|
|
text = '''```bash
|
|
ssh donghyeon@192.168.122.11 'cloud-init schema -c ~/kc-lab-2.yaml && rm ~/kc-lab-2.yaml'
|
|
```
|
|
'''
|
|
self.assertEqual(
|
|
{
|
|
"raw-ssh-ip",
|
|
"compound-remote-shell",
|
|
"cleanup-chained-with-verification",
|
|
"execution-context-implicit",
|
|
},
|
|
self.codes(text),
|
|
)
|
|
result = analyze_commands("record", text)
|
|
severity = {finding["code"]: finding["severity"] for finding in result["findings"]}
|
|
self.assertEqual("minor", severity["compound-remote-shell"])
|
|
self.assertEqual("major", severity["cleanup-chained-with-verification"])
|
|
|
|
def test_reference_mode_keeps_historical_cleanup_as_review_signal(self):
|
|
text = '''<!-- command-mode: reference -->
|
|
```bash
|
|
ssh host 'cloud-init schema -c ~/lab.yaml; rm -f ~/lab.yaml'
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text, mode="operator")
|
|
severity = {
|
|
finding["code"]: finding["severity"]
|
|
for finding in result["findings"]
|
|
}
|
|
self.assertEqual(
|
|
"minor",
|
|
severity["cleanup-chained-with-verification"],
|
|
)
|
|
self.assertEqual("reference", result["blocks"][0]["mode"])
|
|
|
|
def test_label_mode_marker_can_mark_reference_without_html_marker(self):
|
|
text = '''```bash label="[lab host] [reference] historical"
|
|
ssh host 'cloud-init schema -c ~/lab.yaml; rm -f ~/lab.yaml'
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text, mode="operator")
|
|
self.assertEqual("reference", result["blocks"][0]["mode"])
|
|
self.assertTrue(result["blocks"][0]["mode_explicit"])
|
|
severity = {
|
|
finding["code"]: finding["severity"]
|
|
for finding in result["findings"]
|
|
}
|
|
self.assertEqual(
|
|
"minor",
|
|
severity["cleanup-chained-with-verification"],
|
|
)
|
|
|
|
def test_printf_generated_file_and_substitution_are_signals(self):
|
|
text = '''```bash
|
|
printf 'instance-id: kc-lab-1-%s\\nlocal-hostname: kc-lab-1\\n' "$(date +%s)" > meta-kc-lab-1
|
|
```
|
|
'''
|
|
self.assertEqual(
|
|
{"opaque-file-generation", "command-substitution"},
|
|
self.codes(text),
|
|
)
|
|
|
|
def test_hidden_stderr_and_compressed_pipeline_are_signals(self):
|
|
text = '''```bash
|
|
openssl s_client -connect example.com:443 2>/dev/null | grep subject | head -1
|
|
```
|
|
'''
|
|
self.assertEqual(
|
|
{"hidden-stderr", "compressed-pipeline"},
|
|
self.codes(text),
|
|
)
|
|
|
|
def test_simple_operator_commands_are_not_globally_banned(self):
|
|
text = '''```bash
|
|
kubectl get pods
|
|
kubectl describe pod api-0
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text)
|
|
self.assertEqual("PASS", result["result"])
|
|
self.assertFalse(result["requires_editor"])
|
|
self.assertEqual(1, len(result["blocks"]))
|
|
self.assertEqual([], result["findings"])
|
|
|
|
def test_non_shell_fences_are_out_of_scope(self):
|
|
result = analyze_commands("record", '''```yaml
|
|
apiVersion: v1
|
|
kind: Pod
|
|
```
|
|
''')
|
|
self.assertEqual([], result["blocks"])
|
|
self.assertEqual([], result["findings"])
|
|
|
|
def test_block_metadata_is_stable_and_bounded(self):
|
|
text = '''앞 문장
|
|
|
|
```sh
|
|
echo hello
|
|
```
|
|
|
|
뒤 문장
|
|
'''
|
|
result = analyze_commands("record", text)
|
|
block = result["blocks"][0]
|
|
self.assertEqual("command-001", block["id"])
|
|
self.assertEqual("sh", block["language"])
|
|
self.assertEqual(text[block["start"]:block["end"]], block["source"])
|
|
self.assertEqual(64, len(block["source_sha256"]))
|
|
self.assertEqual(64, len(result["source_sha256"]))
|
|
|
|
def test_labeled_shell_fence_uses_first_info_token_as_language(self):
|
|
text = '''```bash label="[lab host] 파드 상태를 본다"
|
|
kubectl get pods
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text)
|
|
self.assertEqual(1, len(result["blocks"]))
|
|
block = result["blocks"][0]
|
|
self.assertEqual("bash", block["language"])
|
|
self.assertEqual('label="[lab host] 파드 상태를 본다"', block["info_string"])
|
|
self.assertEqual("[lab host] 파드 상태를 본다", block["label"])
|
|
|
|
def test_labeled_sh_fence_is_analyzed(self):
|
|
text = '''```sh label="[탐침 파드] 연결을 확인한다"
|
|
curl -fsS http://keycloak:8080/health/ready
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text)
|
|
self.assertEqual(1, len(result["blocks"]))
|
|
self.assertEqual("sh", result["blocks"][0]["language"])
|
|
|
|
def test_language_must_be_the_complete_first_info_token(self):
|
|
result = analyze_commands("record", '''```bashish label="[lab host]"
|
|
echo nope
|
|
```
|
|
''')
|
|
self.assertEqual([], result["blocks"])
|
|
|
|
def test_non_context_label_does_not_hide_remote_context_signal(self):
|
|
text = '''```bash label="[결과 확인] 원격 상태를 본다"
|
|
ssh test-server
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text)
|
|
codes = {finding["code"] for finding in result["findings"]}
|
|
self.assertIn("execution-context-implicit", codes)
|
|
|
|
def test_execution_context_in_label_suppresses_context_signal(self):
|
|
text = '''```bash label="[워크스테이션] test-server에 접속한다"
|
|
ssh test-server
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text)
|
|
codes = {finding["code"] for finding in result["findings"]}
|
|
self.assertNotIn("execution-context-implicit", codes)
|
|
|
|
|
|
def test_inline_command_mode_overrides_document_default(self):
|
|
text = '''일반 참고 문서다.
|
|
|
|
<!-- command-mode: operator -->
|
|
```bash
|
|
echo "$(date +%s)"
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text, mode="reference")
|
|
self.assertEqual("operator", result["blocks"][0]["mode"])
|
|
self.assertIn(
|
|
"command-substitution",
|
|
{finding["code"] for finding in result["findings"]},
|
|
)
|
|
|
|
def test_inline_command_mode_applies_only_to_the_next_block(self):
|
|
text = '''<!-- command-mode: operator -->
|
|
```bash
|
|
echo "$(date +%s)"
|
|
```
|
|
|
|
```bash
|
|
echo "$(date +%s)"
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text, mode="reference")
|
|
self.assertTrue(result["blocks"][0]["mode_explicit"])
|
|
self.assertEqual("operator", result["blocks"][0]["mode"])
|
|
self.assertFalse(result["blocks"][1]["mode_explicit"])
|
|
self.assertEqual("reference", result["blocks"][1]["mode"])
|
|
substitution_blocks = {
|
|
finding["block_id"]
|
|
for finding in result["findings"]
|
|
if finding["code"] == "command-substitution"
|
|
}
|
|
self.assertEqual({"command-001"}, substitution_blocks)
|
|
|
|
def test_text_fence_with_cli_flow_is_reported_as_extension(self):
|
|
text = '''```text
|
|
docker build . → docker save app | gzip → scp app.tar.gz test-server:/tmp/
|
|
→ kubectl set image deployment/app app=local/app:test
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text, mode="reference")
|
|
self.assertEqual([], result["blocks"])
|
|
self.assertEqual([], result["findings"])
|
|
self.assertEqual(
|
|
1,
|
|
len(result["extensions"]["command_like_text_blocks"]),
|
|
)
|
|
|
|
def test_explicit_reference_text_flow_is_not_reported_as_misfenced_command(self):
|
|
text = '''<!-- command-mode: reference -->
|
|
```text
|
|
docker build . → docker save app | gzip → scp app.tar.gz test-server:/tmp/
|
|
→ kubectl set image deployment/app app=local/app:test
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text, mode="reference")
|
|
self.assertEqual([], result["extensions"]["command_like_text_blocks"])
|
|
|
|
def test_cwd_change_followed_by_new_command_group_is_a_signal(self):
|
|
text = '''```bash
|
|
cd src
|
|
./gradlew build
|
|
|
|
# 다른 도구
|
|
python3 scripts/check.py
|
|
```
|
|
'''
|
|
result = analyze_commands("record", text, mode="operator")
|
|
self.assertIn(
|
|
"cwd-transition-crosses-command-group",
|
|
{finding["code"] for finding in result["findings"]},
|
|
)
|
|
|
|
|
|
class CommandPatchBoundaryTest(unittest.TestCase):
|
|
def test_patch_can_replace_only_the_named_command_span(self):
|
|
text = '''앞 문장은 그대로다.\n\n```bash\nprintf 'x=%s\\n' "$(date +%s)" > x.conf\n```\n\n뒤 문장도 그대로다.\n'''
|
|
analysis = analyze_commands("record", text)
|
|
block = analysis["blocks"][0]
|
|
replacement = '''```bash\nnano x.conf\n```\n```text\nx=20260917\n```'''
|
|
repaired = apply_command_patch_set(
|
|
text=text,
|
|
analysis=analysis,
|
|
patch_set={
|
|
"schema_version": "1.0",
|
|
"source_sha256": analysis["source_sha256"],
|
|
"patches": [{
|
|
"block_id": block["id"],
|
|
"expected_sha256": block["source_sha256"],
|
|
"replacement_markdown": replacement,
|
|
}],
|
|
},
|
|
)
|
|
self.assertTrue(repaired.startswith("앞 문장은 그대로다.\n\n"))
|
|
self.assertTrue(repaired.endswith("\n\n뒤 문장도 그대로다.\n"))
|
|
self.assertIn(replacement, repaired)
|
|
self.assertNotIn("printf", repaired)
|
|
|
|
def test_stale_block_hash_is_rejected(self):
|
|
text = '''```bash\nsed -i 's/a/b/' x.conf\n```\n'''
|
|
analysis = analyze_commands("record", text)
|
|
with self.assertRaisesRegex(ValueError, "expected block hash"):
|
|
apply_command_patch_set(
|
|
text=text,
|
|
analysis=analysis,
|
|
patch_set={
|
|
"schema_version": "1.0",
|
|
"source_sha256": analysis["source_sha256"],
|
|
"patches": [{
|
|
"block_id": "command-001",
|
|
"expected_sha256": "0" * 64,
|
|
"replacement_markdown": "```bash\\nnano x.conf\\n```",
|
|
}],
|
|
},
|
|
)
|
|
|
|
def test_patch_cannot_reference_an_unknown_block(self):
|
|
text = '''```bash\necho hello\n```\n'''
|
|
analysis = analyze_commands("record", text)
|
|
with self.assertRaisesRegex(ValueError, "unknown block"):
|
|
apply_command_patch_set(
|
|
text=text,
|
|
analysis=analysis,
|
|
patch_set={
|
|
"schema_version": "1.0",
|
|
"source_sha256": analysis["source_sha256"],
|
|
"patches": [{
|
|
"block_id": "command-999",
|
|
"expected_sha256": "0" * 64,
|
|
"replacement_markdown": "```bash\\necho world\\n```",
|
|
}],
|
|
},
|
|
)
|
|
|
|
|
|
class CommandPatchCliTest(unittest.TestCase):
|
|
def test_cli_applies_a_frozen_patch_set(self):
|
|
script = os.path.join(ROOT, "scripts", "apply-command-pedagogy-patch.py")
|
|
text = "앞\n\n```bash\nprintf 'x\\n' > x.conf\n```\n\n뒤\n"
|
|
analysis = analyze_commands("record", text)
|
|
block = analysis["blocks"][0]
|
|
patch_set = {
|
|
"schema_version": "1.0",
|
|
"source_sha256": analysis["source_sha256"],
|
|
"patches": [{
|
|
"block_id": block["id"],
|
|
"expected_sha256": block["source_sha256"],
|
|
"replacement_markdown": "```bash\nnano x.conf\n```",
|
|
}],
|
|
}
|
|
with tempfile.TemporaryDirectory() as d:
|
|
record = os.path.join(d, "record.md")
|
|
analysis_path = os.path.join(d, "analysis.json")
|
|
patch_path = os.path.join(d, "patch.json")
|
|
output = os.path.join(d, "repaired.md")
|
|
with open(record, "w", encoding="utf-8") as fh:
|
|
fh.write(text)
|
|
with open(analysis_path, "w", encoding="utf-8") as fh:
|
|
json.dump(analysis, fh, ensure_ascii=False)
|
|
with open(patch_path, "w", encoding="utf-8") as fh:
|
|
json.dump(patch_set, fh, ensure_ascii=False)
|
|
p = subprocess.run(
|
|
[sys.executable, script, record, analysis_path, patch_path, "-o", output],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
self.assertEqual(0, p.returncode, p.stdout + p.stderr)
|
|
with open(output, encoding="utf-8") as fh:
|
|
repaired = fh.read()
|
|
self.assertEqual("앞\n\n```bash\nnano x.conf\n```\n\n뒤\n", repaired)
|
|
|
|
|
|
class CommandPedagogyModeTest(unittest.TestCase):
|
|
def test_zsh_fence_is_analyzed(self):
|
|
result = analyze_commands("record", "```zsh\nssh user@192.168.1.10\n```\n")
|
|
self.assertEqual(1, len(result["blocks"]))
|
|
self.assertEqual("zsh", result["blocks"][0]["language"])
|
|
|
|
def test_nested_command_substitution_is_a_distinct_signal(self):
|
|
result = analyze_commands(
|
|
"record",
|
|
"```bash\necho \"$(printf %s \"$(date +%s)\")\"\n```\n",
|
|
)
|
|
codes = {finding["code"] for finding in result["findings"]}
|
|
self.assertIn("nested-command-substitution", codes)
|
|
|
|
def test_automation_mode_does_not_blanket_flag_compact_shell(self):
|
|
text = '''자동화 스크립트는 입력 파일을 갱신한 뒤 결과를 별도 검증한다.\n\n```bash\nsed -i -e "s|__KEY__|$(cat key.pub)|" config.yaml\ncat config.yaml | grep KEY | head -1\n```\n'''
|
|
operator = analyze_commands("record", text, mode="operator")
|
|
automation = analyze_commands("record", text, mode="automation")
|
|
self.assertTrue(operator["findings"])
|
|
self.assertEqual("automation", automation["mode"])
|
|
self.assertEqual([], automation["findings"])
|
|
|
|
def test_unknown_mode_is_rejected(self):
|
|
with self.assertRaisesRegex(ValueError, "unsupported command pedagogy mode"):
|
|
analyze_commands("record", "```bash\necho ok\n```\n", mode="unknown")
|
|
|
|
|
|
def test_remote_command_without_execution_context_is_a_signal(self):
|
|
result = analyze_commands("record", "```bash\nssh test-server\n```\n")
|
|
codes = {finding["code"] for finding in result["findings"]}
|
|
self.assertIn("execution-context-implicit", codes)
|
|
|
|
def test_explicit_execution_context_suppresses_context_signal(self):
|
|
text = "이 명령은 로컬 워크스테이션에서 실행한다.\n\n```bash\nssh test-server\n```\n"
|
|
result = analyze_commands("record", text)
|
|
codes = {finding["code"] for finding in result["findings"]}
|
|
self.assertNotIn("execution-context-implicit", codes)
|
|
|
|
def test_repeated_raw_ssh_ip_is_reported(self):
|
|
text = '''```bash\nssh user@192.168.122.11\n```\n\n```bash\nscp x user@192.168.122.11:~/x\n```\n'''
|
|
result = analyze_commands("record", text)
|
|
codes = [finding["code"] for finding in result["findings"]]
|
|
self.assertIn("repeated-raw-ssh-host", codes)
|
|
|
|
|
|
class CommandArtifactContractTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.text = "```bash\nssh user@192.168.1.10\n```\n"
|
|
self.analysis = analyze_commands("section-a", self.text)
|
|
self.block = self.analysis["blocks"][0]
|
|
|
|
def valid_plan(self):
|
|
return {
|
|
"schema_version": "1.0",
|
|
"section_id": "section-a",
|
|
"source_sha256": self.analysis["source_sha256"],
|
|
"mode": "operator",
|
|
"command_groups": [
|
|
{
|
|
"id": "connect-host",
|
|
"block_id": self.block["id"],
|
|
"source_sha256": self.block["source_sha256"],
|
|
"goal": "대상 호스트 연결을 확인한다.",
|
|
"execution_context": {"host": "local", "cwd": "."},
|
|
"prerequisites": ["SSH key가 준비되어 있다."],
|
|
"steps": [
|
|
{
|
|
"command": "ssh test-server",
|
|
"reason": "별칭으로 연결한다.",
|
|
"expected_result": "원격 셸이 열린다.",
|
|
}
|
|
],
|
|
"cleanup": [],
|
|
}
|
|
],
|
|
}
|
|
|
|
def test_valid_command_plan_passes(self):
|
|
validate_command_plan(self.valid_plan(), analysis=self.analysis)
|
|
|
|
def test_command_plan_requires_supported_mode(self):
|
|
plan = self.valid_plan()
|
|
plan["mode"] = "clever"
|
|
with self.assertRaisesRegex(ValueError, "mode"):
|
|
validate_command_plan(plan, analysis=self.analysis)
|
|
|
|
def test_command_plan_rejects_duplicate_group_ids(self):
|
|
plan = self.valid_plan()
|
|
plan["command_groups"].append(dict(plan["command_groups"][0]))
|
|
with self.assertRaisesRegex(ValueError, "duplicate command group id"):
|
|
validate_command_plan(plan, analysis=self.analysis)
|
|
|
|
def test_command_plan_requires_execution_context(self):
|
|
plan = self.valid_plan()
|
|
del plan["command_groups"][0]["execution_context"]
|
|
with self.assertRaisesRegex(ValueError, "execution_context"):
|
|
validate_command_plan(plan, analysis=self.analysis)
|
|
|
|
def test_patch_set_requires_schema_version_and_known_hashes(self):
|
|
patch = {
|
|
"schema_version": "1.0",
|
|
"source_sha256": self.analysis["source_sha256"],
|
|
"patches": [
|
|
{
|
|
"block_id": self.block["id"],
|
|
"expected_sha256": self.block["source_sha256"],
|
|
"replacement_markdown": "```bash\nssh test-server\n```",
|
|
}
|
|
],
|
|
}
|
|
validate_command_patch_set(patch, analysis=self.analysis)
|
|
bad = dict(patch)
|
|
bad.pop("schema_version")
|
|
with self.assertRaisesRegex(ValueError, "schema_version"):
|
|
validate_command_patch_set(bad, analysis=self.analysis)
|
|
|
|
|
|
class CommandArtifactCliContractTest(unittest.TestCase):
|
|
def test_analysis_cli_accepts_document_mode(self):
|
|
script = os.path.join(ROOT, "scripts", "check-command-pedagogy.py")
|
|
with tempfile.TemporaryDirectory() as d:
|
|
record = os.path.join(d, "record.md")
|
|
output = os.path.join(d, "analysis.json")
|
|
with open(record, "w", encoding="utf-8") as fh:
|
|
fh.write('```bash\nsed -i "s/a/$(cat value)/" config\n```\n')
|
|
p = subprocess.run(
|
|
[sys.executable, script, record, "--mode", "automation", "-o", output],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
self.assertEqual(0, p.returncode, p.stdout + p.stderr)
|
|
with open(output, encoding="utf-8") as fh:
|
|
result = json.load(fh)
|
|
self.assertEqual("automation", result["mode"])
|
|
self.assertEqual([], result["findings"])
|
|
|
|
def test_artifact_validator_cli_checks_plan_against_analysis(self):
|
|
validator = os.path.join(ROOT, "scripts", "validate-command-pedagogy-artifact.py")
|
|
text = "```bash\nssh user@192.168.1.10\n```\n"
|
|
analysis = analyze_commands("section-a", text)
|
|
block = analysis["blocks"][0]
|
|
plan = {
|
|
"schema_version": "1.0",
|
|
"section_id": "section-a",
|
|
"source_sha256": analysis["source_sha256"],
|
|
"mode": "operator",
|
|
"command_groups": [
|
|
{
|
|
"id": "connect",
|
|
"block_id": block["id"],
|
|
"source_sha256": block["source_sha256"],
|
|
"goal": "연결한다.",
|
|
"execution_context": {"host": "local", "cwd": "."},
|
|
"prerequisites": [],
|
|
"steps": [{
|
|
"command": "ssh test-server",
|
|
"reason": "연결 확인",
|
|
"expected_result": "원격 셸",
|
|
}],
|
|
"cleanup": [],
|
|
}
|
|
],
|
|
}
|
|
with tempfile.TemporaryDirectory() as d:
|
|
analysis_path = os.path.join(d, "analysis.json")
|
|
plan_path = os.path.join(d, "plan.json")
|
|
with open(analysis_path, "w", encoding="utf-8") as fh:
|
|
json.dump(analysis, fh)
|
|
with open(plan_path, "w", encoding="utf-8") as fh:
|
|
json.dump(plan, fh)
|
|
p = subprocess.run(
|
|
[sys.executable, validator, "plan", plan_path, "--analysis", analysis_path],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
self.assertEqual(0, p.returncode, p.stdout + p.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|