document-haness/korean-technical-blog-skills-bundle-v1/reducing-ai-like-korean-writing/scripts/validate_skill.py

#!/usr/bin/env python3
from __future__ import annotations

import json
import re
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
REQUIRED = [
    ROOT / "SKILL.md",
    ROOT / "README.md",
    ROOT / "references" / "decision-policy.md",
    ROOT / "references" / "genre-profiles.md",
    ROOT / "references" / "output-modes.md",
    ROOT / "references" / "pattern-catalog.md",
    ROOT / "references" / "source-basis.md",
    ROOT / "tests" / "baseline-observations.md",
    ROOT / "tests" / "cases.json",
    ROOT / "tests" / "evaluation-rubric.md",
    ROOT / "tests" / "pressure-scenarios.md",
]


def fail(message: str) -> None:
    print(f"FAIL: {message}")
    raise SystemExit(1)


def parse_frontmatter(text: str) -> dict[str, str]:
    match = re.match(r"^---\n(.*?)\n---\n", text, re.S)
    if not match:
        fail("SKILL.md must begin with YAML frontmatter")
    block = match.group(1)
    result: dict[str, str] = {}
    for key in ("name", "description"):
        key_match = re.search(rf"(?m)^{key}:\s*(.+)$", block)
        if not key_match:
            fail(f"frontmatter is missing {key!r}")
        result[key] = key_match.group(1).strip().strip('"').strip("'")
    return result


def extract_protected(text: str) -> dict[str, list[str]]:
    return {
        "fenced_code": re.findall(r"```.*?```", text, re.S),
        "inline_code": re.findall(r"(?<!`)`[^`\n]+`(?!`)", text),
        "url": re.findall(r"https?://[^\s<>()]+", text),
        "numbers": re.findall(r"(?<![A-Za-z])\d+(?:\.\d+)?%?", text),
    }


def main() -> None:
    missing = [str(path.relative_to(ROOT)) for path in REQUIRED if not path.exists()]
    if missing:
        fail("missing required files: " + ", ".join(missing))

    skill_text = (ROOT / "SKILL.md").read_text(encoding="utf-8")
    frontmatter = parse_frontmatter(skill_text)
    name = frontmatter["name"]
    description = frontmatter["description"]

    if name != ROOT.name:
        fail(f"frontmatter name {name!r} must match directory {ROOT.name!r}")
    if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name):
        fail("name must use lowercase letters, numbers, and hyphens only")
    if len(name) > 64:
        fail("name exceeds 64 characters")
    if not description.startswith("Use when "):
        fail("description must start with 'Use when '")
    if len((name + description).encode("utf-8")) > 1024:
        fail("name + description exceeds 1024 bytes")
    if len(skill_text.split()) > 500:
        fail(f"SKILL.md exceeds 500 words: {len(skill_text.split())}")
    if "cite" in skill_text or "filecite" in skill_text or re.search(r"turn\d+(?:view|search|file)\d+", skill_text):
        fail("runtime-specific citation markers must not appear in SKILL.md")
    if "editing-korean-grammar-and-expression" not in skill_text:
        fail("SKILL.md must declare the final grammar-review sub-skill")

    catalog = (ROOT / "references" / "pattern-catalog.md").read_text(encoding="utf-8")
    known_patterns = set(re.findall(r"(?m)^###\s+(AIK-(?:[A-Z]+-)+\d{3})\b", catalog))
    if not known_patterns:
        fail("pattern catalog contains no AIK pattern headings")

    cases = json.loads((ROOT / "tests" / "cases.json").read_text(encoding="utf-8"))
    if not isinstance(cases, list) or not cases:
        fail("tests/cases.json must be a non-empty array")

    required_keys = {
        "id", "category", "input", "expected_action", "reference_text",
        "pattern_ids", "required_properties", "forbidden_changes", "explanation"
    }
    allowed_actions = {"rewrite", "keep", "suggest", "review"}
    ids: set[str] = set()
    used_patterns: set[str] = set()

    for index, case in enumerate(cases):
        if not isinstance(case, dict):
            fail(f"case #{index} must be an object")
        missing_keys = required_keys - set(case)
        if missing_keys:
            fail(f"case #{index} missing keys: {sorted(missing_keys)}")
        if case["id"] in ids:
            fail(f"duplicate case id: {case['id']}")
        ids.add(case["id"])
        if case["expected_action"] not in allowed_actions:
            fail(f"invalid expected_action in {case['id']}: {case['expected_action']}")
        if not isinstance(case["pattern_ids"], list):
            fail(f"pattern_ids must be an array in {case['id']}")
        unknown = set(case["pattern_ids"]) - known_patterns
        if unknown:
            fail(f"unknown pattern IDs in {case['id']}: {sorted(unknown)}")
        used_patterns.update(case["pattern_ids"])
        if case["expected_action"] == "keep" and case["reference_text"] != case["input"]:
            fail(f"keep case {case['id']} must preserve input exactly")
        if case["expected_action"] == "rewrite" and case["reference_text"] == case["input"]:
            fail(f"rewrite case {case['id']} must change reference_text")
        for key in ("required_properties", "forbidden_changes"):
            if not isinstance(case[key], list) or not case[key]:
                fail(f"{key} must be a non-empty array in {case['id']}")

        if case["expected_action"] in {"rewrite", "keep"}:
            before = extract_protected(case["input"])
            after = extract_protected(case["reference_text"])
            for kind in ("fenced_code", "inline_code", "url", "numbers"):
                if before[kind] and before[kind] != after[kind]:
                    fail(f"protected {kind} changed in {case['id']}: {before[kind]} -> {after[kind]}")

    uncovered = known_patterns - used_patterns
    if uncovered:
        fail(f"pattern IDs without test coverage: {sorted(uncovered)}")

    print(
        f"PASS: Agent Skill structure valid; {len(cases)} test cases; "
        f"{len(known_patterns)} pattern IDs; SKILL.md words={len(skill_text.split())}"
    )


if __name__ == "__main__":
    main()