#!/usr/bin/env python3 from __future__ import annotations import json import re from pathlib import Path import yaml ROOT = Path(__file__).resolve().parents[1] REQUIRED = [ ROOT / "SKILL.md", ROOT / "README.md", ROOT / "references" / "decision-policy.md", ROOT / "references" / "evidence-and-source-policy.md", ROOT / "references" / "enterprise-blog-patterns.md", ROOT / "references" / "exceptions.md", ROOT / "references" / "output-modes.md", ROOT / "references" / "rule-catalog.md", ROOT / "references" / "source-basis.md", ROOT / "references" / "structure-patterns.md", ROOT / "references" / "titles-introductions-conclusions.md", ROOT / "profiles" / "default-formal.yaml", ROOT / "profiles" / "performance-case-study.yaml", ROOT / "profiles" / "architecture-decision.yaml", ROOT / "profiles" / "migration-case-study.yaml", ROOT / "profiles" / "incident-postmortem.yaml", ROOT / "profiles" / "tooling-adoption.yaml", ROOT / "profiles" / "conversational-tech.yaml", ROOT / "profiles" / "recruitment-tech-content.yaml", ROOT / "profiles" / "tutorial-lab.yaml", ROOT / "lexicons" / "vague-expressions.yaml", ROOT / "lexicons" / "formulaic-openings-and-closings.yaml", ROOT / "lexicons" / "product-names.example.yaml", ROOT / "lexicons" / "protected-identifiers.example.yaml", ROOT / "examples" / "revision-pairs.jsonl", ROOT / "examples" / "end-to-end-performance-case.md", ROOT / "tests" / "baseline-observations.md", ROOT / "tests" / "cases.json", ROOT / "tests" / "evaluation-rubric.md", ROOT / "tests" / "pressure-scenarios.md", ROOT / "tests" / "workflow.jsonl", ROOT / "schemas" / "article-brief.schema.json", ROOT / "schemas" / "article-result.schema.json", ROOT / "schemas" / "rubric.schema.json", ] def fail(message: str) -> None: print(f"FAIL: {message}") raise SystemExit(1) def parse_frontmatter(text: str) -> dict[str, str]: match = re.match(r"^---\n(.*?)\n---\n", text, re.S) if not match: fail("SKILL.md must begin with YAML frontmatter") try: data = yaml.safe_load(match.group(1)) except yaml.YAMLError as exc: fail(f"invalid SKILL.md frontmatter: {exc}") if not isinstance(data, dict): fail("frontmatter must be an object") for key in ("name", "description"): if not isinstance(data.get(key), str) or not data[key].strip(): fail(f"frontmatter is missing non-empty {key!r}") return {"name": data["name"].strip(), "description": data["description"].strip()} def read_jsonl(path: Path) -> list[dict]: records: list[dict] = [] for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): if not raw.strip(): continue try: value = json.loads(raw) except json.JSONDecodeError as exc: fail(f"invalid JSONL in {path.name}:{line_number}: {exc}") if not isinstance(value, dict): fail(f"JSONL record must be object in {path.name}:{line_number}") records.append(value) if not records: fail(f"JSONL file is empty: {path.name}") return records def main() -> None: missing = [str(path.relative_to(ROOT)) for path in REQUIRED if not path.exists()] if missing: fail("missing required files: " + ", ".join(missing)) skill_text = (ROOT / "SKILL.md").read_text(encoding="utf-8") frontmatter = parse_frontmatter(skill_text) name = frontmatter["name"] description = frontmatter["description"] if name != ROOT.name: fail(f"frontmatter name {name!r} must match directory {ROOT.name!r}") if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name): fail("name must use lowercase letters, numbers, and hyphens only") if len(name) > 64: fail("name exceeds 64 characters") if not description.startswith("Use when "): fail("description must start with 'Use when '") if len((name + description).encode("utf-8")) > 1024: fail("name + description exceeds 1024 bytes") words = len(skill_text.split()) if words > 500: fail(f"SKILL.md exceeds 500 words: {words}") if "cite" in skill_text or "filecite" in skill_text or re.search(r"turn\d+(?:view|search|file)\d+", skill_text): fail("runtime-specific citation markers must not appear in SKILL.md") for dependency in ("reducing-ai-like-korean-writing", "editing-korean-grammar-and-expression"): if dependency not in skill_text: fail(f"SKILL.md must declare required sub-skill {dependency}") catalog = (ROOT / "references" / "rule-catalog.md").read_text(encoding="utf-8") known_rules = set(re.findall(r"(?m)^###\s+([A-Z]+-\d{2})\s+—", catalog)) if len(known_rules) < 20: fail(f"rule catalog too small: {len(known_rules)}") cases = json.loads((ROOT / "tests" / "cases.json").read_text(encoding="utf-8")) if not isinstance(cases, list) or not cases: fail("tests/cases.json must be a non-empty array") required_keys = { "id", "category", "mode", "profile", "request", "source_material", "expected_status", "reference_output", "must_include", "must_not_include", "preserve_exact", "rule_ids", "manual_criteria", } allowed_categories = {"general", "hard", "regression"} allowed_status = {"pass", "needs_clarification", "blocked"} ids: set[str] = set() used_rules: set[str] = set() for index, case in enumerate(cases): if not isinstance(case, dict): fail(f"case #{index} must be an object") missing_keys = required_keys - set(case) if missing_keys: fail(f"case #{index} missing keys: {sorted(missing_keys)}") if case["id"] in ids: fail(f"duplicate case id: {case['id']}") ids.add(case["id"]) if case["category"] not in allowed_categories: fail(f"invalid category in {case['id']}") if case["expected_status"] not in allowed_status: fail(f"invalid expected_status in {case['id']}") if not isinstance(case["rule_ids"], list) or not case["rule_ids"]: fail(f"rule_ids must be a non-empty array in {case['id']}") unknown = set(case["rule_ids"]) - known_rules if unknown: fail(f"unknown rule IDs in {case['id']}: {sorted(unknown)}") used_rules.update(case["rule_ids"]) for key in ("must_include", "must_not_include", "preserve_exact", "manual_criteria"): if not isinstance(case[key], list): fail(f"{key} must be an array in {case['id']}") reference = case["reference_output"] for text in case["must_include"]: if text not in reference: fail(f"must_include missing from reference_output in {case['id']}: {text!r}") for text in case["must_not_include"]: if text in reference: fail(f"must_not_include present in reference_output in {case['id']}: {text!r}") for text in case["preserve_exact"]: if text not in case["source_material"] or text not in reference: fail(f"preserve_exact must exist in source and reference in {case['id']}: {text!r}") uncovered = known_rules - used_rules if uncovered: fail(f"rule IDs without test coverage: {sorted(uncovered)}") categories = {c: sum(1 for x in cases if x["category"] == c) for c in allowed_categories} if categories["general"] < 10 or categories["hard"] < 7 or categories["regression"] < 5: fail(f"insufficient test category counts: {categories}") profile_ids: set[str] = set() for path in (ROOT / "profiles").glob("*.yaml"): try: data = yaml.safe_load(path.read_text(encoding="utf-8")) except yaml.YAMLError as exc: fail(f"invalid YAML profile {path.name}: {exc}") if not isinstance(data, dict) or not isinstance(data.get("id"), str): fail(f"profile missing string id: {path.name}") if data["id"] in profile_ids: fail(f"duplicate profile id: {data['id']}") profile_ids.add(data["id"]) unknown_profiles = {case["profile"] for case in cases} - profile_ids if unknown_profiles: fail(f"cases reference unknown profiles: {sorted(unknown_profiles)}") for path in (ROOT / "lexicons").glob("*.yaml"): try: data = yaml.safe_load(path.read_text(encoding="utf-8")) except yaml.YAMLError as exc: fail(f"invalid YAML lexicon {path.name}: {exc}") if data is None: fail(f"empty YAML lexicon: {path.name}") for schema_name in ("article-brief.schema.json", "article-result.schema.json", "rubric.schema.json"): schema = json.loads((ROOT / "schemas" / schema_name).read_text(encoding="utf-8")) if schema.get("type") != "object" or not schema.get("required"): fail(f"invalid schema structure: {schema_name}") example_records = read_jsonl(ROOT / "examples" / "revision-pairs.jsonl") for record in example_records: unknown = set(record.get("rule_ids", [])) - known_rules if unknown: fail(f"unknown rule IDs in revision example {record.get('id')}: {sorted(unknown)}") workflow_records = read_jsonl(ROOT / "tests" / "workflow.jsonl") for record in workflow_records: unknown = set(record.get("rule_ids", [])) - known_rules if unknown: fail(f"unknown rule IDs in workflow case {record.get('id')}: {sorted(unknown)}") pressure_text = (ROOT / "tests" / "pressure-scenarios.md").read_text(encoding="utf-8") pressure_count = len(re.findall(r"(?m)^##\s+\d+\.", pressure_text)) if pressure_count < 8: fail(f"need at least 8 pressure scenarios, found {pressure_count}") print( f"PASS: Agent Skill structure valid; cases={len(cases)} " f"(general={categories['general']}, hard={categories['hard']}, regression={categories['regression']}); " f"workflow={len(workflow_records)}; rules={len(known_rules)}; profiles={len(profile_ids)}; " f"pressure_scenarios={pressure_count}; SKILL.md words={words}" ) if __name__ == "__main__": main()