418 lines
19 KiB
Python
418 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract policy-declared semantic surfaces without performing semantic judgment."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from dataclasses import dataclass
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
from typing import Any, Iterable, Mapping
|
|
|
|
from contract_markdown import as_list, parse_frontmatter
|
|
|
|
|
|
DEFAULT_ROOT = Path(__file__).resolve().parents[2]
|
|
DEFAULT_POLICY = Path("harness/source/document-semantic-surfaces.json")
|
|
RESULT_SCHEMA = "semantic-surface-extractor-result/v1"
|
|
POLICY_SCHEMA = "document-semantic-surfaces/v1"
|
|
SECTION_MARKER_RE = re.compile(r"^\s*<!--\s*section-id:\s*([a-z0-9][a-z0-9-]*)\s*-->\s*$")
|
|
HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
|
|
|
|
|
|
class SemanticSurfaceError(ValueError):
|
|
"""The policy or an input document cannot be interpreted safely."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Section:
|
|
section_id: str | None
|
|
heading: str
|
|
level: int
|
|
line_start: int
|
|
line_end: int
|
|
text: str
|
|
|
|
|
|
def canonical_json_bytes(value: Any) -> bytes:
|
|
return (json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + "\n").encode("utf-8")
|
|
|
|
|
|
def _safe_rel(value: Any, location: str) -> str:
|
|
if not isinstance(value, str) or not value or "\\" in value:
|
|
raise SemanticSurfaceError(f"{location}: expected a non-empty repo-relative POSIX path")
|
|
path = Path(value)
|
|
if path.is_absolute() or ".." in path.parts:
|
|
raise SemanticSurfaceError(f"{location}: path escapes repository")
|
|
return path.as_posix()
|
|
|
|
|
|
def load_policy(root: Path, policy_path: Path = DEFAULT_POLICY) -> Mapping[str, Any]:
|
|
source = policy_path if policy_path.is_absolute() else root / policy_path
|
|
try:
|
|
data = json.loads(source.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
raise SemanticSurfaceError(f"surface policy error: {exc}") from exc
|
|
if not isinstance(data, dict) or data.get("schema_version") != POLICY_SCHEMA:
|
|
raise SemanticSurfaceError(f"expected {POLICY_SCHEMA}")
|
|
if not isinstance(data.get("required_statuses"), list) or not all(isinstance(x, str) for x in data["required_statuses"]):
|
|
raise SemanticSurfaceError("required_statuses must be a string list")
|
|
if not isinstance(data.get("always_required_source_types"), list) or not all(
|
|
isinstance(x, str) for x in data["always_required_source_types"]
|
|
):
|
|
raise SemanticSurfaceError("always_required_source_types must be a string list")
|
|
documents = data.get("documents")
|
|
if not isinstance(documents, dict) or not documents:
|
|
raise SemanticSurfaceError("documents must be a non-empty object")
|
|
for source_type, config in documents.items():
|
|
if not isinstance(source_type, str) or not isinstance(config, dict):
|
|
raise SemanticSurfaceError("invalid document policy")
|
|
if config.get("mode") not in {"local", "hub"}:
|
|
raise SemanticSurfaceError(f"documents.{source_type}.mode must be local or hub")
|
|
roots = config.get("roots")
|
|
surfaces = config.get("required_surfaces")
|
|
if not isinstance(roots, list) or not roots:
|
|
raise SemanticSurfaceError(f"documents.{source_type}.roots must be non-empty")
|
|
for index, value in enumerate(roots):
|
|
_safe_rel(value, f"documents.{source_type}.roots[{index}]")
|
|
if not isinstance(surfaces, list) or not surfaces:
|
|
raise SemanticSurfaceError(f"documents.{source_type}.required_surfaces must be non-empty")
|
|
ids: set[str] = set()
|
|
for index, surface in enumerate(surfaces):
|
|
if not isinstance(surface, dict) or set(surface) not in (
|
|
{"section_id", "legacy_heading"},
|
|
{"section_id", "section_aliases", "legacy_heading"},
|
|
):
|
|
raise SemanticSurfaceError(f"documents.{source_type}.required_surfaces[{index}] has invalid fields")
|
|
section_id = surface.get("section_id")
|
|
pattern = surface.get("legacy_heading")
|
|
if not isinstance(section_id, str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]*", section_id):
|
|
raise SemanticSurfaceError(f"invalid semantic section id: {section_id!r}")
|
|
if section_id in ids:
|
|
raise SemanticSurfaceError(f"duplicate semantic section id: {section_id}")
|
|
ids.add(section_id)
|
|
aliases = surface.get("section_aliases", [])
|
|
if not isinstance(aliases, list) or any(
|
|
not isinstance(alias, str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]*", alias)
|
|
for alias in aliases
|
|
):
|
|
raise SemanticSurfaceError(f"invalid section_aliases for {section_id}")
|
|
try:
|
|
re.compile(str(pattern), re.IGNORECASE)
|
|
except re.error as exc:
|
|
raise SemanticSurfaceError(f"invalid legacy heading regex for {section_id}: {exc}") from exc
|
|
return data
|
|
|
|
|
|
def _sections(text: str) -> tuple[Section, ...]:
|
|
lines = text.splitlines()
|
|
headings: list[tuple[int, int, str, str | None]] = []
|
|
pending_id: str | None = None
|
|
for line_number, line in enumerate(lines, 1):
|
|
marker = SECTION_MARKER_RE.fullmatch(line)
|
|
if marker:
|
|
pending_id = marker.group(1)
|
|
continue
|
|
heading = HEADING_RE.match(line)
|
|
if heading:
|
|
headings.append((line_number, len(heading.group(1)), heading.group(2).strip(), pending_id))
|
|
pending_id = None
|
|
elif line.strip() and not line.lstrip().startswith("<!--"):
|
|
pending_id = None
|
|
result: list[Section] = []
|
|
for index, (start, level, heading, section_id) in enumerate(headings):
|
|
end = len(lines)
|
|
for next_start, next_level, _next_heading, _next_id in headings[index + 1 :]:
|
|
if next_level <= level:
|
|
end = next_start - 1
|
|
break
|
|
result.append(Section(section_id, heading, level, start, end, "\n".join(lines[start - 1 : end])))
|
|
return tuple(result)
|
|
|
|
|
|
def _exclusions(frontmatter: Mapping[str, object], field: str) -> tuple[dict[str, str], list[dict[str, Any]]]:
|
|
values: dict[str, str] = {}
|
|
findings: list[dict[str, Any]] = []
|
|
for raw in as_list(frontmatter.get(field)):
|
|
section_id, separator, reason = raw.partition("|")
|
|
section_id = section_id.strip()
|
|
reason = reason.strip()
|
|
if not separator or not section_id or not reason:
|
|
findings.append({
|
|
"code": "INVALID_SEMANTIC_SURFACE_EXCLUSION",
|
|
"path": "",
|
|
"line": 0,
|
|
"message": f"{field} entries must be '<section-id>|<reason>': {raw!r}",
|
|
"severity": "error",
|
|
})
|
|
continue
|
|
if section_id in values:
|
|
findings.append({
|
|
"code": "DUPLICATE_SEMANTIC_SURFACE_EXCLUSION",
|
|
"path": "",
|
|
"line": 0,
|
|
"message": f"duplicate exclusion: {section_id}",
|
|
"severity": "error",
|
|
})
|
|
continue
|
|
values[section_id] = reason
|
|
return values, findings
|
|
|
|
|
|
def is_required(frontmatter: Mapping[str, object], policy: Mapping[str, Any]) -> bool:
|
|
if str(frontmatter.get("source_type", "")) in set(policy.get("always_required_source_types", [])):
|
|
return True
|
|
explicit_field = str(policy.get("explicit_gate_field", "semantic_gate"))
|
|
explicit = str(frontmatter.get(explicit_field, "")).strip().casefold()
|
|
if explicit in {"required", "true", "yes"}:
|
|
return True
|
|
return str(frontmatter.get("status", "")) in set(policy["required_statuses"])
|
|
|
|
|
|
def _layout(root: Path) -> tuple[str, str]:
|
|
path = root / "harness/source/vault-layout.json"
|
|
if not path.is_file():
|
|
return "compatibility", "vault"
|
|
try:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
raise SemanticSurfaceError(f"vault layout source error: {exc}") from exc
|
|
mode = value.get("mode") if isinstance(value, dict) else None
|
|
vault_root = value.get("vault_root", "vault") if isinstance(value, dict) else "vault"
|
|
if mode not in {"compatibility", "shadow", "canonical"} or not isinstance(vault_root, str) or not vault_root:
|
|
raise SemanticSurfaceError("invalid vault layout semantic authority")
|
|
return mode, vault_root
|
|
|
|
|
|
def document_policy(
|
|
relative: str,
|
|
frontmatter: Mapping[str, object],
|
|
policy: Mapping[str, Any],
|
|
*,
|
|
root: Path | None = None,
|
|
) -> tuple[str, Mapping[str, Any]] | None:
|
|
source_type = str(frontmatter.get("source_type", ""))
|
|
config = policy["documents"].get(source_type)
|
|
if not isinstance(config, Mapping):
|
|
return None
|
|
mode, vault_root = _layout(root) if root is not None else ("compatibility", "vault")
|
|
if mode == "canonical":
|
|
parts = Path(relative).parts
|
|
categories = {Path(value).name for value in config["roots"]}
|
|
canonical_match = bool(parts) and parts[0] == vault_root and bool(categories.intersection(parts))
|
|
# Canonical cutover keeps legacy paths as read-only compatibility
|
|
# symlinks. Embedded semantic audits deliberately retain that stable
|
|
# logical subject, so policy coverage must recognize both spellings
|
|
# while document discovery continues to select only vault authority.
|
|
legacy_match = any(
|
|
relative == policy_root or relative.startswith(f"{policy_root}/")
|
|
for policy_root in config["roots"]
|
|
)
|
|
if not canonical_match and not legacy_match:
|
|
return None
|
|
elif not any(relative == policy_root or relative.startswith(f"{policy_root}/") for policy_root in config["roots"]):
|
|
return None
|
|
return source_type, config
|
|
|
|
|
|
def extract_document(root: Path, path: Path, policy: Mapping[str, Any]) -> dict[str, Any]:
|
|
root = root.resolve(strict=True)
|
|
# Preserve the caller's logical compatibility path while separately
|
|
# validating the resolved target. Canonical cutover symlinks may point to
|
|
# identical bytes under ``vault/`` without changing a certificate's
|
|
# stable subject identity.
|
|
path = Path(os.path.abspath(path))
|
|
try:
|
|
relative = path.relative_to(root).as_posix()
|
|
path.resolve(strict=True).relative_to(root)
|
|
except (ValueError, FileNotFoundError) as exc:
|
|
raise SemanticSurfaceError(f"document escapes repository: {path}") from exc
|
|
if not path.is_file():
|
|
raise SemanticSurfaceError(f"document does not exist: {path}")
|
|
text = path.read_text(encoding="utf-8")
|
|
frontmatter = parse_frontmatter(text)
|
|
matched = document_policy(relative, frontmatter, policy, root=root)
|
|
if matched is None:
|
|
raise SemanticSurfaceError(f"document is not covered by semantic policy: {relative}")
|
|
source_type, config = matched
|
|
sections = _sections(text)
|
|
exclusions, findings = _exclusions(frontmatter, str(policy.get("exclusion_field", "semantic_surface_exclusions")))
|
|
for item in findings:
|
|
item["path"] = relative
|
|
required_ids = {str(item["section_id"]) for item in config["required_surfaces"]}
|
|
for unknown in sorted(set(exclusions) - required_ids):
|
|
findings.append({
|
|
"code": "UNKNOWN_SEMANTIC_SURFACE_EXCLUSION",
|
|
"path": relative,
|
|
"line": 0,
|
|
"message": f"exclusion does not name a required surface: {unknown}",
|
|
"severity": "error",
|
|
})
|
|
surfaces: list[dict[str, Any]] = []
|
|
excluded: list[dict[str, str]] = []
|
|
for requirement in config["required_surfaces"]:
|
|
section_id = str(requirement["section_id"])
|
|
accepted_ids = {section_id, *map(str, requirement.get("section_aliases", []))}
|
|
exact = [section for section in sections if section.section_id in accepted_ids]
|
|
legacy = [
|
|
section for section in sections
|
|
if section.section_id is None and re.search(str(requirement["legacy_heading"]), section.heading, re.IGNORECASE)
|
|
]
|
|
matches = exact if exact else legacy
|
|
if len(matches) > 1:
|
|
findings.append({
|
|
"code": "AMBIGUOUS_SEMANTIC_SURFACE",
|
|
"path": relative,
|
|
"line": matches[0].line_start,
|
|
"message": f"multiple sections match required surface {section_id}",
|
|
"severity": "error",
|
|
})
|
|
continue
|
|
if not matches:
|
|
if section_id in exclusions:
|
|
excluded.append({"section_id": section_id, "reason": exclusions[section_id]})
|
|
else:
|
|
findings.append({
|
|
"code": "SEMANTIC_SURFACE_UNCOVERED",
|
|
"path": relative,
|
|
"line": 0,
|
|
"message": f"required surface is neither extracted nor explicitly excluded: {section_id}",
|
|
"severity": "error",
|
|
})
|
|
continue
|
|
section = matches[0]
|
|
if not exact:
|
|
findings.append({
|
|
"code": "LEGACY_SEMANTIC_SURFACE",
|
|
"path": relative,
|
|
"line": section.line_start,
|
|
"message": f"legacy heading fallback matched {section_id}: {section.heading}",
|
|
"severity": "warning",
|
|
})
|
|
surface_key = f"{relative}\0{section_id}\0{section.line_start}\0{section.line_end}".encode("utf-8")
|
|
surfaces.append({
|
|
"surface_id": "SURF-" + hashlib.sha256(surface_key).hexdigest()[:20].upper(),
|
|
"path": relative,
|
|
"section_id": section_id,
|
|
"heading": section.heading,
|
|
"line_start": section.line_start,
|
|
"line_end": section.line_end,
|
|
"authority": "high" if source_type == "project-note" else "medium",
|
|
"content_sha256": hashlib.sha256(section.text.encode("utf-8")).hexdigest(),
|
|
"text": section.text,
|
|
})
|
|
eligible = len(config["required_surfaces"])
|
|
extracted = len(surfaces)
|
|
excluded_count = len(excluded)
|
|
uncovered = eligible - extracted - excluded_count
|
|
if uncovered != sum(1 for item in findings if item["code"] in {"SEMANTIC_SURFACE_UNCOVERED", "AMBIGUOUS_SEMANTIC_SURFACE"}):
|
|
raise SemanticSurfaceError(f"coverage accounting drift for {relative}")
|
|
return {
|
|
"path": relative,
|
|
"source_type": source_type,
|
|
"mode": config["mode"],
|
|
"document_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
|
"required": is_required(frontmatter, policy),
|
|
"coverage": {
|
|
"eligible_surface_blocks": eligible,
|
|
"extracted_surface_blocks": extracted,
|
|
"explicitly_excluded_blocks": excluded_count,
|
|
"uncovered_surface_blocks": uncovered,
|
|
},
|
|
"surfaces": sorted(surfaces, key=lambda item: item["section_id"]),
|
|
"excluded": sorted(excluded, key=lambda item: item["section_id"]),
|
|
"findings": sorted(findings, key=lambda item: (item["severity"], item["code"], item["line"])),
|
|
}
|
|
|
|
|
|
def eligible_documents(root: Path, policy: Mapping[str, Any], *, required_only: bool = False, mode: str | None = None) -> tuple[Path, ...]:
|
|
root = root.resolve(strict=True)
|
|
found: set[Path] = set()
|
|
layout_mode, vault_root = _layout(root)
|
|
for source_type, config in policy["documents"].items():
|
|
if mode is not None and config["mode"] != mode:
|
|
continue
|
|
scan_roots = [root / vault_root] if layout_mode == "canonical" else [root / value for value in config["roots"]]
|
|
categories = {Path(value).name for value in config["roots"]}
|
|
for directory in scan_roots:
|
|
if not directory.is_dir():
|
|
continue
|
|
for path in directory.rglob("*.md"):
|
|
if layout_mode == "canonical" and not categories.intersection(path.relative_to(root).parts):
|
|
continue
|
|
frontmatter = parse_frontmatter(path.read_text(encoding="utf-8"))
|
|
if str(frontmatter.get("source_type", "")) != source_type:
|
|
continue
|
|
if required_only and not is_required(frontmatter, policy):
|
|
continue
|
|
found.add(path.resolve())
|
|
return tuple(sorted(found))
|
|
|
|
|
|
def check(root: Path, policy_path: Path = DEFAULT_POLICY, *, paths: Iterable[Path] | None = None) -> dict[str, Any]:
|
|
root = root.resolve(strict=True)
|
|
policy = load_policy(root, policy_path)
|
|
selected = tuple(paths) if paths is not None else eligible_documents(root, policy, required_only=True)
|
|
documents = [extract_document(root, path if path.is_absolute() else root / path, policy) for path in selected]
|
|
findings = [item for document in documents for item in document["findings"]]
|
|
# 자동 탐색(paths=None)이 0건을 반환하면 "검사할 게 없어 PASS" 처럼 보이지만, 실제로는
|
|
# 레이아웃 모드 오판·카테고리 필터·frontmatter 탐색이 깨져 *조용히 아무것도 안 검사한*
|
|
# 상태일 수 있다. 명시적 paths=[] 는 호출자가 의도한 공허참(그대로 PASS)이지만, 자동
|
|
# 탐색의 0건은 loud FAIL 로 드러낸다(진짜 빈 저장소면 paths=[] 로 호출).
|
|
if paths is None and not selected:
|
|
findings.append({
|
|
"code": "NO_ELIGIBLE_DOCUMENTS",
|
|
"path": "",
|
|
"line": 0,
|
|
"message": "자동 탐색이 대상 문서를 0건 발견 — 탐색(레이아웃/카테고리/source_type)이 "
|
|
"깨졌을 수 있음. 빈 저장소가 맞다면 paths=[] 로 명시 호출하세요.",
|
|
"severity": "error",
|
|
})
|
|
errors = [item for item in findings if item["severity"] == "error"]
|
|
coverage = {
|
|
key: sum(int(document["coverage"][key]) for document in documents)
|
|
for key in (
|
|
"eligible_surface_blocks",
|
|
"extracted_surface_blocks",
|
|
"explicitly_excluded_blocks",
|
|
"uncovered_surface_blocks",
|
|
)
|
|
}
|
|
if coverage["eligible_surface_blocks"] != coverage["extracted_surface_blocks"] + coverage["explicitly_excluded_blocks"] + coverage["uncovered_surface_blocks"]:
|
|
raise SemanticSurfaceError("global semantic surface coverage invariant failed")
|
|
return {
|
|
"schema_version": RESULT_SCHEMA,
|
|
"status": "PASS" if not errors else "FAIL",
|
|
"policy_sha256": hashlib.sha256(canonical_json_bytes(policy)).hexdigest(),
|
|
"document_count": len(documents),
|
|
"coverage": coverage,
|
|
"documents": documents,
|
|
"findings": sorted(findings, key=lambda item: (item["path"], item["line"], item["code"])),
|
|
}
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
|
|
parser.add_argument("--policy", type=Path, default=DEFAULT_POLICY)
|
|
parser.add_argument("--path", type=Path, action="append")
|
|
parser.add_argument("--check", action="store_true", help="validate without writing (the only supported mode)")
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
result = check(args.root, args.policy, paths=args.path)
|
|
exit_code = 0 if result["status"] == "PASS" else 1
|
|
except (SemanticSurfaceError, OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
result = {"schema_version": RESULT_SCHEMA, "status": "ERROR", "errors": [{"code": "SEMANTIC_SURFACE_ERROR", "message": str(exc)}]}
|
|
exit_code = 2
|
|
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True)
|
|
sys.stdout.write("\n")
|
|
return exit_code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|