init: llm-wiki-haness 하네스 설계

2026-07-24 14:21:35 +09:00
parent 42bf3db4fd
commit 6c53ded9cb
2436 changed files with 194486 additions and 1 deletions
@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import pathlib
+import re
+
+ROOT = pathlib.Path(__file__).resolve().parents[5]
+TOPIC = "2026-07-18-keycloak-branch-note-consistency"
+SPEC = ROOT / "docs/superpowers/specs"
+BASE = SPEC / TOPIC
+MASTER = SPEC / f"{TOPIC}-report.md"
+CONTROLLER = SPEC / f"{TOPIC}-controller-verification.md"
+ADV_ROOT = SPEC / f"{TOPIC}-adversarial-review.md"
+
+scope = sorted(
+    str(path.relative_to(ROOT))
+    for path in (ROOT / "raw/branch-notes").glob("*.md")
+    if "keycloak" in path.name.lower() or "keycloak" in path.read_text(encoding="utf-8").lower()
+)
+matrix = (BASE / "evidence-matrix.md").read_text(encoding="utf-8")
+matrix_rows = []
+malformed = []
+for line in matrix.splitlines():
+    if re.match(r"^\| \x60raw/branch-notes/[^\x60]+\.md\x60 \|", line):
+        cells = [cell.strip() for cell in line.strip().strip("|").split("|")]
+        if len(cells) != 4:
+            malformed.append(line)
+            continue
+        matrix_rows.append((cells[0].strip("\x60"), cells[1], cells[2], cells[3]))
+        if cells[1] not in {"READ_FULL", "READ_PARTIAL", "NOT_READ", "BLOCKED"}:
+            malformed.append(line)
+
+matrix_paths = [row[0] for row in matrix_rows]
+missing = sorted(set(scope) - set(matrix_paths))
+extra = sorted(set(matrix_paths) - set(scope))
+duplicates = sorted({path for path in matrix_paths if matrix_paths.count(path) > 1})
+nonexistent = sorted(path for path in matrix_paths if not (ROOT / path).is_file())
+
+lane_files = sorted((BASE / "lanes").glob("lane-0[1-4]-*.md"))
+sections = {}
+schema_failures = []
+for lane in lane_files:
+    text = lane.read_text(encoding="utf-8")
+    matches = list(re.finditer(r"(?m)^### (L[1-4]-F\d{2})\b[^\n]*", text))
+    for index, match in enumerate(matches):
+        end = matches[index + 1].start() if index + 1 < len(matches) else len(text)
+        sections.setdefault(match.group(1), []).append(text[match.start():end])
+
+expected_ids = (
+    [f"L1-F{i:02d}" for i in range(1, 15)]
+    + [f"L2-F{i:02d}" for i in range(1, 10)]
+    + [f"L3-F{i:02d}" for i in range(1, 17)]
+    + [f"L4-F{i:02d}" for i in range(1, 22)]
+)
+for finding_id, entries in sections.items():
+    section = entries[0]
+    checks = {
+        "source_file": "raw/branch-notes/" in section,
+        "severity": bool(re.search(r"Severity|심각도", section, re.I)),
+        "falsification": bool(re.search(r"Falsification|Falsified|무효|반증", section, re.I)),
+        "recommendation": bool(re.search(r"Required action|Action / why|synthesis recommendation|권고|조치", section, re.I)),
+    }
+    failed = [name for name, ok in checks.items() if not ok]
+    if failed:
+        schema_failures.append(f"{finding_id}:{','.join(failed)}")
+
+per_file = (BASE / "per-file-findings.md").read_text(encoding="utf-8")
+per_file_sections = len(re.findall(r"(?m)^## \x60[^\x60]+\.md\x60$", per_file))
+proofs = (BASE / "sed-proofs.md").read_text(encoding="utf-8")
+proof_ids = re.findall(r"(?m)^\| \x60(L[1-4]-F\d{2})\x60 \|", proofs)
+adversarial = (BASE / "adversarial-review.md").read_text(encoding="utf-8")
+adv_ids = re.findall(r"(?m)^\| (L[1-4]-F\d{2}) \|", adversarial)
+generic = ["수동 보완책이 존재함", "일부 비핵심 경로", "치명적인 영향이 없음"]
+generic_hits = sum(adversarial.count(term) for term in generic)
+priority = (BASE / "priority-recommendations.md").read_text(encoding="utf-8")
+priority_ids = sorted(set(re.findall(r"L[1-4]-F\d{2}", priority)))
+
+required = [
+    MASTER,
+    CONTROLLER,
+    ADV_ROOT,
+    BASE / "evidence-matrix.md",
+    BASE / "per-file-findings.md",
+    BASE / "sed-proofs.md",
+    BASE / "priority-recommendations.md",
+    BASE / "unresolved-risk-register.md",
+    BASE / "adversarial-review.md",
+]
+missing_artifacts = [str(path.relative_to(ROOT)) for path in required if not path.is_file()]
+
+link_files = [path for path in required if path.is_file()] + lane_files
+broken_links = []
+for source in link_files:
+    text = source.read_text(encoding="utf-8")
+    for match in re.finditer(r"\[[^\]]+\]\(([^)]+\.md)(?:#[^)]+)?\)", text):
+        target = pathlib.Path(match.group(1))
+        if str(target).startswith(("http:/", "https:/")):
+            continue
+        resolved = target if target.is_absolute() else (source.parent / target).resolve()
+        if not resolved.is_file():
+            broken_links.append(f"{source.relative_to(ROOT)}->{target}")
+
+banned = [
+    "100%", "완벽", "완전", "극한", "극단", "정밀한", "흔들림 없이",
+    "절대로", "최강", "역사상 가장", "명품", "원천 차단", "보증", "폭사",
+]
+forbidden = []
+for path in [MASTER, CONTROLLER, BASE / "priority-recommendations.md", BASE / "unresolved-risk-register.md"]:
+    if not path.is_file():
+        continue
+    text = re.sub(r"\x60\x60\x60.*?\x60\x60\x60", "", path.read_text(encoding="utf-8"), flags=re.S)
+    text = re.sub(r"\x60[^\x60]*\x60", "", text)
+    for line_number, line in enumerate(text.splitlines(), 1):
+        for term in banned:
+            if term in line:
+                forbidden.append(f"{path.relative_to(ROOT)}:{line_number}:{term}")
+
+claim_texts = [(ROOT / path).read_text(encoding="utf-8") for path in scope]
+values = {
+    "raw_file_count": len(scope),
+    "raw_line_count": sum(len(text.splitlines()) for text in claim_texts),
+    "matrix_rows": len(matrix_rows),
+    "read_full_rows": sum(row[1] == "READ_FULL" for row in matrix_rows),
+    "blocked_rows": sum(row[1] == "BLOCKED" for row in matrix_rows),
+    "missing_paths": len(missing),
+    "extra_paths": len(extra),
+    "duplicate_paths": len(duplicates),
+    "nonexistent_paths": len(nonexistent),
+    "malformed_matrix_rows": len(malformed),
+    "lane_finding_count": sum(len(entries) for entries in sections.values()),
+    "unique_finding_ids": len(sections),
+    "duplicate_finding_ids": sum(len(entries) != 1 for entries in sections.values()),
+    "missing_finding_ids": len(set(expected_ids) - set(sections)),
+    "finding_schema_failures": len(schema_failures),
+    "per_file_sections": per_file_sections,
+    "sed_proof_rows": len(proof_ids),
+    "sed_proof_unique_ids": len(set(proof_ids)),
+    "adversarial_rows": len(adv_ids),
+    "adversarial_unique_ids": len(set(adv_ids)),
+    "adversarial_generic_hits": generic_hits,
+    "priority_unique_ids": len(priority_ids),
+    "unresolved_priority_ids": len(set(priority_ids) - set(expected_ids)),
+    "broken_internal_links": len(broken_links),
+    "forbidden_word_hits": len(forbidden),
+    "missing_required_artifacts": len(missing_artifacts),
+    "decision_evidence_map_files": sum("## Decision Evidence Map" in text for text in claim_texts),
+    "claims_extracted_files": sum("## Claims Extracted" in text for text in claim_texts),
+    "missing_claims_extracted": sum("## Claims Extracted" not in text for text in claim_texts),
+    "unsupported_decision_files": sum("UNSUPPORTED_DECISION" in text for text in claim_texts),
+    "unsupported_decision_occurrences": sum(text.count("UNSUPPORTED_DECISION") for text in claim_texts),
+    "broken_claim_reference_literal_occurrences": sum(text.count("BROKEN_CLAIM_REFERENCE") for text in claim_texts),
+}
+for key, value in values.items():
+    print(f"{key}={value}")
+for label, entries in [
+    ("missing", missing), ("extra", extra), ("duplicates", duplicates),
+    ("nonexistent", nonexistent), ("malformed", malformed),
+    ("schema_failures", schema_failures), ("missing_artifacts", missing_artifacts),
+    ("broken_links", broken_links), ("forbidden_hits", forbidden),
+]:
+    if entries:
+        print(f"{label}=" + " || ".join(entries))
@@ -0,0 +1,11 @@
+# Implementation Plan
+
+1. `rg -il --glob '*.md' 'keycloak' raw/branch-notes`로 범위를 확정한다.
+2. 38개 파일을 10/10/10/8개 lane으로 분할하고 각 파일을 정확히 한 lane에 배정한다.
+3. `wiki_consistency_check.py --all`과 `--packets`로 결정론 finding 및 참조 팩킷을 만든다.
+4. lane별 READ_FULL evidence matrix와 line-verified finding을 수집한다.
+5. cross-lane 후보를 `wiki-consistency-auditor`로 의미 대조한다.
+6. finding이 5개 이상이면 `wiki-adversarial-reviewer`로 전 항목을 반증 시도한다.
+7. controller가 파일 집합·행 수·finding ID·인용·artifact·금지어를 재계산한다.
+8. 원문을 수정하지 않고 위험도·승인 단위가 포함된 fix-plan을 보고한다.
+
@@ -0,0 +1,7 @@
+# Task
+
+- 요청: `raw/branch-notes/`에서 Keycloak 관련 문서를 모두 읽고 설계 간 비일관성을 리뷰한다.
+- 범위 산정: 파일명 또는 본문에 `keycloak`이 포함된 branch-note 38개.
+- 원문 수정: 하지 않음.
+- 산출물: evidence matrix, lane별 finding, semantic edge audit, adversarial review, fix-plan, controller verification, master report.
+
@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import pathlib
+import re
+import shlex
+import subprocess
+
+ROOT = pathlib.Path(__file__).resolve().parents[5]
+LANES = ROOT / "docs/superpowers/specs/2026-07-18-keycloak-branch-note-consistency/lanes"
+FINDING = re.compile(r"^###\s+(L[1-4]-F\d{2})\b")
+commands: dict[str, list[tuple[pathlib.Path, int, str]]] = {}
+inventory: list[tuple[str, str]] = []
+
+for lane in sorted(LANES.glob("lane-0[1-4]-*.md")):
+    current = None
+    for line_number, raw in enumerate(lane.read_text(encoding="utf-8").splitlines(), 1):
+        match = FINDING.match(raw)
+        if match:
+            current = match.group(1)
+            commands.setdefault(current, [])
+            continue
+        row = re.match(r"^\| (?:\x60)?(raw/branch-notes/[^\x60|]+\.md)(?:\x60)? \| (READ_FULL|NOT_READ|BLOCKED) \|", raw)
+        if row:
+            inventory.append((row.group(1), row.group(2)))
+        candidate = raw.strip()
+        if candidate.startswith("$ "):
+            candidate = candidate[2:]
+        if current and candidate.startswith(("grep ", "sed ")):
+            commands[current].append((lane, line_number, candidate))
+
+expected = (
+    [f"L1-F{i:02d}" for i in range(1, 15)]
+    + [f"L2-F{i:02d}" for i in range(1, 10)]
+    + [f"L3-F{i:02d}" for i in range(1, 17)]
+    + [f"L4-F{i:02d}" for i in range(1, 22)]
+)
+passed = 0
+verified_output_lines = 0
+failures = []
+no_commands = []
+
+for finding_id in expected:
+    if not commands.get(finding_id):
+        no_commands.append(finding_id)
+        continue
+    for lane, line_number, command in commands[finding_id]:
+        try:
+            argv = shlex.split(command)
+        except ValueError as exc:
+            failures.append(f"{finding_id}:{lane.name}:{line_number}:parse:{exc}")
+            continue
+        if not argv or argv[0] not in {"grep", "sed"}:
+            failures.append(f"{finding_id}:{lane.name}:{line_number}:unsafe")
+            continue
+        paths = []
+        for token in argv[1:]:
+            if token.endswith(".md"):
+                path = pathlib.Path(token)
+                paths.append((path if path.is_absolute() else ROOT / path).resolve())
+        if not paths or any(ROOT not in path.parents for path in paths):
+            failures.append(f"{finding_id}:{lane.name}:{line_number}:path")
+            continue
+        result = subprocess.run(argv, cwd=ROOT, text=True, capture_output=True)
+        if result.returncode != 0 or not result.stdout.strip():
+            failures.append(f"{finding_id}:{lane.name}:{line_number}:exit={result.returncode}")
+            continue
+        passed += 1
+        verified_output_lines += len([line for line in result.stdout.splitlines() if line.strip()])
+
+print(f"inventory_rows={len(inventory)}")
+print(f"inventory_unique={len(set(path for path, _ in inventory))}")
+print(f"read_full={sum(status == 'READ_FULL' for _, status in inventory)}")
+print(f"finding_ids={len(commands)}")
+print(f"expected_finding_ids={len(expected)}")
+print(f"commands_passed={passed}")
+print(f"commands_failed={len(failures)}")
+print(f"verified_output_lines={verified_output_lines}")
+print(f"findings_without_commands={len(no_commands)}")
+if failures:
+    print("failures=" + " || ".join(failures))
@@ -0,0 +1,11 @@
+# Walkthrough
+
+- 2026-07-18: Keycloak 관련 branch-note 38개, 10,649줄을 식별했다.
+- 2026-07-18: 분할 감사 `10 + 10 + 10 + 8 = 38`, 중복 0, 누락 0을 확인했다.
+- 2026-07-18: 결정론 검사 전체 결과 144건 중 Keycloak 범위의 `BARE_DECISION_REF` 8건을 분리했다.
+- 2026-07-18: 유효 참조 647건과 logical edge 298개를 산출했다.
+- 2026-07-18: lane finding 60건의 107개 인용 명령을 controller가 재실행해 실패 0, source output 158줄을 확인했다.
+- 2026-07-18: semantic risk sample 20건을 대조해 actual edge 9건, `NOT_AN_EDGE` 11건으로 분류했다.
+- 2026-07-18: 적대 리뷰에서 KEEP 21, DOWNGRADE 27, REJECT 12로 판정했으며 retained High는 10건이다.
+- 2026-07-18: 9개 controller gate 중 claim traceability를 포함한 `finding_gate`만 FAIL로 계산해 master Verdict를 PARTIAL로 확정했다.
+- 2026-07-18: branch-note 원문은 수정하지 않고 report artifacts만 생성했다.