3526 lines
134 KiB
Python
3526 lines
134 KiB
Python
#!/usr/bin/env python3
|
||
"""Lint Markdown, logic, terminology, and fidelity contracts."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import bisect
|
||
import hashlib
|
||
import html
|
||
import re
|
||
import sys
|
||
import unicodedata
|
||
from collections import Counter, defaultdict
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Any, Iterable
|
||
from urllib.parse import unquote
|
||
|
||
from harness_common import (
|
||
DEFAULT_RULES_PATH,
|
||
InputError,
|
||
decode_utf8,
|
||
load_json_text,
|
||
load_rules,
|
||
prepare_report_output,
|
||
publish_report_json,
|
||
rule_index,
|
||
schema_version,
|
||
sha256_file,
|
||
snapshot_file,
|
||
utc_now,
|
||
validate_with_schema,
|
||
)
|
||
from markdown_structure import (
|
||
advance_html_block,
|
||
closing_fence,
|
||
crosses_paragraph_boundary,
|
||
fence_container_continues,
|
||
indented_code_container,
|
||
indentation_columns,
|
||
inline_code_spans,
|
||
inside_any_span,
|
||
is_thematic_break,
|
||
mask_closed_fence_candidates,
|
||
list_continuation_indent,
|
||
opening_fence,
|
||
ordered_list_interrupts_paragraph,
|
||
strip_blockquotes,
|
||
)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Heading:
|
||
level: int
|
||
text: str
|
||
line: int
|
||
start: int
|
||
end: int
|
||
slug: str
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class TextSpan:
|
||
start: int
|
||
end: int
|
||
line: int
|
||
text: str
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class FencedBlock:
|
||
start: int
|
||
end: int
|
||
line: int
|
||
text: str
|
||
closed: bool
|
||
kind: str = "fenced_code"
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ProtectedOccurrence:
|
||
value: str
|
||
context: str
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ReferenceDefinition:
|
||
identifier: str
|
||
destination: str
|
||
start: int
|
||
end: int
|
||
target_start: int
|
||
target_end: int
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ReferenceLink:
|
||
start: int
|
||
end: int
|
||
metadata_start: int
|
||
metadata_end: int
|
||
raw_label: str
|
||
identifier: str
|
||
|
||
|
||
ABSOLUTE_URI_PATTERN = re.compile(
|
||
r"(?<![A-Za-z0-9+.-])(?:(?:https?|ftps?|file|ssh|git)://|mailto:)"
|
||
r"[^\s<>\"'\])}]+",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
|
||
def absolute_uri_matches(text: str) -> list[tuple[int, int, str]]:
|
||
"""Return common reader-visible absolute URIs without trailing prose punctuation."""
|
||
|
||
matches: list[tuple[int, int, str]] = []
|
||
for match in ABSOLUTE_URI_PATTERN.finditer(text):
|
||
value = match.group(0).rstrip(".,;:")
|
||
if value:
|
||
matches.append((match.start(), match.start() + len(value), value))
|
||
return matches
|
||
|
||
|
||
REQUIRED_LOGIC_TOP = (
|
||
"schema_version",
|
||
"title",
|
||
"document_kind",
|
||
"core_claim",
|
||
"sections",
|
||
"closure",
|
||
)
|
||
REQUIRED_LOGIC_SECTION = (
|
||
"id",
|
||
"heading",
|
||
"role",
|
||
"depends_on",
|
||
"reader_state_before",
|
||
"question",
|
||
"answer_plain",
|
||
"claim_ids",
|
||
"new_terms",
|
||
"transition_to",
|
||
"reader_state_after",
|
||
)
|
||
REQUIRED_TERM = (
|
||
"id",
|
||
"canonical",
|
||
"plain_definition",
|
||
"why_needed",
|
||
"aliases",
|
||
"first_section",
|
||
"first_use",
|
||
)
|
||
REQUIRED_READER = (
|
||
"schema_version",
|
||
"document_kind",
|
||
"primary_audience",
|
||
"purpose",
|
||
"reader_question",
|
||
"reader_outcome",
|
||
"prerequisites",
|
||
"assumed_known",
|
||
"must_explain",
|
||
"non_goals",
|
||
)
|
||
|
||
|
||
def line_starts(text: str) -> list[int]:
|
||
starts = [0]
|
||
starts.extend(match.end() for match in re.finditer("\n", text))
|
||
return starts
|
||
|
||
|
||
def location(starts: list[int], offset: int) -> tuple[int, int]:
|
||
index = bisect.bisect_right(starts, max(0, offset)) - 1
|
||
return index + 1, offset - starts[index] + 1
|
||
|
||
|
||
def is_backslash_escaped(text: str, position: int) -> bool:
|
||
count = 0
|
||
position -= 1
|
||
while position >= 0 and text[position] == "\\":
|
||
count += 1
|
||
position -= 1
|
||
return count % 2 == 1
|
||
|
||
|
||
def html_tag_spans(text: str) -> list[tuple[int, int]]:
|
||
"""Return real inline/block HTML tag spans, excluding comments/autolinks."""
|
||
|
||
spans: list[tuple[int, int]] = []
|
||
cursor = 0
|
||
while cursor < len(text):
|
||
start = text.find("<", cursor)
|
||
if start < 0:
|
||
break
|
||
name = re.match(r"</?([A-Za-z][A-Za-z0-9-]*)", text[start:])
|
||
if name is None:
|
||
cursor = start + 1
|
||
continue
|
||
prefix_end = start + name.end()
|
||
if prefix_end >= len(text) or text[prefix_end] not in " \t\r\n/>":
|
||
cursor = start + 1
|
||
continue
|
||
quote: str | None = None
|
||
end = prefix_end
|
||
while end < len(text):
|
||
character = text[end]
|
||
if quote is not None:
|
||
if character == quote:
|
||
quote = None
|
||
elif character in "\"'":
|
||
quote = character
|
||
elif character == ">":
|
||
spans.append((start, end + 1))
|
||
end += 1
|
||
break
|
||
end += 1
|
||
cursor = max(start + 1, end)
|
||
return spans
|
||
|
||
|
||
def html_tag_name_and_attributes(raw: str) -> tuple[str, dict[str, str]] | None:
|
||
"""Parse the tag name and explicit attribute values from one HTML tag."""
|
||
|
||
tag = re.match(r"<(?P<closing>/)?(?P<name>[A-Za-z][A-Za-z0-9-]*)", raw)
|
||
if tag is None or tag.group("closing"):
|
||
return None
|
||
attributes: dict[str, str] = {}
|
||
cursor = tag.end()
|
||
while cursor < len(raw):
|
||
while cursor < len(raw) and raw[cursor].isspace():
|
||
cursor += 1
|
||
if cursor >= len(raw) or raw[cursor] in "/>":
|
||
break
|
||
name = re.match(r"[A-Za-z_:][A-Za-z0-9_.:-]*", raw[cursor:])
|
||
if name is None:
|
||
cursor += 1
|
||
continue
|
||
attribute_name = name.group(0).casefold()
|
||
cursor += name.end()
|
||
while cursor < len(raw) and raw[cursor].isspace():
|
||
cursor += 1
|
||
value = ""
|
||
if cursor < len(raw) and raw[cursor] == "=":
|
||
cursor += 1
|
||
while cursor < len(raw) and raw[cursor].isspace():
|
||
cursor += 1
|
||
if cursor < len(raw) and raw[cursor] in "\"'":
|
||
quote = raw[cursor]
|
||
cursor += 1
|
||
start = cursor
|
||
while cursor < len(raw) and raw[cursor] != quote:
|
||
cursor += 1
|
||
value = raw[start:cursor]
|
||
if cursor < len(raw):
|
||
cursor += 1
|
||
else:
|
||
start = cursor
|
||
while cursor < len(raw) and raw[cursor] not in " \t\r\n\"'=<>`":
|
||
cursor += 1
|
||
value = raw[start:cursor]
|
||
attributes.setdefault(attribute_name, value)
|
||
return tag.group("name").casefold(), attributes
|
||
|
||
|
||
def normalize_reference_identifier(value: str) -> str:
|
||
value = re.sub(r"\\([!\"#$%&'()*+,./:;<=>?@\[\\\]^_`{|}~-])", r"\1", value)
|
||
decoded, _ = decode_html_entities_with_offsets(value)
|
||
return " ".join(decoded.split()).casefold()
|
||
|
||
|
||
def reference_definitions(text: str) -> list[ReferenceDefinition]:
|
||
"""Parse CommonMark-style link definitions without exposing their metadata."""
|
||
|
||
definitions: list[ReferenceDefinition] = []
|
||
lines = text.splitlines(keepends=True)
|
||
offsets: list[int] = []
|
||
offset = 0
|
||
for line in lines:
|
||
offsets.append(offset)
|
||
offset += len(line)
|
||
|
||
title_patterns = (
|
||
re.compile(r'^"(?:\\.|[^"\r\n])*"$'),
|
||
re.compile(r"^'(?:\\.|[^'\r\n])*'$"),
|
||
re.compile(r"^\((?:\\.|[^()\r\n])*\)$"),
|
||
)
|
||
|
||
def container_content(content: str) -> tuple[str, int, int, bool]:
|
||
rest, quote_depth = strip_blockquotes(content)
|
||
prefix = len(content) - len(rest)
|
||
item = re.match(
|
||
r"^(?P<leading>[ \t]*)(?:[-+*]|[0-9]{1,9}[.)])(?P<gap>[ \t]+)",
|
||
rest,
|
||
)
|
||
if item:
|
||
prefix += item.end()
|
||
rest = rest[item.end() :]
|
||
return rest, prefix, quote_depth, item is not None
|
||
|
||
def parse_header(content: str) -> tuple[str, int] | None:
|
||
leading = len(content) - len(content.lstrip(" "))
|
||
if leading > 3:
|
||
return None
|
||
cursor = leading
|
||
if cursor >= len(content) or content[cursor] != "[":
|
||
return None
|
||
label_start = cursor + 1
|
||
cursor = label_start
|
||
while cursor < len(content):
|
||
if content[cursor] == "]" and not is_backslash_escaped(content, cursor):
|
||
break
|
||
cursor += 1
|
||
if cursor >= len(content) or not content[label_start:cursor]:
|
||
return None
|
||
label = content[label_start:cursor]
|
||
cursor += 1
|
||
if cursor >= len(content) or content[cursor] != ":":
|
||
return None
|
||
cursor += 1
|
||
while cursor < len(content) and content[cursor] in " \t":
|
||
cursor += 1
|
||
return label, cursor
|
||
|
||
def parse_destination_prefix(
|
||
content: str, cursor: int
|
||
) -> tuple[int, int, int] | None:
|
||
if cursor >= len(content):
|
||
return None
|
||
if content[cursor] == "<":
|
||
target_start = cursor + 1
|
||
target_end = target_start
|
||
while target_end < len(content) and content[target_end] != ">":
|
||
if content[target_end] in "<>" or (
|
||
content[target_end] == "\\" and target_end + 1 >= len(content)
|
||
):
|
||
return None
|
||
target_end += 2 if content[target_end] == "\\" else 1
|
||
if target_end >= len(content) or content[target_end] != ">":
|
||
return None
|
||
tail = target_end + 1
|
||
else:
|
||
target_start = cursor
|
||
depth = 0
|
||
while cursor < len(content) and not content[cursor].isspace():
|
||
if content[cursor] == "\\" and cursor + 1 < len(content):
|
||
cursor += 2
|
||
continue
|
||
if content[cursor] == "(":
|
||
depth += 1
|
||
elif content[cursor] == ")":
|
||
if depth == 0:
|
||
return None
|
||
depth -= 1
|
||
cursor += 1
|
||
if depth:
|
||
return None
|
||
target_end = cursor
|
||
tail = cursor
|
||
return target_start, target_end, tail
|
||
|
||
def parse_destination(
|
||
content: str, cursor: int
|
||
) -> tuple[int, int, bool] | None:
|
||
prefix = parse_destination_prefix(content, cursor)
|
||
if prefix is None:
|
||
return None
|
||
target_start, target_end, tail = prefix
|
||
raw_rest = content[tail:]
|
||
rest = raw_rest.strip()
|
||
if rest and (not raw_rest or raw_rest[0] not in " \t\r\n"):
|
||
return None
|
||
if rest and not any(pattern.fullmatch(rest) for pattern in title_patterns):
|
||
return None
|
||
return target_start, target_end, not rest
|
||
|
||
def parse_line(content: str) -> tuple[str, int, int, bool] | None:
|
||
header = parse_header(content)
|
||
if header is None:
|
||
return None
|
||
label, cursor = header
|
||
destination = parse_destination(content, cursor)
|
||
if destination is None:
|
||
return None
|
||
target_start, target_end, title_may_follow = destination
|
||
return label, target_start, target_end, title_may_follow
|
||
|
||
def consume_multiline_title(
|
||
start_index: int,
|
||
start_content: str,
|
||
opener_position: int,
|
||
quote_depth: int,
|
||
) -> int | None:
|
||
opener = start_content[opener_position]
|
||
closer = ")" if opener == "(" else opener
|
||
line_index = start_index
|
||
content = start_content
|
||
cursor = opener_position + 1
|
||
while True:
|
||
while cursor < len(content):
|
||
if content[cursor] == "\\" and cursor + 1 < len(content):
|
||
cursor += 2
|
||
continue
|
||
if opener == "(" and content[cursor] == "(":
|
||
return None
|
||
if content[cursor] == closer:
|
||
return line_index if not content[cursor + 1 :].strip() else None
|
||
cursor += 1
|
||
line_index += 1
|
||
if line_index >= len(lines):
|
||
return None
|
||
continuation_raw = lines[line_index].rstrip("\r\n")
|
||
(
|
||
content,
|
||
_,
|
||
continuation_quote_depth,
|
||
continuation_list_item,
|
||
) = container_content(continuation_raw)
|
||
if (
|
||
not content.strip()
|
||
or continuation_quote_depth != quote_depth
|
||
or continuation_list_item
|
||
):
|
||
return None
|
||
cursor = 0
|
||
|
||
paragraph_open = False
|
||
active_quote_depth = 0
|
||
skip_through = -1
|
||
for index, line in enumerate(lines):
|
||
if index <= skip_through:
|
||
continue
|
||
raw_content = line.rstrip("\r\n")
|
||
content, prefix, quote_depth, list_item = container_content(raw_content)
|
||
if not raw_content.strip():
|
||
paragraph_open = False
|
||
active_quote_depth = 0
|
||
continue
|
||
list_interrupt = ordered_list_interrupts_paragraph(
|
||
strip_blockquotes(raw_content)[0]
|
||
)
|
||
if quote_depth != active_quote_depth or (
|
||
list_item and (not paragraph_open or list_interrupt)
|
||
):
|
||
paragraph_open = False
|
||
active_quote_depth = quote_depth
|
||
parsed = None if paragraph_open else parse_line(content)
|
||
target_line_index = index
|
||
target_prefix = prefix
|
||
definition_end_index = index
|
||
header = None if paragraph_open else parse_header(content)
|
||
if parsed is None and header is not None:
|
||
destination_prefix = parse_destination_prefix(content, header[1])
|
||
if destination_prefix is not None:
|
||
target_start, target_end, tail = destination_prefix
|
||
raw_rest = content[tail:]
|
||
title_position = tail + (len(raw_rest) - len(raw_rest.lstrip()))
|
||
if (
|
||
raw_rest
|
||
and raw_rest[0].isspace()
|
||
and title_position < len(content)
|
||
and content[title_position] in "\"'("
|
||
):
|
||
title_end = consume_multiline_title(
|
||
index, content, title_position, quote_depth
|
||
)
|
||
if title_end is not None:
|
||
definition_end_index = title_end
|
||
parsed = (header[0], target_start, target_end, False)
|
||
if parsed is None and not paragraph_open:
|
||
if header is not None and header[1] == len(content) and index + 1 < len(lines):
|
||
next_raw = lines[index + 1].rstrip("\r\n")
|
||
(
|
||
next_content,
|
||
next_prefix,
|
||
next_quote_depth,
|
||
next_list_item,
|
||
) = container_content(next_raw)
|
||
next_leading = len(next_content) - len(next_content.lstrip(" "))
|
||
next_destination_prefix = (
|
||
parse_destination_prefix(next_content, next_leading)
|
||
if 1 <= next_leading <= 3
|
||
and next_quote_depth == quote_depth
|
||
and not next_list_item
|
||
else None
|
||
)
|
||
if next_destination_prefix is not None:
|
||
target_start, target_end, tail = next_destination_prefix
|
||
raw_rest = next_content[tail:]
|
||
title_position = tail + (
|
||
len(raw_rest) - len(raw_rest.lstrip())
|
||
)
|
||
title_end: int | None = None
|
||
if not raw_rest.strip():
|
||
parsed = (header[0], target_start, target_end, True)
|
||
elif (
|
||
raw_rest[0].isspace()
|
||
and title_position < len(next_content)
|
||
and next_content[title_position] in "\"'("
|
||
):
|
||
title_end = consume_multiline_title(
|
||
index + 1,
|
||
next_content,
|
||
title_position,
|
||
quote_depth,
|
||
)
|
||
if title_end is not None:
|
||
parsed = (header[0], target_start, target_end, False)
|
||
target_line_index = index + 1
|
||
target_prefix = next_prefix
|
||
definition_end_index = (
|
||
title_end if title_end is not None else index + 1
|
||
)
|
||
if parsed is not None:
|
||
label, target_start, target_end, title_may_follow = parsed
|
||
end_index = definition_end_index
|
||
if title_may_follow and end_index + 1 < len(lines):
|
||
next_raw = lines[end_index + 1].rstrip("\r\n")
|
||
next_content, _, next_quote_depth, next_list_item = container_content(
|
||
next_raw
|
||
)
|
||
next_leading = len(next_content) - len(next_content.lstrip(" "))
|
||
title_end = None
|
||
if (
|
||
next_quote_depth == quote_depth
|
||
and not next_list_item
|
||
and next_leading <= 3
|
||
and next_leading < len(next_content)
|
||
and next_content[next_leading] in "\"'("
|
||
):
|
||
title_end = consume_multiline_title(
|
||
end_index + 1,
|
||
next_content,
|
||
next_leading,
|
||
quote_depth,
|
||
)
|
||
if title_end is not None:
|
||
end_index = title_end
|
||
absolute_start = offsets[index]
|
||
absolute_target_start = (
|
||
offsets[target_line_index] + target_prefix + target_start
|
||
)
|
||
absolute_target_end = (
|
||
offsets[target_line_index] + target_prefix + target_end
|
||
)
|
||
absolute_end = offsets[end_index] + len(lines[end_index])
|
||
destination = text[absolute_target_start:absolute_target_end]
|
||
destination = re.sub(r"\\(.)", r"\1", destination)
|
||
definitions.append(
|
||
ReferenceDefinition(
|
||
normalize_reference_identifier(label),
|
||
decode_html_entities_with_offsets(destination)[0],
|
||
absolute_start,
|
||
absolute_end,
|
||
absolute_target_start,
|
||
absolute_target_end,
|
||
)
|
||
)
|
||
skip_through = end_index
|
||
paragraph_open = False
|
||
continue
|
||
|
||
structural = bool(
|
||
re.match(
|
||
r"^[ ]{0,3}(?:#{1,6}(?:[ \t]+|$)|(?:=+|-+)[ \t]*$|"
|
||
r"(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,}))",
|
||
content,
|
||
)
|
||
or opening_fence(content + "\n") is not None
|
||
or advance_html_block(content + "\n", None, paragraph_open=paragraph_open)[0]
|
||
)
|
||
paragraph_open = bool(content.strip()) and not structural
|
||
raw_html_structure = mask_raw_html_blocks(text)
|
||
return [
|
||
item
|
||
for item in definitions
|
||
if not (
|
||
item.start < len(text)
|
||
and not text[item.start].isspace()
|
||
and raw_html_structure[item.start] == " "
|
||
)
|
||
]
|
||
|
||
|
||
def scan_markdown_visibility(
|
||
text: str,
|
||
) -> tuple[str, str, list[FencedBlock], list[int]]:
|
||
"""Return fence-masked, reader-visible text and structural defects.
|
||
|
||
Markdown comments and fenced blocks are parsed in one pass so a fence-shaped
|
||
line inside an HTML comment is not treated as code, while literal ``<!--``
|
||
inside a real fenced block remains code. All masks preserve byte-independent
|
||
character offsets by replacing non-newline characters with spaces.
|
||
"""
|
||
|
||
lines = text.splitlines(keepends=True)
|
||
fence_masked = list(text)
|
||
reader_masked = list(text)
|
||
blocks: list[FencedBlock] = []
|
||
unclosed_comments: list[int] = []
|
||
offset = 0
|
||
open_start: int | None = None
|
||
open_line = 0
|
||
marker_char = ""
|
||
marker_size = 0
|
||
fence_quote_depth = 0
|
||
fence_close_mode = "top"
|
||
fence_list_indent = 0
|
||
comment_start: int | None = None
|
||
html_state: tuple[str, str] | None = None
|
||
indented_start: int | None = None
|
||
indented_line = 0
|
||
paragraph_open = False
|
||
list_context_indent = 0
|
||
inline_source = mask_closed_fence_candidates(text)
|
||
raw_html_structure = mask_raw_html_blocks(inline_source)
|
||
all_inline_spans = [
|
||
span
|
||
for span in inline_code_spans(inline_source)
|
||
if not (
|
||
span[0] < len(inline_source)
|
||
and not inline_source[span[0]].isspace()
|
||
and raw_html_structure[span[0]] == " "
|
||
)
|
||
]
|
||
non_comment_spans = list(all_inline_spans)
|
||
non_comment_spans.extend((start, end, "") for start, end in html_tag_spans(text))
|
||
non_comment_spans.extend(
|
||
(target_start, link_end, "")
|
||
for link_start, link_end, target_start, _, _ in markdown_inline_links(
|
||
text, include_images=True
|
||
)
|
||
if not (
|
||
link_start < len(text)
|
||
and not text[link_start].isspace()
|
||
and raw_html_structure[link_start] == " "
|
||
)
|
||
)
|
||
definitions = reference_definitions(text)
|
||
non_comment_spans.extend((item.start, item.end, "") for item in definitions)
|
||
|
||
def mask_range(target: list[str], start: int, end: int) -> None:
|
||
for index in range(start, end):
|
||
if target[index] not in "\r\n":
|
||
target[index] = " "
|
||
|
||
for number, line in enumerate(lines, start=1):
|
||
content = line.rstrip("\r\n")
|
||
is_blank = not content.strip()
|
||
indented_info = indented_code_container(line, list_context_indent)
|
||
is_indented_code = indented_info is not None
|
||
if open_start is not None:
|
||
if closing_fence(
|
||
line,
|
||
marker_char,
|
||
marker_size,
|
||
fence_quote_depth,
|
||
fence_close_mode,
|
||
fence_list_indent,
|
||
):
|
||
end = offset + len(line)
|
||
blocks.append(FencedBlock(open_start, end, open_line, text[open_start:end], True))
|
||
mask_range(fence_masked, open_start, end)
|
||
mask_range(reader_masked, open_start, end)
|
||
open_start = None
|
||
marker_char = ""
|
||
marker_size = 0
|
||
fence_quote_depth = 0
|
||
fence_close_mode = "top"
|
||
fence_list_indent = 0
|
||
paragraph_open = False
|
||
offset += len(line)
|
||
continue
|
||
if fence_container_continues(
|
||
line,
|
||
fence_quote_depth,
|
||
fence_close_mode,
|
||
fence_list_indent,
|
||
):
|
||
offset += len(line)
|
||
continue
|
||
blocks.append(
|
||
FencedBlock(
|
||
open_start,
|
||
offset,
|
||
open_line,
|
||
text[open_start:offset],
|
||
False,
|
||
)
|
||
)
|
||
mask_range(fence_masked, open_start, offset)
|
||
mask_range(reader_masked, open_start, offset)
|
||
open_start = None
|
||
marker_char = ""
|
||
marker_size = 0
|
||
fence_quote_depth = 0
|
||
fence_close_mode = "top"
|
||
fence_list_indent = 0
|
||
|
||
if indented_start is not None:
|
||
if is_indented_code or is_blank:
|
||
paragraph_open = False
|
||
offset += len(line)
|
||
continue
|
||
blocks.append(
|
||
FencedBlock(
|
||
indented_start,
|
||
offset,
|
||
indented_line,
|
||
text[indented_start:offset],
|
||
True,
|
||
"indented_code",
|
||
)
|
||
)
|
||
mask_range(fence_masked, indented_start, offset)
|
||
mask_range(reader_masked, indented_start, offset)
|
||
indented_start = None
|
||
indented_line = 0
|
||
|
||
if comment_start is None:
|
||
html_line, html_state = advance_html_block(
|
||
line, html_state, paragraph_open=paragraph_open
|
||
)
|
||
else:
|
||
html_line = True
|
||
if comment_start is None and not html_line:
|
||
opening = opening_fence(line, list_context_indent)
|
||
if opening:
|
||
(
|
||
marker_char,
|
||
marker_size,
|
||
fence_quote_depth,
|
||
fence_close_mode,
|
||
fence_list_indent,
|
||
) = opening
|
||
open_start = offset
|
||
open_line = number
|
||
paragraph_open = False
|
||
offset += len(line)
|
||
continue
|
||
if not paragraph_open and is_indented_code:
|
||
indented_start = offset
|
||
indented_line = number
|
||
paragraph_open = False
|
||
offset += len(line)
|
||
continue
|
||
|
||
cursor = offset
|
||
line_end = offset + len(line)
|
||
while cursor < line_end:
|
||
if comment_start is None:
|
||
opening = text.find("<!--", cursor, line_end)
|
||
if opening < 0:
|
||
break
|
||
if inside_any_span(opening, non_comment_spans):
|
||
cursor = opening + 4
|
||
continue
|
||
comment_start = opening
|
||
cursor = opening + 4
|
||
closing = text.find("-->", cursor, line_end)
|
||
if closing < 0:
|
||
break
|
||
comment_end = closing + 3
|
||
mask_range(reader_masked, comment_start, comment_end)
|
||
comment_start = None
|
||
cursor = comment_end
|
||
list_context_indent = list_continuation_indent(line, list_context_indent)
|
||
paragraph_open = bool(content.strip()) and not (
|
||
html_line
|
||
or comment_start is not None
|
||
or re.match(
|
||
r"^[ ]{0,3}(?:#{1,6}(?:[ \t]+|$)|(?:=+|-+)[ \t]*$|"
|
||
r"(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,}))",
|
||
content,
|
||
)
|
||
)
|
||
offset += len(line)
|
||
if indented_start is not None:
|
||
blocks.append(
|
||
FencedBlock(
|
||
indented_start,
|
||
len(text),
|
||
indented_line,
|
||
text[indented_start:],
|
||
True,
|
||
"indented_code",
|
||
)
|
||
)
|
||
mask_range(fence_masked, indented_start, len(text))
|
||
mask_range(reader_masked, indented_start, len(text))
|
||
if open_start is not None:
|
||
blocks.append(
|
||
FencedBlock(
|
||
open_start,
|
||
len(text),
|
||
open_line,
|
||
text[open_start:],
|
||
False,
|
||
)
|
||
)
|
||
mask_range(fence_masked, open_start, len(text))
|
||
mask_range(reader_masked, open_start, len(text))
|
||
if comment_start is not None:
|
||
unclosed_comments.append(comment_start)
|
||
mask_range(reader_masked, comment_start, len(text))
|
||
return (
|
||
"".join(fence_masked),
|
||
"".join(reader_masked),
|
||
blocks,
|
||
unclosed_comments,
|
||
)
|
||
|
||
|
||
def mask_fenced_blocks(text: str) -> tuple[str, list[FencedBlock]]:
|
||
fence_masked, _, blocks, _ = scan_markdown_visibility(text)
|
||
return fence_masked, blocks
|
||
|
||
|
||
def mask_raw_html_blocks(text: str) -> str:
|
||
"""Mask raw HTML block source for Markdown structural parsing only."""
|
||
|
||
masked = list(text)
|
||
state: tuple[str, str] | None = None
|
||
paragraph_open = False
|
||
offset = 0
|
||
for line in text.splitlines(keepends=True):
|
||
html_line, state = advance_html_block(
|
||
line, state, paragraph_open=paragraph_open
|
||
)
|
||
if html_line:
|
||
for index in range(offset, offset + len(line)):
|
||
if masked[index] not in "\r\n":
|
||
masked[index] = " "
|
||
paragraph_open = False
|
||
else:
|
||
content = line.rstrip("\r\n")
|
||
paragraph_open = bool(content.strip()) and not re.match(
|
||
r"^[ ]{0,3}(?:#{1,6}(?:[ \t]+|$)|(?:=+|-+)[ \t]*$|"
|
||
r"(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,}))",
|
||
content,
|
||
)
|
||
offset += len(line)
|
||
return "".join(masked)
|
||
|
||
|
||
def mask_raw_code_html_blocks(text: str) -> str:
|
||
"""Mask raw HTML containers whose contents are code/data, not prose."""
|
||
|
||
masked = list(text)
|
||
active_tag: str | None = None
|
||
offset = 0
|
||
opener = re.compile(
|
||
r"^[ ]{0,3}<(?P<tag>pre|script|style|textarea)(?:[ \t>]|$)",
|
||
re.IGNORECASE,
|
||
)
|
||
for line in text.splitlines(keepends=True):
|
||
content = line.rstrip("\r\n")
|
||
structural, _ = strip_blockquotes(content)
|
||
list_item = re.match(
|
||
r"^[ \t]*(?:[-+*]|[0-9]{1,9}[.)])[ \t]+",
|
||
structural,
|
||
)
|
||
if list_item:
|
||
structural = structural[list_item.end() :]
|
||
match = opener.match(structural) if active_tag is None else None
|
||
if match is not None:
|
||
active_tag = match.group("tag").casefold()
|
||
if active_tag is not None:
|
||
for index in range(offset, offset + len(line)):
|
||
if masked[index] not in "\r\n":
|
||
masked[index] = " "
|
||
if re.search(
|
||
rf"</{re.escape(active_tag)}[ \t]*>",
|
||
structural,
|
||
re.IGNORECASE,
|
||
):
|
||
active_tag = None
|
||
offset += len(line)
|
||
return "".join(masked)
|
||
|
||
|
||
def github_slug(value: str) -> str:
|
||
# GitHub derives an anchor from the rendered heading label, not from link
|
||
# destinations or HTML attributes that happen to occur in the source.
|
||
value = rendered_markdown_label(value)
|
||
value = unicodedata.normalize("NFKC", value).strip().lower()
|
||
value = re.sub(r"<[^>]+>", "", value)
|
||
value = value.replace("`", "")
|
||
value = re.sub(r"[^\w\- ]+", "", value, flags=re.UNICODE)
|
||
value = re.sub(r"\s+", "-", value).strip("-")
|
||
return value
|
||
|
||
|
||
def markdown_inline_links(
|
||
text: str,
|
||
*,
|
||
include_images: bool = False,
|
||
) -> list[tuple[int, int, int, int, str]]:
|
||
"""Return link/open-close and balanced destination spans (images excluded)."""
|
||
|
||
links: list[tuple[int, int, int, int, str]] = []
|
||
position = 0
|
||
|
||
def escaped(index: int) -> bool:
|
||
count = 0
|
||
index -= 1
|
||
while index >= 0 and text[index] == "\\":
|
||
count += 1
|
||
index -= 1
|
||
return count % 2 == 1
|
||
|
||
def skip_space(index: int) -> int | None:
|
||
newlines = 0
|
||
while index < len(text) and text[index] in " \t\r\n":
|
||
if text[index] == "\n":
|
||
newlines += 1
|
||
if newlines > 1:
|
||
return None
|
||
index += 1
|
||
return index
|
||
|
||
while position < len(text):
|
||
label_start = text.find("[", position)
|
||
if label_start < 0:
|
||
break
|
||
is_image = (
|
||
label_start > 0
|
||
and text[label_start - 1] == "!"
|
||
and not escaped(label_start - 1)
|
||
)
|
||
if escaped(label_start) or (is_image and not include_images):
|
||
position = label_start + 1
|
||
continue
|
||
depth = 1
|
||
cursor = label_start + 1
|
||
while cursor < len(text) and depth:
|
||
if text[cursor] == "\\" and cursor + 1 < len(text):
|
||
cursor += 2
|
||
continue
|
||
if text[cursor] == "[":
|
||
depth += 1
|
||
elif text[cursor] == "]":
|
||
depth -= 1
|
||
cursor += 1
|
||
if depth or cursor >= len(text) or text[cursor] != "(":
|
||
position = label_start + 1
|
||
continue
|
||
label_end = cursor - 1
|
||
raw_label = text[label_start + 1 : label_end]
|
||
if crosses_paragraph_boundary(text, label_start + 1, label_end):
|
||
position = label_start + 1
|
||
continue
|
||
cursor = skip_space(cursor + 1)
|
||
if cursor is None or cursor >= len(text):
|
||
position = label_start + 1
|
||
continue
|
||
if text[cursor] == "<":
|
||
destination_start = cursor + 1
|
||
destination_end = destination_start
|
||
while destination_end < len(text) and text[destination_end] not in ">\r\n":
|
||
if text[destination_end] == "\\" and destination_end + 1 < len(text):
|
||
destination_end += 2
|
||
else:
|
||
destination_end += 1
|
||
if destination_end >= len(text) or text[destination_end] != ">":
|
||
position = label_start + 1
|
||
continue
|
||
tail = destination_end + 1
|
||
else:
|
||
destination_start = cursor
|
||
depth = 0
|
||
while cursor < len(text):
|
||
character = text[cursor]
|
||
if character == "\\" and cursor + 1 < len(text):
|
||
cursor += 2
|
||
continue
|
||
if character == "(":
|
||
depth += 1
|
||
elif character == ")":
|
||
if depth == 0:
|
||
destination_end = cursor
|
||
links.append(
|
||
(
|
||
label_start,
|
||
cursor + 1,
|
||
destination_start,
|
||
destination_end,
|
||
raw_label,
|
||
)
|
||
)
|
||
position = cursor + 1
|
||
break
|
||
depth -= 1
|
||
elif character in " \t\r\n" and depth == 0:
|
||
destination_end = cursor
|
||
tail = cursor
|
||
break
|
||
cursor += 1
|
||
else:
|
||
position = label_start + 1
|
||
continue
|
||
if links and links[-1][0] == label_start:
|
||
continue
|
||
if cursor >= len(text):
|
||
position = label_start + 1
|
||
continue
|
||
|
||
unspaced_tail = tail
|
||
spaced_tail = skip_space(tail)
|
||
if spaced_tail is None:
|
||
position = label_start + 1
|
||
continue
|
||
tail = spaced_tail
|
||
if tail < len(text) and text[tail] in "\"'(":
|
||
if tail == unspaced_tail:
|
||
position = label_start + 1
|
||
continue
|
||
opener = text[tail]
|
||
quote = ")" if opener == "(" else opener
|
||
tail += 1
|
||
title_start = tail
|
||
while tail < len(text) and text[tail] != quote:
|
||
if text[tail] == "\\" and tail + 1 < len(text):
|
||
tail += 2
|
||
continue
|
||
if opener == "(" and text[tail] == "(":
|
||
break
|
||
if re.match(r"\r?\n[ \t]*\r?\n", text[tail:]):
|
||
break
|
||
tail += 1
|
||
if tail >= len(text) or text[tail] != quote:
|
||
position = label_start + 1
|
||
continue
|
||
if re.search(r"\r?\n[ \t]*\r?\n", text[title_start:tail]):
|
||
position = label_start + 1
|
||
continue
|
||
tail += 1
|
||
while tail < len(text) and text[tail] in " \t":
|
||
tail += 1
|
||
if tail < len(text) and text[tail] == ")":
|
||
links.append(
|
||
(
|
||
label_start,
|
||
tail + 1,
|
||
destination_start,
|
||
destination_end,
|
||
raw_label,
|
||
)
|
||
)
|
||
position = tail + 1
|
||
else:
|
||
position = label_start + 1
|
||
return links
|
||
|
||
|
||
def markdown_reference_links(
|
||
text: str,
|
||
definitions: Iterable[ReferenceDefinition] | None = None,
|
||
*,
|
||
include_images: bool = False,
|
||
) -> list[ReferenceLink]:
|
||
"""Return full, collapsed, and resolvable shortcut reference links."""
|
||
|
||
parsed_definitions = list(definitions or reference_definitions(text))
|
||
known = {item.identifier for item in parsed_definitions}
|
||
definition_spans = [(item.start, item.end) for item in parsed_definitions]
|
||
links: list[ReferenceLink] = []
|
||
position = 0
|
||
while position < len(text):
|
||
label_start = text.find("[", position)
|
||
if label_start < 0:
|
||
break
|
||
is_image = (
|
||
label_start > 0
|
||
and text[label_start - 1] == "!"
|
||
and not is_backslash_escaped(text, label_start - 1)
|
||
)
|
||
if is_backslash_escaped(text, label_start) or (
|
||
is_image and not include_images
|
||
):
|
||
position = label_start + 1
|
||
continue
|
||
if any(start <= label_start < end for start, end in definition_spans):
|
||
position = label_start + 1
|
||
continue
|
||
depth = 1
|
||
cursor = label_start + 1
|
||
while cursor < len(text) and depth:
|
||
if text[cursor] == "\\" and cursor + 1 < len(text):
|
||
cursor += 2
|
||
continue
|
||
if text[cursor] == "[":
|
||
depth += 1
|
||
elif text[cursor] == "]":
|
||
depth -= 1
|
||
cursor += 1
|
||
if depth:
|
||
break
|
||
label_end = cursor - 1
|
||
raw_label = text[label_start + 1 : label_end]
|
||
if crosses_paragraph_boundary(text, label_start + 1, label_end):
|
||
position = label_start + 1
|
||
continue
|
||
if cursor < len(text) and text[cursor] == "(":
|
||
position = cursor + 1
|
||
continue
|
||
|
||
if cursor < len(text) and text[cursor] == "[":
|
||
identifier_start = cursor + 1
|
||
identifier_end = identifier_start
|
||
while identifier_end < len(text):
|
||
if text[identifier_end] == "]" and not is_backslash_escaped(
|
||
text, identifier_end
|
||
):
|
||
break
|
||
if text[identifier_end] in "\r\n":
|
||
break
|
||
identifier_end += 1
|
||
if identifier_end >= len(text) or text[identifier_end] != "]":
|
||
position = label_start + 1
|
||
continue
|
||
raw_identifier = text[identifier_start:identifier_end] or raw_label
|
||
identifier = normalize_reference_identifier(raw_identifier)
|
||
if identifier not in known:
|
||
position = identifier_end + 1
|
||
continue
|
||
links.append(
|
||
ReferenceLink(
|
||
label_start,
|
||
identifier_end + 1,
|
||
cursor,
|
||
identifier_end + 1,
|
||
raw_label,
|
||
identifier,
|
||
)
|
||
)
|
||
position = identifier_end + 1
|
||
continue
|
||
|
||
identifier = normalize_reference_identifier(raw_label)
|
||
if identifier in known:
|
||
links.append(
|
||
ReferenceLink(
|
||
label_start,
|
||
label_end + 1,
|
||
label_end + 1,
|
||
label_end + 1,
|
||
raw_label,
|
||
identifier,
|
||
)
|
||
)
|
||
position = label_end + 1
|
||
return links
|
||
|
||
|
||
def mask_markdown_metadata(text: str) -> str:
|
||
"""Mask non-rendered destinations/tags while preserving visible labels/code."""
|
||
|
||
masked = list(text)
|
||
inline_spans = inline_code_spans(text)
|
||
raw_html_structure = mask_raw_html_blocks(text)
|
||
|
||
def inside_raw_html_block(position: int) -> bool:
|
||
return (
|
||
0 <= position < len(text)
|
||
and not text[position].isspace()
|
||
and raw_html_structure[position] == " "
|
||
)
|
||
|
||
for link_start, link_end, target_start, _, _ in markdown_inline_links(
|
||
text, include_images=True
|
||
):
|
||
if inside_any_span(link_start, inline_spans) or inside_raw_html_block(
|
||
link_start
|
||
):
|
||
continue
|
||
metadata_start = text.rfind("(", link_start, target_start)
|
||
for index in range(metadata_start, link_end):
|
||
if masked[index] not in "\r\n":
|
||
masked[index] = " "
|
||
definitions = reference_definitions(text)
|
||
for definition in definitions:
|
||
if inside_any_span(definition.start, inline_spans) or inside_raw_html_block(
|
||
definition.start
|
||
):
|
||
continue
|
||
for index in range(definition.start, definition.end):
|
||
if masked[index] not in "\r\n":
|
||
masked[index] = " "
|
||
for link in markdown_reference_links(text, definitions, include_images=True):
|
||
if inside_any_span(link.start, inline_spans) or inside_raw_html_block(
|
||
link.start
|
||
):
|
||
continue
|
||
for index in range(link.metadata_start, link.metadata_end):
|
||
if masked[index] not in "\r\n":
|
||
masked[index] = " "
|
||
for start, end in html_tag_spans(text):
|
||
if inside_any_span(start, inline_spans):
|
||
continue
|
||
for index in range(start, end):
|
||
if masked[index] not in "\r\n":
|
||
masked[index] = " "
|
||
return "".join(masked)
|
||
|
||
|
||
def rendered_search_projection(
|
||
text: str,
|
||
*,
|
||
include_inline_code_text: bool = False,
|
||
) -> tuple[str, list[int]]:
|
||
"""Return reader-visible searchable prose plus source-offset mapping."""
|
||
|
||
projected = mask_markdown_metadata(text)
|
||
removed: set[int] = set()
|
||
inline_spans = inline_code_spans(text)
|
||
|
||
def remove_label_delimiters(start: int, close: int, image: bool) -> None:
|
||
removed.add(start)
|
||
removed.add(close)
|
||
if image and start > 0:
|
||
removed.add(start - 1)
|
||
|
||
for link_start, _, target_start, _, _ in markdown_inline_links(
|
||
text, include_images=True
|
||
):
|
||
if inside_any_span(link_start, inline_spans):
|
||
continue
|
||
close = text.rfind("]", link_start, target_start)
|
||
if close > link_start:
|
||
remove_label_delimiters(
|
||
link_start,
|
||
close,
|
||
link_start > 0
|
||
and text[link_start - 1] == "!"
|
||
and not is_backslash_escaped(text, link_start - 1),
|
||
)
|
||
definitions = reference_definitions(text)
|
||
for link in markdown_reference_links(text, definitions, include_images=True):
|
||
if inside_any_span(link.start, inline_spans):
|
||
continue
|
||
close = text.find("]", link.start + 1, link.end)
|
||
if close > link.start:
|
||
remove_label_delimiters(
|
||
link.start,
|
||
close,
|
||
link.start > 0
|
||
and text[link.start - 1] == "!"
|
||
and not is_backslash_escaped(text, link.start - 1),
|
||
)
|
||
|
||
for marker in ("***", "___", "**", "__", "~~", "*", "_"):
|
||
pattern = re.compile(
|
||
rf"{re.escape(marker)}(?=\S)"
|
||
rf"(?P<body>(?:(?!\r?\n[ \t]*\r?\n).)+?)"
|
||
rf"(?<=\S){re.escape(marker)}",
|
||
re.DOTALL,
|
||
)
|
||
for match in pattern.finditer(text):
|
||
close_start = match.end() - len(marker)
|
||
if is_backslash_escaped(text, match.start()) or is_backslash_escaped(
|
||
text, close_start
|
||
):
|
||
continue
|
||
if inside_any_span(match.start(), inline_spans) or crosses_paragraph_boundary(
|
||
text, match.start() + len(marker), close_start
|
||
):
|
||
continue
|
||
if marker.startswith("_") and (
|
||
(match.start() > 0 and text[match.start() - 1].isalnum())
|
||
or (match.end() < len(text) and text[match.end()].isalnum())
|
||
):
|
||
continue
|
||
removed.update(range(match.start(), match.start() + len(marker)))
|
||
removed.update(range(close_start, match.end()))
|
||
|
||
if include_inline_code_text:
|
||
for start, end, _ in inline_spans:
|
||
opening = re.match(r"`+", text[start:end])
|
||
closing = re.search(r"`+$", text[start:end])
|
||
if opening is None or closing is None:
|
||
continue
|
||
removed.update(range(start, start + opening.end()))
|
||
removed.update(range(start + closing.start(), end))
|
||
|
||
stripped_chars: list[str] = []
|
||
stripped_offsets: list[int] = []
|
||
for index, character in enumerate(projected):
|
||
if index not in removed:
|
||
stripped_chars.append(character)
|
||
stripped_offsets.append(index)
|
||
decoded, decoded_offsets = decode_html_entities_with_offsets(
|
||
"".join(stripped_chars)
|
||
)
|
||
return decoded, [stripped_offsets[index] for index in decoded_offsets]
|
||
|
||
|
||
def resolve_reference_labels(text: str, known: set[str]) -> str:
|
||
"""Replace resolvable reference syntax with its reader-visible label."""
|
||
|
||
def full(match: re.Match[str]) -> str:
|
||
bracket = match.start() + (1 if match.group(0).startswith("!") else 0)
|
||
if is_backslash_escaped(text, bracket):
|
||
return match.group(0)
|
||
label = match.group("label")
|
||
identifier = match.group("identifier") or label
|
||
return label if normalize_reference_identifier(identifier) in known else match.group(0)
|
||
|
||
text = re.sub(
|
||
r"!?\[(?P<label>[^\]\r\n]+)\]\[(?P<identifier>[^\]\r\n]*)\]",
|
||
full,
|
||
text,
|
||
)
|
||
|
||
def shortcut(match: re.Match[str]) -> str:
|
||
bracket = match.start() + (1 if match.group(0).startswith("!") else 0)
|
||
if is_backslash_escaped(text, bracket):
|
||
return match.group(0)
|
||
label = match.group("label")
|
||
return label if normalize_reference_identifier(label) in known else match.group(0)
|
||
|
||
return re.sub(r"!?\[(?P<label>[^\]\r\n]+)\]", shortcut, text)
|
||
|
||
|
||
def rendered_markdown_label(text: str) -> str:
|
||
"""Return a stable approximation of text a reader sees in a heading/label."""
|
||
|
||
value, _ = decode_html_entities_with_offsets(mask_markdown_metadata(text))
|
||
escaped: list[str] = []
|
||
|
||
def protect_escape(match: re.Match[str]) -> str:
|
||
escaped.append(match.group(1))
|
||
return chr(0xE000 + len(escaped) - 1)
|
||
|
||
value = re.sub(
|
||
r"\\([!\"#$%&'()*+,./:;<=>?@\[\\\]^_`{|}~-])",
|
||
protect_escape,
|
||
value,
|
||
)
|
||
value = re.sub(r"!\[([^\]\r\n]*)\]", r"\1", value)
|
||
value = re.sub(r"\[([^\]\r\n]+)\]", r"\1", value)
|
||
value = re.sub(r"(?<!\w)_{1,3}(?=\S)", "", value)
|
||
value = re.sub(r"(?<=\S)_{1,3}(?!\w)", "", value)
|
||
value = re.sub(r"[`*~]", "", value)
|
||
for index, character in enumerate(escaped):
|
||
value = value.replace(chr(0xE000 + index), character)
|
||
return " ".join(value.split())
|
||
|
||
|
||
def decode_html_entities_with_offsets(text: str) -> tuple[str, list[int]]:
|
||
"""Decode reader-visible entities and map decoded indices to source offsets."""
|
||
|
||
output: list[str] = []
|
||
offsets: list[int] = []
|
||
cursor = 0
|
||
pattern = re.compile(r"&(?:#[xX][0-9A-Fa-f]+|#[0-9]+|[A-Za-z][A-Za-z0-9]+);")
|
||
for match in pattern.finditer(text):
|
||
output.append(text[cursor : match.start()])
|
||
offsets.extend(range(cursor, match.start()))
|
||
decoded = html.unescape(match.group(0))
|
||
if decoded == match.group(0):
|
||
output.append(match.group(0))
|
||
offsets.extend(range(match.start(), match.end()))
|
||
else:
|
||
output.append(decoded)
|
||
offsets.extend([match.start()] * len(decoded))
|
||
cursor = match.end()
|
||
output.append(text[cursor:])
|
||
offsets.extend(range(cursor, len(text)))
|
||
return "".join(output), offsets
|
||
|
||
|
||
def mask_inline_code(text: str) -> str:
|
||
masked = list(text)
|
||
raw_html_structure = mask_raw_html_blocks(text)
|
||
for start, end, _ in inline_code_spans(text):
|
||
if (
|
||
start < len(text)
|
||
and not text[start].isspace()
|
||
and raw_html_structure[start] == " "
|
||
):
|
||
continue
|
||
for index in range(start, end):
|
||
if masked[index] not in "\r\n":
|
||
masked[index] = " "
|
||
return "".join(masked)
|
||
|
||
|
||
def parse_headings(visible: str) -> list[Heading]:
|
||
starts = line_starts(visible)
|
||
structural = visible
|
||
nested_lines: set[int] = set()
|
||
list_indent = 0
|
||
line_offset = 0
|
||
for line in structural.splitlines(keepends=True):
|
||
content = line.rstrip("\r\n")
|
||
rest, quote_depth = strip_blockquotes(content)
|
||
leading = re.match(r"^[ \t]*", rest)
|
||
columns = indentation_columns(leading.group(0) if leading else "")
|
||
if quote_depth or (list_indent and columns >= list_indent):
|
||
nested_lines.add(line_offset)
|
||
list_indent = list_continuation_indent(line, list_indent)
|
||
line_offset += len(line)
|
||
raw_headings: list[tuple[int, int, int, str]] = []
|
||
atx_spans: list[tuple[int, int]] = []
|
||
pattern = re.compile(
|
||
r"^[ ]{0,3}(#{1,6})[ \t]+(.+?)(?:[ \t]+#+[ \t]*)?$",
|
||
re.MULTILINE,
|
||
)
|
||
for match in pattern.finditer(structural):
|
||
if match.start() in nested_lines:
|
||
continue
|
||
title = match.group(2).strip()
|
||
raw_headings.append((match.start(), match.end(), len(match.group(1)), title))
|
||
atx_spans.append(match.span())
|
||
structural_lines = structural.splitlines(keepends=True)
|
||
structural_offsets: list[int] = []
|
||
structural_offset = 0
|
||
for line in structural_lines:
|
||
structural_offsets.append(structural_offset)
|
||
structural_offset += len(line)
|
||
setext_underline = re.compile(r"^[ ]{0,3}(?P<underline>=+|-+)[ \t]*$")
|
||
prior_block = re.compile(
|
||
r"^[ ]{0,3}(?:#{1,6}(?:[ \t]+|$)|>|[-+*][ \t]+|"
|
||
r"`{3,}|~{3,}|"
|
||
r"(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}))"
|
||
)
|
||
for index, line in enumerate(structural_lines):
|
||
content = line.rstrip("\r\n")
|
||
underline = setext_underline.fullmatch(content)
|
||
if underline is None or index == 0:
|
||
continue
|
||
underline_start = structural_offsets[index]
|
||
if underline_start in nested_lines:
|
||
continue
|
||
title_indexes: list[int] = []
|
||
previous = index - 1
|
||
while previous >= 0:
|
||
previous_content = structural_lines[previous].rstrip("\r\n")
|
||
previous_start = structural_offsets[previous]
|
||
ordered_marker = re.match(
|
||
r"^[ ]{0,3}[0-9]{1,9}[.)][ \t]+(?=\S)",
|
||
previous_content,
|
||
)
|
||
ordered_starts_block = bool(
|
||
ordered_marker
|
||
and (
|
||
ordered_list_interrupts_paragraph(previous_content)
|
||
or previous == 0
|
||
or not structural_lines[previous - 1].strip()
|
||
or prior_block.match(
|
||
structural_lines[previous - 1].rstrip("\r\n")
|
||
)
|
||
or re.match(
|
||
r"^[ ]{0,3}[0-9]{1,9}[.)][ \t]+(?=\S)",
|
||
structural_lines[previous - 1].rstrip("\r\n"),
|
||
)
|
||
)
|
||
)
|
||
if (
|
||
not previous_content.strip()
|
||
or previous_start in nested_lines
|
||
or prior_block.match(previous_content)
|
||
or ordered_starts_block
|
||
or setext_underline.fullmatch(previous_content)
|
||
or len(previous_content) - len(previous_content.lstrip(" ")) > 3
|
||
):
|
||
break
|
||
title_indexes.append(previous)
|
||
previous -= 1
|
||
if not title_indexes:
|
||
continue
|
||
title_indexes.reverse()
|
||
heading_start = structural_offsets[title_indexes[0]]
|
||
if any(start <= heading_start < end for start, end in atx_spans):
|
||
continue
|
||
title = "\n".join(
|
||
structural_lines[item].rstrip("\r\n").strip()
|
||
for item in title_indexes
|
||
)
|
||
raw_headings.append(
|
||
(
|
||
heading_start,
|
||
underline_start + len(line.rstrip("\r\n")),
|
||
1 if underline.group("underline").startswith("=") else 2,
|
||
title,
|
||
)
|
||
)
|
||
|
||
headings: list[Heading] = []
|
||
used_slugs: set[str] = set()
|
||
known_references = {
|
||
item.identifier for item in reference_definitions(structural)
|
||
}
|
||
for start, end, level, title in sorted(raw_headings):
|
||
title = resolve_reference_labels(title, known_references)
|
||
base = github_slug(title)
|
||
slug = base
|
||
suffix = 1
|
||
while slug in used_slugs:
|
||
slug = f"{base}-{suffix}"
|
||
suffix += 1
|
||
used_slugs.add(slug)
|
||
line, _ = location(starts, start)
|
||
headings.append(Heading(level, title, line, start, end, slug))
|
||
return headings
|
||
|
||
|
||
def paragraph_spans(visible: str) -> list[TextSpan]:
|
||
# A pipe table is a comparison structure, not one long prose paragraph.
|
||
# Mask complete GFM-style table blocks while preserving offsets/newlines so
|
||
# surrounding prose still receives its normal paragraph budget.
|
||
table_masked = list(visible)
|
||
lines = visible.splitlines(keepends=True)
|
||
line_offsets: list[int] = []
|
||
offset = 0
|
||
for line in lines:
|
||
line_offsets.append(offset)
|
||
offset += len(line)
|
||
|
||
def is_delimiter_row(line: str) -> bool:
|
||
stripped = line.rstrip("\r\n").strip()
|
||
if "|" not in stripped:
|
||
return False
|
||
cells = stripped.strip("|").split("|")
|
||
return bool(cells) and all(
|
||
re.fullmatch(r"[ \t]*:?-{3,}:?[ \t]*", cell) for cell in cells
|
||
)
|
||
|
||
def normalized_table_line(
|
||
line: str,
|
||
expected: tuple[int, int] | None = None,
|
||
) -> tuple[str, tuple[int, int]] | None:
|
||
content = line.rstrip("\r\n")
|
||
rest, quote_depth = strip_blockquotes(content)
|
||
if expected is not None and quote_depth != expected[0]:
|
||
return None
|
||
item = re.match(
|
||
r"^(?P<prefix>[ \t]*(?:[-+*]|[0-9]{1,9}[.)])[ \t]+)",
|
||
rest,
|
||
)
|
||
if item is not None:
|
||
list_indent = indentation_columns(item.group("prefix"))
|
||
if expected is not None and expected[1] not in {0, list_indent}:
|
||
return None
|
||
rest = rest[item.end() :]
|
||
elif expected is not None and expected[1]:
|
||
leading = re.match(r"^[ \t]*", rest)
|
||
raw_indent = leading.group(0) if leading else ""
|
||
if indentation_columns(raw_indent) < expected[1]:
|
||
return None
|
||
consumed = 0
|
||
columns = 0
|
||
while consumed < len(raw_indent) and columns < expected[1]:
|
||
columns += (
|
||
4 - (columns % 4)
|
||
if raw_indent[consumed] == "\t"
|
||
else 1
|
||
)
|
||
consumed += 1
|
||
rest = rest[consumed:]
|
||
list_indent = expected[1]
|
||
else:
|
||
list_indent = 0
|
||
return rest, (quote_depth, list_indent)
|
||
|
||
index = 0
|
||
while index + 1 < len(lines):
|
||
normalized_header = normalized_table_line(lines[index])
|
||
if normalized_header is None:
|
||
index += 1
|
||
continue
|
||
header, container = normalized_header
|
||
normalized_delimiter = normalized_table_line(lines[index + 1], container)
|
||
if (
|
||
"|" not in header
|
||
or normalized_delimiter is None
|
||
or not is_delimiter_row(normalized_delimiter[0])
|
||
):
|
||
index += 1
|
||
continue
|
||
end_index = index + 2
|
||
while end_index < len(lines):
|
||
normalized_row = normalized_table_line(lines[end_index], container)
|
||
if (
|
||
normalized_row is None
|
||
or not normalized_row[0].strip()
|
||
or "|" not in normalized_row[0]
|
||
):
|
||
break
|
||
end_index += 1
|
||
block_start = line_offsets[index]
|
||
block_end = line_offsets[end_index] if end_index < len(lines) else len(visible)
|
||
for position in range(block_start, block_end):
|
||
if table_masked[position] not in "\r\n":
|
||
table_masked[position] = " "
|
||
index = end_index
|
||
|
||
visible = "".join(table_masked)
|
||
headings = parse_headings(mask_raw_html_blocks(visible))
|
||
|
||
# Thematic breaks are block separators, not prose. A `---` line that is
|
||
# already part of a parsed Setext heading remains part of that heading.
|
||
block_masked = list(visible)
|
||
quote_boundaries: list[int] = []
|
||
quote_active = False
|
||
previous_nonblank = False
|
||
offset = 0
|
||
thematic = re.compile(
|
||
r"^[ ]{0,3}(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,})$"
|
||
)
|
||
for line in visible.splitlines(keepends=True):
|
||
content = line.rstrip("\r\n")
|
||
rest, quote_depth = strip_blockquotes(content)
|
||
in_heading = any(
|
||
heading.start <= offset < heading.end for heading in headings
|
||
)
|
||
if not in_heading and thematic.fullmatch(rest):
|
||
for position in range(offset, offset + len(line)):
|
||
if block_masked[position] not in "\r\n":
|
||
block_masked[position] = " "
|
||
quote_active = False
|
||
previous_nonblank = False
|
||
offset += len(line)
|
||
continue
|
||
if not content.strip():
|
||
quote_active = False
|
||
previous_nonblank = False
|
||
elif quote_depth:
|
||
if not quote_active and previous_nonblank:
|
||
quote_boundaries.append(offset)
|
||
quote_active = True
|
||
previous_nonblank = True
|
||
else:
|
||
# An unmarked line immediately following a blockquote is a lazy
|
||
# continuation until a blank/block boundary proves otherwise.
|
||
previous_nonblank = True
|
||
offset += len(line)
|
||
|
||
visible = "".join(block_masked)
|
||
starts = line_starts(visible)
|
||
paragraphs: list[TextSpan] = []
|
||
|
||
def append_segment(start: int, end: int) -> None:
|
||
body = visible[start:end]
|
||
stripped = body.strip()
|
||
if not stripped:
|
||
return
|
||
leading = len(body) - len(body.lstrip())
|
||
segment_start = start + leading
|
||
if re.fullmatch(r"(?:[ \t]*<!--.*?-->[ \t]*\n?)+", stripped, re.DOTALL):
|
||
return
|
||
if all(
|
||
not line.strip()
|
||
or re.match(r"^\s*(?:<!--|-->|\|?\s*:?-{3,})", line)
|
||
for line in stripped.splitlines()
|
||
):
|
||
return
|
||
line, _ = location(starts, segment_start)
|
||
paragraphs.append(TextSpan(segment_start, end, line, visible[segment_start:end]))
|
||
|
||
def append_with_lists(start: int, end: int) -> None:
|
||
body = visible[start:end]
|
||
list_items = list(
|
||
re.finditer(r"(?m)^(?P<indent>[ \t]*)(?:[-+*]|\d+[.)])[ \t]+", body)
|
||
)
|
||
if not list_items:
|
||
append_segment(start, end)
|
||
return
|
||
minimum_indent = min(len(item.group("indent")) for item in list_items)
|
||
boundaries = [
|
||
item.start()
|
||
for item in list_items
|
||
if len(item.group("indent")) == minimum_indent
|
||
]
|
||
if boundaries[0] > 0:
|
||
append_segment(start, start + boundaries[0])
|
||
for index, boundary in enumerate(boundaries):
|
||
item_end = boundaries[index + 1] if index + 1 < len(boundaries) else len(body)
|
||
append_segment(start + boundary, start + item_end)
|
||
|
||
def append_with_headings(start: int, end: int) -> None:
|
||
block_headings = [
|
||
heading
|
||
for heading in headings
|
||
if start <= heading.start and heading.end <= end
|
||
]
|
||
cursor = start
|
||
for heading in block_headings:
|
||
append_with_lists(cursor, heading.start)
|
||
append_segment(heading.start, heading.end)
|
||
cursor = heading.end
|
||
append_with_lists(cursor, end)
|
||
|
||
for match in re.finditer(r"(?:^|\n[ \t]*\n)(?P<body>.*?)(?=\n[ \t]*\n|\Z)", visible, re.DOTALL):
|
||
body_start, body_end = match.span("body")
|
||
cursor = body_start
|
||
for boundary in quote_boundaries:
|
||
if body_start < boundary < body_end:
|
||
append_with_headings(cursor, boundary)
|
||
cursor = boundary
|
||
append_with_headings(cursor, body_end)
|
||
return paragraphs
|
||
|
||
|
||
def sentence_spans(paragraph: TextSpan) -> list[TextSpan]:
|
||
spans: list[TextSpan] = []
|
||
cursor = 0
|
||
pattern = re.compile(r"[.!?。！？]+(?:[\"'”’)]*)\s+")
|
||
for match in pattern.finditer(paragraph.text):
|
||
end = match.end()
|
||
if paragraph.text[cursor:end].strip():
|
||
spans.append(
|
||
TextSpan(
|
||
paragraph.start + cursor,
|
||
paragraph.start + end,
|
||
paragraph.line,
|
||
paragraph.text[cursor:end],
|
||
)
|
||
)
|
||
cursor = end
|
||
if paragraph.text[cursor:].strip():
|
||
spans.append(
|
||
TextSpan(
|
||
paragraph.start + cursor,
|
||
paragraph.end,
|
||
paragraph.line,
|
||
paragraph.text[cursor:],
|
||
)
|
||
)
|
||
return spans
|
||
|
||
|
||
def normalize_space(value: str) -> str:
|
||
return " ".join(value.split())
|
||
|
||
|
||
def normalize_term_name(value: str) -> str:
|
||
return unicodedata.normalize("NFKC", normalize_space(value)).casefold()
|
||
|
||
|
||
def find_flexible(haystack: str, needle: str) -> re.Match[str] | None:
|
||
parts = needle.split()
|
||
if not parts:
|
||
return None
|
||
pattern = r"\s+".join(re.escape(part) for part in parts)
|
||
if re.fullmatch(r"[A-Za-z0-9_$.-]+", needle):
|
||
pattern = rf"(?<![A-Za-z0-9_$]){pattern}(?![A-Za-z0-9_$])"
|
||
return re.search(pattern, haystack)
|
||
|
||
|
||
def find_literal(haystack: str, needle: str) -> int:
|
||
if re.fullmatch(r"[A-Za-z0-9_$.-]+", needle):
|
||
match = re.search(
|
||
rf"(?<![A-Za-z0-9_$]){re.escape(needle)}(?![A-Za-z0-9_$])",
|
||
haystack,
|
||
)
|
||
return match.start() if match else -1
|
||
return haystack.find(needle)
|
||
|
||
|
||
def compile_claim_marker_patterns(templates: list[str]) -> list[re.Pattern[str]]:
|
||
patterns: list[re.Pattern[str]] = []
|
||
claim_group = r"(?P<id>[^\s<>\[\]{}()]+)"
|
||
for template in templates:
|
||
if template.count("{claim_id}") != 1:
|
||
raise InputError(
|
||
"evidence marker에는 {claim_id} placeholder가 정확히 하나 필요합니다."
|
||
)
|
||
try:
|
||
patterns.append(
|
||
re.compile(
|
||
template.replace("{claim_id}", claim_group),
|
||
re.IGNORECASE,
|
||
)
|
||
)
|
||
except re.error as exc:
|
||
raise InputError(f"evidence marker 정규식이 잘못되었습니다: {template}: {exc}") from exc
|
||
return patterns
|
||
|
||
|
||
def extract_claim_markers(
|
||
fence_visible: str,
|
||
reader_visible: str,
|
||
templates: list[str],
|
||
) -> list[tuple[str, int]]:
|
||
"""Extract configured exact comments and reader-visible claim markers."""
|
||
|
||
markers: list[tuple[str, int]] = []
|
||
patterns = compile_claim_marker_patterns(templates)
|
||
marker_visible = mask_markdown_metadata(
|
||
mask_raw_code_html_blocks(reader_visible)
|
||
)
|
||
fence_visible = mask_raw_code_html_blocks(fence_visible)
|
||
inline_spans = inline_code_spans(marker_visible)
|
||
excluded_comment_spans = list(inline_code_spans(fence_visible))
|
||
excluded_comment_spans.extend(
|
||
(start, end, "") for start, end in html_tag_spans(fence_visible)
|
||
)
|
||
excluded_comment_spans.extend(
|
||
(target_start, link_end, "")
|
||
for _, link_end, target_start, _, _ in markdown_inline_links(
|
||
fence_visible, include_images=True
|
||
)
|
||
)
|
||
definitions = reference_definitions(fence_visible)
|
||
excluded_comment_spans.extend(
|
||
(item.start, item.end, "") for item in definitions
|
||
)
|
||
excluded_comment_spans.extend(
|
||
(item.metadata_start, item.metadata_end, "")
|
||
for item in markdown_reference_links(
|
||
fence_visible, definitions, include_images=True
|
||
)
|
||
)
|
||
cursor = 0
|
||
while True:
|
||
start = fence_visible.find("<!--", cursor)
|
||
if start < 0:
|
||
break
|
||
if inside_any_span(start, excluded_comment_spans):
|
||
cursor = start + 4
|
||
continue
|
||
closing = fence_visible.find("-->", start + 4)
|
||
if closing < 0:
|
||
break
|
||
end = closing + 3
|
||
for pattern in patterns:
|
||
match = pattern.fullmatch(fence_visible[start:end])
|
||
if match:
|
||
markers.append((match.group("id"), start))
|
||
break
|
||
cursor = end
|
||
for pattern in patterns:
|
||
for match in pattern.finditer(marker_visible):
|
||
if inside_any_span(match.start(), inline_spans):
|
||
continue
|
||
markers.append((match.group("id"), match.start()))
|
||
return sorted(set(markers), key=lambda item: item[1])
|
||
|
||
|
||
def require_fields(value: dict[str, Any], fields: Iterable[str], label: str) -> None:
|
||
missing = [field for field in fields if field not in value]
|
||
if missing:
|
||
raise InputError(f"{label} 필수 필드가 없습니다: {', '.join(missing)}")
|
||
|
||
|
||
def validate_contracts(
|
||
logic_map: dict[str, Any],
|
||
term_ledger: dict[str, Any],
|
||
reader_contract: dict[str, Any],
|
||
rules: dict[str, Any],
|
||
) -> None:
|
||
for value, label in (
|
||
(logic_map, "logic map"),
|
||
(term_ledger, "term ledger"),
|
||
(reader_contract, "reader contract"),
|
||
):
|
||
schema_version(value, Path(label))
|
||
require_fields(logic_map, REQUIRED_LOGIC_TOP, "logic map")
|
||
require_fields(term_ledger, ("schema_version", "assumed_known", "budgets", "terms"), "term ledger")
|
||
require_fields(reader_contract, REQUIRED_READER, "reader contract")
|
||
for field in ("title", "core_claim", "closure"):
|
||
if not isinstance(logic_map[field], str) or not logic_map[field].strip():
|
||
raise InputError(f"logic map {field}는 비어 있지 않은 문자열이어야 합니다.")
|
||
if not isinstance(logic_map["sections"], list) or not logic_map["sections"]:
|
||
raise InputError("logic map sections는 비어 있지 않은 배열이어야 합니다.")
|
||
section_ids: set[str] = set()
|
||
term_sections: dict[str, list[str]] = defaultdict(list)
|
||
for index, section in enumerate(logic_map["sections"]):
|
||
if not isinstance(section, dict):
|
||
raise InputError(f"logic map sections[{index}]는 객체여야 합니다.")
|
||
require_fields(section, REQUIRED_LOGIC_SECTION, f"logic map sections[{index}]")
|
||
section_id = section["id"]
|
||
if not isinstance(section_id, str) or not section_id.strip() or section_id in section_ids:
|
||
raise InputError(f"logic map section id가 비었거나 중복됩니다: {section_id!r}")
|
||
section_ids.add(section_id)
|
||
for field in ("depends_on", "claim_ids", "new_terms"):
|
||
if not isinstance(section[field], list) or any(
|
||
not isinstance(item, str) or not item.strip() for item in section[field]
|
||
):
|
||
raise InputError(
|
||
f"logic map section {section_id}.{field}는 비어 있지 않은 문자열 배열이어야 합니다."
|
||
)
|
||
for term_id in section["new_terms"]:
|
||
term_sections[term_id].append(section_id)
|
||
for field in ("heading", "role", "reader_state_before", "question", "answer_plain", "reader_state_after"):
|
||
if not isinstance(section[field], str) or not section[field].strip():
|
||
raise InputError(f"logic map section {section_id}.{field}가 비어 있습니다.")
|
||
if section["transition_to"] is not None and not isinstance(section["transition_to"], str):
|
||
raise InputError(f"logic map section {section_id}.transition_to 형식이 잘못되었습니다.")
|
||
if not isinstance(term_ledger["terms"], list):
|
||
raise InputError("term ledger terms는 배열이어야 합니다.")
|
||
term_ids: set[str] = set()
|
||
name_owners: dict[str, tuple[str, str]] = {}
|
||
for index, term in enumerate(term_ledger["terms"]):
|
||
if not isinstance(term, dict):
|
||
raise InputError(f"term ledger terms[{index}]는 객체여야 합니다.")
|
||
require_fields(term, REQUIRED_TERM, f"term ledger terms[{index}]")
|
||
for field in ("id", "canonical", "plain_definition", "why_needed", "first_section", "first_use"):
|
||
if not isinstance(term[field], str) or not term[field].strip():
|
||
raise InputError(f"term ledger terms[{index}].{field}가 비어 있습니다.")
|
||
if term["id"] in term_ids:
|
||
raise InputError(f"term id가 중복됩니다: {term['id']}")
|
||
term_ids.add(term["id"])
|
||
if term["first_section"] not in section_ids:
|
||
raise InputError(f"term {term['id']}의 first_section이 logic map에 없습니다.")
|
||
if not isinstance(term["aliases"], list) or any(
|
||
not isinstance(alias, str) or not alias.strip() for alias in term["aliases"]
|
||
):
|
||
raise InputError(
|
||
f"term {term['id']}.aliases는 비어 있지 않은 문자열 배열이어야 합니다."
|
||
)
|
||
expected_sections = term_sections.get(term["id"], [])
|
||
if expected_sections != [term["first_section"]]:
|
||
raise InputError(
|
||
f"term {term['id']}는 first_section={term['first_section']!r}의 "
|
||
f"new_terms에 정확히 한 번 있어야 합니다: {expected_sections!r}"
|
||
)
|
||
for field, value in (
|
||
("canonical", term["canonical"]),
|
||
*(("alias", alias) for alias in term["aliases"]),
|
||
("english", term.get("english")),
|
||
("abbreviation", term.get("abbreviation")),
|
||
):
|
||
if not isinstance(value, str) or not value.strip():
|
||
continue
|
||
normalized = normalize_term_name(value)
|
||
previous = name_owners.get(normalized)
|
||
if previous is not None:
|
||
raise InputError(
|
||
f"용어 이름 {value!r}이 둘 이상에 배정되었습니다: "
|
||
f"{previous[0]}.{previous[1]}, {term['id']}.{field}"
|
||
)
|
||
name_owners[normalized] = (term["id"], field)
|
||
unknown_term_ids = sorted(set(term_sections) - term_ids)
|
||
if unknown_term_ids:
|
||
raise InputError(f"logic map new_terms에 없는 term id가 있습니다: {unknown_term_ids}")
|
||
expected = rules["thresholds"]["term"]
|
||
budgets = term_ledger.get("budgets")
|
||
if not isinstance(budgets, dict):
|
||
raise InputError("term ledger budgets는 객체여야 합니다.")
|
||
for ledger_key, config_key in (
|
||
("per_sentence", "max_new_terms_per_sentence"),
|
||
("per_paragraph", "max_new_terms_per_paragraph"),
|
||
("per_section", "max_new_terms_per_section"),
|
||
):
|
||
if budgets.get(ledger_key) != expected[config_key]:
|
||
raise InputError(
|
||
f"term ledger budgets.{ledger_key}={budgets.get(ledger_key)!r}가 "
|
||
f"quality rules 값 {expected[config_key]}와 다릅니다."
|
||
)
|
||
for field in ("prerequisites", "assumed_known", "must_explain", "non_goals"):
|
||
if not isinstance(reader_contract[field], list) or any(
|
||
not isinstance(item, str) or not item.strip() for item in reader_contract[field]
|
||
):
|
||
raise InputError(
|
||
f"reader contract {field}는 비어 있지 않은 문자열 배열이어야 합니다."
|
||
)
|
||
for field in ("primary_audience", "purpose", "reader_question", "reader_outcome"):
|
||
if not isinstance(reader_contract[field], str) or not reader_contract[field].strip():
|
||
raise InputError(f"reader contract {field}가 비어 있습니다.")
|
||
assumed = {
|
||
normalize_term_name(str(item)) for item in reader_contract["assumed_known"]
|
||
}
|
||
must_explain = {
|
||
normalize_term_name(str(item)) for item in reader_contract["must_explain"]
|
||
}
|
||
overlap = sorted(assumed & must_explain)
|
||
if overlap:
|
||
raise InputError(
|
||
"reader contract assumed_known와 must_explain은 겹칠 수 없습니다: "
|
||
f"{overlap}"
|
||
)
|
||
explainable: set[str] = set()
|
||
for term in term_ledger["terms"]:
|
||
for value in (
|
||
term["canonical"],
|
||
*term["aliases"],
|
||
term.get("english"),
|
||
term.get("abbreviation"),
|
||
):
|
||
if isinstance(value, str) and value.strip():
|
||
explainable.add(normalize_term_name(value))
|
||
missing_explanations = sorted(
|
||
item
|
||
for item in reader_contract["must_explain"]
|
||
if normalize_term_name(item) not in explainable
|
||
)
|
||
if missing_explanations:
|
||
raise InputError(
|
||
"reader contract must_explain 항목이 term ledger에 등록되지 않았습니다: "
|
||
f"{missing_explanations}"
|
||
)
|
||
|
||
|
||
class Linter:
|
||
def __init__(
|
||
self,
|
||
*,
|
||
document_path: Path,
|
||
text: str,
|
||
logic_map: dict[str, Any],
|
||
term_ledger: dict[str, Any],
|
||
reader_contract: dict[str, Any],
|
||
rules: dict[str, Any],
|
||
) -> None:
|
||
self.document_path = document_path
|
||
self.text = text
|
||
(
|
||
self.visible,
|
||
self.prose_visible,
|
||
self.fenced,
|
||
self.unclosed_comments,
|
||
) = scan_markdown_visibility(text)
|
||
self.markdown_visible = mask_inline_code(self.prose_visible)
|
||
self.metadata_visible = mask_markdown_metadata(
|
||
mask_raw_code_html_blocks(self.markdown_visible)
|
||
)
|
||
contract_source = mask_raw_html_blocks(self.markdown_visible)
|
||
term_source = mask_raw_html_blocks(self.prose_visible)
|
||
logic_source = mask_raw_html_blocks(self.prose_visible)
|
||
self.contract_visible = mask_markdown_metadata(contract_source)
|
||
self.term_visible = mask_markdown_metadata(term_source)
|
||
self.candidate_visible = mask_markdown_metadata(
|
||
mask_raw_html_blocks(self.prose_visible)
|
||
)
|
||
self.metadata_search, self.metadata_offsets = decode_html_entities_with_offsets(
|
||
self.metadata_visible
|
||
)
|
||
self.term_search, self.term_offsets = rendered_search_projection(
|
||
term_source,
|
||
include_inline_code_text=True,
|
||
)
|
||
self.term_inline_spans = inline_code_spans(self.prose_visible)
|
||
self.candidate_search, self.candidate_offsets = decode_html_entities_with_offsets(
|
||
self.candidate_visible
|
||
)
|
||
self.logic_search, self.logic_offsets = rendered_search_projection(
|
||
logic_source,
|
||
include_inline_code_text=True,
|
||
)
|
||
self.logic_inline_spans = inline_code_spans(self.prose_visible)
|
||
self.starts = line_starts(text)
|
||
self.structural_visible = mask_raw_html_blocks(self.prose_visible)
|
||
self.headings = parse_headings(self.structural_visible)
|
||
self.paragraphs = paragraph_spans(mask_raw_html_blocks(self.markdown_visible))
|
||
self.logic_map = logic_map
|
||
self.term_ledger = term_ledger
|
||
self.reader_contract = reader_contract
|
||
self.rules = rules
|
||
self.rule_by_id = rule_index(rules)
|
||
self.findings: list[dict[str, Any]] = []
|
||
self.section_spans: dict[str, tuple[int, int]] = {}
|
||
self.term_positions: dict[str, int] = {}
|
||
self.claim_markers = extract_claim_markers(
|
||
self.visible,
|
||
self.prose_visible,
|
||
self.rules["patterns"]["evidence_markers"],
|
||
)
|
||
|
||
def add(
|
||
self,
|
||
rule_id: str,
|
||
message: str,
|
||
*,
|
||
offset: int | None = None,
|
||
section_id: str | None = None,
|
||
context: str | None = None,
|
||
path: Path | None = None,
|
||
) -> None:
|
||
rule = self.rule_by_id[rule_id]
|
||
line: int | None = None
|
||
column: int | None = None
|
||
if offset is not None:
|
||
line, column = location(self.starts, offset)
|
||
self.findings.append(
|
||
{
|
||
"rule_id": rule_id,
|
||
"severity": rule["severity"],
|
||
"message": message,
|
||
"path": str(path or self.document_path),
|
||
"line": line,
|
||
"column": column,
|
||
"section_id": section_id,
|
||
"context": context,
|
||
}
|
||
)
|
||
|
||
def lint_markdown(self) -> None:
|
||
heading_thresholds = self.rules["thresholds"]["heading"]
|
||
previous: Heading | None = None
|
||
for heading in self.headings:
|
||
if previous and heading.level - previous.level > int(heading_thresholds["max_level_jump"]):
|
||
self.add(
|
||
"DOC-H001",
|
||
f"H{previous.level} 다음에 H{heading.level}이 나옵니다: {heading.text}",
|
||
offset=heading.start,
|
||
context=heading.text,
|
||
)
|
||
previous = heading
|
||
h1 = [heading for heading in self.headings if heading.level == 1]
|
||
required = int(heading_thresholds["required_h1_count"])
|
||
if len(h1) != required:
|
||
offset = h1[1].start if len(h1) > 1 else 0
|
||
self.add(
|
||
"DOC-H002",
|
||
f"H1은 {required}개여야 하지만 {len(h1)}개입니다.",
|
||
offset=offset,
|
||
)
|
||
|
||
for pattern in self.rules["patterns"]["placeholders"]:
|
||
try:
|
||
matches = re.finditer(pattern, self.metadata_search, re.IGNORECASE | re.MULTILINE)
|
||
except re.error as exc:
|
||
raise InputError(f"placeholder 정규식이 잘못되었습니다: {pattern}: {exc}") from exc
|
||
for match in matches:
|
||
self.add(
|
||
"DOC-M001",
|
||
f"미완성 표시가 남아 있습니다: {match.group(0)}",
|
||
offset=self.metadata_offsets[match.start()],
|
||
context=match.group(0),
|
||
)
|
||
|
||
for block in self.fenced:
|
||
if block.kind == "fenced_code" and not block.closed:
|
||
self.add(
|
||
"DOC-M002",
|
||
"닫히지 않은 code fence가 있습니다.",
|
||
offset=block.start,
|
||
)
|
||
for offset in self.unclosed_comments:
|
||
self.add(
|
||
"DOC-M004",
|
||
"닫히지 않은 HTML 주석이 있습니다.",
|
||
offset=offset,
|
||
)
|
||
|
||
heading_anchors = {heading.slug for heading in self.headings if heading.slug}
|
||
explicit_anchors: set[str] = set()
|
||
html_fragment_links: list[tuple[str, int]] = []
|
||
html_spans = html_tag_spans(self.markdown_visible)
|
||
raw_html_structure = mask_raw_html_blocks(self.markdown_visible)
|
||
raw_text_tag: str | None = None
|
||
for start, end in html_spans:
|
||
tag_match = re.match(
|
||
r"<(?P<closing>/)?(?P<name>[A-Za-z][A-Za-z0-9-]*)",
|
||
self.markdown_visible[start:end],
|
||
)
|
||
if tag_match is None:
|
||
continue
|
||
tag_name = tag_match.group("name").casefold()
|
||
if raw_text_tag is not None:
|
||
if tag_match.group("closing") and tag_name == raw_text_tag:
|
||
raw_text_tag = None
|
||
continue
|
||
parsed = html_tag_name_and_attributes(self.markdown_visible[start:end])
|
||
if parsed is None:
|
||
continue
|
||
tag_name, attributes = parsed
|
||
if tag_name in {"pre", "script", "style", "textarea"}:
|
||
raw_text_tag = tag_name
|
||
values = [attributes["id"]] if "id" in attributes else []
|
||
if tag_name == "a" and "name" in attributes:
|
||
values.append(attributes["name"])
|
||
if tag_name == "a" and attributes.get("href", "").startswith("#"):
|
||
html_fragment_links.append((attributes["href"], start))
|
||
for value in values:
|
||
if value:
|
||
explicit_anchors.add(html.unescape(unquote(value)).strip())
|
||
|
||
def validate_fragment(destination: str, offset: int) -> None:
|
||
if not destination.startswith("#"):
|
||
return
|
||
target = re.sub(r"\\(.)", r"\1", destination[1:])
|
||
target = html.unescape(unquote(target)).strip()
|
||
if target not in heading_anchors and target not in explicit_anchors:
|
||
self.add(
|
||
"DOC-M003",
|
||
f"내부 앵커 '#{target}'를 찾을 수 없습니다.",
|
||
offset=offset,
|
||
context=target,
|
||
)
|
||
|
||
for destination, offset in html_fragment_links:
|
||
validate_fragment(destination, offset)
|
||
|
||
definitions = [
|
||
item
|
||
for item in reference_definitions(self.markdown_visible)
|
||
if not (
|
||
item.start < len(self.markdown_visible)
|
||
and not self.markdown_visible[item.start].isspace()
|
||
and raw_html_structure[item.start] == " "
|
||
)
|
||
]
|
||
definition_spans = [(item.start, item.end) for item in definitions]
|
||
for link_start, _, target_start, target_end, _ in markdown_inline_links(
|
||
self.markdown_visible
|
||
):
|
||
if (
|
||
link_start < len(self.markdown_visible)
|
||
and not self.markdown_visible[link_start].isspace()
|
||
and raw_html_structure[link_start] == " "
|
||
):
|
||
continue
|
||
if any(start <= link_start < end for start, end in html_spans):
|
||
continue
|
||
if any(start <= link_start < end for start, end in definition_spans):
|
||
continue
|
||
validate_fragment(
|
||
self.markdown_visible[target_start:target_end],
|
||
target_start,
|
||
)
|
||
|
||
definition_by_id: dict[str, ReferenceDefinition] = {}
|
||
for definition in definitions:
|
||
definition_by_id.setdefault(definition.identifier, definition)
|
||
for link in markdown_reference_links(self.markdown_visible, definitions):
|
||
if (
|
||
link.start < len(self.markdown_visible)
|
||
and not self.markdown_visible[link.start].isspace()
|
||
and raw_html_structure[link.start] == " "
|
||
):
|
||
continue
|
||
if any(start <= link.start < end for start, end in html_spans):
|
||
continue
|
||
definition = definition_by_id[link.identifier]
|
||
validate_fragment(definition.destination, link.start)
|
||
|
||
def lint_logic(self) -> None:
|
||
h1 = [heading for heading in self.headings if heading.level == 1]
|
||
if len(h1) == 1 and rendered_markdown_label(h1[0].text) != rendered_markdown_label(
|
||
self.logic_map["title"]
|
||
):
|
||
self.add(
|
||
"DOC-L001",
|
||
"문서 H1과 logic map title이 일치하지 않습니다.",
|
||
offset=h1[0].start,
|
||
context=f"document={h1[0].text!r}; logic={self.logic_map['title']!r}",
|
||
)
|
||
|
||
document_sections = [heading for heading in self.headings if heading.level == 2]
|
||
logic_sections = self.logic_map["sections"]
|
||
matched: list[tuple[dict[str, Any], Heading]] = []
|
||
for index, section in enumerate(logic_sections):
|
||
heading = document_sections[index] if index < len(document_sections) else None
|
||
if heading is None or rendered_markdown_label(
|
||
heading.text
|
||
) != rendered_markdown_label(section["heading"]):
|
||
self.add(
|
||
"DOC-L001",
|
||
f"logic map 섹션과 같은 위치의 H2가 일치하지 않습니다: {section['heading']}",
|
||
offset=heading.start if heading else None,
|
||
section_id=section["id"],
|
||
context=(
|
||
f"document={heading.text!r}; logic={section['heading']!r}"
|
||
if heading
|
||
else section["heading"]
|
||
),
|
||
)
|
||
continue
|
||
matched.append((section, heading))
|
||
for heading in document_sections[len(logic_sections) :]:
|
||
self.add(
|
||
"DOC-L001",
|
||
f"logic map에 없는 H2 섹션입니다: {heading.text}",
|
||
offset=heading.start,
|
||
context=heading.text,
|
||
)
|
||
for index, (section, heading) in enumerate(matched):
|
||
document_index = document_sections.index(heading)
|
||
end = (
|
||
document_sections[document_index + 1].start
|
||
if document_index + 1 < len(document_sections)
|
||
else len(self.text)
|
||
)
|
||
# A heading is reader-visible section content and may introduce a
|
||
# term. Include it in first-use and section-budget boundaries.
|
||
self.section_spans[section["id"]] = (heading.start, end)
|
||
|
||
core_claim = self.logic_map["core_claim"]
|
||
if not isinstance(core_claim, str) or not core_claim.strip():
|
||
raise InputError("logic map core_claim이 비어 있습니다.")
|
||
match = find_flexible(self.logic_search, core_claim)
|
||
match_source = self.logic_offsets[match.start()] if match else None
|
||
match_inside_code = False
|
||
if match is not None:
|
||
match_source_end = self.logic_offsets[match.end() - 1] + 1
|
||
match_inside_code = any(
|
||
start <= match_source and match_source_end <= end
|
||
for start, end, _ in self.logic_inline_spans
|
||
)
|
||
maximum_front_paragraphs = int(
|
||
self.rules["thresholds"]["logic"]["core_claim_max_reader_paragraphs"]
|
||
)
|
||
heading_spans = [(heading.start, heading.end) for heading in self.headings]
|
||
reader_paragraphs = [
|
||
paragraph
|
||
for paragraph in self.paragraphs
|
||
if not any(
|
||
start <= paragraph.start and paragraph.end <= end
|
||
for start, end in heading_spans
|
||
)
|
||
]
|
||
front_paragraphs = reader_paragraphs[:maximum_front_paragraphs]
|
||
front_limit = front_paragraphs[-1].end if front_paragraphs else 0
|
||
if match is None or match_inside_code or (
|
||
match_source is not None and match_source >= front_limit
|
||
):
|
||
self.add(
|
||
"DOC-L002",
|
||
f"핵심 주장이 독자용 앞 {maximum_front_paragraphs}개 문단 안에 "
|
||
"logic map 문구로 명시되지 않았습니다.",
|
||
offset=match_source if match_source is not None else 0,
|
||
context=core_claim,
|
||
)
|
||
|
||
seen_ids: set[str] = set()
|
||
sections = self.logic_map["sections"]
|
||
for index, section in enumerate(sections):
|
||
section_id = section["id"]
|
||
if index > 0 and not section["depends_on"]:
|
||
self.add(
|
||
"DOC-L004",
|
||
"첫 절이 아닌 절에는 앞선 절을 가리키는 depends_on이 필요합니다.",
|
||
section_id=section_id,
|
||
)
|
||
for dependency in section["depends_on"]:
|
||
if dependency not in seen_ids:
|
||
self.add(
|
||
"DOC-L004",
|
||
f"depends_on '{dependency}'가 앞선 섹션을 가리키지 않습니다.",
|
||
section_id=section_id,
|
||
)
|
||
expected_transition = sections[index + 1]["id"] if index + 1 < len(sections) else None
|
||
if section["transition_to"] != expected_transition:
|
||
self.add(
|
||
"DOC-L004",
|
||
f"transition_to는 {expected_transition!r}이어야 합니다.",
|
||
section_id=section_id,
|
||
)
|
||
seen_ids.add(section_id)
|
||
|
||
span = self.section_spans.get(section_id)
|
||
if span is None:
|
||
continue
|
||
section_text = self.contract_visible[span[0] : span[1]]
|
||
section_claim_ids = {
|
||
claim_id
|
||
for claim_id, position in self.claim_markers
|
||
if span[0] <= position < span[1]
|
||
}
|
||
expected_section_claim_ids = set(section["claim_ids"])
|
||
for claim_id in section["claim_ids"]:
|
||
if claim_id not in section_claim_ids:
|
||
self.add(
|
||
"DOC-L003",
|
||
f"claim marker가 없습니다: {claim_id}",
|
||
offset=span[0],
|
||
section_id=section_id,
|
||
context=str(claim_id),
|
||
)
|
||
for claim_id in sorted(section_claim_ids - expected_section_claim_ids):
|
||
marker_position = next(
|
||
position
|
||
for marker_id, position in self.claim_markers
|
||
if marker_id == claim_id and span[0] <= position < span[1]
|
||
)
|
||
self.add(
|
||
"DOC-L003",
|
||
f"이 절의 logic map에 없는 claim marker가 있습니다: {claim_id}",
|
||
offset=marker_position,
|
||
section_id=section_id,
|
||
context=claim_id,
|
||
)
|
||
for marker in section.get("required_markers", []):
|
||
if marker not in section_text:
|
||
self.add(
|
||
"DOC-L003",
|
||
f"필수 marker가 없습니다: {marker}",
|
||
offset=span[0],
|
||
section_id=section_id,
|
||
context=marker,
|
||
)
|
||
|
||
expected_claim_ids = {
|
||
claim_id
|
||
for section in self.logic_map["sections"]
|
||
for claim_id in section["claim_ids"]
|
||
}
|
||
for claim_id, position in self.claim_markers:
|
||
if claim_id not in expected_claim_ids:
|
||
self.add(
|
||
"DOC-L003",
|
||
f"logic map에 없는 claim marker가 있습니다: {claim_id}",
|
||
offset=position,
|
||
context=claim_id,
|
||
)
|
||
|
||
def _term_mentions(self, term: dict[str, Any]) -> list[tuple[str, int, int]]:
|
||
values = [term["canonical"], *term["aliases"]]
|
||
for key in ("english", "abbreviation"):
|
||
value = term.get(key)
|
||
if isinstance(value, str) and value:
|
||
values.append(value)
|
||
mentions: list[tuple[str, int, int]] = []
|
||
for value in values:
|
||
position = find_literal(self.term_search, value)
|
||
if position >= 0:
|
||
mentions.append((value, position, self.term_offsets[position]))
|
||
return sorted(mentions, key=lambda item: item[1])
|
||
|
||
def lint_terms(self) -> None:
|
||
config = self.rules["thresholds"]["term"]
|
||
ledger_assumed = self.term_ledger.get("assumed_known", [])
|
||
reader_assumed = self.reader_contract.get("assumed_known", [])
|
||
if set(ledger_assumed) != set(reader_assumed):
|
||
self.add(
|
||
"DOC-T009",
|
||
"term ledger와 reader contract의 assumed_known 목록이 다릅니다.",
|
||
context=f"ledger={ledger_assumed!r}; reader={reader_assumed!r}",
|
||
)
|
||
prerequisites = self.reader_contract.get("prerequisites", [])
|
||
max_assumed = int(config["max_assumed_known"])
|
||
max_per_prerequisite = int(config["max_assumed_per_prerequisite"])
|
||
if len(reader_assumed) > max_assumed or (
|
||
len(reader_assumed) > 2
|
||
and (
|
||
not prerequisites
|
||
or len(reader_assumed) > len(prerequisites) * max_per_prerequisite
|
||
)
|
||
):
|
||
self.add(
|
||
"DOC-T008",
|
||
f"assumed_known {len(reader_assumed)}개가 선수지식 {len(prerequisites)}개에 비해 넓습니다.",
|
||
context=", ".join(reader_assumed),
|
||
)
|
||
|
||
for term in self.term_ledger["terms"]:
|
||
term_id = term["id"]
|
||
first_use = term["first_use"]
|
||
first_use_match = find_flexible(
|
||
self.term_search,
|
||
rendered_markdown_label(first_use),
|
||
)
|
||
first_use_inside_code = False
|
||
if first_use_match is not None:
|
||
source_start = self.term_offsets[first_use_match.start()]
|
||
source_end = (
|
||
self.term_offsets[first_use_match.end() - 1] + 1
|
||
if first_use_match.end() > first_use_match.start()
|
||
else source_start
|
||
)
|
||
first_use_inside_code = any(
|
||
start <= source_start and source_end <= end
|
||
for start, end, _ in self.term_inline_spans
|
||
)
|
||
mentions = self._term_mentions(term)
|
||
earliest_search = mentions[0][1] if mentions else -1
|
||
earliest = mentions[0][2] if mentions else -1
|
||
canonical_search = find_literal(self.term_search, term["canonical"])
|
||
canonical_position = (
|
||
self.term_offsets[canonical_search] if canonical_search >= 0 else -1
|
||
)
|
||
canonical_named = find_literal(first_use, term["canonical"]) >= 0
|
||
if not mentions:
|
||
self.add(
|
||
"DOC-T001",
|
||
f"용어 '{term['canonical']}'이 본문에 실제로 등장하지 않습니다.",
|
||
section_id=term["first_section"],
|
||
context=term["canonical"],
|
||
)
|
||
if not canonical_named:
|
||
self.add(
|
||
"DOC-T001",
|
||
f"용어 '{term['canonical']}'의 first_use에 정식 용어가 없습니다.",
|
||
offset=(
|
||
self.term_offsets[first_use_match.start()]
|
||
if first_use_match
|
||
else None
|
||
),
|
||
section_id=term["first_section"],
|
||
context=first_use,
|
||
)
|
||
intro_valid = (
|
||
first_use_match is not None
|
||
and not first_use_inside_code
|
||
and earliest_search >= 0
|
||
and canonical_search >= 0
|
||
and canonical_named
|
||
and first_use_match.start() <= earliest_search
|
||
)
|
||
if not intro_valid:
|
||
self.add(
|
||
"DOC-T001",
|
||
f"용어 '{term['canonical']}'의 first_use 문구가 첫 등장에 없습니다.",
|
||
offset=earliest if earliest >= 0 else None,
|
||
section_id=term["first_section"],
|
||
context=first_use,
|
||
)
|
||
position = canonical_position
|
||
position_search = canonical_search
|
||
else:
|
||
position_search = first_use_match.start()
|
||
position = self.term_offsets[position_search]
|
||
definition = normalize_space(term["plain_definition"])
|
||
mention_position = (
|
||
earliest_search if earliest_search >= 0 else first_use_match.end()
|
||
)
|
||
window_size = int(config["definition_window_chars"])
|
||
window_start = max(0, mention_position - window_size)
|
||
window_end = min(
|
||
len(self.term_search),
|
||
max(first_use_match.end(), mention_position + window_size),
|
||
)
|
||
definition_window = normalize_space(
|
||
self.term_search[window_start:window_end]
|
||
)
|
||
if definition not in definition_window:
|
||
self.add(
|
||
"DOC-T001",
|
||
f"용어 '{term['canonical']}'의 첫 등장 {window_size}자 안에 쉬운 정의가 없습니다.",
|
||
offset=position,
|
||
section_id=term["first_section"],
|
||
context=term["plain_definition"],
|
||
)
|
||
expected_span = self.section_spans.get(term["first_section"])
|
||
if expected_span and not (expected_span[0] <= position < expected_span[1]):
|
||
self.add(
|
||
"DOC-T001",
|
||
f"용어 '{term['canonical']}'이 first_section 밖에서 처음 소개됩니다.",
|
||
offset=position,
|
||
section_id=term["first_section"],
|
||
)
|
||
if position >= 0:
|
||
self.term_positions[term_id] = position
|
||
|
||
for alias in term["aliases"]:
|
||
alias_search = find_literal(self.term_search, alias)
|
||
alias_position = self.term_offsets[alias_search] if alias_search >= 0 else -1
|
||
if alias_search >= 0 and (
|
||
position_search < 0 or alias_search < position_search
|
||
):
|
||
self.add(
|
||
"DOC-T002",
|
||
f"별칭 '{alias}'가 정식 용어 '{term['canonical']}'의 설명보다 먼저 나옵니다.",
|
||
offset=alias_position,
|
||
section_id=term["first_section"],
|
||
context=alias,
|
||
)
|
||
abbreviation = term.get("abbreviation")
|
||
if isinstance(abbreviation, str) and abbreviation:
|
||
abbreviation_search = find_literal(self.term_search, abbreviation)
|
||
abbreviation_position = (
|
||
self.term_offsets[abbreviation_search]
|
||
if abbreviation_search >= 0
|
||
else -1
|
||
)
|
||
full_name_present = term["canonical"] in first_use or (
|
||
isinstance(term.get("english"), str) and term["english"] in first_use
|
||
)
|
||
definition_present = normalize_space(term["plain_definition"]) in normalize_space(first_use)
|
||
if abbreviation_search >= 0 and (
|
||
position_search < 0
|
||
or abbreviation_search < position_search
|
||
or abbreviation not in first_use
|
||
or not full_name_present
|
||
or not definition_present
|
||
):
|
||
self.add(
|
||
"DOC-T003",
|
||
f"약어 '{abbreviation}'를 정식 이름과 쉬운 뜻보다 먼저 사용했습니다.",
|
||
offset=abbreviation_position,
|
||
section_id=term["first_section"],
|
||
context=abbreviation,
|
||
)
|
||
|
||
positions = list(self.term_positions.values())
|
||
self._budget_findings(
|
||
self.paragraphs,
|
||
positions,
|
||
int(config["max_new_terms_per_paragraph"]),
|
||
"DOC-T004",
|
||
"문단",
|
||
)
|
||
sentences = [sentence for paragraph in self.paragraphs for sentence in sentence_spans(paragraph)]
|
||
self._budget_findings(
|
||
sentences,
|
||
positions,
|
||
int(config["max_new_terms_per_sentence"]),
|
||
"DOC-T005",
|
||
"문장",
|
||
)
|
||
for section_id, (start, end) in self.section_spans.items():
|
||
count = sum(start <= position < end for position in positions)
|
||
maximum = int(config["max_new_terms_per_section"])
|
||
if count > maximum:
|
||
self.add(
|
||
"DOC-T006",
|
||
f"절에서 새 용어 {count}개를 소개해 예산 {maximum}개를 넘었습니다.",
|
||
offset=start,
|
||
section_id=section_id,
|
||
)
|
||
self.lint_unregistered_candidates()
|
||
|
||
def _budget_findings(
|
||
self,
|
||
spans: list[TextSpan],
|
||
positions: list[int],
|
||
maximum: int,
|
||
rule_id: str,
|
||
label: str,
|
||
) -> None:
|
||
for span in spans:
|
||
count = sum(span.start <= position < span.end for position in positions)
|
||
if count > maximum:
|
||
self.add(
|
||
rule_id,
|
||
f"{label}에서 새 용어 {count}개를 소개해 예산 {maximum}개를 넘었습니다.",
|
||
offset=span.start,
|
||
context=normalize_space(span.text)[:160],
|
||
)
|
||
|
||
def lint_unregistered_candidates(self) -> None:
|
||
registered: set[str] = set()
|
||
registered_values: list[str] = []
|
||
for section in self.logic_map["sections"]:
|
||
for claim_id in section["claim_ids"]:
|
||
registered.add(claim_id.casefold())
|
||
registered_values.append(claim_id)
|
||
for term in self.term_ledger["terms"]:
|
||
for key in ("canonical", "english", "abbreviation"):
|
||
value = term.get(key)
|
||
if isinstance(value, str) and value:
|
||
registered.add(value.casefold())
|
||
registered_values.append(value)
|
||
for alias in term["aliases"]:
|
||
registered.add(str(alias).casefold())
|
||
registered_values.append(str(alias))
|
||
for value in self.reader_contract["assumed_known"]:
|
||
registered.add(str(value).casefold())
|
||
registered_values.append(str(value))
|
||
allowlist_values = [
|
||
str(value)
|
||
for value in self.rules["patterns"].get(
|
||
"technical_candidate_allowlist", []
|
||
)
|
||
]
|
||
allowlist = {value.casefold() for value in allowlist_values}
|
||
|
||
prose = list(self.candidate_search)
|
||
for start, end, _ in absolute_uri_matches(self.candidate_search):
|
||
for index in range(start, end):
|
||
prose[index] = " "
|
||
metadata_patterns = (
|
||
r"(?<![A-Za-z0-9.!#$%&'*+/=?^_`{|}~-])"
|
||
r"[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@"
|
||
r"(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,}"
|
||
r"(?![A-Za-z0-9-])",
|
||
)
|
||
for pattern in metadata_patterns:
|
||
for match in re.finditer(pattern, self.candidate_search):
|
||
for index in range(match.start(), match.end()):
|
||
prose[index] = " "
|
||
prose_text = "".join(prose)
|
||
registered_spans: list[tuple[int, int]] = []
|
||
for value in [*registered_values, *allowlist_values]:
|
||
parts = value.split()
|
||
if not parts:
|
||
continue
|
||
phrase = r"\s+".join(re.escape(part) for part in parts)
|
||
if re.match(r"[A-Za-z0-9_$]", value):
|
||
phrase = rf"(?<![A-Za-z0-9_$]){phrase}"
|
||
if re.search(r"[A-Za-z0-9_$]$", value):
|
||
phrase = rf"{phrase}(?![A-Za-z0-9_$])"
|
||
registered_spans.extend(
|
||
match.span() for match in re.finditer(phrase, prose_text, re.IGNORECASE)
|
||
)
|
||
lowercase_candidates = [
|
||
str(value)
|
||
for value in self.rules["patterns"].get(
|
||
"technical_lowercase_candidates", []
|
||
)
|
||
]
|
||
candidate_patterns = [
|
||
r"(?<![A-Za-z0-9_])[A-Z][A-Z0-9_]{1,}(?![A-Za-z0-9_])",
|
||
r"(?<![A-Za-z0-9_])(?:[A-Z][a-z0-9]+){2,}(?![A-Za-z0-9_])",
|
||
r"(?<![A-Za-z0-9_])[a-z][a-z0-9]*(?:[A-Z][A-Za-z0-9]*)+(?![A-Za-z0-9_])",
|
||
r"(?<![A-Za-z0-9_])[A-Za-z_$][A-Za-z0-9_$]*(?:\.[A-Za-z_$][A-Za-z0-9_$]*)+(?![A-Za-z0-9_])",
|
||
r"(?<![A-Za-z0-9_])[a-z][a-z0-9]*(?:[-_][a-z0-9]+)+(?![A-Za-z0-9_])",
|
||
]
|
||
if lowercase_candidates:
|
||
alternatives = "|".join(
|
||
re.escape(value)
|
||
for value in sorted(lowercase_candidates, key=len, reverse=True)
|
||
)
|
||
candidate_patterns.append(
|
||
rf"(?<![A-Za-z0-9_])(?:{alternatives})(?![A-Za-z0-9_])"
|
||
)
|
||
candidates: dict[str, int] = {}
|
||
for pattern in candidate_patterns:
|
||
for match in re.finditer(pattern, prose_text):
|
||
candidates.setdefault(match.group(0), match.start())
|
||
for match in re.finditer(r"`([^`\n]+)`", prose_text):
|
||
value = match.group(1).strip()
|
||
if re.fullmatch(r"[A-Za-z_$][\w$]*(?:[.:/#@-][A-Za-z0-9_$-]+)*", value):
|
||
candidates.setdefault(value, match.start(1))
|
||
|
||
for candidate, offset in sorted(candidates.items(), key=lambda item: item[1]):
|
||
folded = candidate.casefold()
|
||
if folded in registered or folded in allowlist:
|
||
continue
|
||
if any(
|
||
start <= offset and offset + len(candidate) <= end
|
||
for start, end in registered_spans
|
||
):
|
||
continue
|
||
self.add(
|
||
"DOC-T007",
|
||
f"미등록 기술 용어 후보입니다: {candidate}",
|
||
offset=self.candidate_offsets[offset],
|
||
context=candidate,
|
||
)
|
||
|
||
def lint_paragraphs(self) -> None:
|
||
thresholds = self.rules["thresholds"]["paragraph"]
|
||
for paragraph in self.paragraphs:
|
||
compact = normalize_space(paragraph.text)
|
||
if len(compact) > int(thresholds["max_chars"]):
|
||
self.add(
|
||
"DOC-P001",
|
||
f"문단 길이 {len(compact)}자가 상한 {thresholds['max_chars']}자를 넘었습니다.",
|
||
offset=paragraph.start,
|
||
context=compact[:160],
|
||
)
|
||
sentence_count = len(sentence_spans(paragraph))
|
||
if sentence_count > int(thresholds["max_sentences"]):
|
||
self.add(
|
||
"DOC-P002",
|
||
f"문단 문장 수 {sentence_count}개가 상한 {thresholds['max_sentences']}개를 넘었습니다.",
|
||
offset=paragraph.start,
|
||
context=compact[:160],
|
||
)
|
||
|
||
|
||
def protected_context(visible: str, start: int, end: int) -> str:
|
||
"""Bind a protected value to nearby semantic labels, not just its raw spelling."""
|
||
|
||
def words(fragment: str) -> list[str]:
|
||
return [
|
||
match.group(0).casefold()
|
||
for match in re.finditer(r"(?<!\w)[^\W\d_][\w.-]*", fragment, re.UNICODE)
|
||
if not match.group(0).casefold().startswith(
|
||
("http", "ftp", "file", "mailto", "ssh", "git", "www")
|
||
)
|
||
]
|
||
|
||
left = words(visible[max(0, start - 120) : start])[-3:]
|
||
right = words(visible[end : min(len(visible), end + 120)])[:3]
|
||
return "\u241e".join(left) + "\u241f" + "\u241e".join(right)
|
||
|
||
|
||
def reader_visible_double_quotes(visible: str) -> list[ProtectedOccurrence]:
|
||
"""Extract literal/entity double quotes from rendered prose, including hard wraps."""
|
||
|
||
projection = mask_markdown_metadata(
|
||
mask_raw_code_html_blocks(mask_inline_code(visible))
|
||
)
|
||
decoded, _ = decode_html_entities_with_offsets(projection)
|
||
occurrences: list[ProtectedOccurrence] = []
|
||
|
||
def quote_context(start: int, end: int) -> str:
|
||
line_start = decoded.rfind("\n", 0, start) + 1
|
||
if re.match(r"^[ ]{0,3}(?:>[ \t]?)+", decoded[line_start:start]):
|
||
return ""
|
||
return protected_context(decoded, start, end)
|
||
|
||
def canonical_body(body: str) -> str | None:
|
||
if re.search(r"\r?\n[ \t]*\r?\n", body):
|
||
return None
|
||
lines: list[str] = []
|
||
for index, raw_line in enumerate(body.splitlines() or [body]):
|
||
line = raw_line
|
||
if index > 0:
|
||
line = re.sub(r"^[ ]{0,3}(?:>[ \t]?)+", "", line)
|
||
structural = line.lstrip()
|
||
if is_thematic_break(structural) or re.match(
|
||
r"(?:#{1,6}(?:[ \t]+|$)|(?:[-+*]|\d+[.)])[ \t]+|"
|
||
r"`{3,}|~{3,}|(?:={3,}|-{3,}|\*{3,}|_{3,})[ \t]*$|"
|
||
r"<[/!?A-Za-z])",
|
||
structural,
|
||
):
|
||
return None
|
||
lines.append(line.strip())
|
||
value = "\n".join(lines).strip()
|
||
return value if len(normalize_space(value)) >= 2 else None
|
||
|
||
for opener, closer in (("\"", "\""), ("“", "”")):
|
||
position = 0
|
||
while position < len(decoded):
|
||
start = decoded.find(opener, position)
|
||
if start < 0:
|
||
break
|
||
end = decoded.find(closer, start + len(opener))
|
||
if end < 0:
|
||
break
|
||
body = canonical_body(decoded[start + len(opener) : end])
|
||
if body is not None:
|
||
value = f"{opener}{body}{closer}"
|
||
occurrences.append(
|
||
ProtectedOccurrence(
|
||
value,
|
||
quote_context(start, end + len(closer)),
|
||
)
|
||
)
|
||
position = end + len(closer)
|
||
return occurrences
|
||
|
||
|
||
def protected_occurrences(text: str) -> dict[str, list[ProtectedOccurrence]]:
|
||
_, visible, fenced, _ = scan_markdown_visibility(text)
|
||
occurrences: dict[str, list[ProtectedOccurrence]] = {
|
||
"fenced_code": [],
|
||
"indented_code": [],
|
||
"inline_code": [],
|
||
"url": [],
|
||
"link_destination": [],
|
||
"number_unit": [],
|
||
"number_range": [],
|
||
"number": [],
|
||
"date": [],
|
||
"version": [],
|
||
"quote": [],
|
||
}
|
||
for block in fenced:
|
||
if block.closed:
|
||
occurrences[block.kind].append(ProtectedOccurrence(block.text, ""))
|
||
for start, end, value in inline_code_spans(visible):
|
||
if value.strip():
|
||
occurrences["inline_code"].append(
|
||
ProtectedOccurrence(
|
||
value,
|
||
protected_context(visible, start, end),
|
||
)
|
||
)
|
||
for start, end, value in absolute_uri_matches(visible):
|
||
occurrences["url"].append(
|
||
ProtectedOccurrence(value, protected_context(visible, start, end))
|
||
)
|
||
inline_spans = inline_code_spans(visible)
|
||
for link_start, _, target_start, target_end, raw_label in markdown_inline_links(
|
||
visible, include_images=True
|
||
):
|
||
if inside_any_span(link_start, inline_spans):
|
||
continue
|
||
target = visible[target_start:target_end].strip("<>")
|
||
label = rendered_markdown_label(raw_label)
|
||
occurrences["link_destination"].append(
|
||
ProtectedOccurrence(
|
||
target,
|
||
f"label:{label}\u241f{protected_context(visible, target_start, target_end)}",
|
||
)
|
||
)
|
||
for definition in reference_definitions(visible):
|
||
if inside_any_span(definition.start, inline_spans):
|
||
continue
|
||
occurrences["link_destination"].append(
|
||
ProtectedOccurrence(
|
||
definition.destination,
|
||
f"label:{definition.identifier}\u241f"
|
||
f"{protected_context(visible, definition.target_start, definition.target_end)}",
|
||
)
|
||
)
|
||
occupied_numeric_spans: list[tuple[int, int]] = []
|
||
typed_numeric_patterns = {
|
||
"date": re.compile(
|
||
r"(?<![A-Za-z0-9_.])\d{4}-\d{2}-\d{2}(?![A-Za-z0-9_.])"
|
||
),
|
||
"version": re.compile(
|
||
r"(?<![A-Za-z0-9_.])v?\d+(?:\.\d+){1,3}(?:[-+][0-9A-Za-z.-]+)?(?![A-Za-z0-9_.])"
|
||
),
|
||
}
|
||
for kind, pattern in typed_numeric_patterns.items():
|
||
for match in pattern.finditer(visible):
|
||
occupied_numeric_spans.append(match.span())
|
||
occurrences[kind].append(
|
||
ProtectedOccurrence(
|
||
match.group(0),
|
||
protected_context(visible, match.start(), match.end()),
|
||
)
|
||
)
|
||
qualifier = (
|
||
r"(?:(?:최대|최소|약|대략|약간|approximately|about|at[ \t]+most|"
|
||
r"at[ \t]+least|less[ \t]+than|more[ \t]+than)[ \t]*)?"
|
||
)
|
||
operator = r"(?:(?:<=|>=|==|!=|<|>|≤|≥|=|≈|±)[ \t]*)?"
|
||
number_atom = (
|
||
r"[-+]?(?:0[xX][0-9A-Fa-f]+|0[bB][01]+|0[oO][0-7]+|"
|
||
r"(?:\d+(?:[.,]\d+)?|[.,]\d+)(?:[eE][-+]?\d+)?)"
|
||
)
|
||
unit = (
|
||
r"(?:%|℃|°C|ns|us|µs|μs|ms|sec|s|min|h|Hz|kHz|MHz|GHz|THz|"
|
||
r"bit|B|KB|MB|GB|TB|PB|KiB|MiB|GiB|TiB|bps|Kbps|Mbps|Gbps|Tbps|"
|
||
r"B/s|KB/s|MB/s|GB/s|px|mm|cm|m|km|in|ft|V|mV|A|mA|W|kW|Wh|kWh|"
|
||
r"rpm|rps|IOPS|ops/s|req/s|requests/s|cores?|threads?|"
|
||
r"초|분|시간|일|주|개월|년|개|건|회|배|자|줄|명|원|달러)"
|
||
)
|
||
suffix = r"(?:[ \t]*(?:이하|이상|미만|초과))?"
|
||
range_pattern = re.compile(
|
||
rf"(?<![A-Za-z0-9_.]){qualifier}{operator}{number_atom}[ \t]*"
|
||
rf"(?P<range_first_unit>{unit})?[ \t]*"
|
||
rf"(?:\.\.|[-–—~∼/]|에서|부터)[ \t]*{number_atom}[ \t]*"
|
||
rf"(?P<range_second_unit>{unit}){suffix}"
|
||
rf"(?![A-Za-z0-9_])",
|
||
re.IGNORECASE,
|
||
)
|
||
unit_pattern = re.compile(
|
||
rf"(?<![A-Za-z0-9_.]){qualifier}{operator}{number_atom}[ \t]*{unit}{suffix}"
|
||
rf"(?![A-Za-z0-9_])",
|
||
re.IGNORECASE,
|
||
)
|
||
for kind, pattern in (
|
||
("number_range", range_pattern),
|
||
("number_unit", unit_pattern),
|
||
):
|
||
for match in pattern.finditer(visible):
|
||
if kind == "number_range":
|
||
first_unit = match.group("range_first_unit")
|
||
second_unit = match.group("range_second_unit")
|
||
if first_unit is not None and (
|
||
first_unit.replace("µ", "μ").casefold()
|
||
!= second_unit.replace("µ", "μ").casefold()
|
||
):
|
||
continue
|
||
if any(
|
||
start < match.end() and match.start() < end
|
||
for start, end in occupied_numeric_spans
|
||
):
|
||
continue
|
||
occupied_numeric_spans.append(match.span())
|
||
occurrences[kind].append(
|
||
ProtectedOccurrence(
|
||
match.group(0),
|
||
protected_context(visible, match.start(), match.end()),
|
||
)
|
||
)
|
||
standalone_number = re.compile(
|
||
rf"(?<![A-Za-z0-9_.]){qualifier}{operator}{number_atom}{suffix}"
|
||
rf"(?![A-Za-z0-9_.])",
|
||
re.IGNORECASE,
|
||
)
|
||
for match in standalone_number.finditer(visible):
|
||
if any(start < match.end() and match.start() < end for start, end in occupied_numeric_spans):
|
||
continue
|
||
occurrences["number"].append(
|
||
ProtectedOccurrence(
|
||
match.group(0),
|
||
protected_context(visible, match.start(), match.end()),
|
||
)
|
||
)
|
||
occurrences["quote"].extend(reader_visible_double_quotes(visible))
|
||
quote_lines: list[tuple[int, int, str]] = []
|
||
lazy_quote_continuation = False
|
||
offset = 0
|
||
for line in visible.splitlines(keepends=True):
|
||
match = re.match(r"^[ ]{0,3}(?:>[ \t]?)+(?P<body>.*?)(?:\r?\n)?$", line)
|
||
if match:
|
||
body = match.group("body")
|
||
quote_lines.append((offset, offset + len(line), body))
|
||
lazy_quote_continuation = (
|
||
bool(body.strip())
|
||
and not is_thematic_break(body)
|
||
and not re.match(
|
||
r"^[ ]{0,3}(?:#{1,6}(?:[ \t]+|$)|(?:[-+*]|\d+[.)])[ \t]+|"
|
||
r"(?:={1,}|-{1,}|(?:\*[ \t]*){3,}|(?:_[ \t]*){3,})[ \t]*$|"
|
||
r"(?:`{3,}|~{3,})|<[/!?A-Za-z])",
|
||
body,
|
||
)
|
||
)
|
||
elif (
|
||
quote_lines
|
||
and lazy_quote_continuation
|
||
and line.strip()
|
||
and not is_thematic_break(line.rstrip("\r\n"))
|
||
and not re.match(
|
||
r"^[ ]{0,3}(?:#{1,6}(?:[ \t]+|$)|(?:[-+*]|\d+[.)])[ \t]+|"
|
||
r"(?:={3,}|-{3,}|\*{3,}|_{3,})[ \t]*$|<[/!?A-Za-z])",
|
||
line.rstrip("\r\n"),
|
||
)
|
||
):
|
||
quote_lines.append((offset, offset + len(line), line.rstrip("\r\n")))
|
||
elif quote_lines:
|
||
value = "\n".join(item[2].strip() for item in quote_lines).strip()
|
||
if value:
|
||
occurrences["quote"].append(
|
||
ProtectedOccurrence(value, "")
|
||
)
|
||
quote_lines = []
|
||
lazy_quote_continuation = False
|
||
offset += len(line)
|
||
if quote_lines:
|
||
value = "\n".join(item[2].strip() for item in quote_lines).strip()
|
||
if value:
|
||
occurrences["quote"].append(
|
||
ProtectedOccurrence(value, "")
|
||
)
|
||
return occurrences
|
||
|
||
|
||
def protected_items(text: str) -> dict[str, list[str]]:
|
||
return {
|
||
kind: [occurrence.value for occurrence in occurrences]
|
||
for kind, occurrences in protected_occurrences(text).items()
|
||
}
|
||
|
||
|
||
PROTECTED_CONTEXT_MIN_OVERLAP = 2 / 3
|
||
|
||
|
||
def protected_context_features(context: str) -> Counter[str]:
|
||
"""Represent a protected occurrence's bounded left/right semantic context."""
|
||
|
||
if not context:
|
||
return Counter()
|
||
parts = context.split("\u241f")
|
||
features: Counter[str] = Counter()
|
||
if parts[0].startswith("label:"):
|
||
label = normalize_space(parts[0][len("label:") :]).casefold()
|
||
if label:
|
||
features[f"label:{label}"] += 1
|
||
left = parts[-2] if len(parts) >= 3 else ""
|
||
right = parts[-1] if len(parts) >= 2 else ""
|
||
else:
|
||
left = parts[0]
|
||
right = parts[1] if len(parts) >= 2 else ""
|
||
for side, value in (("left", left), ("right", right)):
|
||
for token in value.split("\u241e"):
|
||
normalized = unicodedata.normalize("NFKC", token).strip().casefold()
|
||
if normalized:
|
||
features[f"{side}:{normalized}"] += 1
|
||
return features
|
||
|
||
|
||
def protected_context_overlap(left: Counter[str], right: Counter[str]) -> float:
|
||
"""Return overlap of the smaller bounded context, preserving word direction."""
|
||
|
||
left_size = sum(left.values())
|
||
right_size = sum(right.values())
|
||
if left_size == 0 or right_size == 0:
|
||
return 1.0 if left_size == right_size else 0.0
|
||
return sum((left & right).values()) / min(left_size, right_size)
|
||
|
||
|
||
def context_bound_missing(
|
||
expected: list[ProtectedOccurrence],
|
||
actual: list[ProtectedOccurrence],
|
||
) -> dict[str, int]:
|
||
"""Match every exact value to a compatible local context without reusing copies."""
|
||
|
||
expected_by_value: dict[str, list[ProtectedOccurrence]] = defaultdict(list)
|
||
actual_by_value: dict[str, list[ProtectedOccurrence]] = defaultdict(list)
|
||
for item in expected:
|
||
expected_by_value[item.value].append(item)
|
||
for item in actual:
|
||
actual_by_value[item.value].append(item)
|
||
|
||
missing_by_value: dict[str, int] = {}
|
||
for value, expected_items in expected_by_value.items():
|
||
actual_items = actual_by_value[value]
|
||
expected_features = [
|
||
protected_context_features(item.context) for item in expected_items
|
||
]
|
||
actual_features = [
|
||
protected_context_features(item.context) for item in actual_items
|
||
]
|
||
adjacency: list[list[int]] = []
|
||
for features in expected_features:
|
||
candidates = [
|
||
(protected_context_overlap(features, other), index)
|
||
for index, other in enumerate(actual_features)
|
||
]
|
||
adjacency.append(
|
||
[
|
||
index
|
||
for score, index in sorted(
|
||
candidates, key=lambda candidate: (-candidate[0], candidate[1])
|
||
)
|
||
if score >= PROTECTED_CONTEXT_MIN_OVERLAP
|
||
]
|
||
)
|
||
|
||
# Find a maximum-cardinality bipartite match. The iterative augmenting
|
||
# path avoids recursion limits when a document repeats the same value.
|
||
left_match: dict[int, int] = {}
|
||
right_match: dict[int, int] = {}
|
||
for start in sorted(
|
||
range(len(expected_items)), key=lambda item: len(adjacency[item])
|
||
):
|
||
queue = [start]
|
||
visited_left = {start}
|
||
visited_right: set[int] = set()
|
||
parent_right: dict[int, int] = {}
|
||
free_right: int | None = None
|
||
cursor = 0
|
||
while cursor < len(queue) and free_right is None:
|
||
left_index = queue[cursor]
|
||
cursor += 1
|
||
for right_index in adjacency[left_index]:
|
||
if right_index in visited_right:
|
||
continue
|
||
visited_right.add(right_index)
|
||
parent_right[right_index] = left_index
|
||
owner = right_match.get(right_index)
|
||
if owner is None:
|
||
free_right = right_index
|
||
break
|
||
if owner not in visited_left:
|
||
visited_left.add(owner)
|
||
queue.append(owner)
|
||
if free_right is None:
|
||
continue
|
||
right_index: int | None = free_right
|
||
while right_index is not None:
|
||
left_index = parent_right[right_index]
|
||
previous_right = left_match.get(left_index)
|
||
left_match[left_index] = right_index
|
||
right_match[right_index] = left_index
|
||
right_index = previous_right
|
||
|
||
missing = len(expected_items) - len(left_match)
|
||
if missing:
|
||
missing_by_value[value] = missing
|
||
return missing_by_value
|
||
|
||
|
||
def bounded_change_rate(before: str, after: str) -> float:
|
||
"""Linear-time change estimate that also treats paragraph moves as non-local edits."""
|
||
|
||
def tokens(value: str) -> list[str]:
|
||
return re.findall(r"\w+|[^\w\s]", value, flags=re.UNICODE)
|
||
|
||
before_tokens = tokens(before)
|
||
after_tokens = tokens(after)
|
||
|
||
def multiset_rate(left: Counter[str], right: Counter[str]) -> float:
|
||
total = sum(left.values()) + sum(right.values())
|
||
overlap = sum((left & right).values())
|
||
return 0.0 if total == 0 else 1.0 - (2.0 * overlap / total)
|
||
|
||
lexical_rate = multiset_rate(Counter(before_tokens), Counter(after_tokens))
|
||
before_pairs = Counter(
|
||
"\u241f".join(before_tokens[index : index + 2])
|
||
for index in range(max(0, len(before_tokens) - 1))
|
||
)
|
||
after_pairs = Counter(
|
||
"\u241f".join(after_tokens[index : index + 2])
|
||
for index in range(max(0, len(after_tokens) - 1))
|
||
)
|
||
sequence_rate = multiset_rate(before_pairs, after_pairs)
|
||
|
||
def paragraph_hashes(value: str) -> list[str]:
|
||
paragraphs = [normalize_space(item) for item in re.split(r"\n\s*\n", value)]
|
||
return [hashlib.sha256(item.encode("utf-8")).hexdigest() for item in paragraphs if item]
|
||
|
||
def paragraph_tokens(value: str) -> list[list[str]]:
|
||
paragraphs = [normalize_space(item) for item in re.split(r"\n\s*\n", value)]
|
||
return [re.findall(r"\w+", item.casefold(), flags=re.UNICODE) for item in paragraphs if item]
|
||
|
||
def simhash(values: list[str]) -> int:
|
||
if not values:
|
||
return 0
|
||
weights = [0] * 64
|
||
for value in values:
|
||
hashed = int.from_bytes(
|
||
hashlib.blake2b(value.encode("utf-8"), digest_size=8).digest(), "big"
|
||
)
|
||
for bit in range(64):
|
||
weights[bit] += 1 if hashed & (1 << bit) else -1
|
||
result = 0
|
||
for bit, weight in enumerate(weights):
|
||
if weight >= 0:
|
||
result |= 1 << bit
|
||
return result
|
||
|
||
before_blocks = paragraph_hashes(before)
|
||
after_blocks = paragraph_hashes(after)
|
||
before_block_tokens = paragraph_tokens(before)
|
||
after_block_tokens = paragraph_tokens(after)
|
||
denominator = max(len(before_blocks), len(after_blocks), 1)
|
||
before_counts = Counter(before_blocks)
|
||
after_counts = Counter(after_blocks)
|
||
exact_after: dict[str, list[int]] = defaultdict(list)
|
||
for index, value in enumerate(after_blocks):
|
||
exact_after[value].append(index)
|
||
matched_pairs: list[tuple[int, int]] = []
|
||
used_after: set[int] = set()
|
||
unmatched_before: list[int] = []
|
||
for before_index, value in enumerate(before_blocks):
|
||
candidates = exact_after.get(value, [])
|
||
after_index = next((item for item in candidates if item not in used_after), None)
|
||
if after_index is None:
|
||
unmatched_before.append(before_index)
|
||
continue
|
||
used_after.add(after_index)
|
||
matched_pairs.append((before_index, after_index))
|
||
|
||
unmatched_after = [index for index in range(len(after_blocks)) if index not in used_after]
|
||
token_index: dict[str, list[int]] = defaultdict(list)
|
||
for after_index in unmatched_after:
|
||
for token in set(after_block_tokens[after_index]):
|
||
token_index[token].append(after_index)
|
||
after_simhash = {index: simhash(after_block_tokens[index]) for index in unmatched_after}
|
||
band_index: dict[tuple[int, int], list[int]] = defaultdict(list)
|
||
for after_index, signature in after_simhash.items():
|
||
for band in range(4):
|
||
band_index[(band, (signature >> (band * 16)) & 0xFFFF)].append(after_index)
|
||
|
||
for before_index in unmatched_before:
|
||
values = before_block_tokens[before_index]
|
||
candidate_set: set[int] = set()
|
||
for token in set(values):
|
||
occurrences = token_index.get(token, [])
|
||
if 0 < len(occurrences) <= 3:
|
||
candidate_set.update(occurrences)
|
||
signature = simhash(values)
|
||
if not candidate_set:
|
||
for band in range(4):
|
||
candidate_set.update(
|
||
band_index.get((band, (signature >> (band * 16)) & 0xFFFF), [])
|
||
)
|
||
candidate_set.difference_update(used_after)
|
||
if not candidate_set:
|
||
continue
|
||
before_set = set(values)
|
||
scored: list[tuple[float, int, int]] = []
|
||
for after_index in candidate_set:
|
||
after_set = set(after_block_tokens[after_index])
|
||
union = before_set | after_set
|
||
jaccard = len(before_set & after_set) / max(len(union), 1)
|
||
distance = (signature ^ after_simhash[after_index]).bit_count()
|
||
scored.append((-jaccard, distance, after_index))
|
||
_, distance, after_index = min(scored)
|
||
after_set = set(after_block_tokens[after_index])
|
||
jaccard = len(before_set & after_set) / max(len(before_set | after_set), 1)
|
||
if jaccard < 0.65 and distance > 14:
|
||
continue
|
||
used_after.add(after_index)
|
||
matched_pairs.append((before_index, after_index))
|
||
|
||
moved = 0
|
||
for before_index, after_index in matched_pairs:
|
||
before_position = before_index / max(len(before_blocks) - 1, 1)
|
||
after_position = after_index / max(len(after_blocks) - 1, 1)
|
||
if abs(before_position - after_position) > 0.08:
|
||
moved += 1
|
||
block_move_rate = min(1.0, moved / denominator)
|
||
return round(max(lexical_rate, sequence_rate * 0.5, block_move_rate), 6)
|
||
|
||
|
||
def fidelity_findings(
|
||
linter: Linter,
|
||
baseline_path: Path | None,
|
||
draft_baseline_path: Path | None,
|
||
) -> dict[str, Any]:
|
||
summary: dict[str, Any] = {
|
||
"baseline": None,
|
||
"draft_baseline": None,
|
||
"protected_total": 0,
|
||
"preserved": 0,
|
||
"missing": 0,
|
||
"by_type": {},
|
||
"finalization_change_rate": None,
|
||
"max_finalization_change_rate": linter.rules["thresholds"]["finalization"][
|
||
"max_change_rate"
|
||
],
|
||
}
|
||
if baseline_path is not None:
|
||
baseline_inventory, baseline_bytes = snapshot_file(baseline_path, "baseline")
|
||
baseline = decode_utf8(baseline_bytes, "baseline")
|
||
summary["baseline"] = {
|
||
"path": baseline_inventory["resolved_path"],
|
||
"sha256": baseline_inventory["sha256"],
|
||
}
|
||
rule_for = {
|
||
"fenced_code": "DOC-F001",
|
||
"indented_code": "DOC-F001",
|
||
"inline_code": "DOC-F002",
|
||
"url": "DOC-F003",
|
||
"link_destination": "DOC-F003",
|
||
"number_unit": "DOC-F004",
|
||
"number_range": "DOC-F004",
|
||
"number": "DOC-F004",
|
||
"date": "DOC-F004",
|
||
"version": "DOC-F004",
|
||
"quote": "DOC-F005",
|
||
}
|
||
inventory = protected_occurrences(baseline)
|
||
final_inventory = protected_occurrences(linter.text)
|
||
for kind, items in inventory.items():
|
||
actual_items = final_inventory[kind]
|
||
total = len(items)
|
||
missing_by_value = context_bound_missing(items, actual_items)
|
||
missing = sum(missing_by_value.values())
|
||
summary["by_type"][kind] = {
|
||
"total": total,
|
||
"preserved": total - missing,
|
||
"missing": missing,
|
||
}
|
||
summary["protected_total"] += total
|
||
summary["missing"] += missing
|
||
summary["preserved"] += total - missing
|
||
for value, missing_count in missing_by_value.items():
|
||
if missing_count:
|
||
display = normalize_space(value)
|
||
linter.add(
|
||
rule_for[kind],
|
||
f"기준 문서의 {kind} 보호 항목 또는 주변 의미 연결이 {missing_count}개 보존되지 않았습니다: {display[:120]}",
|
||
offset=0,
|
||
context=display[:200],
|
||
)
|
||
if draft_baseline_path is not None:
|
||
draft_inventory, draft_bytes = snapshot_file(
|
||
draft_baseline_path, "draft baseline"
|
||
)
|
||
draft = decode_utf8(draft_bytes, "draft baseline")
|
||
rate = bounded_change_rate(draft, linter.text)
|
||
maximum = float(linter.rules["thresholds"]["finalization"]["max_change_rate"])
|
||
summary["draft_baseline"] = {
|
||
"path": draft_inventory["resolved_path"],
|
||
"sha256": draft_inventory["sha256"],
|
||
}
|
||
summary["finalization_change_rate"] = rate
|
||
if rate > maximum:
|
||
linter.add(
|
||
"FNL-001",
|
||
f"finalizer 변경률 {rate:.1%}가 상한 {maximum:.1%}를 넘었습니다.",
|
||
offset=0,
|
||
context=f"change_rate={rate}",
|
||
)
|
||
return summary
|
||
|
||
|
||
def lint(args: argparse.Namespace) -> tuple[dict[str, Any], int]:
|
||
document_inventory, document_bytes = snapshot_file(Path(args.document), "document")
|
||
logic_inventory, logic_bytes = snapshot_file(Path(args.logic_map), "logic map")
|
||
term_inventory, term_bytes = snapshot_file(Path(args.term_ledger), "term ledger")
|
||
reader_inventory, reader_bytes = snapshot_file(
|
||
Path(args.reader_contract), "reader contract"
|
||
)
|
||
rules_inventory, _ = snapshot_file(
|
||
Path(args.rules) if args.rules else DEFAULT_RULES_PATH,
|
||
"quality rules",
|
||
)
|
||
document_path = Path(document_inventory["resolved_path"])
|
||
logic_path = Path(logic_inventory["resolved_path"])
|
||
term_path = Path(term_inventory["resolved_path"])
|
||
reader_path = Path(reader_inventory["resolved_path"])
|
||
rules_path = Path(rules_inventory["resolved_path"])
|
||
rules_sha256 = rules_inventory["sha256"]
|
||
rules = load_rules(rules_path)
|
||
if sha256_file(rules_path) != rules_sha256:
|
||
raise InputError("quality rules가 읽는 동안 변경되었습니다.")
|
||
text = decode_utf8(document_bytes, "document")
|
||
logic_text = decode_utf8(logic_bytes, "logic map")
|
||
term_text = decode_utf8(term_bytes, "term ledger")
|
||
reader_text = decode_utf8(reader_bytes, "reader contract")
|
||
logic_map = load_json_text(logic_text, str(logic_path))
|
||
term_ledger = load_json_text(term_text, str(term_path))
|
||
reader_contract = load_json_text(reader_text, str(reader_path))
|
||
validate_with_schema(logic_map, "logic-map.schema.json", "logic map")
|
||
validate_with_schema(term_ledger, "term-ledger.schema.json", "term ledger")
|
||
validate_with_schema(
|
||
reader_contract,
|
||
"reader-contract.schema.json",
|
||
"reader contract",
|
||
)
|
||
validate_contracts(logic_map, term_ledger, reader_contract, rules)
|
||
linter = Linter(
|
||
document_path=document_path,
|
||
text=text,
|
||
logic_map=logic_map,
|
||
term_ledger=term_ledger,
|
||
reader_contract=reader_contract,
|
||
rules=rules,
|
||
)
|
||
linter.lint_markdown()
|
||
linter.lint_logic()
|
||
linter.lint_terms()
|
||
linter.lint_paragraphs()
|
||
fidelity = fidelity_findings(
|
||
linter,
|
||
Path(args.baseline) if args.baseline else None,
|
||
Path(args.draft_baseline) if args.draft_baseline else None,
|
||
)
|
||
severity_order = {"error": 0, "warning": 1, "info": 2}
|
||
linter.findings.sort(
|
||
key=lambda item: (
|
||
item["line"] if item["line"] is not None else 10**9,
|
||
severity_order[item["severity"]],
|
||
item["rule_id"],
|
||
item["message"],
|
||
)
|
||
)
|
||
counts = Counter(item["severity"] for item in linter.findings)
|
||
failed = counts["error"] > 0 or (args.fail_on == "warning" and counts["warning"] > 0)
|
||
report = {
|
||
"schema_version": "1.0",
|
||
"rules_version": rules["rules_version"],
|
||
"rules_sha256": rules_sha256,
|
||
"tool": "lint_document",
|
||
"generated_at": utc_now(),
|
||
"document": {"path": str(document_path), "sha256": document_inventory["sha256"]},
|
||
"logic_map_sha256": logic_inventory["sha256"],
|
||
"term_ledger_sha256": term_inventory["sha256"],
|
||
"reader_contract_sha256": reader_inventory["sha256"],
|
||
"fail_on": args.fail_on,
|
||
"verdict": "fail" if failed else "pass",
|
||
"summary": {
|
||
"errors": counts["error"],
|
||
"warnings": counts["warning"],
|
||
"info": counts["info"],
|
||
"total": len(linter.findings),
|
||
},
|
||
"fidelity": fidelity,
|
||
"limitations": [
|
||
"한국어 전문용어 후보를 형태만으로 완전하게 검출할 수 없습니다. "
|
||
"영문 약어·코드형 식별자와 설정된 소문자 기술어 목록을 보조 신호로 검사합니다."
|
||
],
|
||
"findings": linter.findings,
|
||
}
|
||
return report, 1 if failed else 0
|
||
|
||
|
||
def input_error_report(args: argparse.Namespace, message: str) -> dict[str, Any]:
|
||
return {
|
||
"schema_version": "1.0",
|
||
"rules_version": "unknown",
|
||
"rules_sha256": None,
|
||
"tool": "lint_document",
|
||
"generated_at": utc_now(),
|
||
"document": {"path": str(args.document), "sha256": "0" * 64},
|
||
"fail_on": args.fail_on,
|
||
"verdict": "input_error",
|
||
"summary": {"errors": 1, "warnings": 0, "info": 0, "total": 1},
|
||
"fidelity": {},
|
||
"limitations": [],
|
||
"findings": [
|
||
{
|
||
"rule_id": "INPUT",
|
||
"severity": "error",
|
||
"message": message,
|
||
"path": str(args.document),
|
||
"line": None,
|
||
"column": None,
|
||
"section_id": None,
|
||
"context": None,
|
||
}
|
||
],
|
||
}
|
||
|
||
|
||
def parser() -> argparse.ArgumentParser:
|
||
value = argparse.ArgumentParser(description=__doc__)
|
||
value.add_argument("--document", required=True)
|
||
value.add_argument("--logic-map", required=True)
|
||
value.add_argument("--term-ledger", required=True)
|
||
value.add_argument("--reader-contract", required=True)
|
||
value.add_argument("--baseline", help="revise/review 원본 보호 항목 기준")
|
||
value.add_argument("--draft-baseline", help="finalizer 변경률 기준 07_draft.md")
|
||
value.add_argument("--output", required=True)
|
||
value.add_argument("--fail-on", choices=("error", "warning"), default="error")
|
||
value.add_argument("--rules", help=argparse.SUPPRESS)
|
||
return value
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = parser().parse_args(argv)
|
||
protected = [
|
||
Path(args.document),
|
||
Path(args.logic_map),
|
||
Path(args.term_ledger),
|
||
Path(args.reader_contract),
|
||
Path(args.rules) if args.rules else DEFAULT_RULES_PATH,
|
||
]
|
||
if args.baseline:
|
||
protected.append(Path(args.baseline))
|
||
if args.draft_baseline:
|
||
protected.append(Path(args.draft_baseline))
|
||
try:
|
||
output = prepare_report_output(
|
||
Path(args.output),
|
||
protected_paths=protected,
|
||
expected_tool="lint_document",
|
||
schema_name="lint-report.schema.json",
|
||
)
|
||
except InputError as exc:
|
||
print(f"input error: {exc}", file=sys.stderr)
|
||
return 2
|
||
try:
|
||
report, exit_code = lint(args)
|
||
except InputError as exc:
|
||
report = input_error_report(args, str(exc))
|
||
exit_code = 2
|
||
except (OSError, TypeError, ValueError, KeyError) as exc:
|
||
report = input_error_report(args, f"입력을 검사할 수 없습니다: {exc}")
|
||
exit_code = 2
|
||
try:
|
||
output_path = publish_report_json(output, report)
|
||
except (InputError, OSError) as exc:
|
||
print(
|
||
f"input error: lint report를 쓸 수 없습니다: {output.path}: {exc}",
|
||
file=sys.stderr,
|
||
)
|
||
return 2
|
||
print(
|
||
f"{report['verdict']}: errors={report['summary']['errors']} "
|
||
f"warnings={report['summary']['warnings']} output={output_path}"
|
||
)
|
||
return exit_code
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|