461 lines
16 KiB
Python
461 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Shared Markdown container, fence, and inline-code recognition."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
|
|
OPEN_FENCE_RE = re.compile(r"^([ \t]*)(`{3,}|~{3,})(.*?)(?:\r?\n)?$")
|
|
LIST_FENCE_RE = re.compile(
|
|
r"^(?P<leading>[ \t]*)(?P<list>[-+*]|[0-9]{1,9}[.)])"
|
|
r"(?P<gap>[ \t]+)(?P<fence>`{3,}|~{3,})(?P<info>.*)$"
|
|
)
|
|
BLOCKQUOTE_PREFIX_RE = re.compile(r"^ {0,3}>[ \t]?")
|
|
RAW_HTML_TAG_RE = re.compile(
|
|
r"^[ ]{0,3}<(?P<tag>pre|script|style|textarea)(?:[ \t>]|$)",
|
|
re.IGNORECASE,
|
|
)
|
|
BLOCK_HTML_TAG_RE = re.compile(
|
|
r"^[ ]{0,3}</?(?:address|article|aside|base|basefont|blockquote|body|caption|center|"
|
|
r"col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|"
|
|
r"form|frame|frameset|h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|"
|
|
r"menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|"
|
|
r"tbody|td|tfoot|th|thead|title|tr|track|ul)(?:[ \t>/]|$)",
|
|
re.IGNORECASE,
|
|
)
|
|
COMPLETE_HTML_TAG_RE = re.compile(
|
|
r"^[ ]{0,3}</?[A-Za-z][A-Za-z0-9-]*"
|
|
r"(?:[ \t]+[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \t]*=[ \t]*(?:[^ \t\"'=<>`]+|'[^']*'|\"[^\"]*\"))?)*"
|
|
r"[ \t]*/?>[ \t]*$"
|
|
)
|
|
THEMATIC_BREAK_RE = re.compile(
|
|
r"^[ ]{0,3}(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,})$"
|
|
)
|
|
|
|
|
|
def html_tag_spans(text: str) -> list[tuple[int, int]]:
|
|
"""Return HTML tag spans while excluding comments and URI autolinks."""
|
|
|
|
spans: list[tuple[int, int]] = []
|
|
cursor = 0
|
|
while cursor < len(text):
|
|
start = text.find("<", cursor)
|
|
if start < 0:
|
|
break
|
|
name = re.match(r"</?([A-Za-z][A-Za-z0-9-]*)", text[start:])
|
|
if name is None:
|
|
cursor = start + 1
|
|
continue
|
|
prefix_end = start + name.end()
|
|
if prefix_end >= len(text) or text[prefix_end] not in " \t\r\n/>":
|
|
cursor = start + 1
|
|
continue
|
|
quote: str | None = None
|
|
end = prefix_end
|
|
while end < len(text):
|
|
character = text[end]
|
|
if quote is not None:
|
|
if character == quote:
|
|
quote = None
|
|
elif character in "\"'":
|
|
quote = character
|
|
elif character == ">":
|
|
spans.append((start, end + 1))
|
|
end += 1
|
|
break
|
|
end += 1
|
|
cursor = max(start + 1, end)
|
|
return spans
|
|
|
|
|
|
def indentation_columns(value: str) -> int:
|
|
columns = 0
|
|
for character in value:
|
|
columns += 4 - (columns % 4) if character == "\t" else 1
|
|
return columns
|
|
|
|
|
|
def strip_blockquotes(value: str, required_depth: int | None = None) -> tuple[str, int]:
|
|
depth = 0
|
|
rest = value
|
|
while required_depth is None or depth < required_depth:
|
|
match = BLOCKQUOTE_PREFIX_RE.match(rest)
|
|
if match is None:
|
|
break
|
|
rest = rest[match.end() :]
|
|
depth += 1
|
|
return rest, depth
|
|
|
|
|
|
def list_continuation_indent(line: str, current: int = 0) -> int:
|
|
"""Track the indentation owned by the current simple list item."""
|
|
|
|
content = line.rstrip("\r\n")
|
|
rest, _ = strip_blockquotes(content)
|
|
item = re.match(
|
|
r"^(?P<leading>[ \t]*)(?P<marker>[-+*]|[0-9]{1,9}[.)])(?P<gap>[ \t]+)",
|
|
rest,
|
|
)
|
|
if item:
|
|
prefix = item.group("leading") + item.group("marker") + item.group("gap")
|
|
return indentation_columns(prefix)
|
|
if not rest.strip():
|
|
return current
|
|
leading = re.match(r"^[ \t]*", rest)
|
|
columns = indentation_columns(leading.group(0) if leading else "")
|
|
return current if current and columns >= current else 0
|
|
|
|
|
|
def indented_code_container(
|
|
line: str,
|
|
list_indent: int = 0,
|
|
) -> tuple[int, int] | None:
|
|
"""Return blockquote depth/list indent for an indented code line."""
|
|
|
|
content = line.rstrip("\r\n")
|
|
rest, quote_depth = strip_blockquotes(content)
|
|
leading = re.match(r"^[ \t]*", rest)
|
|
columns = indentation_columns(leading.group(0) if leading else "")
|
|
required = list_indent + 4 if list_indent else 4
|
|
return (quote_depth, list_indent) if rest.strip() and columns >= required else None
|
|
|
|
|
|
def opening_fence(
|
|
line: str,
|
|
list_context_indent: int = 0,
|
|
) -> tuple[str, int, int, str, int] | None:
|
|
"""Return marker char/len, quote depth, close mode, and list indentation."""
|
|
|
|
content = line.rstrip("\r\n")
|
|
rest, quote_depth = strip_blockquotes(content)
|
|
list_match = LIST_FENCE_RE.match(rest)
|
|
if list_match:
|
|
marker = list_match.group("fence")
|
|
if marker.startswith("`") and "`" in list_match.group("info"):
|
|
return None
|
|
continuation = (
|
|
list_match.group("leading")
|
|
+ list_match.group("list")
|
|
+ list_match.group("gap")
|
|
)
|
|
return marker[0], len(marker), quote_depth, "list", indentation_columns(
|
|
continuation
|
|
)
|
|
match = OPEN_FENCE_RE.match(rest)
|
|
if match is None:
|
|
return None
|
|
indentation = match.group(1)
|
|
marker = match.group(2)
|
|
if marker.startswith("`") and "`" in match.group(3):
|
|
return None
|
|
columns = indentation_columns(indentation)
|
|
if list_context_indent and list_context_indent <= columns <= list_context_indent + 3:
|
|
return marker[0], len(marker), quote_depth, "list", list_context_indent
|
|
if indentation.replace(" ", "") == "" and len(indentation) <= 3:
|
|
return marker[0], len(marker), quote_depth, "top", 0
|
|
# Four or more columns outside a list are indented code, not a fence.
|
|
return None
|
|
|
|
|
|
def closing_fence(
|
|
line: str,
|
|
marker_char: str,
|
|
marker_len: int,
|
|
quote_depth: int,
|
|
close_mode: str,
|
|
list_indent: int,
|
|
) -> bool:
|
|
content = line.rstrip("\r\n")
|
|
rest, actual_quote_depth = strip_blockquotes(content, quote_depth)
|
|
if actual_quote_depth != quote_depth:
|
|
return False
|
|
if close_mode not in {"top", "list"}:
|
|
if not rest.startswith(close_mode):
|
|
return False
|
|
rest = rest[len(close_mode) :]
|
|
match = re.match(
|
|
rf"^(?P<indent>[ \t]*){re.escape(marker_char)}{{{marker_len},}}[ \t]*$",
|
|
rest,
|
|
)
|
|
if match is None:
|
|
return False
|
|
columns = indentation_columns(match.group("indent"))
|
|
if close_mode == "top":
|
|
return columns <= 3
|
|
if close_mode == "list":
|
|
return list_indent <= columns <= list_indent + 3
|
|
return columns <= 3
|
|
|
|
|
|
def fence_container_continues(
|
|
line: str,
|
|
quote_depth: int,
|
|
close_mode: str,
|
|
list_indent: int,
|
|
) -> bool:
|
|
"""Whether an open fence's blockquote/list container owns this line."""
|
|
|
|
content = line.rstrip("\r\n")
|
|
rest, actual_quote_depth = strip_blockquotes(content, quote_depth)
|
|
if actual_quote_depth != quote_depth:
|
|
return False
|
|
if close_mode != "list" or not rest.strip():
|
|
return True
|
|
leading = re.match(r"^[ \t]*", rest)
|
|
return indentation_columns(leading.group(0) if leading else "") >= list_indent
|
|
|
|
|
|
def advance_html_block(
|
|
line: str,
|
|
state: tuple[str, str] | None,
|
|
paragraph_open: bool = False,
|
|
) -> tuple[bool, tuple[str, str] | None]:
|
|
"""Classify reader-visible raw HTML lines where Markdown fences are inert."""
|
|
|
|
content = line.rstrip("\r\n")
|
|
structural, _ = strip_blockquotes(content)
|
|
list_item = re.match(
|
|
r"^[ \t]*(?:[-+*]|[0-9]{1,9}[.)])[ \t]+",
|
|
structural,
|
|
)
|
|
if list_item:
|
|
structural = structural[list_item.end() :]
|
|
if state is not None:
|
|
kind, value = state
|
|
if kind == "blank":
|
|
if not structural.strip():
|
|
return False, None
|
|
return True, state
|
|
if kind == "tag":
|
|
closing = re.search(
|
|
rf"</{re.escape(value)}[ \t]*>", structural, re.IGNORECASE
|
|
)
|
|
return True, None if closing else state
|
|
return True, None if value in structural else state
|
|
|
|
raw = RAW_HTML_TAG_RE.match(structural)
|
|
if raw:
|
|
tag = raw.group("tag").lower()
|
|
closing = re.search(
|
|
rf"</{re.escape(tag)}[ \t]*>", structural, re.IGNORECASE
|
|
)
|
|
return True, None if closing else ("tag", tag)
|
|
if BLOCK_HTML_TAG_RE.match(structural):
|
|
return True, ("blank", "")
|
|
stripped = (
|
|
structural.lstrip(" ")
|
|
if len(structural) - len(structural.lstrip(" ")) <= 3
|
|
else structural
|
|
)
|
|
for opener, closer in (("<?", "?>"), ("<![CDATA[", "]]>") ):
|
|
if stripped.startswith(opener):
|
|
return True, None if closer in stripped[len(opener) :] else ("token", closer)
|
|
if re.match(r"^<![A-Z]", stripped):
|
|
return True, None if ">" in stripped[2:] else ("token", ">")
|
|
# CommonMark HTML block type 7 cannot interrupt an open paragraph.
|
|
if not paragraph_open and COMPLETE_HTML_TAG_RE.match(structural):
|
|
return True, ("blank", "")
|
|
return False, None
|
|
|
|
|
|
def ordered_list_interrupts_paragraph(value: str) -> bool:
|
|
"""Return whether a CommonMark ordered marker can interrupt prose."""
|
|
|
|
match = re.match(
|
|
r"^[ ]{0,3}(?P<number>[0-9]{1,9})[.)][ \t]+(?=\S)",
|
|
value,
|
|
)
|
|
return bool(match and int(match.group("number")) == 1)
|
|
|
|
|
|
def is_thematic_break(value: str) -> bool:
|
|
return THEMATIC_BREAK_RE.fullmatch(value.rstrip("\r\n")) is not None
|
|
|
|
|
|
def line_interrupts_paragraph(line: str, quote_depth: int = 0) -> bool:
|
|
"""Recognize block starts that terminate multiline inline constructs."""
|
|
|
|
content = line.rstrip("\r\n")
|
|
rest, actual_quote_depth = strip_blockquotes(content)
|
|
if actual_quote_depth != quote_depth or not rest.strip():
|
|
return True
|
|
if re.match(r"^[ ]{0,3}#{1,6}(?:[ \t]+|$)", rest):
|
|
return True
|
|
if re.match(r"^[ ]{0,3}[-+*][ \t]+(?=\S)", rest):
|
|
return True
|
|
if ordered_list_interrupts_paragraph(rest):
|
|
return True
|
|
if is_thematic_break(rest) or re.fullmatch(r"[ ]{0,3}(?:=+|-+)[ \t]*", rest):
|
|
return True
|
|
if opening_fence(line) is not None:
|
|
return True
|
|
stripped = rest.lstrip(" ")
|
|
if len(rest) - len(stripped) <= 3 and stripped.startswith("<!--"):
|
|
return True
|
|
html_line, _ = advance_html_block(line, None, paragraph_open=True)
|
|
return html_line
|
|
|
|
|
|
def crosses_paragraph_boundary(text: str, start: int, end: int) -> bool:
|
|
"""Return whether ``text[start:end]`` crosses a Markdown block boundary."""
|
|
|
|
first_newline = text.find("\n", start, end)
|
|
if first_newline < 0:
|
|
return False
|
|
opener_line_start = text.rfind("\n", 0, start) + 1
|
|
_, opener_quote_depth = strip_blockquotes(text[opener_line_start:start])
|
|
line_start = first_newline + 1
|
|
while line_start <= end:
|
|
line_end = text.find("\n", line_start)
|
|
if line_end < 0:
|
|
line_end = len(text)
|
|
if line_interrupts_paragraph(
|
|
text[line_start:line_end], opener_quote_depth
|
|
):
|
|
return True
|
|
if line_end >= end:
|
|
break
|
|
line_start = line_end + 1
|
|
return False
|
|
|
|
|
|
def inline_code_spans(text: str) -> list[tuple[int, int, str]]:
|
|
"""Return paired CommonMark-style backtick spans with stable offsets."""
|
|
|
|
spans: list[tuple[int, int, str]] = []
|
|
def escaped(position: int) -> bool:
|
|
backslashes = 0
|
|
cursor = position - 1
|
|
while cursor >= 0 and text[cursor] == "\\":
|
|
backslashes += 1
|
|
cursor -= 1
|
|
return backslashes % 2 == 1
|
|
|
|
runs = [match for match in re.finditer(r"`+", text) if not escaped(match.start())]
|
|
|
|
index = 0
|
|
while index < len(runs):
|
|
opener = runs[index]
|
|
size = len(opener.group(0))
|
|
opener_line_start = text.rfind("\n", 0, opener.start()) + 1
|
|
opener_line_end = text.find("\n", opener.start())
|
|
if opener_line_end < 0:
|
|
opener_line_end = len(text)
|
|
_, opener_quote_depth = strip_blockquotes(
|
|
text[opener_line_start : opener.start()]
|
|
)
|
|
opener_line, _ = strip_blockquotes(
|
|
text[opener_line_start:opener_line_end]
|
|
)
|
|
opener_in_atx_heading = bool(
|
|
re.match(r"^[ ]{0,3}#{1,6}(?:[ \t]+|$)", opener_line)
|
|
)
|
|
closing_index: int | None = None
|
|
for candidate in range(index + 1, len(runs)):
|
|
between = text[opener.end() : runs[candidate].start()]
|
|
if opener_in_atx_heading and "\n" in between:
|
|
break
|
|
if crosses_paragraph_boundary(
|
|
text, opener.end(), runs[candidate].start()
|
|
):
|
|
break
|
|
if len(runs[candidate].group(0)) == size:
|
|
closing_index = candidate
|
|
break
|
|
if closing_index is None:
|
|
index += 1
|
|
continue
|
|
closer = runs[closing_index]
|
|
raw = text[opener.end() : closer.start()]
|
|
raw_lines = raw.replace("\r", "").split("\n")
|
|
if opener_quote_depth:
|
|
normalized_lines = [raw_lines[0]]
|
|
for line in raw_lines[1:]:
|
|
rest, depth = strip_blockquotes(line, opener_quote_depth)
|
|
normalized_lines.append(rest if depth == opener_quote_depth else line)
|
|
value = " ".join(normalized_lines)
|
|
else:
|
|
value = " ".join(raw_lines)
|
|
if value.startswith(" ") and value.endswith(" ") and value.strip():
|
|
value = value[1:-1]
|
|
spans.append((opener.start(), closer.end(), value))
|
|
index = closing_index + 1
|
|
return spans
|
|
|
|
|
|
def inside_any_span(offset: int, spans: list[tuple[int, int, str]]) -> bool:
|
|
return any(start <= offset < end for start, end, _ in spans)
|
|
|
|
|
|
def mask_closed_fence_candidates(text: str) -> str:
|
|
"""Mask closed container-aware fences solely for inline-code discovery."""
|
|
|
|
lines = text.splitlines(keepends=True)
|
|
masked = list(text)
|
|
offset = 0
|
|
open_start: int | None = None
|
|
marker_char = ""
|
|
marker_size = 0
|
|
quote_depth = 0
|
|
close_mode = "top"
|
|
list_indent = 0
|
|
html_state: tuple[str, str] | None = None
|
|
list_context_indent = 0
|
|
paragraph_open = False
|
|
for line in lines:
|
|
html_line = False
|
|
if open_start is not None:
|
|
explicit_close = closing_fence(
|
|
line,
|
|
marker_char,
|
|
marker_size,
|
|
quote_depth,
|
|
close_mode,
|
|
list_indent,
|
|
)
|
|
implicit_close = not explicit_close and not fence_container_continues(
|
|
line, quote_depth, close_mode, list_indent
|
|
)
|
|
if explicit_close or implicit_close:
|
|
end = offset + len(line) if explicit_close else offset
|
|
for index in range(open_start, end):
|
|
if masked[index] not in "\r\n":
|
|
masked[index] = " "
|
|
open_start = None
|
|
marker_char = ""
|
|
marker_size = 0
|
|
quote_depth = 0
|
|
close_mode = "top"
|
|
list_indent = 0
|
|
if explicit_close:
|
|
offset += len(line)
|
|
continue
|
|
if open_start is None:
|
|
html_line, html_state = advance_html_block(
|
|
line, html_state, paragraph_open=paragraph_open
|
|
)
|
|
if not html_line:
|
|
opening = opening_fence(line, list_context_indent)
|
|
if opening:
|
|
marker_char, marker_size, quote_depth, close_mode, list_indent = opening
|
|
open_start = offset
|
|
paragraph_open = False
|
|
else:
|
|
paragraph_open = False
|
|
if open_start is None:
|
|
list_context_indent = list_continuation_indent(line, list_context_indent)
|
|
content = line.rstrip("\r\n")
|
|
if not html_line:
|
|
paragraph_open = bool(content.strip()) and not re.match(
|
|
r"^[ ]{0,3}(?:#{1,6}(?:[ \t]+|$)|(?:=+|-+)[ \t]*$|"
|
|
r"(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,}))",
|
|
content,
|
|
)
|
|
offset += len(line)
|
|
if open_start is not None and (quote_depth > 0 or close_mode == "list"):
|
|
for index in range(open_start, len(text)):
|
|
if masked[index] not in "\r\n":
|
|
masked[index] = " "
|
|
return "".join(masked)
|