1726 lines
57 KiB
Bash
1726 lines
57 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -o pipefail
|
|
|
|
readonly PLATFORM_SENSITIVE_SOURCE_EXACT_PLATFORM_ROOT=/home/donghyeon/workspace/platform
|
|
readonly PLATFORM_SENSITIVE_SOURCE_EXACT_DOCS_ROOT=/home/donghyeon/workspace/docs/platform
|
|
readonly PLATFORM_SENSITIVE_SOURCE_EXACT_SCANNER="$PLATFORM_SENSITIVE_SOURCE_EXACT_PLATFORM_ROOT/scripts/validate/scan-platform-sensitive-source.sh"
|
|
|
|
_sensitive_source_fail() {
|
|
printf 'ERROR: %s\n' "$*" >&2
|
|
return 1
|
|
}
|
|
|
|
_sensitive_source_usage() {
|
|
cat <<'USAGE'
|
|
Usage:
|
|
bash scripts/validate/scan-platform-sensitive-source.sh
|
|
|
|
Scans the complete platform source and central platform documentation trees.
|
|
Only .git and .helm directory contents, plus the exact platform-root
|
|
.superpowers/sdd scratch subtree, are excluded. Diagnostics contain file names
|
|
and finding classes only; matched values and lines are never printed.
|
|
|
|
The production run invokes all five renderer entrypoints in private temporary
|
|
contexts. Four output-producing renderers publish private handoffs. The access
|
|
renderer uses its output-free Grafana verifier because it does not require
|
|
historical metric inventories; its exact two-line success contract is checked.
|
|
It still traverses the pinned core renderer, whose separately published handoff
|
|
contains the only allowlisted credential-free Secret and is decoded here.
|
|
USAGE
|
|
}
|
|
|
|
_sensitive_source_is_test_file() {
|
|
case "$(basename -- "$1")" in
|
|
test-*.sh|test_*.sh|test-*.py)
|
|
return 0
|
|
;;
|
|
esac
|
|
return 1
|
|
}
|
|
|
|
_sensitive_source_report_path() {
|
|
local path=$1 platform_root=$2 docs_root=$3 relative
|
|
if [[ "$path" == "$platform_root"/* ]]; then
|
|
relative="platform/${path#"$platform_root"/}"
|
|
elif [[ "$path" == "$docs_root"/* ]]; then
|
|
relative="docs/${path#"$docs_root"/}"
|
|
else
|
|
relative="$(basename -- "$path")"
|
|
fi
|
|
# Preserve filename boundaries without allowing embedded control characters
|
|
# to create fake diagnostic records.
|
|
printf '%q' "$relative"
|
|
}
|
|
|
|
_sensitive_source_rg_matches() {
|
|
local pattern=$1 file=$2 rc
|
|
rg --quiet --no-messages -P -- "$pattern" "$file"
|
|
rc=$?
|
|
case "$rc" in
|
|
0) return 0 ;;
|
|
1) return 1 ;;
|
|
*) return "$rc" ;;
|
|
esac
|
|
}
|
|
|
|
_sensitive_source_python_scan() {
|
|
local file=$1 test_file=$2 scanner_source=$3
|
|
python3 - "$file" "$test_file" "$scanner_source" <<'PY'
|
|
import ast
|
|
import io
|
|
import re
|
|
import sys
|
|
import tokenize
|
|
from pathlib import Path
|
|
|
|
path = sys.argv[1]
|
|
test_file = sys.argv[2] == "1"
|
|
scanner_source = sys.argv[3]
|
|
suffix = Path(path).suffix.lower()
|
|
structured_source = suffix in (".yaml", ".yml", ".json")
|
|
python_source = suffix == ".py"
|
|
shell_source = suffix in (".sh", ".bash", ".zsh") or not suffix
|
|
prose_source = suffix in (".md", ".markdown", ".txt", ".rst")
|
|
embedded_python_source = path in (
|
|
"/home/donghyeon/workspace/platform/scripts/validate/observability-smoke.sh",
|
|
"/home/donghyeon/workspace/platform/scripts/validate/test-scan-platform-sensitive-source.sh",
|
|
"/home/donghyeon/workspace/platform/scripts/validate/test-configure-keycloak-grafana-oidc.sh",
|
|
"/home/donghyeon/workspace/platform/scripts/bootstrap/create-observability-secrets.sh",
|
|
"/home/donghyeon/workspace/platform/scripts/bootstrap/configure-keycloak-grafana-oidc.sh",
|
|
)
|
|
|
|
try:
|
|
raw = open(path, "rb").read()
|
|
except OSError:
|
|
raise SystemExit(2)
|
|
|
|
if python_source:
|
|
try:
|
|
encoding, _ = tokenize.detect_encoding(io.BytesIO(raw).readline)
|
|
text = raw.decode(encoding)
|
|
except (SyntaxError, LookupError, UnicodeDecodeError):
|
|
raise SystemExit(2)
|
|
else:
|
|
if b"\0" in raw:
|
|
raise SystemExit(0)
|
|
try:
|
|
text = raw.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
raise SystemExit(0)
|
|
|
|
url = "https://hooks." + "slack.com/services/"
|
|
slack = re.compile(re.escape(url) + r"[^/\s'\"]+/[^/\s'\"]+/[^/\s'\"]+")
|
|
credential = re.compile(
|
|
r"(?i)(?:^|[^A-Za-z0-9_])(?P<key>cloudflare[^\r\n:=]{0,24}token|"
|
|
r"client[_-]?secret|password|passwd|access[_-]?key|secret[_-]?key|webhook)"
|
|
r"[\"']?[ \t]*[:=][ \t]*(?P<rhs>[^\r\n#]+)"
|
|
)
|
|
synthetic = re.compile(
|
|
r"(?:fixture|synthetic|do-not-leak|should[-_ ]?never|example|t123|b456|s789|"
|
|
r"temp[-_ ]?admin|^grafanapassword-[0-9]+$)",
|
|
re.I,
|
|
)
|
|
status_value = re.compile(
|
|
r"^(?:CREATE_CONFIRMED|REUSED_UNCHANGED|PASS|FAIL|ABSENT|PRESENT|REQUIRED|BLOCKED)$"
|
|
)
|
|
unfinished = ("TO" + "DO", "TB" + "D", "나중에" + "채움", "UNRESOLVED_" + "PLACEHOLDER")
|
|
definition_line = (
|
|
'unfinished = ("TO" + "DO", "TB" + "D", "나중에" + "채움", '
|
|
'"UNRESOLVED_" + "PLACEHOLDER")'
|
|
)
|
|
|
|
def test_context_allowed(value):
|
|
return test_file and synthetic.search(value) is not None
|
|
|
|
def first_rhs(raw):
|
|
value = raw.lstrip()
|
|
if not value:
|
|
return ""
|
|
if value[0] in "\"'":
|
|
quote = value[0]
|
|
escaped = False
|
|
for index, character in enumerate(value[1:], start=1):
|
|
if escaped:
|
|
escaped = False
|
|
elif character == "\\":
|
|
escaped = True
|
|
elif character == quote:
|
|
trailing = value[index + 1 :].strip()
|
|
if trailing and trailing not in (",", ")", ";", ");", "}", "},"):
|
|
return value
|
|
return value[1:index].strip()
|
|
return value
|
|
result = []
|
|
stack = []
|
|
quote = None
|
|
escaped = False
|
|
pairs = {")": "(", "]": "[", "}": "{"}
|
|
for character in value:
|
|
if quote is not None:
|
|
result.append(character)
|
|
if escaped:
|
|
escaped = False
|
|
elif character == "\\":
|
|
escaped = True
|
|
elif character == quote:
|
|
quote = None
|
|
continue
|
|
if character in "\"'":
|
|
quote = character
|
|
result.append(character)
|
|
elif character in "([{":
|
|
stack.append(character)
|
|
result.append(character)
|
|
elif character in ")]}":
|
|
if stack and stack[-1] == pairs[character]:
|
|
stack.pop()
|
|
result.append(character)
|
|
else:
|
|
break
|
|
elif not stack and character == "#":
|
|
break
|
|
else:
|
|
result.append(character)
|
|
return "".join(result).strip().rstrip("\\\"'").strip()
|
|
|
|
shell_reference = re.compile(
|
|
r"(?:\$[A-Za-z_][A-Za-z0-9_]*|\$\{[^{}\r\n]+\}|\$\([^\r\n]+\))"
|
|
)
|
|
base64_expression = re.compile(
|
|
r"base64\.b64encode\("
|
|
r"(?:b?(?:\"[^\"\r\n]*\"|'[^'\r\n]*')|"
|
|
r"[A-Za-z_][A-Za-z0-9_]*(?:\.encode\((?:\"(?:ascii|utf-8)\"|'(?:ascii|utf-8)')?\))?)"
|
|
r"\)\.decode\((?:\"ascii\"|'ascii')?\)"
|
|
)
|
|
pathlib_read_expression = re.compile(
|
|
r"pathlib\.Path\([A-Za-z_][A-Za-z0-9_]*\)\.read_text\(\)"
|
|
)
|
|
explicit_placeholder = re.compile(
|
|
r"^<(?:(?:redacted)|(?:non-credential-placeholder)|(?:runtime-only))>$", re.I
|
|
)
|
|
|
|
def exact_runtime_reference(value):
|
|
if explicit_placeholder.fullmatch(value):
|
|
return True
|
|
if shell_reference.fullmatch(value):
|
|
return True
|
|
match = shell_reference.match(value)
|
|
return bool(match and re.fullmatch(r"(?:/[A-Za-z0-9._-]+)+", value[match.end():]))
|
|
|
|
def exact_python_runtime_expression(value):
|
|
if not (python_source or embedded_python_source):
|
|
return False
|
|
string_literals = re.findall(r"(?i)(?:[bruf]{0,2})([\"'])(.*?)\1", value)
|
|
for _, literal in string_literals:
|
|
literal = literal.strip()
|
|
if len(literal) >= 12 and not test_context_allowed(literal):
|
|
return False
|
|
expression_shape = re.fullmatch(r"[A-Za-z0-9_.'\"(), -]+", value)
|
|
if expression_shape is None:
|
|
return False
|
|
return (
|
|
base64_expression.fullmatch(value) is not None
|
|
or pathlib_read_expression.fullmatch(value) is not None
|
|
)
|
|
|
|
def exact_python_mapping_reference(line, match, value):
|
|
if not (python_source or embedded_python_source):
|
|
return False
|
|
if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value) is None:
|
|
return False
|
|
before = line[:match.start("rhs")]
|
|
return (
|
|
"{" in before
|
|
and re.search(r"[\"'][^\"']+[\"']\s*:\s*$", before) is not None
|
|
and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*\s*[,}]", match.group("rhs").strip())
|
|
is not None
|
|
)
|
|
|
|
def exact_python_nonliteral_reference(line, match, value):
|
|
if python_source or embedded_python_source:
|
|
before_rhs = line[:match.start("rhs")]
|
|
typed_parameter = re.fullmatch(
|
|
r"[A-Za-z_][A-Za-z0-9_.]*"
|
|
r"(?:\s*\|\s*(?:[A-Za-z_][A-Za-z0-9_.]*|None))*"
|
|
r"(?:\s*=\s*None)?\s*,?",
|
|
value,
|
|
)
|
|
if (
|
|
re.fullmatch(
|
|
r"\s*[A-Za-z_][A-Za-z0-9_]*[\"']?\s*:\s*",
|
|
before_rhs,
|
|
)
|
|
and typed_parameter is not None
|
|
):
|
|
return True
|
|
|
|
if not python_source:
|
|
return False
|
|
|
|
if re.fullmatch(
|
|
r"runtime\s*/\s*[\"']input-webhook[\"']\s*[,;)]?",
|
|
match.group("rhs").strip(),
|
|
):
|
|
return True
|
|
|
|
try:
|
|
expression = ast.parse(value, mode="eval").body
|
|
except SyntaxError:
|
|
return False
|
|
|
|
def literal_free_call(node):
|
|
if isinstance(node, ast.Name):
|
|
return isinstance(node.ctx, ast.Load)
|
|
if isinstance(node, ast.Attribute):
|
|
return isinstance(node.ctx, ast.Load) and literal_free_call(node.value)
|
|
if isinstance(node, ast.Call):
|
|
return (
|
|
not node.keywords
|
|
and literal_free_call(node.func)
|
|
and all(
|
|
not isinstance(argument, ast.Starred)
|
|
and literal_free_call(argument)
|
|
for argument in node.args
|
|
)
|
|
)
|
|
return False
|
|
|
|
return isinstance(expression, ast.Call) and literal_free_call(expression)
|
|
|
|
python_string_verdicts = {}
|
|
if python_source:
|
|
try:
|
|
python_tokens = list(tokenize.generate_tokens(io.StringIO(text).readline))
|
|
python_tree = ast.parse(text)
|
|
except (SyntaxError, UnicodeDecodeError, tokenize.TokenError):
|
|
raise SystemExit(2)
|
|
|
|
python_lines = text.splitlines(keepends=True)
|
|
python_parents = {}
|
|
for python_parent in ast.walk(python_tree):
|
|
for python_child in ast.iter_child_nodes(python_parent):
|
|
python_parents[id(python_child)] = python_parent
|
|
|
|
def token_contains(line_number, column, token):
|
|
return (
|
|
(line_number, column) >= token.start
|
|
and (line_number, column) < token.end
|
|
)
|
|
|
|
def ast_contains(line_number, byte_column, node):
|
|
return (
|
|
(line_number, byte_column) >= (node.lineno, node.col_offset)
|
|
and (line_number, byte_column) < (node.end_lineno, node.end_col_offset)
|
|
)
|
|
|
|
def token_raw_offset(token, line_number, column):
|
|
if line_number == token.start[0]:
|
|
return column - token.start[1]
|
|
offset = len(python_lines[token.start[0] - 1][token.start[1] :])
|
|
for source_line in range(token.start[0], line_number - 1):
|
|
offset += len(python_lines[source_line])
|
|
return offset + column
|
|
|
|
def decoded_constant(value):
|
|
if isinstance(value, bytes):
|
|
return value.decode("utf-8")
|
|
if isinstance(value, str):
|
|
return value
|
|
raise ValueError("non-string constant")
|
|
|
|
def python_string_verdict(line_number, line, match):
|
|
key_column = match.start("key")
|
|
token = next(
|
|
(
|
|
candidate
|
|
for candidate in python_tokens
|
|
if candidate.type == tokenize.STRING
|
|
and token_contains(line_number, key_column, candidate)
|
|
),
|
|
None,
|
|
)
|
|
if token is None:
|
|
return None
|
|
string_open = re.match(r"(?i)[bruf]{0,3}(\"\"\"|'''|[\"'])", token.string)
|
|
if string_open is None or not token.string.endswith(string_open.group(1)):
|
|
return "unknown"
|
|
raw_key_start = token_raw_offset(token, line_number, key_column)
|
|
raw_end = len(token.string) - len(string_open.group(1))
|
|
if not (string_open.end() <= raw_key_start < raw_end):
|
|
return "unknown"
|
|
key_byte_column = len(line[:key_column].encode("utf-8"))
|
|
constants = [
|
|
node
|
|
for node in ast.walk(python_tree)
|
|
if isinstance(node, ast.Constant)
|
|
and isinstance(node.value, (str, bytes))
|
|
and ast_contains(line_number, key_byte_column, node)
|
|
]
|
|
if not constants:
|
|
return "unknown"
|
|
constant = min(
|
|
constants,
|
|
key=lambda node: (
|
|
node.end_lineno - node.lineno,
|
|
node.end_col_offset - node.col_offset,
|
|
),
|
|
)
|
|
own_rhs = token.string[raw_key_start:raw_end]
|
|
if re.fullmatch(
|
|
re.escape(match.group("key")) + r"[\"']?[ \t]*:[ \t]*",
|
|
own_rhs,
|
|
re.I,
|
|
) is None:
|
|
mapping = python_parents.get(id(constant))
|
|
if isinstance(mapping, ast.Dict):
|
|
for index, mapping_key in enumerate(mapping.keys):
|
|
if mapping_key is not constant:
|
|
continue
|
|
mapping_value = mapping.values[index]
|
|
if not any(
|
|
isinstance(leaf, ast.Constant)
|
|
and isinstance(leaf.value, (str, bytes))
|
|
and decoded_constant(leaf.value)
|
|
for leaf in ast.walk(mapping_value)
|
|
):
|
|
return "safe"
|
|
return "unsafe"
|
|
return "unsafe"
|
|
try:
|
|
prompt = decoded_constant(constant.value)
|
|
except UnicodeDecodeError:
|
|
return "unknown"
|
|
if re.search(
|
|
re.escape(match.group("key")) + r"[\"']?[ \t]*:[ \t]*$",
|
|
prompt,
|
|
re.I,
|
|
) is None:
|
|
return "unsafe"
|
|
expression = constant
|
|
while isinstance(python_parents.get(id(expression)), ast.expr):
|
|
expression = python_parents[id(expression)]
|
|
nonempty_constants = 0
|
|
try:
|
|
for leaf in ast.walk(expression):
|
|
if (
|
|
isinstance(leaf, ast.Constant)
|
|
and isinstance(leaf.value, (str, bytes))
|
|
and decoded_constant(leaf.value)
|
|
):
|
|
nonempty_constants += 1
|
|
except (UnicodeDecodeError, ValueError):
|
|
return "unknown"
|
|
return "safe" if nonempty_constants == 1 else "unsafe"
|
|
|
|
for python_line_number, python_line in enumerate(text.splitlines(), start=1):
|
|
for python_match in credential.finditer(python_line):
|
|
verdict = python_string_verdict(
|
|
python_line_number, python_line, python_match
|
|
)
|
|
if verdict is not None:
|
|
python_string_verdicts[
|
|
(python_line_number, python_match.start("key"))
|
|
] = verdict
|
|
if verdict == "unknown":
|
|
raise SystemExit(2)
|
|
|
|
def exact_shell_argument_reference(line, match):
|
|
if not shell_source:
|
|
return None
|
|
prefix = line[:match.start()]
|
|
if "--from-file=" not in prefix:
|
|
return None
|
|
rhs = match.group("rhs").lstrip()
|
|
closing_quote = re.search(r"[\"']", rhs)
|
|
if closing_quote is None:
|
|
return None
|
|
token_value = rhs[:closing_quote.start()].strip()
|
|
if not exact_runtime_reference(token_value):
|
|
return None
|
|
return match.start("rhs") + closing_quote.end()
|
|
|
|
for line in text.splitlines():
|
|
if line == definition_line:
|
|
if path == scanner_source:
|
|
continue
|
|
print("UNFINISHED_MARKER")
|
|
break
|
|
hits = [marker for marker in unfinished if marker in line]
|
|
if hits:
|
|
print("UNFINISHED_MARKER")
|
|
break
|
|
|
|
for line in text.splitlines():
|
|
matches = list(slack.finditer(line))
|
|
if matches and not all(test_context_allowed(match.group(0)) for match in matches):
|
|
print("SLACK_WEBHOOK")
|
|
break
|
|
|
|
for line_number, line in enumerate(text.splitlines(), start=1):
|
|
offset = 0
|
|
while offset < len(line):
|
|
match = credential.search(line, offset)
|
|
if match is None:
|
|
break
|
|
next_offset = max(match.start() + 1, match.end())
|
|
python_string_verdict = python_string_verdicts.get(
|
|
(line_number, match.start("key"))
|
|
)
|
|
if python_string_verdict == "safe":
|
|
offset = next_offset
|
|
continue
|
|
if python_string_verdict == "unsafe":
|
|
print("CREDENTIAL_LITERAL")
|
|
raise SystemExit(0)
|
|
if python_string_verdict == "unknown":
|
|
raise SystemExit(2)
|
|
prefix = line[:match.start()]
|
|
statement_start = max(prefix.rfind(";"), prefix.rfind("&&"), prefix.rfind("||")) + 1
|
|
statement_prefix = prefix[statement_start:]
|
|
regex_literal_context = (
|
|
not statement_prefix.strip()
|
|
and "[[:space:]]" in match.group(0)
|
|
and ("|" in match.group(0) or "`" in match.group(0) or "[^" in match.group(0))
|
|
)
|
|
detector_context = (
|
|
re.search(r"(?i)(?:rg|grep|check_pattern|regex|pattern)", statement_prefix)
|
|
and ("[[:" in match.group(0) or "\\" in match.group(0) or "|" in match.group(0))
|
|
)
|
|
quoted_regex_context = (
|
|
re.search(r"[\"'][^\"']*$", statement_prefix)
|
|
and re.search(r"[\"'](?:\s*\\)?\s*$", match.group("rhs"))
|
|
and ("[[:" in match.group(0) or "\\" in match.group(0) or "|" in match.group(0))
|
|
)
|
|
if regex_literal_context or detector_context or quoted_regex_context:
|
|
delimiter = re.search(r"(?:;|&&|\|\|)", match.group("rhs"))
|
|
quoted_end = re.search(r"[\"']", match.group("rhs"))
|
|
if delimiter is not None:
|
|
offset = match.start("rhs") + delimiter.end()
|
|
elif quoted_end is not None:
|
|
offset = match.start("rhs") + quoted_end.end()
|
|
else:
|
|
break
|
|
continue
|
|
value = first_rhs(match.group("rhs"))
|
|
value = re.sub(r"\\[nrt]$", "", value)
|
|
if not value or exact_runtime_reference(value):
|
|
offset = next_offset
|
|
continue
|
|
if exact_python_runtime_expression(value):
|
|
offset = next_offset
|
|
continue
|
|
if exact_python_nonliteral_reference(line, match, value):
|
|
offset = next_offset
|
|
continue
|
|
if exact_python_mapping_reference(line, match, value):
|
|
offset = next_offset
|
|
continue
|
|
shell_argument_end = exact_shell_argument_reference(line, match)
|
|
if shell_argument_end is not None:
|
|
offset = shell_argument_end
|
|
continue
|
|
line_starts_with_assignment = re.match(
|
|
r"^[ \t]*(?:cloudflare[^:=]{0,24}token|client[_-]?secret|password|passwd|"
|
|
r"access[_-]?key|secret[_-]?key|webhook)[\"']?[ \t]*[:=]",
|
|
line,
|
|
re.I,
|
|
) is not None
|
|
if (
|
|
prose_source
|
|
and not line_starts_with_assignment
|
|
and ":" in match.group(0)
|
|
and re.match(r"^[ \t]*(?:[-*+]\s+|\d+[.)]\s+|[^:=`]*\s+)", line)
|
|
):
|
|
offset = next_offset
|
|
continue
|
|
if len(value) < 12:
|
|
offset = next_offset
|
|
continue
|
|
if status_value.fullmatch(value):
|
|
offset = next_offset
|
|
continue
|
|
if test_context_allowed(value):
|
|
offset = next_offset
|
|
continue
|
|
print("CREDENTIAL_LITERAL")
|
|
raise SystemExit(0)
|
|
PY
|
|
}
|
|
|
|
_sensitive_source_yaml_secret_scan() {
|
|
local file=$1
|
|
python3 - "$file" <<'PY'
|
|
import sys
|
|
|
|
try:
|
|
import yaml
|
|
from yaml.constructor import ConstructorError
|
|
from yaml.nodes import MappingNode
|
|
from yaml.resolver import BaseResolver
|
|
except Exception:
|
|
raise SystemExit(2)
|
|
|
|
|
|
class StrictSourceLoader(yaml.BaseLoader):
|
|
pass
|
|
|
|
|
|
def construct_unique_mapping(loader, node, deep=False):
|
|
if not isinstance(node, MappingNode):
|
|
raise ConstructorError(None, None, "mapping node required", node.start_mark)
|
|
result = {}
|
|
for key_node, value_node in node.value:
|
|
key = loader.construct_object(key_node, deep=deep)
|
|
if not isinstance(key, str):
|
|
raise ConstructorError(None, None, "scalar mapping key required", key_node.start_mark)
|
|
if key in result:
|
|
raise ConstructorError(None, None, "duplicate mapping key", key_node.start_mark)
|
|
result[key] = loader.construct_object(value_node, deep=deep)
|
|
return result
|
|
|
|
|
|
StrictSourceLoader.add_constructor(BaseResolver.DEFAULT_MAPPING_TAG, construct_unique_mapping)
|
|
|
|
path = sys.argv[1]
|
|
try:
|
|
with open(path, encoding="utf-8") as stream:
|
|
for item in yaml.load_all(stream, Loader=StrictSourceLoader):
|
|
if item is None or not isinstance(item, dict) or item.get("kind") != "Secret":
|
|
continue
|
|
data = item.get("data") or {}
|
|
string_data = item.get("stringData") or {}
|
|
if not isinstance(data, dict) or not isinstance(string_data, dict):
|
|
print("SECRET_SOURCE_PAYLOAD", flush=True)
|
|
continue
|
|
if data or string_data:
|
|
print("SECRET_SOURCE_PAYLOAD", flush=True)
|
|
except (OSError, UnicodeError, yaml.YAMLError):
|
|
print("SECRET_SOURCE_MALFORMED", flush=True)
|
|
PY
|
|
}
|
|
|
|
_sensitive_source_structured_credential_scan() {
|
|
local file=$1 test_file=$2
|
|
python3 - "$file" "$test_file" <<'PY'
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
import yaml
|
|
from yaml.constructor import ConstructorError
|
|
from yaml.nodes import MappingNode
|
|
from yaml.resolver import BaseResolver
|
|
except Exception:
|
|
raise SystemExit(2)
|
|
|
|
|
|
class StrictCredentialLoader(yaml.BaseLoader):
|
|
pass
|
|
|
|
|
|
def construct_unique_mapping(loader, node, deep=False):
|
|
if not isinstance(node, MappingNode):
|
|
raise ConstructorError(None, None, "mapping node required", node.start_mark)
|
|
result = {}
|
|
for key_node, value_node in node.value:
|
|
key = loader.construct_object(key_node, deep=deep)
|
|
if not isinstance(key, str):
|
|
raise ConstructorError(None, None, "scalar mapping key required", key_node.start_mark)
|
|
if key in result:
|
|
raise ConstructorError(None, None, "duplicate mapping key", key_node.start_mark)
|
|
result[key] = loader.construct_object(value_node, deep=deep)
|
|
return result
|
|
|
|
|
|
StrictCredentialLoader.add_constructor(
|
|
BaseResolver.DEFAULT_MAPPING_TAG, construct_unique_mapping
|
|
)
|
|
|
|
path = Path(sys.argv[1])
|
|
test_file = sys.argv[2] == "1"
|
|
credential_segments = (
|
|
("password",),
|
|
("passwd",),
|
|
("client", "secret"),
|
|
("access", "key"),
|
|
("secret", "key"),
|
|
("webhook",),
|
|
)
|
|
synthetic = re.compile(
|
|
r"(?:fixture|synthetic|do-not-leak|should[-_ ]?never|example|t123|b456|s789|"
|
|
r"temp[-_ ]?admin|^grafanapassword-[0-9]+$)",
|
|
re.I,
|
|
)
|
|
status_value = re.compile(
|
|
r"^(?:CREATE_CONFIRMED|REUSED_UNCHANGED|PASS|FAIL|ABSENT|PRESENT|REQUIRED|BLOCKED)$"
|
|
)
|
|
runtime_reference = re.compile(
|
|
r"^(?:\$[A-Za-z_][A-Za-z0-9_]*|\$\{[^{}\r\n]+\}|\$\([^\r\n]+\)|"
|
|
r"<(?:(?:redacted)|(?:non-credential-placeholder)|(?:runtime-only))>)$",
|
|
re.I,
|
|
)
|
|
|
|
|
|
def unsafe_scalar(value):
|
|
if not isinstance(value, str):
|
|
return False
|
|
value = value.strip()
|
|
return (
|
|
len(value) >= 12
|
|
and runtime_reference.fullmatch(value) is None
|
|
and status_value.fullmatch(value) is None
|
|
and not (test_file and synthetic.search(value))
|
|
)
|
|
|
|
|
|
def normalized_key(value):
|
|
separated = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", value)
|
|
separated = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", separated)
|
|
return tuple(
|
|
segment
|
|
for segment in separated.lower()
|
|
.replace("-", "_")
|
|
.split("_")
|
|
if segment
|
|
)
|
|
|
|
|
|
def is_credential_key(value):
|
|
segments = normalized_key(value)
|
|
if segments and segments[-1] == "token" and "cloudflare" in segments:
|
|
return True
|
|
return any(
|
|
len(segments) >= len(candidate)
|
|
and segments[-len(candidate) :] == candidate
|
|
for candidate in credential_segments
|
|
)
|
|
|
|
|
|
def permitted_reference_container(key, value):
|
|
key_segments = normalized_key(key)
|
|
if not (
|
|
len(key_segments) >= 4
|
|
and key_segments[:2] == ("gf", "auth")
|
|
and key_segments[-2:] == ("client", "secret")
|
|
):
|
|
return False
|
|
if not isinstance(value, dict) or set(value) != {"secretKeyRef"}:
|
|
return False
|
|
reference = value["secretKeyRef"]
|
|
return (
|
|
isinstance(reference, dict)
|
|
and set(reference) == {"name", "key"}
|
|
and all(isinstance(reference[field], str) and reference[field] for field in reference)
|
|
)
|
|
|
|
|
|
def permitted_webhook_feature(path, key, value):
|
|
return (
|
|
tuple(str(part) for part in path) == ("operators", "object-store")
|
|
and key == "webhook"
|
|
and isinstance(value, dict)
|
|
and set(value) == {"enabled", "replicas"}
|
|
and str(value["enabled"]).lower() in ("true", "false")
|
|
and str(value["replicas"]).isdigit()
|
|
)
|
|
|
|
|
|
def strict_pairs(pairs):
|
|
result = {}
|
|
exact = set()
|
|
normalized = set()
|
|
for key, value in pairs:
|
|
if not isinstance(key, str):
|
|
raise ValueError("non-string JSON key")
|
|
key_normalized = normalized_key(key)
|
|
if key in exact or key_normalized in normalized:
|
|
raise ValueError("duplicate JSON key")
|
|
exact.add(key)
|
|
normalized.add(key_normalized)
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def walk(value, path=()):
|
|
if isinstance(value, dict):
|
|
for key, child in value.items():
|
|
if isinstance(key, str) and is_credential_key(key):
|
|
if permitted_webhook_feature(path, key, child):
|
|
pass
|
|
elif permitted_reference_container(key, child):
|
|
pass
|
|
elif not isinstance(child, str) or unsafe_scalar(child):
|
|
return True
|
|
if walk(child, path + (key,)):
|
|
return True
|
|
elif isinstance(value, list):
|
|
return any(walk(child, path + (index,)) for index, child in enumerate(value))
|
|
return False
|
|
|
|
|
|
try:
|
|
with path.open(encoding="utf-8") as stream:
|
|
if path.suffix.lower() == ".json":
|
|
documents = (json.load(stream, object_pairs_hook=strict_pairs),)
|
|
else:
|
|
documents = yaml.load_all(stream, Loader=StrictCredentialLoader)
|
|
for document in documents:
|
|
if walk(document):
|
|
print("CREDENTIAL_LITERAL", flush=True)
|
|
break
|
|
except (OSError, UnicodeError, ValueError, json.JSONDecodeError, yaml.YAMLError):
|
|
raise SystemExit(2)
|
|
PY
|
|
}
|
|
|
|
_sensitive_source_rendered_secret_scan() {
|
|
local manifest=$1 label=${2:-any} artifact=${3:-any}
|
|
python3 - "$manifest" "$label" "$artifact" <<'PY'
|
|
import base64
|
|
import binascii
|
|
import re
|
|
import sys
|
|
|
|
try:
|
|
import yaml
|
|
from yaml.constructor import ConstructorError
|
|
from yaml.nodes import MappingNode
|
|
from yaml.resolver import BaseResolver
|
|
except Exception:
|
|
raise SystemExit(2)
|
|
|
|
|
|
class StrictBaseLoader(yaml.BaseLoader):
|
|
pass
|
|
|
|
|
|
def construct_unique_mapping(loader, node, deep=False):
|
|
if not isinstance(node, MappingNode):
|
|
raise ConstructorError(None, None, "mapping node required", node.start_mark)
|
|
result = {}
|
|
for key_node, value_node in node.value:
|
|
key = loader.construct_object(key_node, deep=deep)
|
|
if not isinstance(key, str):
|
|
raise ConstructorError(None, None, "scalar mapping key required", key_node.start_mark)
|
|
if key in result:
|
|
raise ConstructorError(None, None, "duplicate mapping key", key_node.start_mark)
|
|
result[key] = loader.construct_object(value_node, deep=deep)
|
|
return result
|
|
|
|
|
|
StrictBaseLoader.add_constructor(BaseResolver.DEFAULT_MAPPING_TAG, construct_unique_mapping)
|
|
|
|
path = sys.argv[1]
|
|
label = sys.argv[2]
|
|
artifact = sys.argv[3]
|
|
|
|
gitea_common = {
|
|
"gitea": frozenset(("assertions", "config_environment.sh")),
|
|
"gitea-init": frozenset(
|
|
(
|
|
"configure_gitea.sh",
|
|
"configure_gpg_environment.sh",
|
|
"init_directory_structure.sh",
|
|
)
|
|
),
|
|
}
|
|
gitea_inline_baseline = frozenset(
|
|
(
|
|
"_generals_",
|
|
"actions",
|
|
"cache",
|
|
"database",
|
|
"indexer",
|
|
"metrics",
|
|
"packages",
|
|
"queue",
|
|
"repository",
|
|
"security",
|
|
"server",
|
|
"service",
|
|
"session",
|
|
)
|
|
)
|
|
gitea_inline_oidc = gitea_inline_baseline | {"oauth2_client"}
|
|
|
|
slack_url = re.compile(
|
|
re.escape("https://hooks." + "slack.com/services/")
|
|
+ r"[^/\s'\"]+/[^/\s'\"]+/[^/\s'\"]+"
|
|
)
|
|
private_key = re.compile(
|
|
"-----BEGIN (?:[A-Z0-9 ]+ )?PRI" + "VATE KEY-----|"
|
|
"-----BEGIN OPENSSH PRI" + "VATE KEY-----"
|
|
)
|
|
credential_assignment = re.compile(
|
|
r"(?i)(?:^|[^A-Za-z0-9_])"
|
|
r"(?:cloudflare[^\r\n:=]{0,24}token|client[_-]?secret|password|passwd|token|"
|
|
r"api[_-]?url|webhook|access[_-]?key|secret[_-]?key|authorization)"
|
|
r"[\"']?[ \t]*[:=][ \t]*([^\r\n#]+)"
|
|
)
|
|
safe_reference = re.compile(
|
|
r"^(?:\$[A-Za-z_][A-Za-z0-9_]*|\$\{[^{}\r\n]+\}|\$\([^\r\n]+\)|"
|
|
r"<(?:(?:redacted)|(?:non-credential-placeholder)|(?:runtime-only))>)$",
|
|
re.I,
|
|
)
|
|
|
|
|
|
def first_assignment_rhs(raw):
|
|
value = raw.strip()
|
|
if not value:
|
|
return ""
|
|
if value[0] in "\"'":
|
|
quote = value[0]
|
|
escaped = False
|
|
result = []
|
|
for index, character in enumerate(value[1:], start=1):
|
|
if escaped:
|
|
result.append(character)
|
|
escaped = False
|
|
elif character == "\\":
|
|
result.append(character)
|
|
escaped = True
|
|
elif character == quote:
|
|
trailing = value[index + 1 :].strip()
|
|
if trailing and trailing not in (",", ")", ";", ");"):
|
|
return value
|
|
return "".join(result).strip()
|
|
else:
|
|
result.append(character)
|
|
return "".join(result).strip()
|
|
return value.rstrip(",;)\\").rstrip().rstrip("\"'").strip()
|
|
|
|
|
|
def contains_literal_credential(raw):
|
|
try:
|
|
text = raw.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
return True
|
|
if slack_url.search(text) or private_key.search(text):
|
|
return True
|
|
for line in text.splitlines():
|
|
for match in credential_assignment.finditer(line):
|
|
value = first_assignment_rhs(match.group(1))
|
|
if not value or safe_reference.fullmatch(value):
|
|
continue
|
|
if len(value) >= 12:
|
|
return True
|
|
return False
|
|
|
|
|
|
def decoded_values(data, string_data):
|
|
result = []
|
|
for value in data.values():
|
|
if not isinstance(value, str):
|
|
raise ValueError("non-scalar data value")
|
|
try:
|
|
result.append(base64.b64decode(value, validate=True))
|
|
except (binascii.Error, ValueError, TypeError):
|
|
raise ValueError("invalid base64")
|
|
for value in string_data.values():
|
|
if not isinstance(value, str):
|
|
raise ValueError("non-scalar stringData value")
|
|
result.append(value.encode("utf-8"))
|
|
return result
|
|
|
|
|
|
def allowlisted_secret(metadata, secret_type, data, string_data):
|
|
namespace = metadata.get("namespace")
|
|
name = metadata.get("name")
|
|
if namespace == "observability" and name == (
|
|
"alertmanager-observability-core-kube-pr-alertmanager"
|
|
):
|
|
return (
|
|
secret_type in (None, "Opaque")
|
|
and set(data) == {"alertmanager.yaml"}
|
|
and not string_data
|
|
)
|
|
if namespace != "gitea" or secret_type != "Opaque" or data:
|
|
return False
|
|
keys = frozenset(string_data)
|
|
if name in gitea_common:
|
|
return keys == gitea_common[name]
|
|
if name == "gitea-inline-config":
|
|
if label == "phase1" and artifact == "gitea":
|
|
return keys == gitea_inline_baseline
|
|
if label == "phase1" and artifact == "gitea-oidc":
|
|
return keys == gitea_inline_oidc
|
|
return keys in (gitea_inline_baseline, gitea_inline_oidc)
|
|
return False
|
|
|
|
|
|
def expected_secret_identities():
|
|
if label == "phase1" and artifact == "gitea":
|
|
return {
|
|
("gitea", "gitea"),
|
|
("gitea", "gitea-init"),
|
|
("gitea", "gitea-inline-config"),
|
|
}
|
|
if label == "phase1" and artifact == "gitea-oidc":
|
|
return {
|
|
("gitea", "gitea"),
|
|
("gitea", "gitea-init"),
|
|
("gitea", "gitea-inline-config"),
|
|
}
|
|
if label == "observability-core" and artifact in ("kps", "aggregate"):
|
|
return {
|
|
("observability", "alertmanager-observability-core-kube-pr-alertmanager"),
|
|
}
|
|
return set()
|
|
|
|
|
|
try:
|
|
identities = []
|
|
with open(path, encoding="utf-8") as stream:
|
|
for item in yaml.load_all(stream, Loader=StrictBaseLoader):
|
|
if item is None:
|
|
continue
|
|
if not isinstance(item, dict) or item.get("kind") != "Secret":
|
|
continue
|
|
metadata = item.get("metadata") or {}
|
|
data = item.get("data") or {}
|
|
string_data = item.get("stringData") or {}
|
|
if not isinstance(metadata, dict) or not isinstance(data, dict) or not isinstance(
|
|
string_data, dict
|
|
):
|
|
print("RENDERED_SECRET_MALFORMED")
|
|
continue
|
|
if not allowlisted_secret(metadata, item.get("type"), data, string_data):
|
|
print("RENDERED_SECRET_NOT_ALLOWLISTED")
|
|
continue
|
|
identities.append((metadata.get("namespace"), metadata.get("name")))
|
|
try:
|
|
values = decoded_values(data, string_data)
|
|
except ValueError:
|
|
print("RENDERED_SECRET_MALFORMED")
|
|
continue
|
|
if any(contains_literal_credential(value) for value in values):
|
|
print("RENDERED_SECRET_CREDENTIAL")
|
|
continue
|
|
print("RENDERED_SECRET_ALLOWLISTED")
|
|
if label != "any":
|
|
expected = expected_secret_identities()
|
|
if len(identities) != len(expected) or set(identities) != expected:
|
|
print("RENDERED_REQUIRED_SECRET_MISSING")
|
|
except (OSError, UnicodeError, yaml.YAMLError):
|
|
raise SystemExit(2)
|
|
PY
|
|
}
|
|
|
|
_sensitive_source_render_results_are_safe() {
|
|
local results=$1 line
|
|
while IFS= read -r line; do
|
|
[[ -n "$line" ]] || continue
|
|
[[ "$line" == RENDERED_SECRET_ALLOWLISTED ]] || return 1
|
|
done <<<"$results"
|
|
}
|
|
|
|
_sensitive_source_access_log_is_safe() {
|
|
_sensitive_source_renderer_log_is_safe observability-access "$1"
|
|
}
|
|
|
|
_sensitive_source_renderer_log_is_safe() {
|
|
local label=$1 log=$2
|
|
python3 - "$label" "$log" <<'PY'
|
|
import os
|
|
import re
|
|
import stat
|
|
import sys
|
|
|
|
label, path = sys.argv[1:]
|
|
|
|
|
|
def exact(value):
|
|
return re.compile(re.escape(value))
|
|
|
|
|
|
def rendered(name, width=None, digest=False, korean=False):
|
|
prefix = "렌더" if korean else "Rendered"
|
|
suffix = r" bytes SHA-256 [0-9a-f]{64}" if digest else r" bytes"
|
|
return re.compile(re.escape(prefix) + r" +" + re.escape(name) + r" +[0-9]+" + suffix)
|
|
|
|
|
|
def verified(name, korean=False):
|
|
prefix = "검증" if korean else "Verified"
|
|
return re.compile(re.escape(prefix) + r" +" + re.escape(name) + r" +SHA-256 +[0-9a-f]{64}")
|
|
|
|
|
|
def phase1(published):
|
|
result = [
|
|
rendered("namespaces"),
|
|
rendered("ssd-local-pv"),
|
|
verified("cloudnative-pg-chart"),
|
|
rendered("cnpg-operator"),
|
|
rendered("platform-postgres"),
|
|
verified("gitea-chart"),
|
|
rendered("gitea"),
|
|
rendered("gitea-oidc"),
|
|
]
|
|
if published:
|
|
result.append(exact("Preserved six verified manifests for the apply handoff."))
|
|
result.extend(
|
|
(
|
|
exact("Phase 1 baseline and Gitea OIDC desired rendering invariants passed."),
|
|
exact("Temporary rendered manifests and generated chart caches will be removed on exit."),
|
|
)
|
|
)
|
|
return result
|
|
|
|
|
|
patterns = {}
|
|
patterns["phase1"] = phase1(True)
|
|
patterns["phase2"] = [
|
|
exact("Validating the Phase 1 baseline and Gitea OIDC desired profile first."),
|
|
*phase1(False),
|
|
*(rendered(name) for name in (
|
|
"phase2-namespaces",
|
|
"aistor-local-pv",
|
|
"aistor-network-policies",
|
|
"keycloak-operator",
|
|
"platform-postgres-keycloak",
|
|
"keycloak",
|
|
)),
|
|
verified("aistor-operator-chart"),
|
|
rendered("aistor-operator"),
|
|
verified("aistor-objectstore-chart"),
|
|
rendered("minio-aistor"),
|
|
exact("Preserved eight verified manifests for the AIStor apply handoff."),
|
|
exact("Phase 2 rendering and source invariants passed."),
|
|
exact("No live ObjectStore CRD or Kubernetes cluster access was required."),
|
|
exact("Temporary Phase 2 rendered manifests and generated chart caches will be removed on exit."),
|
|
]
|
|
patterns["admin"] = [
|
|
exact("Pulled: docker.io/dpage/pgadmin4-helm:9.16.0"),
|
|
re.compile(r"Digest: sha256:[0-9a-f]{64}"),
|
|
verified("pgadmin", korean=True),
|
|
verified("aistor-objectstore", korean=True),
|
|
*(rendered(name, digest=True, korean=True) for name in (
|
|
"admin-namespace",
|
|
"pgadmin-local-pv",
|
|
"coredns-custom",
|
|
"aistor-admin-oidc",
|
|
"pgadmin",
|
|
)),
|
|
exact("ADMIN SERVICES STATIC RENDER PASS"),
|
|
]
|
|
patterns["observability-core"] = [
|
|
*(verified(name) for name in (
|
|
"prometheus-operator-crds",
|
|
"kube-prometheus-stack",
|
|
"loki",
|
|
"tempo",
|
|
"alloy",
|
|
"prometheus-node-exporter",
|
|
"grafana",
|
|
"prometheus-blackbox-exporter",
|
|
)),
|
|
*(rendered(name, digest=True) for name in (
|
|
"namespaces",
|
|
"crds",
|
|
"storage",
|
|
"kps",
|
|
"loki",
|
|
"tempo",
|
|
"alloy",
|
|
"node-exporter",
|
|
"grafana",
|
|
"blackbox",
|
|
"core-policies",
|
|
"targets",
|
|
"core-rules",
|
|
"alerting",
|
|
"dashboards",
|
|
"aggregate",
|
|
)),
|
|
exact("OBSERVABILITY CORE STATIC RENDER PASS"),
|
|
]
|
|
patterns["observability-access"] = [
|
|
*patterns["observability-core"],
|
|
exact("OBSERVABILITY ACCESS STATIC RENDER PASS"),
|
|
]
|
|
|
|
try:
|
|
metadata = os.lstat(path)
|
|
if (
|
|
not stat.S_ISREG(metadata.st_mode)
|
|
or stat.S_ISLNK(metadata.st_mode)
|
|
or metadata.st_uid != os.getuid()
|
|
or stat.S_IMODE(metadata.st_mode) != 0o600
|
|
or metadata.st_nlink != 1
|
|
):
|
|
raise ValueError("unsafe log")
|
|
with open(path, "rb") as stream:
|
|
raw = stream.read(65537)
|
|
if len(raw) > 65536 or not raw or b"\0" in raw or b"\r" in raw or not raw.endswith(b"\n"):
|
|
raise ValueError("unsafe log bytes")
|
|
lines = raw.decode("utf-8").splitlines()
|
|
expected = patterns[label]
|
|
if len(lines) != len(expected):
|
|
raise ValueError("renderer log line count differs")
|
|
if any(pattern.fullmatch(line) is None for pattern, line in zip(expected, lines)):
|
|
raise ValueError("renderer log grammar differs")
|
|
except (KeyError, OSError, UnicodeError, ValueError):
|
|
raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
_sensitive_source_validate_handoff_entries() {
|
|
local label=$1 directory=$2
|
|
shift 2
|
|
python3 - "$label" "$directory" "$@" <<'PY'
|
|
import os
|
|
import stat
|
|
import sys
|
|
|
|
label, directory, *expected = sys.argv[1:]
|
|
try:
|
|
metadata = os.lstat(directory)
|
|
if (
|
|
not stat.S_ISDIR(metadata.st_mode)
|
|
or stat.S_ISLNK(metadata.st_mode)
|
|
or metadata.st_uid != os.getuid()
|
|
or stat.S_IMODE(metadata.st_mode) != 0o700
|
|
):
|
|
raise ValueError("unsafe handoff directory")
|
|
entries = os.listdir(directory)
|
|
if len(entries) != len(expected) or set(entries) != set(expected):
|
|
raise ValueError("handoff entry set differs")
|
|
descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
try:
|
|
for name in expected:
|
|
item = os.stat(name, dir_fd=descriptor, follow_symlinks=False)
|
|
if (
|
|
not stat.S_ISREG(item.st_mode)
|
|
or item.st_uid != os.getuid()
|
|
or stat.S_IMODE(item.st_mode) != 0o600
|
|
or item.st_nlink != 1
|
|
):
|
|
raise ValueError("unsafe handoff entry")
|
|
finally:
|
|
os.close(descriptor)
|
|
except (OSError, ValueError):
|
|
raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
_sensitive_source_validate_core_index() {
|
|
local directory=$1
|
|
python3 - "$directory" <<'PY'
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
try:
|
|
import yaml
|
|
from yaml.constructor import ConstructorError
|
|
from yaml.nodes import MappingNode
|
|
from yaml.resolver import BaseResolver
|
|
except Exception:
|
|
raise SystemExit(2)
|
|
|
|
|
|
class StrictIndexLoader(yaml.SafeLoader):
|
|
pass
|
|
|
|
|
|
def construct_unique_mapping(loader, node, deep=False):
|
|
if not isinstance(node, MappingNode):
|
|
raise ConstructorError(None, None, "mapping node required", node.start_mark)
|
|
result = {}
|
|
for key_node, value_node in node.value:
|
|
key = loader.construct_object(key_node, deep=deep)
|
|
if key in result:
|
|
raise ConstructorError(None, None, "duplicate mapping key", key_node.start_mark)
|
|
result[key] = loader.construct_object(value_node, deep=deep)
|
|
return result
|
|
|
|
|
|
StrictIndexLoader.add_constructor(BaseResolver.DEFAULT_MAPPING_TAG, construct_unique_mapping)
|
|
StrictIndexLoader.add_constructor(
|
|
"tag:yaml.org,2002:value", lambda loader, node: loader.construct_scalar(node)
|
|
)
|
|
|
|
artifacts = (
|
|
"namespaces",
|
|
"crds",
|
|
"storage",
|
|
"kps",
|
|
"loki",
|
|
"tempo",
|
|
"alloy",
|
|
"node-exporter",
|
|
"grafana",
|
|
"blackbox",
|
|
"core-policies",
|
|
"targets",
|
|
"core-rules",
|
|
"alerting",
|
|
"dashboards",
|
|
)
|
|
header = (
|
|
"order",
|
|
"artifact",
|
|
"apiVersion",
|
|
"kind",
|
|
"namespace",
|
|
"name",
|
|
"canonicalSha256",
|
|
)
|
|
|
|
|
|
def expected_rows(directory):
|
|
rows = []
|
|
identities = set()
|
|
for artifact in artifacts:
|
|
path = os.path.join(directory, artifact + ".yaml")
|
|
with open(path, encoding="utf-8") as stream:
|
|
for document in yaml.load_all(stream, Loader=StrictIndexLoader):
|
|
if document is None:
|
|
continue
|
|
if not isinstance(document, dict):
|
|
raise ValueError("non-object manifest")
|
|
metadata = document.get("metadata") or {}
|
|
if not isinstance(metadata, dict):
|
|
raise ValueError("malformed metadata")
|
|
identity = (
|
|
str(document.get("apiVersion", "")),
|
|
str(document.get("kind", "")),
|
|
str(metadata.get("namespace", "")),
|
|
str(metadata.get("name", "")),
|
|
)
|
|
if not identity[0] or not identity[1] or not identity[3] or identity in identities:
|
|
raise ValueError("invalid resource identity")
|
|
identities.add(identity)
|
|
canonical = json.dumps(
|
|
document, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
|
)
|
|
digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
rows.append((str(len(rows) + 1), artifact, *identity, digest))
|
|
return rows
|
|
|
|
|
|
def actual_rows(directory):
|
|
path = os.path.join(directory, "resource-index.tsv")
|
|
with open(path, "rb") as stream:
|
|
raw = stream.read()
|
|
if not raw or b"\0" in raw or not raw.endswith(b"\n"):
|
|
raise ValueError("unsafe index bytes")
|
|
text = raw.decode("utf-8")
|
|
lines = text.splitlines()
|
|
if not lines or tuple(lines[0].split("\t")) != header:
|
|
raise ValueError("index header differs")
|
|
rows = []
|
|
identities = set()
|
|
for position, line in enumerate(lines[1:], 1):
|
|
fields = tuple(line.split("\t"))
|
|
if len(fields) != 7 or fields[0] != str(position):
|
|
raise ValueError("index shape differs")
|
|
if (
|
|
fields[1] not in artifacts
|
|
or not fields[2]
|
|
or not fields[3]
|
|
or not fields[5]
|
|
or re.fullmatch(r"[0-9a-f]{64}", fields[6]) is None
|
|
):
|
|
raise ValueError("index field differs")
|
|
identity = fields[2:6]
|
|
if identity in identities:
|
|
raise ValueError("duplicate index identity")
|
|
identities.add(identity)
|
|
rows.append(fields)
|
|
return rows
|
|
|
|
|
|
try:
|
|
directory = sys.argv[1]
|
|
if actual_rows(directory) != expected_rows(directory):
|
|
raise ValueError("canonical index mismatch")
|
|
except (OSError, UnicodeError, ValueError, yaml.YAMLError):
|
|
raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
scan_platform_sensitive_roots() (
|
|
local platform_root=$1 docs_root=$2
|
|
shift 2
|
|
local argument rendered file test_file finding rc report
|
|
local -a rendered_manifests=()
|
|
local -a files=()
|
|
local failed=false
|
|
local list_file=''
|
|
|
|
cleanup_sensitive_source_scan() {
|
|
case "$list_file" in
|
|
/tmp/platform-sensitive-source-files.??????)
|
|
rm -f -- "$list_file"
|
|
;;
|
|
esac
|
|
}
|
|
trap cleanup_sensitive_source_scan EXIT HUP INT TERM
|
|
|
|
while (( $# > 0 )); do
|
|
argument=$1
|
|
case "$argument" in
|
|
--rendered-manifest)
|
|
(( $# >= 2 )) || return 2
|
|
rendered_manifests+=("$2")
|
|
shift 2
|
|
;;
|
|
*)
|
|
return 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
for argument in rg find mktemp python3 rm stat; do
|
|
command -v "$argument" >/dev/null 2>&1 || {
|
|
printf 'SCANNER_ERROR: required command unavailable: %s\n' "$argument" >&2
|
|
return 1
|
|
}
|
|
done
|
|
[[ "$platform_root" == /* && "$docs_root" == /* && "$platform_root" != "$docs_root" ]] ||
|
|
_sensitive_source_fail 'scan roots must be distinct absolute paths' || return
|
|
for argument in "$platform_root" "$docs_root"; do
|
|
[[ -d "$argument" && ! -L "$argument" && "$(cd -- "$argument" && pwd -P)" == "$argument" ]] ||
|
|
_sensitive_source_fail "scan root is not a physical directory: $argument" || return
|
|
done
|
|
|
|
list_file="$(mktemp /tmp/platform-sensitive-source-files.XXXXXX)" || {
|
|
printf 'SCANNER_ERROR: file inventory allocation failed\n' >&2
|
|
return 1
|
|
}
|
|
chmod 0600 "$list_file" || {
|
|
printf 'SCANNER_ERROR: file inventory hardening failed\n' >&2
|
|
return 1
|
|
}
|
|
if ! find "$platform_root" "$docs_root" \
|
|
\( -type d -path "$platform_root/.superpowers/sdd" -prune \) -o \
|
|
\( -type d \( -name .git -o -name .helm \) -prune \) -o \
|
|
\( -type f -o -type l \) -print0 >"$list_file"; then
|
|
printf 'SCANNER_ERROR: find failed\n' >&2
|
|
return 1
|
|
fi
|
|
while IFS= read -r -d '' file; do
|
|
files+=("$file")
|
|
done <"$list_file"
|
|
|
|
for file in "${files[@]}"; do
|
|
report="$(_sensitive_source_report_path "$file" "$platform_root" "$docs_root")"
|
|
if [[ -L "$file" ]]; then
|
|
printf 'SENSITIVE_SOURCE_FINDING=SYMLINK FILE=%s\n' "$report" >&2
|
|
failed=true
|
|
continue
|
|
fi
|
|
[[ -f "$file" ]] || continue
|
|
test_file=0
|
|
_sensitive_source_is_test_file "$file" && test_file=1
|
|
|
|
private_key_pattern='-----BEGIN (?:[A-Z0-9 ]+ )?PRI''VATE KEY-----|-----BEGIN OPENSSH PRI''VATE KEY-----'
|
|
if _sensitive_source_rg_matches "$private_key_pattern" "$file"; then
|
|
printf 'SENSITIVE_SOURCE_FINDING=PRIVATE_KEY FILE=%s\n' "$report" >&2
|
|
failed=true
|
|
else
|
|
rc=$?
|
|
if (( rc > 1 )); then
|
|
printf 'SCANNER_ERROR: rg failed for FILE=%s\n' "$report" >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
finding="$(_sensitive_source_python_scan \
|
|
"$file" "$test_file" "$PLATFORM_SENSITIVE_SOURCE_EXACT_SCANNER")" || {
|
|
printf 'SCANNER_ERROR: content scan failed for FILE=%s\n' "$report" >&2
|
|
return 1
|
|
}
|
|
if [[ -n "$finding" ]]; then
|
|
printf 'SENSITIVE_SOURCE_FINDING=%s FILE=%s\n' "${finding%%$'\n'*}" "$report" >&2
|
|
failed=true
|
|
fi
|
|
|
|
case "${file,,}" in
|
|
*.yaml|*.yml|*.json)
|
|
finding="$(_sensitive_source_structured_credential_scan "$file" "$test_file")" || {
|
|
printf 'SCANNER_ERROR: structured credential scan failed for FILE=%s\n' "$report" >&2
|
|
return 1
|
|
}
|
|
if [[ -n "$finding" ]]; then
|
|
printf 'SENSITIVE_SOURCE_FINDING=%s FILE=%s\n' "${finding%%$'\n'*}" "$report" >&2
|
|
failed=true
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
case "${file,,}" in
|
|
*.yaml|*.yml)
|
|
finding="$(_sensitive_source_yaml_secret_scan "$file")" || {
|
|
printf 'SCANNER_ERROR: Secret source scan failed for FILE=%s\n' "$report" >&2
|
|
return 1
|
|
}
|
|
if [[ -n "$finding" ]] && ! _sensitive_source_render_results_are_safe "$finding"; then
|
|
printf 'SENSITIVE_SOURCE_FINDING=%s FILE=%s\n' "${finding%%$'\n'*}" "$report" >&2
|
|
failed=true
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
if [[ "$file" == "$platform_root/services/observability/"* ]]; then
|
|
future_target_pattern='(?i)(?:spring[ -]?boot|\bjvm\b|\bkafka\b|consumer[ _-]?lag|(?:target|monitor|dashboard|alert|rule)[^\r\n]{0,40}\b(?:batch|backup)\b|\b(?:batch|backup)\b[^\r\n]{0,40}(?:target|monitor|dashboard|alert|rule))'
|
|
if _sensitive_source_rg_matches "$future_target_pattern" "$file"; then
|
|
printf 'SENSITIVE_SOURCE_FINDING=FUTURE_OBSERVABILITY_TARGET FILE=%s\n' "$report" >&2
|
|
failed=true
|
|
else
|
|
rc=$?
|
|
if (( rc > 1 )); then
|
|
printf 'SCANNER_ERROR: rg failed for FILE=%s\n' "$report" >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
|
|
for file in "${rendered_manifests[@]}"; do
|
|
[[ "$file" == /* && -f "$file" && ! -L "$file" ]] || {
|
|
printf 'SCANNER_ERROR: rendered manifest is not a regular absolute file\n' >&2
|
|
return 1
|
|
}
|
|
report="$(_sensitive_source_report_path "$file" "$platform_root" "$docs_root")"
|
|
finding="$(_sensitive_source_rendered_secret_scan "$file")" || {
|
|
printf 'SCANNER_ERROR: rendered Secret scan failed for FILE=%s\n' "$report" >&2
|
|
return 1
|
|
}
|
|
if [[ -n "$finding" ]] && ! _sensitive_source_render_results_are_safe "$finding"; then
|
|
printf 'SENSITIVE_SOURCE_FINDING=%s FILE=%s\n' "${finding%%$'\n'*}" "$report" >&2
|
|
failed=true
|
|
fi
|
|
done
|
|
|
|
[[ "$failed" == false ]] || return 1
|
|
printf 'PLATFORM SENSITIVE SOURCE SCAN PASS\n'
|
|
)
|
|
|
|
scan_platform_renderer_secrets() (
|
|
set -o pipefail
|
|
local platform_root=$1 renderer label output_prefix publishes log file results line rc entry artifact
|
|
local helm_bin list_file=''
|
|
local failed=false
|
|
local -a renderers=(
|
|
'phase1|render-phase1.sh|platform-phase1-apply|yes'
|
|
'phase2|render-phase2.sh|platform-phase2-apply|yes'
|
|
'admin|render-admin-services.sh|platform-admin-apply|yes'
|
|
'observability-core|render-observability-core.sh|platform-observability-core-apply|yes'
|
|
'observability-access|render-observability-access.sh|-|no'
|
|
)
|
|
local -a outputs=()
|
|
local -a logs=()
|
|
local -a expected_entries=()
|
|
local -a manifest_files=()
|
|
local -a manifest_labels=()
|
|
local -a manifest_artifacts=()
|
|
|
|
cleanup_renderer_scan() {
|
|
local path
|
|
for path in "${outputs[@]}"; do
|
|
case "$path" in
|
|
/tmp/platform-phase1-apply.??????|\
|
|
/tmp/platform-phase2-apply.??????|\
|
|
/tmp/platform-admin-apply.??????|\
|
|
/tmp/platform-observability-core-apply.??????|\
|
|
/tmp/platform-observability-metrics.??????)
|
|
rm -rf -- "$path"
|
|
;;
|
|
esac
|
|
done
|
|
for path in "${logs[@]}"; do
|
|
case "$path" in
|
|
/tmp/platform-sensitive-source-render-log.??????)
|
|
rm -f -- "$path"
|
|
;;
|
|
esac
|
|
done
|
|
case "$list_file" in
|
|
/tmp/platform-sensitive-source-render-files.??????)
|
|
rm -f -- "$list_file"
|
|
;;
|
|
esac
|
|
}
|
|
trap cleanup_renderer_scan EXIT HUP INT TERM
|
|
|
|
(( $# == 1 )) || return 2
|
|
for label in bash chmod find mktemp python3 rm timeout; do
|
|
command -v "$label" >/dev/null 2>&1 || {
|
|
printf 'SCANNER_ERROR: renderer command unavailable: %s\n' "$label" >&2
|
|
return 1
|
|
}
|
|
done
|
|
[[ "$platform_root" == /* && -d "$platform_root" && ! -L "$platform_root" &&
|
|
"$(cd -- "$platform_root" && pwd -P)" == "$platform_root" ]] || {
|
|
printf 'SCANNER_ERROR: renderer root is not a physical absolute directory\n' >&2
|
|
return 1
|
|
}
|
|
helm_bin="$(command -v helm 2>/dev/null)" || {
|
|
printf 'SCANNER_ERROR: Helm is unavailable for renderer Secret validation\n' >&2
|
|
return 1
|
|
}
|
|
[[ "$helm_bin" == /* && -f "$helm_bin" && -x "$helm_bin" && ! -L "$helm_bin" ]] || {
|
|
printf 'SCANNER_ERROR: Helm path is unsafe\n' >&2
|
|
return 1
|
|
}
|
|
|
|
for renderer in "${renderers[@]}"; do
|
|
IFS='|' read -r label renderer output_prefix publishes <<<"$renderer"
|
|
renderer="$platform_root/scripts/validate/$renderer"
|
|
[[ -f "$renderer" && ! -L "$renderer" ]] || {
|
|
printf 'SCANNER_ERROR: required renderer is unsafe: %s\n' "$label" >&2
|
|
return 1
|
|
}
|
|
output=''
|
|
if [[ "$publishes" == yes ]]; then
|
|
output="$(mktemp -d "/tmp/$output_prefix.XXXXXX")" || {
|
|
printf 'SCANNER_ERROR: renderer output allocation failed: %s\n' "$label" >&2
|
|
return 1
|
|
}
|
|
outputs+=("$output")
|
|
chmod 0700 "$output" || {
|
|
printf 'SCANNER_ERROR: renderer output hardening failed: %s\n' "$label" >&2
|
|
return 1
|
|
}
|
|
elif [[ "$publishes" != no || "$label" != observability-access ]]; then
|
|
printf 'SCANNER_ERROR: invalid renderer publication contract: %s\n' "$label" >&2
|
|
return 1
|
|
fi
|
|
log="$(mktemp /tmp/platform-sensitive-source-render-log.XXXXXX)" || {
|
|
printf 'SCANNER_ERROR: renderer log allocation failed: %s\n' "$label" >&2
|
|
return 1
|
|
}
|
|
logs+=("$log")
|
|
chmod 0600 "$log" || return 1
|
|
|
|
if [[ "$label" == observability-access ]]; then
|
|
PLATFORM_HELM_BIN="$helm_bin" timeout --signal=TERM --kill-after=5s 300s \
|
|
bash "$renderer" --component grafana >"$log" 2>&1 || rc=$?
|
|
else
|
|
PLATFORM_HELM_BIN="$helm_bin" timeout --signal=TERM --kill-after=5s 300s \
|
|
bash "$renderer" --verified-output-dir "$output" >"$log" 2>&1 || rc=$?
|
|
fi
|
|
if (( ${rc:-0} != 0 )); then
|
|
printf 'SCANNER_ERROR: renderer failed: %s RC=%s\n' "$label" "$rc" >&2
|
|
return 1
|
|
fi
|
|
unset rc
|
|
if [[ "$label" == observability-access ]]; then
|
|
_sensitive_source_access_log_is_safe "$log" || {
|
|
printf 'SCANNER_ERROR: unexpected observability-access output contract\n' >&2
|
|
return 1
|
|
}
|
|
else
|
|
_sensitive_source_renderer_log_is_safe "$label" "$log" || {
|
|
printf 'SCANNER_ERROR: unexpected renderer output contract: %s\n' "$label" >&2
|
|
return 1
|
|
}
|
|
fi
|
|
if [[ "$publishes" == yes ]]; then
|
|
expected_entries=()
|
|
case "$label" in
|
|
phase1)
|
|
expected_entries=(
|
|
namespaces.yaml ssd-local-pv.yaml cnpg-operator.yaml
|
|
platform-postgres.yaml gitea.yaml gitea-oidc.yaml
|
|
)
|
|
;;
|
|
phase2)
|
|
expected_entries=(
|
|
phase2-namespaces.yaml aistor-local-pv.yaml keycloak-operator.yaml
|
|
platform-postgres-keycloak.yaml keycloak.yaml aistor-operator.yaml
|
|
minio-aistor.yaml aistor-network-policies.yaml
|
|
)
|
|
;;
|
|
admin)
|
|
expected_entries=(
|
|
admin-namespace.yaml pgadmin-local-pv.yaml coredns-custom.yaml
|
|
aistor-admin-oidc.yaml pgadmin.yaml
|
|
)
|
|
;;
|
|
observability-core)
|
|
expected_entries=(
|
|
namespaces.yaml crds.yaml storage.yaml kps.yaml loki.yaml tempo.yaml
|
|
alloy.yaml node-exporter.yaml grafana.yaml blackbox.yaml
|
|
core-policies.yaml targets.yaml core-rules.yaml alerting.yaml
|
|
dashboards.yaml aggregate.yaml resource-index.tsv
|
|
)
|
|
;;
|
|
*)
|
|
printf 'SCANNER_ERROR: unknown renderer handoff contract: %s\n' "$label" >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
_sensitive_source_validate_handoff_entries "$label" "$output" "${expected_entries[@]}" || {
|
|
printf 'SCANNER_ERROR: renderer handoff entry contract failed: %s\n' "$label" >&2
|
|
return 1
|
|
}
|
|
if [[ "$label" == observability-core ]]; then
|
|
_sensitive_source_validate_core_index "$output" || {
|
|
printf 'SCANNER_ERROR: observability-core resource index contract failed\n' >&2
|
|
return 1
|
|
}
|
|
fi
|
|
for entry in "${expected_entries[@]}"; do
|
|
[[ "$entry" == *.yaml ]] || continue
|
|
manifest_files+=("$output/$entry")
|
|
manifest_labels+=("$label")
|
|
manifest_artifacts+=("${entry%.yaml}")
|
|
done
|
|
fi
|
|
printf 'SENSITIVE_SOURCE_RENDERER=%s PASS\n' "$label"
|
|
done
|
|
|
|
for ((rc=0; rc < ${#manifest_files[@]}; rc++)); do
|
|
file=${manifest_files[$rc]}
|
|
label=${manifest_labels[$rc]}
|
|
artifact=${manifest_artifacts[$rc]}
|
|
if [[ -L "$file" || ! -f "$file" ]]; then
|
|
printf 'SENSITIVE_SOURCE_FINDING=RENDERED_SYMLINK FILE=%q\n' "$(basename -- "$file")" >&2
|
|
failed=true
|
|
continue
|
|
fi
|
|
results="$(_sensitive_source_rendered_secret_scan "$file" "$label" "$artifact")" || {
|
|
printf 'SCANNER_ERROR: rendered Secret scan failed: FILE=%q\n' "$(basename -- "$file")" >&2
|
|
return 1
|
|
}
|
|
while IFS= read -r line; do
|
|
[[ -n "$line" ]] || continue
|
|
if [[ "$line" == RENDERED_SECRET_ALLOWLISTED ]]; then
|
|
continue
|
|
else
|
|
printf 'SENSITIVE_SOURCE_FINDING=%s FILE=%q\n' "$line" "$(basename -- "$file")" >&2
|
|
failed=true
|
|
fi
|
|
done <<<"$results"
|
|
done
|
|
|
|
[[ "$failed" == false ]] || return 1
|
|
printf 'PLATFORM RENDERED SECRET SCAN PASS\n'
|
|
)
|
|
|
|
scan_platform_sensitive_source_main() {
|
|
(( $# == 0 )) || {
|
|
_sensitive_source_usage >&2
|
|
return 2
|
|
}
|
|
scan_platform_sensitive_roots \
|
|
"$PLATFORM_SENSITIVE_SOURCE_EXACT_PLATFORM_ROOT" \
|
|
"$PLATFORM_SENSITIVE_SOURCE_EXACT_DOCS_ROOT" >/dev/null || return
|
|
scan_platform_renderer_secrets "$PLATFORM_SENSITIVE_SOURCE_EXACT_PLATFORM_ROOT" || return
|
|
printf 'PLATFORM SENSITIVE SOURCE SCAN PASS\n'
|
|
}
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
set -Eeuo pipefail
|
|
set +x
|
|
umask 077
|
|
scan_platform_sensitive_source_main "$@"
|
|
fi
|