Files
platform-core/scripts/bootstrap/prepare-observability-local-paths.sh
T

565 lines
20 KiB
Bash

#!/usr/bin/env bash
# Intentionally sourceable. Privileged filesystem work crosses exactly one
# helper seam and runs descriptor-relative inside one process.
_olp_error() {
printf 'ERROR: %s\n' "$*" >&2
return 1
}
_olp_target_rows() {
local base="$1"
printf '%s\t%s\t%s\n' \
"$base/observability/prometheus" 1000 2000 \
"$base/observability/grafana" 472 472 \
"$base/observability/alertmanager" 1000 2000 \
"$base/observability/alloy" 473 473 \
"$base/observability/loki" 10001 10001 \
"$base/observability/tempo" 10001 10001
}
_olp_helper_program() {
/usr/bin/cat <<'PY'
import errno
import os
import signal
import stat
import sys
TARGETS = (
("prometheus", 1000, 2000),
("grafana", 472, 472),
("alertmanager", 1000, 2000),
("alloy", 473, 473),
("loki", 10001, 10001),
("tempo", 10001, 10001),
)
MIN_AVAILABLE = 50 * 1024 * 1024 * 1024
MAX_USE = 1024 * 1024 * 1024
MAX_INTEGER = (1 << 63) - 1
OPEN_DIR = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW
SIGNALS = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM)
class Rejected(Exception):
pass
class Interrupted(Exception):
def __init__(self, signum):
self.signum = signum
def reject(message):
raise Rejected(message)
if len(sys.argv) != 5 or sys.argv[1] not in {"preflight", "apply"}:
print("REJECT: helper argv differs", file=sys.stderr)
raise SystemExit(97)
action, root_path, ssd_path, aistor_path = sys.argv[1:]
def normalized_absolute(path):
return os.path.isabs(path) and os.path.normpath(path) == path and "//" not in path
for candidate in (root_path, ssd_path, aistor_path):
if not normalized_absolute(candidate):
reject("path is not normalized absolute")
fixture_root = os.environ.get("OLP_HELPER_FIXTURE_ROOT", "")
fixture = bool(fixture_root) and os.geteuid() != 0
if fixture:
if not normalized_absolute(fixture_root):
reject("fixture root is invalid")
for candidate in (root_path, ssd_path, aistor_path):
if os.path.commonpath((fixture_root, candidate)) != fixture_root:
reject("fixture path escaped fixture root")
elif (root_path, ssd_path, aistor_path) != ("/", "/srv/k3s/ssd", "/srv/k3s/aistor"):
reject("production helper paths differ from exact contract")
def fixture_value(name, default=""):
return os.environ.get(name, default) if fixture else default
audit_path = fixture_value("OLP_HELPER_FIXTURE_AUDIT")
if audit_path and os.path.commonpath((fixture_root, audit_path)) != fixture_root:
reject("fixture audit escaped fixture root")
def audit(message):
if audit_path:
with open(audit_path, "a", encoding="utf-8") as stream:
stream.write(message + "\n")
def open_physical_absolute(path):
descriptor = os.open("/", OPEN_DIR)
try:
for component in path.split("/")[1:]:
if not component:
continue
child = os.open(component, OPEN_DIR, dir_fd=descriptor)
os.close(descriptor)
descriptor = child
return descriptor
except BaseException:
os.close(descriptor)
raise
def open_child(parent, name, missing_ok=False):
try:
return os.open(name, OPEN_DIR, dir_fd=parent)
except FileNotFoundError:
if missing_ok:
return None
raise
def identity(descriptor):
metadata = os.fstat(descriptor)
return metadata.st_dev, metadata.st_ino
def entry_identity(parent, name):
metadata = os.stat(name, dir_fd=parent, follow_symlinks=False)
if not stat.S_ISDIR(metadata.st_mode):
reject("entry is not a physical directory")
return metadata.st_dev, metadata.st_ino
def directory_use(descriptor):
total = os.fstat(descriptor).st_blocks * 512
for name in os.listdir(descriptor):
metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False)
if stat.S_ISLNK(metadata.st_mode):
reject("symbolic link below target is forbidden")
if stat.S_ISDIR(metadata.st_mode):
child = os.open(name, OPEN_DIR, dir_fd=descriptor)
try:
total += directory_use(child)
finally:
os.close(child)
else:
total += metadata.st_blocks * 512
if total > MAX_USE:
return total
return total
def filesystem_snapshot(root_fd, ssd_fd, aistor_fd, obs_fd):
root_dev = os.fstat(root_fd).st_dev
if os.fstat(ssd_fd).st_dev != root_dev:
reject("SSD base is not on root filesystem")
aistor_dev = os.fstat(aistor_fd).st_dev
if fixture_value("OLP_HELPER_FIXTURE_VIRTUAL_AISTOR") == "1":
aistor_dev = root_dev + 1
if aistor_dev == root_dev:
reject("AIStor must use another filesystem")
fs = os.fstatvfs(root_fd)
available = fs.f_bavail * fs.f_frsize
if available < 0 or available > MAX_INTEGER or available < MIN_AVAILABLE:
reject("root available bytes fail boundary")
uses = []
if obs_fd is None:
uses = [0] * len(TARGETS)
else:
for name, _, _ in TARGETS:
target_fd = open_child(obs_fd, name, True)
if target_fd is None:
uses.append(0)
continue
try:
value = directory_use(target_fd)
if value < 0 or value > MAX_USE:
reject("individual target use exceeds 1GiB")
uses.append(value)
if os.listdir(target_fd):
reject("existing target is non-empty")
finally:
os.close(target_fd)
total = 0
for value in uses:
if value < 0 or value > MAX_USE:
reject("individual target use exceeds 1GiB")
total += value
if total > MAX_USE:
reject("aggregate target use exceeds 1GiB")
return available, uses
root_fd = ssd_fd = aistor_fd = obs_fd = None
created = []
restored = []
active_created = None
def close_all():
seen = set()
values = [record[2] for record in created]
values.extend(record[1] for record in restored)
values.extend((obs_fd, aistor_fd, ssd_fd, root_fd))
for descriptor in values:
if descriptor is None or descriptor in seen:
continue
seen.add(descriptor)
try:
os.close(descriptor)
except OSError:
pass
def rollback():
global active_created
failed = False
if active_created is not None:
parent_fd, name, expected = active_created
try:
if entry_identity(parent_fd, name) == expected:
descriptor = open_child(parent_fd, name)
try:
if os.listdir(descriptor):
failed = True
else:
os.rmdir(name, dir_fd=parent_fd)
audit(f"remove-active-created {name}")
finally:
os.close(descriptor)
else:
failed = True
except OSError:
failed = True
active_created = None
for parent_fd, name, target_fd, expected in reversed(created):
try:
if entry_identity(parent_fd, name) != expected:
failed = True
continue
if os.listdir(target_fd):
failed = True
continue
os.rmdir(name, dir_fd=parent_fd)
audit(f"remove-created {name}")
except OSError:
failed = True
for name, target_fd, uid, gid, mode, _expected in reversed(restored):
try:
os.fchown(target_fd, uid, gid)
audit(f"restore-chown {name} {uid} {gid}")
os.fchmod(target_fd, mode)
audit(f"restore-chmod {name} {mode:04o}")
except OSError:
failed = True
return not failed
def signal_handler(signum, _frame):
raise Interrupted(signum)
def block_signals():
return signal.pthread_sigmask(signal.SIG_BLOCK, SIGNALS)
def restore_signal_mask(previous):
signal.pthread_sigmask(signal.SIG_SETMASK, previous)
def requested_owner(uid, gid):
if fixture_value("OLP_HELPER_FIXTURE_OWNERS") == "1":
return os.getuid(), os.getgid()
return uid, gid
def maybe_fixture_race(kind, name=""):
race = fixture_value("OLP_HELPER_FIXTURE_RACE")
external = fixture_value("OLP_HELPER_FIXTURE_EXTERNAL")
if not race:
return
if not external or os.path.commonpath((fixture_root, external)) != fixture_root:
reject("fixture race external path is invalid")
if kind == "ancestor" and race == "ancestor":
os.rename("observability", "observability.pinned", src_dir_fd=ssd_fd, dst_dir_fd=ssd_fd)
os.symlink(external, "observability", dir_fd=ssd_fd)
audit("fixture-race ancestor")
elif kind == "target" and race == f"target:{name}":
os.rename(name, f"{name}.pinned", src_dir_fd=obs_fd, dst_dir_fd=obs_fd)
os.symlink(external, name, dir_fd=obs_fd)
audit(f"fixture-race target {name}")
def maybe_fixture_signal(name):
configured = fixture_value("OLP_HELPER_FIXTURE_SIGNAL")
target = fixture_value("OLP_HELPER_FIXTURE_SIGNAL_TARGET")
if configured and target == name:
signum = {"HUP": signal.SIGHUP, "INT": signal.SIGINT, "TERM": signal.SIGTERM}.get(configured)
if signum is None:
reject("fixture signal is invalid")
os.kill(os.getpid(), signum)
exit_status = 0
try:
root_fd = open_physical_absolute(root_path)
ssd_fd = open_physical_absolute(ssd_path)
aistor_fd = open_physical_absolute(aistor_path)
obs_fd = open_child(ssd_fd, "observability", True)
available, uses = filesystem_snapshot(root_fd, ssd_fd, aistor_fd, obs_fd)
if action == "preflight":
print("\t".join(["PREFLIGHT", str(available), *(str(value) for value in uses)]))
else:
for signum in SIGNALS:
signal.signal(signum, signal_handler)
if obs_fd is None:
previous = block_signals()
try:
os.mkdir("observability", 0o770, dir_fd=ssd_fd)
active_created = (ssd_fd, "observability", entry_identity(ssd_fd, "observability"))
obs_fd = open_child(ssd_fd, "observability")
if identity(obs_fd) != active_created[2]:
reject("new observability ancestor identity changed before open")
created.append((ssd_fd, "observability", obs_fd, identity(obs_fd)))
active_created = None
finally:
restore_signal_mask(previous)
obs_expected = identity(obs_fd)
maybe_fixture_race("ancestor")
for name, requested_uid, requested_gid in TARGETS:
target_fd = open_child(obs_fd, name, True)
if target_fd is None:
previous = block_signals()
try:
os.mkdir(name, 0o770, dir_fd=obs_fd)
active_created = (obs_fd, name, entry_identity(obs_fd, name))
target_fd = open_child(obs_fd, name)
if identity(target_fd) != active_created[2]:
reject("new target identity changed before open")
created.append((obs_fd, name, target_fd, identity(target_fd)))
active_created = None
actual_uid, actual_gid = requested_owner(requested_uid, requested_gid)
os.fchown(target_fd, actual_uid, actual_gid)
os.fchmod(target_fd, 0o770)
audit(f"set-owner-mode {name} {requested_uid} {requested_gid} 0770")
finally:
restore_signal_mask(previous)
maybe_fixture_signal(name)
else:
metadata = os.fstat(target_fd)
if os.listdir(target_fd):
reject("existing target is non-empty")
restored.append((
name,
target_fd,
metadata.st_uid,
metadata.st_gid,
stat.S_IMODE(metadata.st_mode),
identity(target_fd),
))
maybe_fixture_race("target", name)
actual_uid, actual_gid = requested_owner(requested_uid, requested_gid)
os.fchown(target_fd, actual_uid, actual_gid)
os.fchmod(target_fd, 0o770)
audit(f"set-owner-mode {name} {requested_uid} {requested_gid} 0770")
metadata = os.fstat(target_fd)
actual_uid, actual_gid = requested_owner(requested_uid, requested_gid)
if (metadata.st_uid, metadata.st_gid, stat.S_IMODE(metadata.st_mode)) != (actual_uid, actual_gid, 0o770):
reject("post-create owner or mode differs")
for parent_fd, name, target_fd, expected in created:
if entry_identity(parent_fd, name) != expected:
reject("created entry identity changed")
for name, _target_fd, _uid, _gid, _mode, expected in restored:
if entry_identity(obs_fd, name) != expected:
reject("pre-existing target entry identity changed")
if entry_identity(ssd_fd, "observability") != obs_expected:
reject("observability ancestor identity changed")
print("APPLIED")
except Interrupted as error:
for signum in SIGNALS:
signal.signal(signum, signal.SIG_IGN)
rollback()
print(f"REJECT: interrupted by signal {error.signum}", file=sys.stderr)
exit_status = 128 + error.signum
except (Rejected, OSError) as error:
for signum in SIGNALS:
signal.signal(signum, signal.SIG_IGN)
rollback()
message = str(error) if str(error) else "privileged helper failed"
print(f"REJECT: {message}", file=sys.stderr)
exit_status = 23
finally:
close_all()
raise SystemExit(exit_status)
PY
}
_olp_privileged_helper() {
if (( $# != 4 )) || [[ "$1" != preflight && "$1" != apply ]]; then
return 97
fi
[[ "$2" == / && "$3" == /srv/k3s/ssd && "$4" == /srv/k3s/aistor ]] || \
return 97
local program
program="$(_olp_helper_program)" || return
/usr/bin/printf '%s\n' "$program" | \
/usr/bin/sudo -- /usr/bin/python3 - "$@"
}
_olp_root_facts() {
/usr/bin/findmnt --kernel --first-only --noheadings --output SOURCE,FSTYPE \
--target "$1" | /usr/bin/awk 'NF == 2 { print $1 "\t" $2 }'
}
_olp_available_bytes() {
/usr/bin/df --block-size=1 --output=avail "$1" | /usr/bin/tail -n 1 | \
/usr/bin/tr -d '[:space:]'
}
_olp_current_context() {
local kubectl_bin
kubectl_bin="$(command -v kubectl)" || return 1
"$kubectl_bin" config current-context
}
_olp_node_names() {
local kubectl_bin
kubectl_bin="$(command -v kubectl)" || return 1
"$kubectl_bin" get nodes \
-o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}'
}
_olp_read_confirmation() {
local value
printf 'Type APPLY default to prepare the six Local PV paths: ' >&2
IFS= read -r value </dev/tty || return 1
printf '%s\n' "$value"
}
_olp_is_normalized_absolute() {
local path="$1"
[[ "$path" == /* && "$path" != *'//' && "$path" != */. && \
"$path" != */.. && "$path" != *'/./'* && "$path" != *'/../'* ]]
}
_olp_is_canonical_decimal() {
[[ "$1" =~ ^(0|[1-9][0-9]*)$ ]]
}
_olp_decimal_le() {
local left="$1" right="$2" LC_ALL=C
(( ${#left} < ${#right} )) || \
{ (( ${#left} == ${#right} )) && [[ "$left" == "$right" || "$left" < "$right" ]]; }
}
_olp_validate_preflight_reply() {
local reply="$1" tag available value total=0
local -a fields=()
[[ "$reply" != *$'\n'* && "$reply" != *$'\r'* ]] || return 1
IFS=$'\t' read -r -a fields <<<"$reply"
(( ${#fields[@]} == 8 )) || return 1
tag="${fields[0]}"
available="${fields[1]}"
[[ "$tag" == PREFLIGHT ]] || return 1
_olp_is_canonical_decimal "$available" || return 1
_olp_decimal_le "$available" 9223372036854775807 || return 1
_olp_decimal_le 53687091200 "$available" || return 1
for value in "${fields[@]:2}"; do
_olp_is_canonical_decimal "$value" || return 1
_olp_decimal_le "$value" 9223372036854775807 || return 1
_olp_decimal_le "$value" 1073741824 || return 1
total=$((total + 10#$value))
(( total <= 1073741824 )) || return 1
done
}
observability_local_paths_dry_run() {
local root_path="$1" ssd_base="$2" aistor_base="$3"
local facts device filesystem available target uid gid
_olp_is_normalized_absolute "$root_path" || \
_olp_error "root path is not normalized" || return
_olp_is_normalized_absolute "$ssd_base" || \
_olp_error "SSD base path is not normalized" || return
_olp_is_normalized_absolute "$aistor_base" || \
_olp_error "AIStor path is not normalized" || return
facts="$(_olp_root_facts "$root_path")" || \
_olp_error "could not read root SSD device/filesystem facts" || return
IFS=$'\t' read -r device filesystem <<<"$facts"
[[ "$device" =~ ^[A-Za-z0-9._/+:=-]+$ && \
"$filesystem" =~ ^[A-Za-z0-9._+-]+$ ]] || \
_olp_error "root SSD facts are not safely printable" || return
available="$(_olp_available_bytes "$root_path")" || \
_olp_error "could not read root SSD available bytes" || return
_olp_is_canonical_decimal "$available" || \
_olp_error "root SSD available bytes are invalid" || return
_olp_decimal_le "$available" 9223372036854775807 || \
_olp_error "root SSD available bytes are invalid" || return
printf 'Root SSD device: %s\n' "$device"
printf 'Root SSD filesystem: %s\n' "$filesystem"
printf 'Root SSD available bytes: %s\n' "$available"
while IFS=$'\t' read -r target uid gid; do
printf 'Planned path: %s\n' "$target"
done < <(_olp_target_rows "$ssd_base")
}
_olp_execute_preflight() {
local root_path="$1" ssd_base="$2" aistor_base="$3"
local reply context nodes
reply="$(_olp_privileged_helper preflight "$root_path" "$ssd_base" "$aistor_base" \
2>/dev/null)" || _olp_error "privileged filesystem preflight failed" || return
_olp_validate_preflight_reply "$reply" || \
_olp_error "privileged preflight protocol or capacity boundary failed" || return
context="$(_olp_current_context)" || \
_olp_error "could not read Kubernetes context" || return
[[ "$context" == default ]] || \
_olp_error "Kubernetes context must be exactly default" || return
nodes="$(_olp_node_names)" || \
_olp_error "could not read Kubernetes node names" || return
[[ "$nodes" == donghyeon-system-product-name ]] || \
_olp_error "Kubernetes node must be exactly donghyeon-system-product-name" || return
}
observability_local_paths_execute() {
local root_path="$1" ssd_base="$2" aistor_base="$3" confirmation reply
_olp_execute_preflight "$root_path" "$ssd_base" "$aistor_base" || return
confirmation="$(_olp_read_confirmation)" || \
_olp_error "TTY confirmation was not read" || return
[[ "$confirmation" == 'APPLY default' ]] || \
_olp_error "confirmation did not match APPLY default" || return
_olp_execute_preflight "$root_path" "$ssd_base" "$aistor_base" || return
reply="$(_olp_privileged_helper apply "$root_path" "$ssd_base" "$aistor_base")" || \
_olp_error "descriptor-relative path preparation failed" || return
[[ "$reply" == APPLIED ]] || \
_olp_error "privileged apply protocol differed" || return
printf 'Prepared six observability Local PV paths on the root filesystem.\n'
printf 'No Kubernetes resources were applied.\n'
}
_olp_usage() {
printf 'Usage: bash scripts/bootstrap/prepare-observability-local-paths.sh [--execute]\n' >&2
}
_olp_main() {
if (( $# == 0 )); then
observability_local_paths_dry_run / /srv/k3s/ssd /srv/k3s/aistor
elif (( $# == 1 )) && [[ "$1" == --execute ]]; then
observability_local_paths_execute / /srv/k3s/ssd /srv/k3s/aistor
else
_olp_usage
return 2
fi
}
_olp_entry() (
set -Eeuo pipefail
_olp_main "$@"
)
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
_olp_entry "$@"
fi