#!/usr/bin/env bash # Intentionally sourceable. Privileged filesystem work crosses exactly one # helper seam and runs descriptor-relative inside one process. _olp_error() { printf 'ERROR: %s\n' "$*" >&2 return 1 } _olp_target_rows() { local base="$1" printf '%s\t%s\t%s\n' \ "$base/observability/prometheus" 1000 2000 \ "$base/observability/grafana" 472 472 \ "$base/observability/alertmanager" 1000 2000 \ "$base/observability/alloy" 473 473 \ "$base/observability/loki" 10001 10001 \ "$base/observability/tempo" 10001 10001 } _olp_helper_program() { /usr/bin/cat <<'PY' import errno import os import signal import stat import sys TARGETS = ( ("prometheus", 1000, 2000), ("grafana", 472, 472), ("alertmanager", 1000, 2000), ("alloy", 473, 473), ("loki", 10001, 10001), ("tempo", 10001, 10001), ) MIN_AVAILABLE = 50 * 1024 * 1024 * 1024 MAX_USE = 1024 * 1024 * 1024 MAX_INTEGER = (1 << 63) - 1 OPEN_DIR = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW SIGNALS = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM) class Rejected(Exception): pass class Interrupted(Exception): def __init__(self, signum): self.signum = signum def reject(message): raise Rejected(message) if len(sys.argv) != 5 or sys.argv[1] not in {"preflight", "apply"}: print("REJECT: helper argv differs", file=sys.stderr) raise SystemExit(97) action, root_path, ssd_path, aistor_path = sys.argv[1:] def normalized_absolute(path): return os.path.isabs(path) and os.path.normpath(path) == path and "//" not in path for candidate in (root_path, ssd_path, aistor_path): if not normalized_absolute(candidate): reject("path is not normalized absolute") fixture_root = os.environ.get("OLP_HELPER_FIXTURE_ROOT", "") fixture = bool(fixture_root) and os.geteuid() != 0 if fixture: if not normalized_absolute(fixture_root): reject("fixture root is invalid") for candidate in (root_path, ssd_path, aistor_path): if os.path.commonpath((fixture_root, candidate)) != fixture_root: reject("fixture path escaped fixture root") elif (root_path, ssd_path, aistor_path) != ("/", "/srv/k3s/ssd", "/srv/k3s/aistor"): reject("production helper paths differ from exact contract") def fixture_value(name, default=""): return os.environ.get(name, default) if fixture else default audit_path = fixture_value("OLP_HELPER_FIXTURE_AUDIT") if audit_path and os.path.commonpath((fixture_root, audit_path)) != fixture_root: reject("fixture audit escaped fixture root") def audit(message): if audit_path: with open(audit_path, "a", encoding="utf-8") as stream: stream.write(message + "\n") def open_physical_absolute(path): descriptor = os.open("/", OPEN_DIR) try: for component in path.split("/")[1:]: if not component: continue child = os.open(component, OPEN_DIR, dir_fd=descriptor) os.close(descriptor) descriptor = child return descriptor except BaseException: os.close(descriptor) raise def open_child(parent, name, missing_ok=False): try: return os.open(name, OPEN_DIR, dir_fd=parent) except FileNotFoundError: if missing_ok: return None raise def identity(descriptor): metadata = os.fstat(descriptor) return metadata.st_dev, metadata.st_ino def entry_identity(parent, name): metadata = os.stat(name, dir_fd=parent, follow_symlinks=False) if not stat.S_ISDIR(metadata.st_mode): reject("entry is not a physical directory") return metadata.st_dev, metadata.st_ino def directory_use(descriptor): total = os.fstat(descriptor).st_blocks * 512 for name in os.listdir(descriptor): metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) if stat.S_ISLNK(metadata.st_mode): reject("symbolic link below target is forbidden") if stat.S_ISDIR(metadata.st_mode): child = os.open(name, OPEN_DIR, dir_fd=descriptor) try: total += directory_use(child) finally: os.close(child) else: total += metadata.st_blocks * 512 if total > MAX_USE: return total return total def filesystem_snapshot(root_fd, ssd_fd, aistor_fd, obs_fd): root_dev = os.fstat(root_fd).st_dev if os.fstat(ssd_fd).st_dev != root_dev: reject("SSD base is not on root filesystem") aistor_dev = os.fstat(aistor_fd).st_dev if fixture_value("OLP_HELPER_FIXTURE_VIRTUAL_AISTOR") == "1": aistor_dev = root_dev + 1 if aistor_dev == root_dev: reject("AIStor must use another filesystem") fs = os.fstatvfs(root_fd) available = fs.f_bavail * fs.f_frsize if available < 0 or available > MAX_INTEGER or available < MIN_AVAILABLE: reject("root available bytes fail boundary") uses = [] if obs_fd is None: uses = [0] * len(TARGETS) else: for name, _, _ in TARGETS: target_fd = open_child(obs_fd, name, True) if target_fd is None: uses.append(0) continue try: value = directory_use(target_fd) if value < 0 or value > MAX_USE: reject("individual target use exceeds 1GiB") uses.append(value) if os.listdir(target_fd): reject("existing target is non-empty") finally: os.close(target_fd) total = 0 for value in uses: if value < 0 or value > MAX_USE: reject("individual target use exceeds 1GiB") total += value if total > MAX_USE: reject("aggregate target use exceeds 1GiB") return available, uses root_fd = ssd_fd = aistor_fd = obs_fd = None created = [] restored = [] active_created = None def close_all(): seen = set() values = [record[2] for record in created] values.extend(record[1] for record in restored) values.extend((obs_fd, aistor_fd, ssd_fd, root_fd)) for descriptor in values: if descriptor is None or descriptor in seen: continue seen.add(descriptor) try: os.close(descriptor) except OSError: pass def rollback(): global active_created failed = False if active_created is not None: parent_fd, name, expected = active_created try: if entry_identity(parent_fd, name) == expected: descriptor = open_child(parent_fd, name) try: if os.listdir(descriptor): failed = True else: os.rmdir(name, dir_fd=parent_fd) audit(f"remove-active-created {name}") finally: os.close(descriptor) else: failed = True except OSError: failed = True active_created = None for parent_fd, name, target_fd, expected in reversed(created): try: if entry_identity(parent_fd, name) != expected: failed = True continue if os.listdir(target_fd): failed = True continue os.rmdir(name, dir_fd=parent_fd) audit(f"remove-created {name}") except OSError: failed = True for name, target_fd, uid, gid, mode, _expected in reversed(restored): try: os.fchown(target_fd, uid, gid) audit(f"restore-chown {name} {uid} {gid}") os.fchmod(target_fd, mode) audit(f"restore-chmod {name} {mode:04o}") except OSError: failed = True return not failed def signal_handler(signum, _frame): raise Interrupted(signum) def block_signals(): return signal.pthread_sigmask(signal.SIG_BLOCK, SIGNALS) def restore_signal_mask(previous): signal.pthread_sigmask(signal.SIG_SETMASK, previous) def requested_owner(uid, gid): if fixture_value("OLP_HELPER_FIXTURE_OWNERS") == "1": return os.getuid(), os.getgid() return uid, gid def maybe_fixture_race(kind, name=""): race = fixture_value("OLP_HELPER_FIXTURE_RACE") external = fixture_value("OLP_HELPER_FIXTURE_EXTERNAL") if not race: return if not external or os.path.commonpath((fixture_root, external)) != fixture_root: reject("fixture race external path is invalid") if kind == "ancestor" and race == "ancestor": os.rename("observability", "observability.pinned", src_dir_fd=ssd_fd, dst_dir_fd=ssd_fd) os.symlink(external, "observability", dir_fd=ssd_fd) audit("fixture-race ancestor") elif kind == "target" and race == f"target:{name}": os.rename(name, f"{name}.pinned", src_dir_fd=obs_fd, dst_dir_fd=obs_fd) os.symlink(external, name, dir_fd=obs_fd) audit(f"fixture-race target {name}") def maybe_fixture_signal(name): configured = fixture_value("OLP_HELPER_FIXTURE_SIGNAL") target = fixture_value("OLP_HELPER_FIXTURE_SIGNAL_TARGET") if configured and target == name: signum = {"HUP": signal.SIGHUP, "INT": signal.SIGINT, "TERM": signal.SIGTERM}.get(configured) if signum is None: reject("fixture signal is invalid") os.kill(os.getpid(), signum) exit_status = 0 try: root_fd = open_physical_absolute(root_path) ssd_fd = open_physical_absolute(ssd_path) aistor_fd = open_physical_absolute(aistor_path) obs_fd = open_child(ssd_fd, "observability", True) available, uses = filesystem_snapshot(root_fd, ssd_fd, aistor_fd, obs_fd) if action == "preflight": print("\t".join(["PREFLIGHT", str(available), *(str(value) for value in uses)])) else: for signum in SIGNALS: signal.signal(signum, signal_handler) if obs_fd is None: previous = block_signals() try: os.mkdir("observability", 0o770, dir_fd=ssd_fd) active_created = (ssd_fd, "observability", entry_identity(ssd_fd, "observability")) obs_fd = open_child(ssd_fd, "observability") if identity(obs_fd) != active_created[2]: reject("new observability ancestor identity changed before open") created.append((ssd_fd, "observability", obs_fd, identity(obs_fd))) active_created = None finally: restore_signal_mask(previous) obs_expected = identity(obs_fd) maybe_fixture_race("ancestor") for name, requested_uid, requested_gid in TARGETS: target_fd = open_child(obs_fd, name, True) if target_fd is None: previous = block_signals() try: os.mkdir(name, 0o770, dir_fd=obs_fd) active_created = (obs_fd, name, entry_identity(obs_fd, name)) target_fd = open_child(obs_fd, name) if identity(target_fd) != active_created[2]: reject("new target identity changed before open") created.append((obs_fd, name, target_fd, identity(target_fd))) active_created = None actual_uid, actual_gid = requested_owner(requested_uid, requested_gid) os.fchown(target_fd, actual_uid, actual_gid) os.fchmod(target_fd, 0o770) audit(f"set-owner-mode {name} {requested_uid} {requested_gid} 0770") finally: restore_signal_mask(previous) maybe_fixture_signal(name) else: metadata = os.fstat(target_fd) if os.listdir(target_fd): reject("existing target is non-empty") restored.append(( name, target_fd, metadata.st_uid, metadata.st_gid, stat.S_IMODE(metadata.st_mode), identity(target_fd), )) maybe_fixture_race("target", name) actual_uid, actual_gid = requested_owner(requested_uid, requested_gid) os.fchown(target_fd, actual_uid, actual_gid) os.fchmod(target_fd, 0o770) audit(f"set-owner-mode {name} {requested_uid} {requested_gid} 0770") metadata = os.fstat(target_fd) actual_uid, actual_gid = requested_owner(requested_uid, requested_gid) if (metadata.st_uid, metadata.st_gid, stat.S_IMODE(metadata.st_mode)) != (actual_uid, actual_gid, 0o770): reject("post-create owner or mode differs") for parent_fd, name, target_fd, expected in created: if entry_identity(parent_fd, name) != expected: reject("created entry identity changed") for name, _target_fd, _uid, _gid, _mode, expected in restored: if entry_identity(obs_fd, name) != expected: reject("pre-existing target entry identity changed") if entry_identity(ssd_fd, "observability") != obs_expected: reject("observability ancestor identity changed") print("APPLIED") except Interrupted as error: for signum in SIGNALS: signal.signal(signum, signal.SIG_IGN) rollback() print(f"REJECT: interrupted by signal {error.signum}", file=sys.stderr) exit_status = 128 + error.signum except (Rejected, OSError) as error: for signum in SIGNALS: signal.signal(signum, signal.SIG_IGN) rollback() message = str(error) if str(error) else "privileged helper failed" print(f"REJECT: {message}", file=sys.stderr) exit_status = 23 finally: close_all() raise SystemExit(exit_status) PY } _olp_privileged_helper() { if (( $# != 4 )) || [[ "$1" != preflight && "$1" != apply ]]; then return 97 fi [[ "$2" == / && "$3" == /srv/k3s/ssd && "$4" == /srv/k3s/aistor ]] || \ return 97 local program program="$(_olp_helper_program)" || return /usr/bin/printf '%s\n' "$program" | \ /usr/bin/sudo -- /usr/bin/python3 - "$@" } _olp_root_facts() { /usr/bin/findmnt --kernel --first-only --noheadings --output SOURCE,FSTYPE \ --target "$1" | /usr/bin/awk 'NF == 2 { print $1 "\t" $2 }' } _olp_available_bytes() { /usr/bin/df --block-size=1 --output=avail "$1" | /usr/bin/tail -n 1 | \ /usr/bin/tr -d '[:space:]' } _olp_current_context() { local kubectl_bin kubectl_bin="$(command -v kubectl)" || return 1 "$kubectl_bin" config current-context } _olp_node_names() { local kubectl_bin kubectl_bin="$(command -v kubectl)" || return 1 "$kubectl_bin" get nodes \ -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}' } _olp_read_confirmation() { local value printf 'Type APPLY default to prepare the six Local PV paths: ' >&2 IFS= read -r value /dev/null)" || _olp_error "privileged filesystem preflight failed" || return _olp_validate_preflight_reply "$reply" || \ _olp_error "privileged preflight protocol or capacity boundary failed" || return context="$(_olp_current_context)" || \ _olp_error "could not read Kubernetes context" || return [[ "$context" == default ]] || \ _olp_error "Kubernetes context must be exactly default" || return nodes="$(_olp_node_names)" || \ _olp_error "could not read Kubernetes node names" || return [[ "$nodes" == donghyeon-system-product-name ]] || \ _olp_error "Kubernetes node must be exactly donghyeon-system-product-name" || return } observability_local_paths_execute() { local root_path="$1" ssd_base="$2" aistor_base="$3" confirmation reply _olp_execute_preflight "$root_path" "$ssd_base" "$aistor_base" || return confirmation="$(_olp_read_confirmation)" || \ _olp_error "TTY confirmation was not read" || return [[ "$confirmation" == 'APPLY default' ]] || \ _olp_error "confirmation did not match APPLY default" || return _olp_execute_preflight "$root_path" "$ssd_base" "$aistor_base" || return reply="$(_olp_privileged_helper apply "$root_path" "$ssd_base" "$aistor_base")" || \ _olp_error "descriptor-relative path preparation failed" || return [[ "$reply" == APPLIED ]] || \ _olp_error "privileged apply protocol differed" || return printf 'Prepared six observability Local PV paths on the root filesystem.\n' printf 'No Kubernetes resources were applied.\n' } _olp_usage() { printf 'Usage: bash scripts/bootstrap/prepare-observability-local-paths.sh [--execute]\n' >&2 } _olp_main() { if (( $# == 0 )); then observability_local_paths_dry_run / /srv/k3s/ssd /srv/k3s/aistor elif (( $# == 1 )) && [[ "$1" == --execute ]]; then observability_local_paths_execute / /srv/k3s/ssd /srv/k3s/aistor else _olp_usage return 2 fi } _olp_entry() ( set -Eeuo pipefail _olp_main "$@" ) if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then _olp_entry "$@" fi