From 457d88e2932478f802336697fd1f2c1ba46d6b8c Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 00:49:58 +0900 Subject: [PATCH] feat(bootstrap): apply the k3s restart bundle with one restart The restart bundle needs root, touches k3s, kubelet and systemd files, and must not leave the cluster half-configured. The W1 time-sync drop-ins need a restart window too. apply-k3s-restart-bundle.sh --check parses the files and prints the install plan without root. --execute checks the cluster is Ready, backs up every file it replaces plus a sqlite .backup of the datastore and the token, installs the W6 files and W1's two time-sync drop-ins as they are, enables systemd-time-wait-sync, restarts k3s once with --no-block (the unit has TimeoutStartSec=0), then verifies the kubelet configz values, the audit log, that the default namespace rejects a privileged pod, three health URLs and the time-sync unit. Any failure restores the backup and restarts again. Last, it creates the local-storage .skip file and clears the local-path default annotation. Risk: --execute was not run (owner sudo step). The API is unavailable for one to two minutes; containers keep running (KillMode=process). The script refuses to run until W1's host files are on main. Verification (desktop, non-root): - --check without the W1 files: MISSING ... (W1), exit 1. - --check with the W1 files overlaid: parse and invariants pass, plan printed, exit 0. - --execute as non-root: "--execute needs root (sudo)". --- scripts/bootstrap/apply-k3s-restart-bundle.sh | 238 ++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 scripts/bootstrap/apply-k3s-restart-bundle.sh diff --git a/scripts/bootstrap/apply-k3s-restart-bundle.sh b/scripts/bootstrap/apply-k3s-restart-bundle.sh new file mode 100644 index 0000000..a5864dc --- /dev/null +++ b/scripts/bootstrap/apply-k3s-restart-bundle.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +# Install the k3s restart bundle and restart k3s exactly once. +# +# bash scripts/bootstrap/apply-k3s-restart-bundle.sh --check # no root, changes nothing +# sudo bash scripts/bootstrap/apply-k3s-restart-bundle.sh --execute +# +# The bundle (infrastructure/host/k3s/README.md): +# W6: kubelet reservations, memory eviction and graceful shutdown (kubelet drop-in), +# API server audit log and cluster Pod Security default (k3s drop-in + two files), +# local-path StorageClass no longer the default (manifest .skip + annotation). +# W1: k3s starts after time sync (infrastructure/host/systemd/*, installed as is). +# +# --execute backs up every file it replaces plus a datastore copy, installs, restarts +# k3s, verifies, and on any failure restores the previous files and restarts again. +set -Eeuo pipefail +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly KCFG=/etc/rancher/k3s/k3s.yaml +readonly SERVER=/var/lib/rancher/k3s/server +readonly SKIP_FILE="$SERVER/manifests/local-storage.yaml.skip" +readonly AUDIT_DIR="$SERVER/logs" +readonly BACKUP_ROOT=/var/backups/platform-k3s-restart-bundle + +# source | destination | mode | owner of the source file +readonly -a FILES=( + "infrastructure/host/k3s/kubelet.conf.d/50-platform-resources.conf|/var/lib/rancher/k3s/agent/etc/kubelet.conf.d/50-platform-resources.conf|0600|W6" + "infrastructure/host/k3s/config.yaml.d/60-platform-apiserver.yaml|/etc/rancher/k3s/config.yaml.d/60-platform-apiserver.yaml|0644|W6" + "infrastructure/host/k3s/server/audit.yaml|$SERVER/audit.yaml|0600|W6" + "infrastructure/host/k3s/server/psa.yaml|$SERVER/psa.yaml|0600|W6" + "infrastructure/host/systemd/k3s.service.d/10-after-time-sync.conf|/etc/systemd/system/k3s.service.d/10-after-time-sync.conf|0644|W1" + "infrastructure/host/systemd/systemd-time-wait-sync.service.d/10-timeout.conf|/etc/systemd/system/systemd-time-wait-sync.service.d/10-timeout.conf|0644|W1" +) +readonly -a HEALTH_URLS=( + https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration + https://hyeonworks.com/api/healthcheck + https://git.learn.hyeonworks.com/api/healthz +) + +say() { printf '%s\n' "$*"; } +fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +mode="${1:-}" +[[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || { + sed -n '2,15p' "${BASH_SOURCE[0]}" >&2 + exit 2 +} + +check_sources() { + local entry src dst perm owner missing=0 + for entry in "${FILES[@]}"; do + IFS='|' read -r src dst perm owner <<<"$entry" + if [[ ! -f "$ROOT/$src" ]]; then + say "MISSING $src (${owner})" + missing=1 + fi + done + (( missing == 0 )) || fail "run this from a platform-core checkout that has both the W6 and the W1 host files on main" + python3 -I - "$ROOT" <<'PY' || fail "bundle files do not parse" +import sys, pathlib, yaml +root = pathlib.Path(sys.argv[1]) / "infrastructure/host/k3s" +k = yaml.safe_load((root / "kubelet.conf.d/50-platform-resources.conf").read_text()) +assert k["kind"] == "KubeletConfiguration", "kubelet drop-in kind" +assert set(k["evictionHard"]) == {"memory.available", "nodefs.available", "imagefs.available"}, "evictionHard keys" +assert k["shutdownGracePeriod"] == "60s" and k["shutdownGracePeriodCriticalPods"] == "20s", "shutdown periods" +a = yaml.safe_load((root / "config.yaml.d/60-platform-apiserver.yaml").read_text()) +assert list(a) == ["kube-apiserver-arg+"], "apiserver drop-in must only append kube-apiserver-arg" +args = dict(x.split("=", 1) for x in a["kube-apiserver-arg+"]) +assert args["audit-policy-file"] == "/var/lib/rancher/k3s/server/audit.yaml" +assert args["admission-control-config-file"] == "/var/lib/rancher/k3s/server/psa.yaml" +p = yaml.safe_load((root / "server/audit.yaml").read_text()) +assert p["kind"] == "Policy" and p["rules"][-1] == {"level": "Metadata"}, "audit policy ends with Metadata" +assert all(r["level"] in ("None", "Metadata") for r in p["rules"]), "audit policy must not log bodies" +s = yaml.safe_load((root / "server/psa.yaml").read_text()) +d = s["plugins"][0]["configuration"]["defaults"] +assert d["enforce"] == "baseline", "PSA default enforce" +assert s["plugins"][0]["configuration"]["exemptions"]["namespaces"] == ["kube-system"], "PSA exemptions" +print("bundle files parse and keep their invariants") +PY +} + +plan() { + local entry src dst perm owner state + for entry in "${FILES[@]}"; do + IFS='|' read -r src dst perm owner <<<"$entry" + if [[ -d "$(dirname "$dst")" && ! -r "$(dirname "$dst")" ]] || + [[ ! -e "$(dirname "$dst")" && ! -r "$(dirname "$(dirname "$dst")")" ]]; then + state="unknown (needs root to read)" + elif [[ ! -e "$dst" ]]; then + state="new" + elif cmp -s "$ROOT/$src" "$dst"; then + state="same" + else + state="replace" + fi + say " ${owner} ${state} ${dst}" + done + say " W6 touch $SKIP_FILE" + say " W6 mkdir $AUDIT_DIR (0700)" + say " W1 systemctl enable systemd-time-wait-sync.service" + say " -- systemctl daemon-reload && systemctl restart k3s (once)" +} + +say "== check" +check_sources +say "== plan" +plan +[[ "$mode" == "--execute" ]] || { say "CHECK ONLY: nothing changed"; exit 0; } + +[[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)" +command -v sqlite3 >/dev/null || fail "sqlite3 is required for the pre-restart datastore copy" +kubectl() { command k3s kubectl --kubeconfig "$KCFG" "$@"; } + +wait_ready() { + local i + for i in $(seq 1 60); do + if kubectl get --raw=/readyz >/dev/null 2>&1 && + [[ "$(kubectl get nodes -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' 2>/dev/null)" == "True" ]]; then + return 0 + fi + sleep 5 + done + return 1 +} + +say "== preflight" +wait_ready || fail "the cluster is not Ready before the change; not touching it" +for url in "${HEALTH_URLS[@]}"; do + code="$(curl -s -o /dev/null -m 15 -w '%{http_code}' "$url" || true)" + say " before $code $url" +done + +ts="$(date -u +%Y%m%dT%H%M%SZ)" +backup="$BACKUP_ROOT/$ts" +install -d -m 0700 "$backup" +time_wait_was_enabled="$(systemctl is-enabled systemd-time-wait-sync.service 2>/dev/null || true)" +say "== backup to $backup" +for entry in "${FILES[@]}"; do + IFS='|' read -r src dst perm owner <<<"$entry" + if [[ -e "$dst" ]]; then + install -D -m 0600 "$dst" "$backup/files$dst" + fi +done +sqlite3 "$SERVER/db/state.db" ".backup '$backup/state.db'" +install -m 0600 "$SERVER/token" "$backup/token" +say " datastore copy: $(sqlite3 "$backup/state.db" 'PRAGMA integrity_check;') $(du -h "$backup/state.db" | cut -f1)" + +rollback() { + local entry src dst perm owner + trap - ERR + say "== ROLLBACK" + set +e + for entry in "${FILES[@]}"; do + IFS='|' read -r src dst perm owner <<<"$entry" + if [[ -e "$backup/files$dst" ]]; then + install -D -m "$perm" "$backup/files$dst" "$dst" + else + rm -f -- "$dst" + fi + done + rm -f -- "$SKIP_FILE" + [[ "$time_wait_was_enabled" == "enabled" ]] || systemctl disable systemd-time-wait-sync.service + systemctl daemon-reload + systemctl restart --no-block k3s + sleep 10 + if wait_ready; then say "ROLLBACK done: previous files restored, k3s Ready"; else say "ROLLBACK: k3s still not Ready; see journalctl -u k3s"; fi + say "backup kept at $backup" + exit 1 +} +trap rollback ERR + +say "== install" +for entry in "${FILES[@]}"; do + IFS='|' read -r src dst perm owner <<<"$entry" + install -D -o root -g root -m "$perm" "$ROOT/$src" "$dst" +done +install -d -o root -g root -m 0700 "$AUDIT_DIR" +install -D -o root -g root -m 0600 /dev/null "$SKIP_FILE" +systemctl daemon-reload +systemctl enable systemd-time-wait-sync.service + +say "== restart k3s" +# k3s.service has TimeoutStartSec=0, so a blocking restart could wait forever. +systemctl restart --no-block k3s +sleep 10 +wait_ready || { say "k3s did not become Ready in 5 minutes"; false; } + +say "== verify" +node="$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')" +kubectl get --raw "/api/v1/nodes/$node/proxy/configz" | python3 -I -c ' +import json, sys +k = json.load(sys.stdin)["kubeletconfig"] +want = { + ("systemReserved", "memory"): "2Gi", ("kubeReserved", "memory"): "1Gi", + ("evictionHard", "memory.available"): "500Mi", ("evictionHard", "nodefs.available"): "5%", + ("evictionHard", "imagefs.available"): "5%", ("evictionSoft", "memory.available"): "1Gi", +} +bad = [f"{a}.{b}={k.get(a, {}).get(b)}" for (a, b), v in want.items() if k.get(a, {}).get(b) != v] +if k.get("shutdownGracePeriod") != "1m0s": bad.append(f"shutdownGracePeriod={k.get(\"shutdownGracePeriod\")}") +if bad: sys.exit("kubelet config not applied: " + ", ".join(bad)) +print(" kubelet: reservations, eviction and graceful shutdown applied") +' +for i in $(seq 1 12); do [[ -s "$AUDIT_DIR/audit.log" ]] && break; sleep 5; done +[[ -s "$AUDIT_DIR/audit.log" ]] || { say "audit log is empty"; false; } +say " audit log: $(wc -l <"$AUDIT_DIR/audit.log") lines" +psa_out="$(kubectl apply --dry-run=server -f - 2>&1 <<'YAML' || true +apiVersion: v1 +kind: Pod +metadata: + name: psa-default-probe + namespace: default +spec: + containers: + - name: probe + image: registry.k8s.io/pause:3.10 + securityContext: + privileged: true +YAML +)" +grep -q 'violates PodSecurity "baseline' <<<"$psa_out" || { say "PSA default did not reject a privileged pod: $psa_out"; false; } +say " pod security: default namespace rejects a privileged pod (baseline)" +for url in "${HEALTH_URLS[@]}"; do + ok=0 + for i in $(seq 1 30); do + [[ "$(curl -s -o /dev/null -m 15 -w '%{http_code}' "$url" || true)" == "200" ]] && { ok=1; break; } + sleep 10 + done + (( ok == 1 )) || { say "health check failed: $url"; false; } + say " after 200 $url" +done +say " time sync: $(systemctl is-enabled systemd-time-wait-sync.service), $(systemctl show systemd-time-wait-sync.service -p TimeoutStartUSec)" +trap - ERR + +say "== local-path is no longer the default StorageClass" +kubectl annotate storageclass local-path storageclass.kubernetes.io/is-default-class=false --overwrite +kubectl get storageclass local-path -o jsonpath=' local-path default={.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}{"\n"}' + +say "K3S RESTART BUNDLE APPLIED (backup: $backup)"