#!/usr/bin/env bash # Install the k3s restart bundle and restart k3s exactly once. # # bash scripts/bootstrap/apply-k3s-restart-bundle.sh --check # no root, changes nothing # sudo bash scripts/bootstrap/apply-k3s-restart-bundle.sh --execute # # The bundle (infrastructure/host/k3s/README.md): # W6: kubelet reservations, memory eviction and graceful shutdown (kubelet drop-in), # API server audit log and cluster Pod Security default (k3s drop-in + two files), # local-path StorageClass no longer the default (manifest .skip + annotation). # W1: k3s starts after time sync (infrastructure/host/systemd/*, installed as is). # # --execute backs up every file it replaces plus a datastore copy, installs, restarts # k3s, verifies, and on any failure restores the previous files and restarts again. set -Eeuo pipefail umask 077 readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" readonly KCFG=/etc/rancher/k3s/k3s.yaml readonly SERVER=/var/lib/rancher/k3s/server readonly SKIP_FILE="$SERVER/manifests/local-storage.yaml.skip" readonly AUDIT_DIR="$SERVER/logs" readonly BACKUP_ROOT=/var/backups/platform-k3s-restart-bundle # source | destination | mode | owner of the source file readonly -a FILES=( "infrastructure/host/k3s/kubelet.conf.d/50-platform-resources.conf|/var/lib/rancher/k3s/agent/etc/kubelet.conf.d/50-platform-resources.conf|0600|W6" "infrastructure/host/k3s/config.yaml.d/60-platform-apiserver.yaml|/etc/rancher/k3s/config.yaml.d/60-platform-apiserver.yaml|0644|W6" "infrastructure/host/k3s/server/audit.yaml|$SERVER/audit.yaml|0600|W6" "infrastructure/host/k3s/server/psa.yaml|$SERVER/psa.yaml|0600|W6" "infrastructure/host/systemd/k3s.service.d/10-after-time-sync.conf|/etc/systemd/system/k3s.service.d/10-after-time-sync.conf|0644|W1" "infrastructure/host/systemd/systemd-time-wait-sync.service.d/10-timeout.conf|/etc/systemd/system/systemd-time-wait-sync.service.d/10-timeout.conf|0644|W1" ) readonly -a HEALTH_URLS=( https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration https://hyeonworks.com/api/healthcheck https://git.learn.hyeonworks.com/api/healthz ) say() { printf '%s\n' "$*"; } fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } mode="${1:-}" [[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || { sed -n '2,15p' "${BASH_SOURCE[0]}" >&2 exit 2 } check_sources() { local entry src dst perm owner missing=0 for entry in "${FILES[@]}"; do IFS='|' read -r src dst perm owner <<<"$entry" if [[ ! -f "$ROOT/$src" ]]; then say "MISSING $src (${owner})" missing=1 fi done (( missing == 0 )) || fail "run this from a platform-core checkout that has both the W6 and the W1 host files on main" python3 -I - "$ROOT" <<'PY' || fail "bundle files do not parse" import sys, pathlib, yaml root = pathlib.Path(sys.argv[1]) / "infrastructure/host/k3s" k = yaml.safe_load((root / "kubelet.conf.d/50-platform-resources.conf").read_text()) assert k["kind"] == "KubeletConfiguration", "kubelet drop-in kind" assert set(k["evictionHard"]) == {"memory.available", "nodefs.available", "imagefs.available"}, "evictionHard keys" assert k["shutdownGracePeriod"] == "60s" and k["shutdownGracePeriodCriticalPods"] == "20s", "shutdown periods" a = yaml.safe_load((root / "config.yaml.d/60-platform-apiserver.yaml").read_text()) assert list(a) == ["kube-apiserver-arg+"], "apiserver drop-in must only append kube-apiserver-arg" args = dict(x.split("=", 1) for x in a["kube-apiserver-arg+"]) assert args["audit-policy-file"] == "/var/lib/rancher/k3s/server/audit.yaml" assert args["admission-control-config-file"] == "/var/lib/rancher/k3s/server/psa.yaml" p = yaml.safe_load((root / "server/audit.yaml").read_text()) assert p["kind"] == "Policy" and p["rules"][-1] == {"level": "Metadata"}, "audit policy ends with Metadata" assert all(r["level"] in ("None", "Metadata") for r in p["rules"]), "audit policy must not log bodies" s = yaml.safe_load((root / "server/psa.yaml").read_text()) d = s["plugins"][0]["configuration"]["defaults"] assert d["enforce"] == "baseline", "PSA default enforce" assert s["plugins"][0]["configuration"]["exemptions"]["namespaces"] == ["kube-system"], "PSA exemptions" print("bundle files parse and keep their invariants") PY } plan() { local entry src dst perm owner state for entry in "${FILES[@]}"; do IFS='|' read -r src dst perm owner <<<"$entry" if [[ -d "$(dirname "$dst")" && ! -r "$(dirname "$dst")" ]] || [[ ! -e "$(dirname "$dst")" && ! -r "$(dirname "$(dirname "$dst")")" ]]; then state="unknown (needs root to read)" elif [[ ! -e "$dst" ]]; then state="new" elif cmp -s "$ROOT/$src" "$dst"; then state="same" else state="replace" fi say " ${owner} ${state} ${dst}" done say " W6 touch $SKIP_FILE" say " W6 mkdir $AUDIT_DIR (0700)" say " W1 systemctl enable systemd-time-wait-sync.service" say " -- systemctl daemon-reload && systemctl restart k3s (once)" } say "== check" check_sources say "== plan" plan [[ "$mode" == "--execute" ]] || { say "CHECK ONLY: nothing changed"; exit 0; } [[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)" command -v sqlite3 >/dev/null || fail "sqlite3 is required for the pre-restart datastore copy" kubectl() { command k3s kubectl --kubeconfig "$KCFG" "$@"; } wait_ready() { local i for i in $(seq 1 60); do if kubectl get --raw=/readyz >/dev/null 2>&1 && [[ "$(kubectl get nodes -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' 2>/dev/null)" == "True" ]]; then return 0 fi sleep 5 done return 1 } say "== preflight" wait_ready || fail "the cluster is not Ready before the change; not touching it" for url in "${HEALTH_URLS[@]}"; do code="$(curl -s -o /dev/null -m 15 -w '%{http_code}' "$url" || true)" say " before $code $url" done ts="$(date -u +%Y%m%dT%H%M%SZ)" backup="$BACKUP_ROOT/$ts" install -d -m 0700 "$backup" time_wait_was_enabled="$(systemctl is-enabled systemd-time-wait-sync.service 2>/dev/null || true)" say "== backup to $backup" for entry in "${FILES[@]}"; do IFS='|' read -r src dst perm owner <<<"$entry" if [[ -e "$dst" ]]; then install -D -m 0600 "$dst" "$backup/files$dst" fi done sqlite3 "$SERVER/db/state.db" ".backup '$backup/state.db'" install -m 0600 "$SERVER/token" "$backup/token" say " datastore copy: $(sqlite3 "$backup/state.db" 'PRAGMA integrity_check;') $(du -h "$backup/state.db" | cut -f1)" rollback() { local entry src dst perm owner trap - ERR say "== ROLLBACK" set +e for entry in "${FILES[@]}"; do IFS='|' read -r src dst perm owner <<<"$entry" if [[ -e "$backup/files$dst" ]]; then install -D -m "$perm" "$backup/files$dst" "$dst" else rm -f -- "$dst" fi done rm -f -- "$SKIP_FILE" [[ "$time_wait_was_enabled" == "enabled" ]] || systemctl disable systemd-time-wait-sync.service systemctl daemon-reload systemctl restart --no-block k3s sleep 10 if wait_ready; then say "ROLLBACK done: previous files restored, k3s Ready"; else say "ROLLBACK: k3s still not Ready; see journalctl -u k3s"; fi say "backup kept at $backup" exit 1 } trap rollback ERR say "== install" for entry in "${FILES[@]}"; do IFS='|' read -r src dst perm owner <<<"$entry" install -D -o root -g root -m "$perm" "$ROOT/$src" "$dst" done install -d -o root -g root -m 0700 "$AUDIT_DIR" install -D -o root -g root -m 0600 /dev/null "$SKIP_FILE" systemctl daemon-reload systemctl enable systemd-time-wait-sync.service say "== restart k3s" # k3s.service has TimeoutStartSec=0, so a blocking restart could wait forever. systemctl restart --no-block k3s sleep 10 wait_ready || { say "k3s did not become Ready in 5 minutes"; false; } say "== verify" node="$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')" kubectl get --raw "/api/v1/nodes/$node/proxy/configz" | python3 -I -c ' import json, sys k = json.load(sys.stdin)["kubeletconfig"] want = { ("systemReserved", "memory"): "2Gi", ("kubeReserved", "memory"): "1Gi", ("evictionHard", "memory.available"): "500Mi", ("evictionHard", "nodefs.available"): "5%", ("evictionHard", "imagefs.available"): "5%", ("evictionSoft", "memory.available"): "1Gi", } bad = [f"{a}.{b}={k.get(a, {}).get(b)}" for (a, b), v in want.items() if k.get(a, {}).get(b) != v] if k.get("shutdownGracePeriod") != "1m0s": bad.append(f"shutdownGracePeriod={k.get(\"shutdownGracePeriod\")}") if bad: sys.exit("kubelet config not applied: " + ", ".join(bad)) print(" kubelet: reservations, eviction and graceful shutdown applied") ' for i in $(seq 1 12); do [[ -s "$AUDIT_DIR/audit.log" ]] && break; sleep 5; done [[ -s "$AUDIT_DIR/audit.log" ]] || { say "audit log is empty"; false; } say " audit log: $(wc -l <"$AUDIT_DIR/audit.log") lines" psa_out="$(kubectl apply --dry-run=server -f - 2>&1 <<'YAML' || true apiVersion: v1 kind: Pod metadata: name: psa-default-probe namespace: default spec: containers: - name: probe image: registry.k8s.io/pause:3.10 securityContext: privileged: true YAML )" grep -q 'violates PodSecurity "baseline' <<<"$psa_out" || { say "PSA default did not reject a privileged pod: $psa_out"; false; } say " pod security: default namespace rejects a privileged pod (baseline)" for url in "${HEALTH_URLS[@]}"; do ok=0 for i in $(seq 1 30); do [[ "$(curl -s -o /dev/null -m 15 -w '%{http_code}' "$url" || true)" == "200" ]] && { ok=1; break; } sleep 10 done (( ok == 1 )) || { say "health check failed: $url"; false; } say " after 200 $url" done say " time sync: $(systemctl is-enabled systemd-time-wait-sync.service), $(systemctl show systemd-time-wait-sync.service -p TimeoutStartUSec)" trap - ERR say "== local-path is no longer the default StorageClass" kubectl annotate storageclass local-path storageclass.kubernetes.io/is-default-class=false --overwrite kubectl get storageclass local-path -o jsonpath=' local-path default={.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}{"\n"}' say "K3S RESTART BUNDLE APPLIED (backup: $backup)"