The restart bundle needs root, touches k3s, kubelet and systemd files, and must not leave the cluster half-configured. The W1 time-sync drop-ins need a restart window too. apply-k3s-restart-bundle.sh --check parses the files and prints the install plan without root. --execute checks the cluster is Ready, backs up every file it replaces plus a sqlite .backup of the datastore and the token, installs the W6 files and W1's two time-sync drop-ins as they are, enables systemd-time-wait-sync, restarts k3s once with --no-block (the unit has TimeoutStartSec=0), then verifies the kubelet configz values, the audit log, that the default namespace rejects a privileged pod, three health URLs and the time-sync unit. Any failure restores the backup and restarts again. Last, it creates the local-storage .skip file and clears the local-path default annotation. Risk: --execute was not run (owner sudo step). The API is unavailable for one to two minutes; containers keep running (KillMode=process). The script refuses to run until W1's host files are on main. Verification (desktop, non-root): - --check without the W1 files: MISSING ... (W1), exit 1. - --check with the W1 files overlaid: parse and invariants pass, plan printed, exit 0. - --execute as non-root: "--execute needs root (sudo)".
239 lines
9.9 KiB
Bash
239 lines
9.9 KiB
Bash
#!/usr/bin/env bash
|
|
# Install the k3s restart bundle and restart k3s exactly once.
|
|
#
|
|
# bash scripts/bootstrap/apply-k3s-restart-bundle.sh --check # no root, changes nothing
|
|
# sudo bash scripts/bootstrap/apply-k3s-restart-bundle.sh --execute
|
|
#
|
|
# The bundle (infrastructure/host/k3s/README.md):
|
|
# W6: kubelet reservations, memory eviction and graceful shutdown (kubelet drop-in),
|
|
# API server audit log and cluster Pod Security default (k3s drop-in + two files),
|
|
# local-path StorageClass no longer the default (manifest .skip + annotation).
|
|
# W1: k3s starts after time sync (infrastructure/host/systemd/*, installed as is).
|
|
#
|
|
# --execute backs up every file it replaces plus a datastore copy, installs, restarts
|
|
# k3s, verifies, and on any failure restores the previous files and restarts again.
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly KCFG=/etc/rancher/k3s/k3s.yaml
|
|
readonly SERVER=/var/lib/rancher/k3s/server
|
|
readonly SKIP_FILE="$SERVER/manifests/local-storage.yaml.skip"
|
|
readonly AUDIT_DIR="$SERVER/logs"
|
|
readonly BACKUP_ROOT=/var/backups/platform-k3s-restart-bundle
|
|
|
|
# source | destination | mode | owner of the source file
|
|
readonly -a FILES=(
|
|
"infrastructure/host/k3s/kubelet.conf.d/50-platform-resources.conf|/var/lib/rancher/k3s/agent/etc/kubelet.conf.d/50-platform-resources.conf|0600|W6"
|
|
"infrastructure/host/k3s/config.yaml.d/60-platform-apiserver.yaml|/etc/rancher/k3s/config.yaml.d/60-platform-apiserver.yaml|0644|W6"
|
|
"infrastructure/host/k3s/server/audit.yaml|$SERVER/audit.yaml|0600|W6"
|
|
"infrastructure/host/k3s/server/psa.yaml|$SERVER/psa.yaml|0600|W6"
|
|
"infrastructure/host/systemd/k3s.service.d/10-after-time-sync.conf|/etc/systemd/system/k3s.service.d/10-after-time-sync.conf|0644|W1"
|
|
"infrastructure/host/systemd/systemd-time-wait-sync.service.d/10-timeout.conf|/etc/systemd/system/systemd-time-wait-sync.service.d/10-timeout.conf|0644|W1"
|
|
)
|
|
readonly -a HEALTH_URLS=(
|
|
https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration
|
|
https://hyeonworks.com/api/healthcheck
|
|
https://git.learn.hyeonworks.com/api/healthz
|
|
)
|
|
|
|
say() { printf '%s\n' "$*"; }
|
|
fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
|
|
|
mode="${1:-}"
|
|
[[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || {
|
|
sed -n '2,15p' "${BASH_SOURCE[0]}" >&2
|
|
exit 2
|
|
}
|
|
|
|
check_sources() {
|
|
local entry src dst perm owner missing=0
|
|
for entry in "${FILES[@]}"; do
|
|
IFS='|' read -r src dst perm owner <<<"$entry"
|
|
if [[ ! -f "$ROOT/$src" ]]; then
|
|
say "MISSING $src (${owner})"
|
|
missing=1
|
|
fi
|
|
done
|
|
(( missing == 0 )) || fail "run this from a platform-core checkout that has both the W6 and the W1 host files on main"
|
|
python3 -I - "$ROOT" <<'PY' || fail "bundle files do not parse"
|
|
import sys, pathlib, yaml
|
|
root = pathlib.Path(sys.argv[1]) / "infrastructure/host/k3s"
|
|
k = yaml.safe_load((root / "kubelet.conf.d/50-platform-resources.conf").read_text())
|
|
assert k["kind"] == "KubeletConfiguration", "kubelet drop-in kind"
|
|
assert set(k["evictionHard"]) == {"memory.available", "nodefs.available", "imagefs.available"}, "evictionHard keys"
|
|
assert k["shutdownGracePeriod"] == "60s" and k["shutdownGracePeriodCriticalPods"] == "20s", "shutdown periods"
|
|
a = yaml.safe_load((root / "config.yaml.d/60-platform-apiserver.yaml").read_text())
|
|
assert list(a) == ["kube-apiserver-arg+"], "apiserver drop-in must only append kube-apiserver-arg"
|
|
args = dict(x.split("=", 1) for x in a["kube-apiserver-arg+"])
|
|
assert args["audit-policy-file"] == "/var/lib/rancher/k3s/server/audit.yaml"
|
|
assert args["admission-control-config-file"] == "/var/lib/rancher/k3s/server/psa.yaml"
|
|
p = yaml.safe_load((root / "server/audit.yaml").read_text())
|
|
assert p["kind"] == "Policy" and p["rules"][-1] == {"level": "Metadata"}, "audit policy ends with Metadata"
|
|
assert all(r["level"] in ("None", "Metadata") for r in p["rules"]), "audit policy must not log bodies"
|
|
s = yaml.safe_load((root / "server/psa.yaml").read_text())
|
|
d = s["plugins"][0]["configuration"]["defaults"]
|
|
assert d["enforce"] == "baseline", "PSA default enforce"
|
|
assert s["plugins"][0]["configuration"]["exemptions"]["namespaces"] == ["kube-system"], "PSA exemptions"
|
|
print("bundle files parse and keep their invariants")
|
|
PY
|
|
}
|
|
|
|
plan() {
|
|
local entry src dst perm owner state
|
|
for entry in "${FILES[@]}"; do
|
|
IFS='|' read -r src dst perm owner <<<"$entry"
|
|
if [[ -d "$(dirname "$dst")" && ! -r "$(dirname "$dst")" ]] ||
|
|
[[ ! -e "$(dirname "$dst")" && ! -r "$(dirname "$(dirname "$dst")")" ]]; then
|
|
state="unknown (needs root to read)"
|
|
elif [[ ! -e "$dst" ]]; then
|
|
state="new"
|
|
elif cmp -s "$ROOT/$src" "$dst"; then
|
|
state="same"
|
|
else
|
|
state="replace"
|
|
fi
|
|
say " ${owner} ${state} ${dst}"
|
|
done
|
|
say " W6 touch $SKIP_FILE"
|
|
say " W6 mkdir $AUDIT_DIR (0700)"
|
|
say " W1 systemctl enable systemd-time-wait-sync.service"
|
|
say " -- systemctl daemon-reload && systemctl restart k3s (once)"
|
|
}
|
|
|
|
say "== check"
|
|
check_sources
|
|
say "== plan"
|
|
plan
|
|
[[ "$mode" == "--execute" ]] || { say "CHECK ONLY: nothing changed"; exit 0; }
|
|
|
|
[[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)"
|
|
command -v sqlite3 >/dev/null || fail "sqlite3 is required for the pre-restart datastore copy"
|
|
kubectl() { command k3s kubectl --kubeconfig "$KCFG" "$@"; }
|
|
|
|
wait_ready() {
|
|
local i
|
|
for i in $(seq 1 60); do
|
|
if kubectl get --raw=/readyz >/dev/null 2>&1 &&
|
|
[[ "$(kubectl get nodes -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' 2>/dev/null)" == "True" ]]; then
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
say "== preflight"
|
|
wait_ready || fail "the cluster is not Ready before the change; not touching it"
|
|
for url in "${HEALTH_URLS[@]}"; do
|
|
code="$(curl -s -o /dev/null -m 15 -w '%{http_code}' "$url" || true)"
|
|
say " before $code $url"
|
|
done
|
|
|
|
ts="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
backup="$BACKUP_ROOT/$ts"
|
|
install -d -m 0700 "$backup"
|
|
time_wait_was_enabled="$(systemctl is-enabled systemd-time-wait-sync.service 2>/dev/null || true)"
|
|
say "== backup to $backup"
|
|
for entry in "${FILES[@]}"; do
|
|
IFS='|' read -r src dst perm owner <<<"$entry"
|
|
if [[ -e "$dst" ]]; then
|
|
install -D -m 0600 "$dst" "$backup/files$dst"
|
|
fi
|
|
done
|
|
sqlite3 "$SERVER/db/state.db" ".backup '$backup/state.db'"
|
|
install -m 0600 "$SERVER/token" "$backup/token"
|
|
say " datastore copy: $(sqlite3 "$backup/state.db" 'PRAGMA integrity_check;') $(du -h "$backup/state.db" | cut -f1)"
|
|
|
|
rollback() {
|
|
local entry src dst perm owner
|
|
trap - ERR
|
|
say "== ROLLBACK"
|
|
set +e
|
|
for entry in "${FILES[@]}"; do
|
|
IFS='|' read -r src dst perm owner <<<"$entry"
|
|
if [[ -e "$backup/files$dst" ]]; then
|
|
install -D -m "$perm" "$backup/files$dst" "$dst"
|
|
else
|
|
rm -f -- "$dst"
|
|
fi
|
|
done
|
|
rm -f -- "$SKIP_FILE"
|
|
[[ "$time_wait_was_enabled" == "enabled" ]] || systemctl disable systemd-time-wait-sync.service
|
|
systemctl daemon-reload
|
|
systemctl restart --no-block k3s
|
|
sleep 10
|
|
if wait_ready; then say "ROLLBACK done: previous files restored, k3s Ready"; else say "ROLLBACK: k3s still not Ready; see journalctl -u k3s"; fi
|
|
say "backup kept at $backup"
|
|
exit 1
|
|
}
|
|
trap rollback ERR
|
|
|
|
say "== install"
|
|
for entry in "${FILES[@]}"; do
|
|
IFS='|' read -r src dst perm owner <<<"$entry"
|
|
install -D -o root -g root -m "$perm" "$ROOT/$src" "$dst"
|
|
done
|
|
install -d -o root -g root -m 0700 "$AUDIT_DIR"
|
|
install -D -o root -g root -m 0600 /dev/null "$SKIP_FILE"
|
|
systemctl daemon-reload
|
|
systemctl enable systemd-time-wait-sync.service
|
|
|
|
say "== restart k3s"
|
|
# k3s.service has TimeoutStartSec=0, so a blocking restart could wait forever.
|
|
systemctl restart --no-block k3s
|
|
sleep 10
|
|
wait_ready || { say "k3s did not become Ready in 5 minutes"; false; }
|
|
|
|
say "== verify"
|
|
node="$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')"
|
|
kubectl get --raw "/api/v1/nodes/$node/proxy/configz" | python3 -I -c '
|
|
import json, sys
|
|
k = json.load(sys.stdin)["kubeletconfig"]
|
|
want = {
|
|
("systemReserved", "memory"): "2Gi", ("kubeReserved", "memory"): "1Gi",
|
|
("evictionHard", "memory.available"): "500Mi", ("evictionHard", "nodefs.available"): "5%",
|
|
("evictionHard", "imagefs.available"): "5%", ("evictionSoft", "memory.available"): "1Gi",
|
|
}
|
|
bad = [f"{a}.{b}={k.get(a, {}).get(b)}" for (a, b), v in want.items() if k.get(a, {}).get(b) != v]
|
|
if k.get("shutdownGracePeriod") != "1m0s": bad.append(f"shutdownGracePeriod={k.get(\"shutdownGracePeriod\")}")
|
|
if bad: sys.exit("kubelet config not applied: " + ", ".join(bad))
|
|
print(" kubelet: reservations, eviction and graceful shutdown applied")
|
|
'
|
|
for i in $(seq 1 12); do [[ -s "$AUDIT_DIR/audit.log" ]] && break; sleep 5; done
|
|
[[ -s "$AUDIT_DIR/audit.log" ]] || { say "audit log is empty"; false; }
|
|
say " audit log: $(wc -l <"$AUDIT_DIR/audit.log") lines"
|
|
psa_out="$(kubectl apply --dry-run=server -f - 2>&1 <<'YAML' || true
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: psa-default-probe
|
|
namespace: default
|
|
spec:
|
|
containers:
|
|
- name: probe
|
|
image: registry.k8s.io/pause:3.10
|
|
securityContext:
|
|
privileged: true
|
|
YAML
|
|
)"
|
|
grep -q 'violates PodSecurity "baseline' <<<"$psa_out" || { say "PSA default did not reject a privileged pod: $psa_out"; false; }
|
|
say " pod security: default namespace rejects a privileged pod (baseline)"
|
|
for url in "${HEALTH_URLS[@]}"; do
|
|
ok=0
|
|
for i in $(seq 1 30); do
|
|
[[ "$(curl -s -o /dev/null -m 15 -w '%{http_code}' "$url" || true)" == "200" ]] && { ok=1; break; }
|
|
sleep 10
|
|
done
|
|
(( ok == 1 )) || { say "health check failed: $url"; false; }
|
|
say " after 200 $url"
|
|
done
|
|
say " time sync: $(systemctl is-enabled systemd-time-wait-sync.service), $(systemctl show systemd-time-wait-sync.service -p TimeoutStartUSec)"
|
|
trap - ERR
|
|
|
|
say "== local-path is no longer the default StorageClass"
|
|
kubectl annotate storageclass local-path storageclass.kubernetes.io/is-default-class=false --overwrite
|
|
kubectl get storageclass local-path -o jsonpath=' local-path default={.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}{"\n"}'
|
|
|
|
say "K3S RESTART BUNDLE APPLIED (backup: $backup)"
|