feat(bootstrap): apply the k3s restart bundle with one restart
The restart bundle needs root, touches k3s, kubelet and systemd files, and must not leave the cluster half-configured. The W1 time-sync drop-ins need a restart window too. apply-k3s-restart-bundle.sh --check parses the files and prints the install plan without root. --execute checks the cluster is Ready, backs up every file it replaces plus a sqlite .backup of the datastore and the token, installs the W6 files and W1's two time-sync drop-ins as they are, enables systemd-time-wait-sync, restarts k3s once with --no-block (the unit has TimeoutStartSec=0), then verifies the kubelet configz values, the audit log, that the default namespace rejects a privileged pod, three health URLs and the time-sync unit. Any failure restores the backup and restarts again. Last, it creates the local-storage .skip file and clears the local-path default annotation. Risk: --execute was not run (owner sudo step). The API is unavailable for one to two minutes; containers keep running (KillMode=process). The script refuses to run until W1's host files are on main. Verification (desktop, non-root): - --check without the W1 files: MISSING ... (W1), exit 1. - --check with the W1 files overlaid: parse and invariants pass, plan printed, exit 0. - --execute as non-root: "--execute needs root (sudo)".
This commit is contained in:
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install the k3s restart bundle and restart k3s exactly once.
|
||||
#
|
||||
# bash scripts/bootstrap/apply-k3s-restart-bundle.sh --check # no root, changes nothing
|
||||
# sudo bash scripts/bootstrap/apply-k3s-restart-bundle.sh --execute
|
||||
#
|
||||
# The bundle (infrastructure/host/k3s/README.md):
|
||||
# W6: kubelet reservations, memory eviction and graceful shutdown (kubelet drop-in),
|
||||
# API server audit log and cluster Pod Security default (k3s drop-in + two files),
|
||||
# local-path StorageClass no longer the default (manifest .skip + annotation).
|
||||
# W1: k3s starts after time sync (infrastructure/host/systemd/*, installed as is).
|
||||
#
|
||||
# --execute backs up every file it replaces plus a datastore copy, installs, restarts
|
||||
# k3s, verifies, and on any failure restores the previous files and restarts again.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
readonly KCFG=/etc/rancher/k3s/k3s.yaml
|
||||
readonly SERVER=/var/lib/rancher/k3s/server
|
||||
readonly SKIP_FILE="$SERVER/manifests/local-storage.yaml.skip"
|
||||
readonly AUDIT_DIR="$SERVER/logs"
|
||||
readonly BACKUP_ROOT=/var/backups/platform-k3s-restart-bundle
|
||||
|
||||
# source | destination | mode | owner of the source file
|
||||
readonly -a FILES=(
|
||||
"infrastructure/host/k3s/kubelet.conf.d/50-platform-resources.conf|/var/lib/rancher/k3s/agent/etc/kubelet.conf.d/50-platform-resources.conf|0600|W6"
|
||||
"infrastructure/host/k3s/config.yaml.d/60-platform-apiserver.yaml|/etc/rancher/k3s/config.yaml.d/60-platform-apiserver.yaml|0644|W6"
|
||||
"infrastructure/host/k3s/server/audit.yaml|$SERVER/audit.yaml|0600|W6"
|
||||
"infrastructure/host/k3s/server/psa.yaml|$SERVER/psa.yaml|0600|W6"
|
||||
"infrastructure/host/systemd/k3s.service.d/10-after-time-sync.conf|/etc/systemd/system/k3s.service.d/10-after-time-sync.conf|0644|W1"
|
||||
"infrastructure/host/systemd/systemd-time-wait-sync.service.d/10-timeout.conf|/etc/systemd/system/systemd-time-wait-sync.service.d/10-timeout.conf|0644|W1"
|
||||
)
|
||||
readonly -a HEALTH_URLS=(
|
||||
https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration
|
||||
https://hyeonworks.com/api/healthcheck
|
||||
https://git.learn.hyeonworks.com/api/healthz
|
||||
)
|
||||
|
||||
say() { printf '%s\n' "$*"; }
|
||||
fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
mode="${1:-}"
|
||||
[[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || {
|
||||
sed -n '2,15p' "${BASH_SOURCE[0]}" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
check_sources() {
|
||||
local entry src dst perm owner missing=0
|
||||
for entry in "${FILES[@]}"; do
|
||||
IFS='|' read -r src dst perm owner <<<"$entry"
|
||||
if [[ ! -f "$ROOT/$src" ]]; then
|
||||
say "MISSING $src (${owner})"
|
||||
missing=1
|
||||
fi
|
||||
done
|
||||
(( missing == 0 )) || fail "run this from a platform-core checkout that has both the W6 and the W1 host files on main"
|
||||
python3 -I - "$ROOT" <<'PY' || fail "bundle files do not parse"
|
||||
import sys, pathlib, yaml
|
||||
root = pathlib.Path(sys.argv[1]) / "infrastructure/host/k3s"
|
||||
k = yaml.safe_load((root / "kubelet.conf.d/50-platform-resources.conf").read_text())
|
||||
assert k["kind"] == "KubeletConfiguration", "kubelet drop-in kind"
|
||||
assert set(k["evictionHard"]) == {"memory.available", "nodefs.available", "imagefs.available"}, "evictionHard keys"
|
||||
assert k["shutdownGracePeriod"] == "60s" and k["shutdownGracePeriodCriticalPods"] == "20s", "shutdown periods"
|
||||
a = yaml.safe_load((root / "config.yaml.d/60-platform-apiserver.yaml").read_text())
|
||||
assert list(a) == ["kube-apiserver-arg+"], "apiserver drop-in must only append kube-apiserver-arg"
|
||||
args = dict(x.split("=", 1) for x in a["kube-apiserver-arg+"])
|
||||
assert args["audit-policy-file"] == "/var/lib/rancher/k3s/server/audit.yaml"
|
||||
assert args["admission-control-config-file"] == "/var/lib/rancher/k3s/server/psa.yaml"
|
||||
p = yaml.safe_load((root / "server/audit.yaml").read_text())
|
||||
assert p["kind"] == "Policy" and p["rules"][-1] == {"level": "Metadata"}, "audit policy ends with Metadata"
|
||||
assert all(r["level"] in ("None", "Metadata") for r in p["rules"]), "audit policy must not log bodies"
|
||||
s = yaml.safe_load((root / "server/psa.yaml").read_text())
|
||||
d = s["plugins"][0]["configuration"]["defaults"]
|
||||
assert d["enforce"] == "baseline", "PSA default enforce"
|
||||
assert s["plugins"][0]["configuration"]["exemptions"]["namespaces"] == ["kube-system"], "PSA exemptions"
|
||||
print("bundle files parse and keep their invariants")
|
||||
PY
|
||||
}
|
||||
|
||||
plan() {
|
||||
local entry src dst perm owner state
|
||||
for entry in "${FILES[@]}"; do
|
||||
IFS='|' read -r src dst perm owner <<<"$entry"
|
||||
if [[ -d "$(dirname "$dst")" && ! -r "$(dirname "$dst")" ]] ||
|
||||
[[ ! -e "$(dirname "$dst")" && ! -r "$(dirname "$(dirname "$dst")")" ]]; then
|
||||
state="unknown (needs root to read)"
|
||||
elif [[ ! -e "$dst" ]]; then
|
||||
state="new"
|
||||
elif cmp -s "$ROOT/$src" "$dst"; then
|
||||
state="same"
|
||||
else
|
||||
state="replace"
|
||||
fi
|
||||
say " ${owner} ${state} ${dst}"
|
||||
done
|
||||
say " W6 touch $SKIP_FILE"
|
||||
say " W6 mkdir $AUDIT_DIR (0700)"
|
||||
say " W1 systemctl enable systemd-time-wait-sync.service"
|
||||
say " -- systemctl daemon-reload && systemctl restart k3s (once)"
|
||||
}
|
||||
|
||||
say "== check"
|
||||
check_sources
|
||||
say "== plan"
|
||||
plan
|
||||
[[ "$mode" == "--execute" ]] || { say "CHECK ONLY: nothing changed"; exit 0; }
|
||||
|
||||
[[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)"
|
||||
command -v sqlite3 >/dev/null || fail "sqlite3 is required for the pre-restart datastore copy"
|
||||
kubectl() { command k3s kubectl --kubeconfig "$KCFG" "$@"; }
|
||||
|
||||
wait_ready() {
|
||||
local i
|
||||
for i in $(seq 1 60); do
|
||||
if kubectl get --raw=/readyz >/dev/null 2>&1 &&
|
||||
[[ "$(kubectl get nodes -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' 2>/dev/null)" == "True" ]]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
say "== preflight"
|
||||
wait_ready || fail "the cluster is not Ready before the change; not touching it"
|
||||
for url in "${HEALTH_URLS[@]}"; do
|
||||
code="$(curl -s -o /dev/null -m 15 -w '%{http_code}' "$url" || true)"
|
||||
say " before $code $url"
|
||||
done
|
||||
|
||||
ts="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
backup="$BACKUP_ROOT/$ts"
|
||||
install -d -m 0700 "$backup"
|
||||
time_wait_was_enabled="$(systemctl is-enabled systemd-time-wait-sync.service 2>/dev/null || true)"
|
||||
say "== backup to $backup"
|
||||
for entry in "${FILES[@]}"; do
|
||||
IFS='|' read -r src dst perm owner <<<"$entry"
|
||||
if [[ -e "$dst" ]]; then
|
||||
install -D -m 0600 "$dst" "$backup/files$dst"
|
||||
fi
|
||||
done
|
||||
sqlite3 "$SERVER/db/state.db" ".backup '$backup/state.db'"
|
||||
install -m 0600 "$SERVER/token" "$backup/token"
|
||||
say " datastore copy: $(sqlite3 "$backup/state.db" 'PRAGMA integrity_check;') $(du -h "$backup/state.db" | cut -f1)"
|
||||
|
||||
rollback() {
|
||||
local entry src dst perm owner
|
||||
trap - ERR
|
||||
say "== ROLLBACK"
|
||||
set +e
|
||||
for entry in "${FILES[@]}"; do
|
||||
IFS='|' read -r src dst perm owner <<<"$entry"
|
||||
if [[ -e "$backup/files$dst" ]]; then
|
||||
install -D -m "$perm" "$backup/files$dst" "$dst"
|
||||
else
|
||||
rm -f -- "$dst"
|
||||
fi
|
||||
done
|
||||
rm -f -- "$SKIP_FILE"
|
||||
[[ "$time_wait_was_enabled" == "enabled" ]] || systemctl disable systemd-time-wait-sync.service
|
||||
systemctl daemon-reload
|
||||
systemctl restart --no-block k3s
|
||||
sleep 10
|
||||
if wait_ready; then say "ROLLBACK done: previous files restored, k3s Ready"; else say "ROLLBACK: k3s still not Ready; see journalctl -u k3s"; fi
|
||||
say "backup kept at $backup"
|
||||
exit 1
|
||||
}
|
||||
trap rollback ERR
|
||||
|
||||
say "== install"
|
||||
for entry in "${FILES[@]}"; do
|
||||
IFS='|' read -r src dst perm owner <<<"$entry"
|
||||
install -D -o root -g root -m "$perm" "$ROOT/$src" "$dst"
|
||||
done
|
||||
install -d -o root -g root -m 0700 "$AUDIT_DIR"
|
||||
install -D -o root -g root -m 0600 /dev/null "$SKIP_FILE"
|
||||
systemctl daemon-reload
|
||||
systemctl enable systemd-time-wait-sync.service
|
||||
|
||||
say "== restart k3s"
|
||||
# k3s.service has TimeoutStartSec=0, so a blocking restart could wait forever.
|
||||
systemctl restart --no-block k3s
|
||||
sleep 10
|
||||
wait_ready || { say "k3s did not become Ready in 5 minutes"; false; }
|
||||
|
||||
say "== verify"
|
||||
node="$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')"
|
||||
kubectl get --raw "/api/v1/nodes/$node/proxy/configz" | python3 -I -c '
|
||||
import json, sys
|
||||
k = json.load(sys.stdin)["kubeletconfig"]
|
||||
want = {
|
||||
("systemReserved", "memory"): "2Gi", ("kubeReserved", "memory"): "1Gi",
|
||||
("evictionHard", "memory.available"): "500Mi", ("evictionHard", "nodefs.available"): "5%",
|
||||
("evictionHard", "imagefs.available"): "5%", ("evictionSoft", "memory.available"): "1Gi",
|
||||
}
|
||||
bad = [f"{a}.{b}={k.get(a, {}).get(b)}" for (a, b), v in want.items() if k.get(a, {}).get(b) != v]
|
||||
if k.get("shutdownGracePeriod") != "1m0s": bad.append(f"shutdownGracePeriod={k.get(\"shutdownGracePeriod\")}")
|
||||
if bad: sys.exit("kubelet config not applied: " + ", ".join(bad))
|
||||
print(" kubelet: reservations, eviction and graceful shutdown applied")
|
||||
'
|
||||
for i in $(seq 1 12); do [[ -s "$AUDIT_DIR/audit.log" ]] && break; sleep 5; done
|
||||
[[ -s "$AUDIT_DIR/audit.log" ]] || { say "audit log is empty"; false; }
|
||||
say " audit log: $(wc -l <"$AUDIT_DIR/audit.log") lines"
|
||||
psa_out="$(kubectl apply --dry-run=server -f - 2>&1 <<'YAML' || true
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: psa-default-probe
|
||||
namespace: default
|
||||
spec:
|
||||
containers:
|
||||
- name: probe
|
||||
image: registry.k8s.io/pause:3.10
|
||||
securityContext:
|
||||
privileged: true
|
||||
YAML
|
||||
)"
|
||||
grep -q 'violates PodSecurity "baseline' <<<"$psa_out" || { say "PSA default did not reject a privileged pod: $psa_out"; false; }
|
||||
say " pod security: default namespace rejects a privileged pod (baseline)"
|
||||
for url in "${HEALTH_URLS[@]}"; do
|
||||
ok=0
|
||||
for i in $(seq 1 30); do
|
||||
[[ "$(curl -s -o /dev/null -m 15 -w '%{http_code}' "$url" || true)" == "200" ]] && { ok=1; break; }
|
||||
sleep 10
|
||||
done
|
||||
(( ok == 1 )) || { say "health check failed: $url"; false; }
|
||||
say " after 200 $url"
|
||||
done
|
||||
say " time sync: $(systemctl is-enabled systemd-time-wait-sync.service), $(systemctl show systemd-time-wait-sync.service -p TimeoutStartUSec)"
|
||||
trap - ERR
|
||||
|
||||
say "== local-path is no longer the default StorageClass"
|
||||
kubectl annotate storageclass local-path storageclass.kubernetes.io/is-default-class=false --overwrite
|
||||
kubectl get storageclass local-path -o jsonpath=' local-path default={.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}{"\n"}'
|
||||
|
||||
say "K3S RESTART BUNDLE APPLIED (backup: $backup)"
|
||||
Reference in New Issue
Block a user