Backups, the daily drift check and the k3s serving certificates have no alert. k3s renews certificates only when it starts within 120 days of expiry, and the two kube-prometheus-stack kubelet certificate alerts can never fire because the k3s kubelet exports no kubelet_certificate_manager_* metric. Add a PrometheusRule (platform-host-checks) for backup age, failed runs, missing metrics, drift left for 6h, a stale drift check, and serving certificate expiry (warning under 100 days, critical under 14). Add the two remaining host timers (certificate expiry via openssl on 6443 and 10250; the drift check against origin/main) and an installer that copies the scripts to a root-owned path, creates the textfile and work directories and enables all four timers. Disable the two dead kubelet certificate rules in kube-prometheus-stack. Risk: the metrics need the node-exporter textfile collector, a one-line change in W1's node-exporter values that was asked of W1. Until then PlatformHostBackupMetricsAbsent fires after a day, which is the point. Verification (desktop): - test-platform-host-checks-alerts.sh: promtool check and unit tests pass; with the k3s threshold changed from 2 to 5 days they fail. - render-observability-core.sh passes; the render has no Kubelet* certificate alerts and still has both KubeClientCertificateExpiration. - k3s-cert-expiry.sh as the desktop user reads both certificates. - install-host-checks.sh --check: systemd-analyze verify is clean.
79 lines
3.9 KiB
Bash
79 lines
3.9 KiB
Bash
#!/usr/bin/env bash
|
|
# Install the host backup and check timers (infrastructure/host/backup).
|
|
#
|
|
# bash scripts/bootstrap/install-host-checks.sh --check # no root, changes nothing
|
|
# sudo bash scripts/bootstrap/install-host-checks.sh --execute # install, enable, first runs
|
|
#
|
|
# Scripts are copied to /usr/local/lib/platform-backup (root-owned) so the root
|
|
# timers never execute files the desktop user can edit.
|
|
set -Eeuo pipefail
|
|
umask 022
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly LIB=/usr/local/lib/platform-backup
|
|
readonly UNIT_DIR=/etc/systemd/system
|
|
readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector
|
|
readonly WORK_ROOT=/var/lib/platform-backup/work
|
|
readonly USER_NAME=donghyeon
|
|
readonly -a SCRIPTS=(
|
|
scripts/lib/host-backup.sh:lib/host-backup.sh
|
|
scripts/backup/k3s-datastore-backup.sh:backup/k3s-datastore-backup.sh
|
|
scripts/backup/gitea-backup.sh:backup/gitea-backup.sh
|
|
scripts/validate/live-drift-check-main.sh:validate/live-drift-check-main.sh
|
|
scripts/validate/k3s-cert-expiry.sh:validate/k3s-cert-expiry.sh
|
|
services/platform-host-checks/README.md:README.md
|
|
)
|
|
readonly -a TIMERS=(platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check platform-k3s-cert-expiry)
|
|
|
|
fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
|
mode="${1:-}"
|
|
[[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || { sed -n '2,8p' "$0" >&2; exit 2; }
|
|
|
|
echo "== check"
|
|
for pair in "${SCRIPTS[@]}"; do
|
|
src="${pair%%:*}"
|
|
[[ -f "$ROOT/$src" ]] || fail "missing $src"
|
|
[[ "$src" != *.sh ]] || bash -n "$ROOT/$src" || fail "syntax error in $src"
|
|
done
|
|
for t in "${TIMERS[@]}"; do
|
|
for u in service timer; do [[ -f "$ROOT/infrastructure/host/backup/$t.$u" ]] || fail "missing $t.$u"; done
|
|
done
|
|
if command -v systemd-analyze >/dev/null; then
|
|
verify_dir="$(mktemp -d)"
|
|
trap 'rm -rf -- "$verify_dir"' EXIT
|
|
cp "$ROOT"/infrastructure/host/backup/*.service "$ROOT"/infrastructure/host/backup/*.timer "$verify_dir/"
|
|
# ExecStart paths do not exist before installation; only syntax and keys are checked.
|
|
out="$(cd "$verify_dir" && systemd-analyze verify ./*.timer ./*.service 2>&1 | grep -v -E 'is not executable|No such file|Command .* is not executable' || true)"
|
|
[[ -z "$out" ]] || { printf '%s\n' "$out" >&2; fail "systemd-analyze verify reported problems"; }
|
|
echo " units: systemd-analyze verify clean"
|
|
fi
|
|
echo "== plan"
|
|
for pair in "${SCRIPTS[@]}"; do echo " install ${pair%%:*} -> $LIB/${pair##*:}"; done
|
|
for t in "${TIMERS[@]}"; do echo " install $UNIT_DIR/$t.service, $UNIT_DIR/$t.timer; enable --now $t.timer"; done
|
|
echo " mkdir $TEXTFILE_DIR (root:$USER_NAME 0775), $WORK_ROOT (root 0700)"
|
|
echo " first runs: k3s-cert-expiry, k3s-datastore-backup, gitea-backup, live-drift-check"
|
|
[[ "$mode" == "--execute" ]] || { echo "CHECK ONLY: nothing changed"; exit 0; }
|
|
|
|
[[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)"
|
|
[[ -f /etc/platform-backup/aistor.env ]] || fail "run scripts/bootstrap/create-host-backup-credentials.sh first"
|
|
echo "== install"
|
|
for pair in "${SCRIPTS[@]}"; do
|
|
install -D -o root -g root -m 0755 "$ROOT/${pair%%:*}" "$LIB/${pair##*:}"
|
|
done
|
|
install -d -o root -g "$USER_NAME" -m 0775 "$TEXTFILE_DIR"
|
|
install -d -o root -g root -m 0700 "$WORK_ROOT"
|
|
for t in "${TIMERS[@]}"; do
|
|
install -o root -g root -m 0644 "$ROOT/infrastructure/host/backup/$t.service" "$ROOT/infrastructure/host/backup/$t.timer" "$UNIT_DIR/"
|
|
done
|
|
systemctl daemon-reload
|
|
for t in "${TIMERS[@]}"; do systemctl enable --now "$t.timer"; done
|
|
|
|
echo "== first runs"
|
|
for t in platform-k3s-cert-expiry platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check; do
|
|
if systemctl start "$t.service"; then echo " $t: ok"; else echo " $t: FAILED (journalctl -u $t)"; fi
|
|
done
|
|
echo "== metrics"
|
|
grep -h -E '^platform_' "$TEXTFILE_DIR"/*.prom || echo " no metrics yet"
|
|
systemctl list-timers 'platform-*' --no-pager
|
|
echo "HOST CHECKS INSTALLED"
|