donghyeon.kang/platform-core
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Files
16c337bcc91b27ac0f074154b8337e4c8dc17ebb
platform-core/scripts/bootstrap/apply-host-nginx-observability.sh
T

1133 lines
42 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
set -Eeuo pipefail
set +x
umask 077
readonly HOST_NGINX_PRODUCTION_PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
PATH=$HOST_NGINX_PRODUCTION_PATH
export PATH
hash -r
readonly ROOT="$(cd -- "${BASH_SOURCE[0]%/*}/../.." && pwd -P)"
declare -Ar HOST_NGINX_PRODUCTION_COMMANDS=(
[awk]=/usr/bin/awk [bash]=/usr/bin/bash [cat]=/usr/bin/cat
[curl]=/usr/bin/curl [date]=/usr/bin/date [dig]=/usr/bin/dig
[dirname]=/usr/bin/dirname [grep]=/usr/bin/grep [head]=/usr/bin/head
[id]=/usr/bin/id [install]=/usr/bin/install [jq]=/usr/bin/jq
[kubectl]=/usr/local/bin/kubectl [mktemp]=/usr/bin/mktemp
[openssl]=/usr/bin/openssl [python3]=/usr/bin/python3
[readlink]=/usr/bin/readlink [rm]=/usr/bin/rm [sed]=/usr/bin/sed
[sha256sum]=/usr/bin/sha256sum [sleep]=/usr/bin/sleep
[sort]=/usr/bin/sort [ss]=/usr/bin/ss [stat]=/usr/bin/stat
[tail]=/usr/bin/tail [test]=/usr/bin/test [tr]=/usr/bin/tr
)
readonly HOST_NGINX_SOURCE="$ROOT/infrastructure/networking/host-nginx"
readonly METRICS_CANDIDATE="$HOST_NGINX_SOURCE/learn-services-metrics-guard.conf"
readonly DENY_CANDIDATE="$HOST_NGINX_SOURCE/learn-services-grafana-deny-guard.conf"
readonly FULL_CANDIDATE="$HOST_NGINX_SOURCE/learn-services-observability.conf"
readonly ADMIN_CANDIDATE="$HOST_NGINX_SOURCE/learn-services-admin.conf"
readonly SOURCE_TEST="$ROOT/scripts/validate/test-host-nginx-observability-source.sh"
readonly ACTIVE="${PLATFORM_HOST_NGINX_ACTIVE:-/etc/nginx/sites-available/learn-services}"
readonly ENABLED="${PLATFORM_HOST_NGINX_ENABLED:-/etc/nginx/sites-enabled/learn-services}"
readonly ROLLBACK_BASE="${PLATFORM_HOST_NGINX_ROLLBACK_BASE:-/var/lib/hyeonworks/platform-rollbacks}"
readonly GRAFANA_HOST=grafana.learn.hyeonworks.com
readonly GITEA_HOST=git.learn.hyeonworks.com
readonly KEYCLOAK_HOST=id.learn.hyeonworks.com
readonly STORAGE_HOST=storage-admin.learn.hyeonworks.com
readonly DB_HOST=db-admin.learn.hyeonworks.com
readonly KEYCLOAK_ISSUER="https://$KEYCLOAK_HOST/realms/hyeonworks"
readonly LAN_IP=192.168.0.107
readonly TAIL_IP=100.92.240.34
readonly CERT_NAME="$GRAFANA_HOST"
readonly CERT_DIR="${PLATFORM_HOST_NGINX_CERT_DIR:-/etc/letsencrypt/live/$CERT_NAME}"
readonly CREDENTIALS="${PLATFORM_HOST_NGINX_CREDENTIALS:-/home/donghyeon/.secrets/certbot/cloudflare.ini}"
readonly NGINX_BIN="${PLATFORM_HOST_NGINX_NGINX_BIN:-/usr/sbin/nginx}"
readonly SYSTEMCTL_BIN="${PLATFORM_HOST_NGINX_SYSTEMCTL_BIN:-/usr/bin/systemctl}"
readonly SUDO_BIN="${PLATFORM_HOST_NGINX_SUDO_BIN:-/usr/bin/sudo}"
readonly TEST_BIN="${PLATFORM_HOST_NGINX_TEST_BIN:-/usr/bin/test}"
readonly CERTBOT_BIN="${PLATFORM_HOST_NGINX_CERTBOT_BIN:-/snap/bin/certbot}"
readonly PREFLIGHT_COMMAND="${PLATFORM_HOST_NGINX_PREFLIGHT_COMMAND:-}"
readonly POSTCHECK_COMMAND="${PLATFORM_HOST_NGINX_POSTCHECK_COMMAND:-}"
readonly TEST_MODE="${PLATFORM_HOST_NGINX_TEST_MODE:-0}"
readonly RULES_ACCEPTANCE_SCHEMA=platform-observability-rules-alerts-v2
readonly SOURCE_PROOF_SCHEMA=platform-blackbox-source-v1
readonly STAGE_LEDGER_HEADER=$'stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc'
execute=false
mode=full
mode_count=0
certbot_email=''
verified_output_dir=''
rollback_armed=false
rollback_payload=''
rollback_previous_sha=''
rollback_stage=''
timestamp_backup=''
fail() {
printf 'ERROR: %s\n' "$*" >&2
return 1
}
usage() {
cat <<'USAGE'
Usage:
bash scripts/bootstrap/apply-host-nginx-observability.sh
bash scripts/bootstrap/apply-host-nginx-observability.sh --execute --metrics-guard-only
bash scripts/bootstrap/apply-host-nginx-observability.sh \
--execute --certificate-only --certbot-email you@example.com
bash scripts/bootstrap/apply-host-nginx-observability.sh --execute --grafana-deny-guard-only
bash scripts/bootstrap/apply-host-nginx-observability.sh \
--execute --verified-output-dir /tmp/platform-observability-metrics.XXXXXX
Dry-run prints the exact stage hashes and does not call sudo, Nginx, systemd,
Certbot, Kubernetes, or network probes. Execute modes are interactive. The full
cutover additionally requires source proof, rules-alerts acceptance, and the two
accepted metric inventories for the same rollback ID.
USAGE
}
sha_of() {
sha256sum -- "$1" | awk '{print $1}'
}
is_sha256() {
[[ "$1" =~ ^[0-9a-f]{64}$ ]]
}
require_regular_source() {
[[ -f "$1" && ! -L "$1" ]] || fail "unsafe or missing source: $1"
}
root_cat() {
"$SUDO_BIN" -n cat -- "$1"
}
root_stat() {
"$SUDO_BIN" -n stat "$@"
}
expected_owner_mode() {
local path=$1 mode=$2 metadata
metadata="$(root_stat -c '%U:%G %a' -- "$path")" || return 1
if [[ "$TEST_MODE" == 1 ]]; then
[[ "${metadata##* }" == "$mode" ]]
else
[[ "$metadata" == "root:root $mode" ]]
fi
}
ensure_root_directory_no_follow() {
local path=$1 metadata type owner mode
if "$SUDO_BIN" -n "$TEST_BIN" -L "$path"; then
fail "root directory path is a symlink: $path"
return 1
fi
if ! "$SUDO_BIN" -n "$TEST_BIN" -e "$path"; then
if ! "$SUDO_BIN" install -d -o root -g root -m 0700 "$path"; then
fail "cannot create root directory: $path"
return 1
fi
fi
if ! metadata="$("$SUDO_BIN" -n stat -c '%F|%U:%G|%a' -- "$path")"; then
fail "cannot lstat root directory: $path"
return 1
fi
IFS='|' read -r type owner mode <<<"$metadata"
if [[ "$type" != directory || "$mode" != 700 ]]; then
fail "root directory type or mode is unsafe: $path"
return 1
fi
if [[ "$TEST_MODE" == 0 && "$owner" != root:root ]]; then
fail "root directory owner is unsafe: $path"
return 1
fi
return 0
}
validate_root_file_no_follow() {
local path=$1 expected_mode=$2 metadata type owner mode links
if "$SUDO_BIN" -n "$TEST_BIN" -L "$path"; then
fail "root file path is a symlink: $path"
return 1
fi
if ! metadata="$("$SUDO_BIN" -n stat -c '%F|%U:%G|%a|%h' -- "$path")"; then
fail "cannot lstat root file: $path"
return 1
fi
IFS='|' read -r type owner mode links <<<"$metadata"
if [[ "$type" != 'regular file' || "$mode" != "$expected_mode" || "$links" != 1 ]]; then
fail "root file type, mode, or link count is unsafe: $path"
return 1
fi
if [[ "$TEST_MODE" == 0 && "$owner" != root:root ]]; then
fail "root file owner is unsafe: $path"
return 1
fi
return 0
}
validate_test_command_boundary() {
local label=$1 command_path=$2 fixture_bin=$3 resolved metadata type owner mode mode_value
if [[ "$command_path" == /usr/bin/sudo ]]; then
fail 'test command boundary is unsafe: real /usr/bin/sudo is forbidden'
return 1
fi
if ! resolved="$(readlink -f -- "$command_path" 2>/dev/null)" || [[ "$resolved" != "$command_path" ]]; then
fail "test command boundary is unsafe: $label is not canonical"
return 1
fi
if [[ "$(dirname -- "$resolved")" != "$fixture_bin" ]]; then
fail "test command boundary is unsafe: $label escapes the fixture bin"
return 1
fi
if [[ ! -f "$command_path" || -L "$command_path" || ! -x "$command_path" ]]; then
fail "test command boundary is unsafe: $label is not a regular executable"
return 1
fi
if ! metadata="$(stat -c '%F|%u|%a' -- "$command_path")"; then
fail "test command boundary is unsafe: cannot stat $label"
return 1
fi
IFS='|' read -r type owner mode <<<"$metadata"
mode_value=$((8#$mode))
if [[ "$type" != 'regular file' || "$owner" != "$(id -u)" ||
$((mode_value & 0022)) != 0 || $((mode_value & 07000)) != 0 ]]; then
fail "test command boundary is unsafe: $label owner or mode"
return 1
fi
return 0
}
validate_test_command_fixture_directory() {
local path=$1 label=$2 metadata type owner mode mode_value
if [[ ! -d "$path" || -L "$path" ]]; then
fail "test command fixture $label is not a regular directory"
return 1
fi
if ! metadata="$(stat -c '%F|%u|%a' -- "$path")"; then
fail "cannot stat test command fixture $label"
return 1
fi
IFS='|' read -r type owner mode <<<"$metadata"
mode_value=$((8#$mode))
if [[ "$type" != directory || "$owner" != "$(id -u)" ||
$((mode_value & 0022)) != 0 || $((mode_value & 07000)) != 0 ]]; then
fail "test command fixture $label owner or mode is unsafe"
return 1
fi
return 0
}
production_executable_metadata_is_safe() {
local label=$1 lexical=$2 canonical=$3 type=$4 uid=$5 gid=$6 mode=$7 links=$8 policy=$9
local mode_value
[[ "$type" == 'regular file' && "$uid" == 0 && "$gid" == 0 && "$links" == 1 ]] || return 1
[[ "$mode" =~ ^[0-7]{3,4}$ ]] || return 1
mode_value=$((8#$mode))
(( (mode_value & 0111) != 0 )) || return 1
(( (mode_value & 0022) == 0 )) || return 1
case "$policy" in
sudo)
[[ "$label" == sudo && "$lexical" == /usr/bin/sudo && "$canonical" == /usr/bin/sudo ]] || return 1
(( mode_value == 04755 )) || return 1
;;
ordinary)
(( (mode_value & 06000) == 0 )) || return 1
;;
*) return 1 ;;
esac
}
validate_production_executable_boundary() {
local label=$1 lexical=$2 policy=$3 canonical metadata type uid gid mode links
[[ "$lexical" == /* ]] || {
fail "production command path is not absolute: $label"
return 1
}
if ! canonical="$(/usr/bin/readlink -f -- "$lexical" 2>/dev/null)"; then
fail "production command target does not resolve: $label"
return 1
fi
if ! metadata="$(/usr/bin/stat -Lc '%F|%u|%g|%a|%h' -- "$lexical" 2>/dev/null)"; then
fail "production command metadata is unreadable: $label"
return 1
fi
IFS='|' read -r type uid gid mode links <<<"$metadata"
if ! production_executable_metadata_is_safe "$label" "$lexical" "$canonical" \
"$type" "$uid" "$gid" "$mode" "$links" "$policy"; then
fail "production command boundary is unsafe: $label"
return 1
fi
}
validate_production_path_directory() {
local path=$1 canonical metadata type uid gid mode mode_value
if ! canonical="$(/usr/bin/readlink -f -- "$path" 2>/dev/null)" ||
! metadata="$(/usr/bin/stat -Lc '%F|%u|%g|%a' -- "$path" 2>/dev/null)"; then
fail "production PATH directory does not resolve: $path"
return 1
fi
IFS='|' read -r type uid gid mode <<<"$metadata"
mode_value=$((8#$mode))
if [[ "$canonical" != /* || "$type" != directory || "$uid" != 0 || "$gid" != 0 ||
$((mode_value & 0022)) != 0 ]]; then
fail "production PATH directory is unsafe: $path"
return 1
fi
}
validate_production_command_boundaries() {
local name expected resolved path_directory
local -a production_path_directories=()
[[ "$PATH" == "$HOST_NGINX_PRODUCTION_PATH" ]] || {
fail 'production PATH is not exact'
return 1
}
IFS=: read -r -a production_path_directories <<<"$HOST_NGINX_PRODUCTION_PATH"
for path_directory in "${production_path_directories[@]}"; do
validate_production_path_directory "$path_directory" || return 1
done
for name in "${!HOST_NGINX_PRODUCTION_COMMANDS[@]}"; do
expected=${HOST_NGINX_PRODUCTION_COMMANDS[$name]}
if [[ "$name" == test ]]; then
resolved=$expected
elif ! resolved="$(command -v -- "$name")" || [[ "$resolved" != "$expected" ]]; then
fail "production command resolution drifted: $name"
return 1
fi
validate_production_executable_boundary "$name" "$expected" ordinary || return 1
done
validate_production_executable_boundary nginx "$NGINX_BIN" ordinary || return 1
validate_production_executable_boundary systemctl "$SYSTEMCTL_BIN" ordinary || return 1
validate_production_executable_boundary certbot "$CERTBOT_BIN" ordinary || return 1
validate_production_executable_boundary sudo "$SUDO_BIN" sudo || return 1
}
validate_test_mode() {
local override_name active_fixture_root rollback_fixture_root fixture_bin fixture_root
if [[ "$TEST_MODE" != 0 && "$TEST_MODE" != 1 ]]; then
fail 'invalid test-mode flag'
return 1
fi
if [[ "$TEST_MODE" == 0 ]]; then
for override_name in ${!PLATFORM_HOST_NGINX_@}; do
fail "production rejects Host Nginx override: $override_name"
return 1
done
if [[ "$ACTIVE" != /etc/nginx/sites-available/learn-services ||
"$ENABLED" != /etc/nginx/sites-enabled/learn-services ||
"$ROLLBACK_BASE" != /var/lib/hyeonworks/platform-rollbacks ||
"$CERT_DIR" != /etc/letsencrypt/live/grafana.learn.hyeonworks.com ||
"$CREDENTIALS" != /home/donghyeon/.secrets/certbot/cloudflare.ini ||
"$NGINX_BIN" != /usr/sbin/nginx || "$SYSTEMCTL_BIN" != /usr/bin/systemctl ||
"$SUDO_BIN" != /usr/bin/sudo || "$TEST_BIN" != /usr/bin/test || "$CERTBOT_BIN" != /snap/bin/certbot ||
-n "$PREFLIGHT_COMMAND" || -n "$POSTCHECK_COMMAND" ]]; then
fail 'production Host Nginx constants are not exact'
return 1
fi
return 0
fi
case "$ACTIVE:$ROLLBACK_BASE" in
/tmp/platform-host-nginx-test.*:/tmp/platform-host-nginx-test.*) ;;
*)
fail 'test mode is restricted to an isolated /tmp/platform-host-nginx-test.* tree'
return 1
;;
esac
if [[ "$SUDO_BIN" == /usr/bin/sudo ]]; then
fail 'test command boundary is unsafe: real /usr/bin/sudo is forbidden'
return 1
fi
if ! active_fixture_root="$(readlink -f -- "$(dirname -- "$(dirname -- "$ACTIVE")")" 2>/dev/null)" ||
! rollback_fixture_root="$(readlink -f -- "$(dirname -- "$ROLLBACK_BASE")" 2>/dev/null)"; then
fail 'test fixture root does not resolve'
return 1
fi
fixture_root=$active_fixture_root
fixture_bin="$fixture_root/bin"
if [[ "$fixture_root" != /tmp/platform-host-nginx-test.* ||
"$ACTIVE" != "$fixture_root/active/learn-services" ||
"$ENABLED" != "$fixture_root/enabled/learn-services" ||
"$ROLLBACK_BASE" != "$fixture_root/rollbacks" ||
"$rollback_fixture_root" != "$fixture_root" ||
"$(readlink -f -- "$fixture_root" 2>/dev/null)" != "$fixture_root" ]]; then
fail 'test command fixture bin is not tied to the active and rollback fixture tree'
return 1
fi
if ! validate_test_command_fixture_directory "$fixture_root" root ||
! validate_test_command_fixture_directory "$fixture_bin" bin; then
return 1
fi
validate_test_command_boundary sudo "$SUDO_BIN" "$fixture_bin" || return 1
validate_test_command_boundary test "$TEST_BIN" "$fixture_bin" || return 1
validate_test_command_boundary nginx "$NGINX_BIN" "$fixture_bin" || return 1
validate_test_command_boundary systemctl "$SYSTEMCTL_BIN" "$fixture_bin" || return 1
validate_test_command_boundary certbot "$CERTBOT_BIN" "$fixture_bin" || return 1
validate_test_command_boundary preflight "$PREFLIGHT_COMMAND" "$fixture_bin" || return 1
validate_test_command_boundary postcheck "$POSTCHECK_COMMAND" "$fixture_bin" || return 1
PATH="$fixture_bin:$HOST_NGINX_PRODUCTION_PATH"
export PATH
hash -r
return 0
}
parse_args() {
while (( $# > 0 )); do
case "$1" in
--execute)
execute=true
shift
;;
--metrics-guard-only)
mode=metrics
mode_count=$((mode_count + 1))
shift
;;
--certificate-only)
mode=certificate
mode_count=$((mode_count + 1))
shift
;;
--grafana-deny-guard-only)
mode=deny
mode_count=$((mode_count + 1))
shift
;;
--certbot-email)
(( $# >= 2 )) || fail '--certbot-email requires a value'
certbot_email=$2
shift 2
;;
--verified-output-dir)
(( $# >= 2 )) || fail '--verified-output-dir requires a value'
verified_output_dir=$2
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
usage >&2
fail "unsupported argument: $1"
;;
esac
done
(( mode_count <= 1 )) || fail 'execute modes are mutually exclusive'
[[ -z "$certbot_email" || "$mode" == certificate ]] || fail '--certbot-email is certificate-only'
[[ -z "$verified_output_dir" || "$mode" == full ]] || fail '--verified-output-dir is full-mode only'
}
source_contract() {
require_regular_source "$ADMIN_CANDIDATE"
require_regular_source "$METRICS_CANDIDATE"
require_regular_source "$DENY_CANDIDATE"
require_regular_source "$FULL_CANDIDATE"
require_regular_source "$SOURCE_TEST"
bash "$SOURCE_TEST" >/dev/null
python3 - "$ADMIN_CANDIDATE" "$METRICS_CANDIDATE" <<'PY'
import pathlib
import sys
prior, guard = map(lambda value: pathlib.Path(value).read_text(), sys.argv[1:])
needle = " location / {\n"
addition = " location = /metrics {\n return 404;\n }\n\n"
if prior.count(needle) < 1 or guard != prior.replace(needle, addition + needle, 1):
raise SystemExit("metrics guard differs from the admin source outside exact /metrics")
PY
}
print_plan() {
local active_sha=unreadable
[[ -f "$ACTIVE" && ! -L "$ACTIVE" && -r "$ACTIVE" ]] && active_sha="$(sha_of "$ACTIVE")"
printf 'HOST_NGINX_ACTIVE_SHA256=%s\n' "$active_sha"
printf 'HOST_NGINX_ADMIN_SHA256=%s\n' "$(sha_of "$ADMIN_CANDIDATE")"
printf 'HOST_NGINX_METRICS_GUARD_SHA256=%s\n' "$(sha_of "$METRICS_CANDIDATE")"
printf 'HOST_NGINX_GRAFANA_DENY_GUARD_SHA256=%s\n' "$(sha_of "$DENY_CANDIDATE")"
printf 'HOST_NGINX_FULL_SHA256=%s\n' "$(sha_of "$FULL_CANDIDATE")"
printf 'HOST_NGINX_CERTIFICATE_EXPECTED_SAN=%s\n' "$GRAFANA_HOST"
printf 'HOST_NGINX_CERTIFICATE_SAN=NOT_CHECKED_DRY_RUN\n'
printf 'HOST_NGINX_GRAFANA_PUBLIC_DNS=NOT_CHECKED_DRY_RUN\n'
printf 'HOST_NGINX_OBSERVABILITY_DRY_RUN=PASS\n'
}
validate_host_files() {
"$SUDO_BIN" -n "$TEST_BIN" -f "$ACTIVE" || fail "active site is missing: $ACTIVE"
! "$SUDO_BIN" -n "$TEST_BIN" -L "$ACTIVE" || fail "active site is a symlink: $ACTIVE"
"$SUDO_BIN" -n "$TEST_BIN" -L "$ENABLED" || fail "enabled site is not a symlink: $ENABLED"
[[ "$("$SUDO_BIN" -n readlink -f -- "$ENABLED")" == "$("$SUDO_BIN" -n readlink -f -- "$ACTIVE")" ]] ||
fail 'enabled site target changed'
expected_owner_mode "$ACTIVE" 644 || fail 'active site must be root:root 0644'
}
validate_rollback_root() {
local id=${PLATFORM_OBSERVABILITY_ROLLBACK_ID:-}
[[ "$id" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || fail 'PLATFORM_OBSERVABILITY_ROLLBACK_ID format is invalid'
ROLLBACK_ROOT="$ROLLBACK_BASE/observability-$id"
"$SUDO_BIN" -n "$TEST_BIN" -d "$ROLLBACK_ROOT" || fail "rollback root is missing: $ROLLBACK_ROOT"
! "$SUDO_BIN" -n "$TEST_BIN" -L "$ROLLBACK_ROOT" || fail "rollback root is a symlink: $ROLLBACK_ROOT"
expected_owner_mode "$ROLLBACK_ROOT" 700 || fail 'rollback root must be root:root 0700'
readonly ROLLBACK_ROOT
}
declare -A PARSED_ENV=()
read_exact_env() {
local file=$1
shift
local line key value allowed expected=$# count=0
PARSED_ENV=()
if ! "$SUDO_BIN" -n "$TEST_BIN" -f "$file"; then
fail "evidence file is missing: $file"
return 1
fi
if "$SUDO_BIN" -n "$TEST_BIN" -L "$file"; then
fail "evidence file is a symlink: $file"
return 1
fi
if ! expected_owner_mode "$file" 600; then
fail "evidence file must be root:root 0600: $file"
return 1
fi
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ "$line" != *=* || "$line" == *$'\t'* || "$line" == *$'\r'* ]]; then
fail "malformed evidence line: $file"
return 1
fi
key=${line%%=*}
value=${line#*=}
if [[ -z "$key" || -z "$value" || -n "${PARSED_ENV[$key]+x}" ]]; then
fail "duplicate or empty evidence field: $file"
return 1
fi
allowed=false
for expected_key in "$@"; do
[[ "$key" == "$expected_key" ]] && allowed=true
done
if [[ "$allowed" != true ]]; then
fail "unexpected evidence field $key: $file"
return 1
fi
PARSED_ENV[$key]=$value
count=$((count + 1))
done < <(root_cat "$file")
if (( count != expected )); then
fail "evidence field count is not exact: $file"
return 1
fi
for expected_key in "$@"; do
if [[ -z "${PARSED_ENV[$expected_key]:-}" ]]; then
fail "missing evidence field $expected_key: $file"
return 1
fi
done
return 0
}
validate_slack_acceptance_tuple() {
local tuple
tuple="${PARSED_ENV[slack_deployment_gate]}|${PARSED_ENV[slack_gate_approval_ref]}|${PARSED_ENV[slack_gate_accepted_by_uid]}"
case "$tuple" in
'RECOVERY|strict-recovery-evidence-v1|not-applicable'|\
'RISK_ACCEPTED|2026-08-14-observability-slack-recovery-risk-acceptance-design|1000')
return 0
;;
*)
fail 'rules-alerts Slack deployment gate tuple mismatch'
return 1
;;
esac
}
validate_recent_utc() {
local value=$1 label=$2 then now age
if [[ ! "$value" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then
fail "$label timestamp format is invalid"
return 1
fi
if ! then="$(date -u -d "$value" +%s 2>/dev/null)"; then
fail "$label timestamp is invalid"
return 1
fi
if ! now="$(date -u +%s)"; then
fail "cannot read current UTC time for $label"
return 1
fi
age=$((now - then))
if (( age < -300 || age > 86400 )); then
fail "$label evidence is stale or from the future"
return 1
fi
return 0
}
validate_utc() {
local value=$1 label=$2
if [[ ! "$value" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then
fail "$label timestamp format is invalid"
return 1
fi
if ! date -u -d "$value" +%s >/dev/null 2>&1; then
fail "$label timestamp is invalid"
return 1
fi
return 0
}
inventory_sha() {
local root=$1 phase=$2 directory checksum actual declared filename extra directory_mode json_mode checksum_mode
directory="$root/$phase"
checksum="$directory/inventory.sha256"
if [[ ! -d "$directory" || -L "$directory" ]]; then
fail "inventory phase is missing or unsafe: $phase"
return 1
fi
if [[ ! -f "$directory/inventory.json" || -L "$directory/inventory.json" ]]; then
fail "inventory JSON is unsafe: $phase"
return 1
fi
if [[ ! -f "$checksum" || -L "$checksum" ]]; then
fail "inventory checksum is unsafe: $phase"
return 1
fi
if ! directory_mode="$(stat -c %a -- "$directory")"; then
fail "cannot stat inventory directory: $phase"
return 1
fi
if [[ "$directory_mode" != 700 ]]; then
fail "inventory directory mode is not 0700: $phase"
return 1
fi
if ! json_mode="$(stat -c %a -- "$directory/inventory.json")" ||
! checksum_mode="$(stat -c %a -- "$checksum")"; then
fail "cannot stat inventory files: $phase"
return 1
fi
if [[ "$json_mode" != 600 || "$checksum_mode" != 600 ]]; then
fail "inventory file mode is not 0600: $phase"
return 1
fi
if ! read -r declared filename extra <"$checksum"; then
fail "cannot read inventory checksum: $phase"
return 1
fi
if [[ "$filename" != inventory.json || -n "${extra:-}" ]]; then
fail "inventory checksum shape is invalid: $phase"
return 1
fi
if ! is_sha256 "$declared"; then
fail "inventory checksum is invalid: $phase"
return 1
fi
if ! actual="$(sha_of "$directory/inventory.json")"; then
fail "cannot hash inventory JSON: $phase"
return 1
fi
if [[ "$actual" != "$declared" ]]; then
fail "inventory checksum mismatch: $phase"
return 1
fi
printf '%s' "$actual" || return 1
return 0
}
validate_full_evidence() {
local deny_sha=$1 acceptance proof initial_sha post_sha canonical_metric_root
local metric_root_mode
if [[ -z "$verified_output_dir" ]]; then
fail 'full execute requires --verified-output-dir'
return 1
fi
if ! canonical_metric_root="$(readlink -f -- "$verified_output_dir" 2>/dev/null)"; then
fail 'verified output dir does not resolve'
return 1
fi
if [[ "$canonical_metric_root" != "$verified_output_dir" || ! -d "$verified_output_dir" || -L "$verified_output_dir" ]]; then
fail 'verified output dir must be a canonical regular directory'
return 1
fi
if ! metric_root_mode="$(stat -c %a -- "$verified_output_dir")"; then
fail 'cannot stat verified output root'
return 1
fi
if [[ "$metric_root_mode" != 700 ]]; then
fail 'verified output root mode must be 0700'
return 1
fi
if ! initial_sha="$(inventory_sha "$verified_output_dir" target-initial)"; then
return 1
fi
if ! post_sha="$(inventory_sha "$verified_output_dir" post-substrate)"; then
return 1
fi
proof="$ROLLBACK_ROOT/blackbox-source-proof.env"
if ! read_exact_env "$proof" schema rollback_id nginx_sha256 tested_at_utc \
grafana_remote_addr grafana_status storage_admin_remote_addr storage_admin_status \
db_admin_remote_addr db_admin_status; then
return 1
fi
if [[ "${PARSED_ENV[schema]}" != "$SOURCE_PROOF_SCHEMA" ]]; then
fail 'source-proof schema mismatch'
return 1
fi
if [[ "${PARSED_ENV[rollback_id]}" != "$PLATFORM_OBSERVABILITY_ROLLBACK_ID" ]]; then
fail 'source-proof rollback ID mismatch'
return 1
fi
if [[ "${PARSED_ENV[nginx_sha256]}" != "$deny_sha" ]]; then
fail 'source-proof Nginx hash mismatch'
return 1
fi
if [[ "${PARSED_ENV[grafana_status]}" != 403 || "${PARSED_ENV[storage_admin_status]}" != 403 ||
"${PARSED_ENV[db_admin_status]}" != 403 ]]; then
fail 'source-proof statuses are not exact 403'
return 1
fi
if ! python3 - "${PARSED_ENV[grafana_remote_addr]}" "${PARSED_ENV[storage_admin_remote_addr]}" \
"${PARSED_ENV[db_admin_remote_addr]}" <<'PY'
import ipaddress
import sys
for value in sys.argv[1:]:
ipaddress.ip_address(value)
PY
then
fail 'source-proof remote address is invalid'
return 1
fi
if ! validate_recent_utc "${PARSED_ENV[tested_at_utc]}" source-proof; then
return 1
fi
acceptance="$ROLLBACK_ROOT/access-rules-alerts/acceptance.env"
if ! read_exact_env "$acceptance" \
schema rollback_id target_initial_sha256 post_substrate_sha256 \
slack_deployment_gate slack_gate_approval_ref slack_gate_accepted_by_uid \
accepted_at_utc state; then
return 1
fi
if [[ "${PARSED_ENV[schema]}" != "$RULES_ACCEPTANCE_SCHEMA" ]]; then
fail 'rules-alerts acceptance schema mismatch'
return 1
fi
if [[ "${PARSED_ENV[rollback_id]}" != "$PLATFORM_OBSERVABILITY_ROLLBACK_ID" ]]; then
fail 'rules-alerts rollback ID mismatch'
return 1
fi
if [[ "${PARSED_ENV[target_initial_sha256]}" != "$initial_sha" ]]; then
fail 'target-initial acceptance hash mismatch'
return 1
fi
if [[ "${PARSED_ENV[post_substrate_sha256]}" != "$post_sha" ]]; then
fail 'post-substrate acceptance hash mismatch'
return 1
fi
if ! validate_slack_acceptance_tuple; then
return 1
fi
if [[ "${PARSED_ENV[state]}" != accepted ]]; then
fail 'rules-alerts state is not accepted'
return 1
fi
if ! validate_utc "${PARSED_ENV[accepted_at_utc]}" rules-alerts; then
return 1
fi
return 0
}
validate_exact_certificate() {
local san_output san_set
if ! "$SUDO_BIN" -n "$TEST_BIN" -s "$CERT_DIR/fullchain.pem"; then
fail 'Grafana certificate is missing'
return 1
fi
if ! "$SUDO_BIN" -n "$TEST_BIN" -s "$CERT_DIR/privkey.pem"; then
fail 'Grafana private key is missing'
return 1
fi
if ! "$SUDO_BIN" -n openssl x509 -in "$CERT_DIR/fullchain.pem" -noout -checkend 0 >/dev/null; then
fail 'Grafana certificate is expired'
return 1
fi
if ! san_output="$("$SUDO_BIN" -n openssl x509 -in "$CERT_DIR/fullchain.pem" -noout -ext subjectAltName)"; then
fail 'cannot read Grafana certificate SAN'
return 1
fi
if ! san_set="$(printf '%s\n' "$san_output" | grep -oE 'DNS:[^,[:space:]]+' | sed 's/^DNS://' | sort -u)"; then
fail 'cannot parse Grafana certificate SAN'
return 1
fi
if [[ "$san_set" != "$GRAFANA_HOST" ]]; then
fail 'Grafana certificate SAN set is not exact'
return 1
fi
return 0
}
validate_cloudflare_credentials() {
local contract
"$SUDO_BIN" -n "$TEST_BIN" -f "$CREDENTIALS" || fail "Cloudflare credential is missing: $CREDENTIALS"
expected_owner_mode "$CREDENTIALS" 600 || fail 'Cloudflare credential must be root:root 0600'
contract="$("$SUDO_BIN" -n awk -F= '
BEGIN { count=0; valid=0; unexpected=0 }
/^[[:space:]]*($|#)/ { next }
/^[[:space:]]*dns_cloudflare_api_token[[:space:]]*=/ {
count++; value=$0; sub(/^[^=]*=[[:space:]]*/, "", value); sub(/[[:space:]]+$/, "", value)
if (length(value) >= 20) valid++; next
}
{ unexpected++ }
END { printf "%d:%d:%d", count, valid, unexpected }
' "$CREDENTIALS")"
[[ "$contract" == 1:1:0 ]] || fail 'Cloudflare credential must contain one nonempty API token key only'
}
certificate_mode() {
if validate_exact_certificate 2>/dev/null; then
printf 'HOST_NGINX_GRAFANA_CERTIFICATE=READY\n'
return 0
fi
[[ "$certbot_email" == *@* ]] || fail 'missing exact certificate; --certbot-email is required for issuance'
validate_cloudflare_credentials
[[ -x "$CERTBOT_BIN" ]] || fail "supported Certbot is missing: $CERTBOT_BIN"
"$CERTBOT_BIN" plugins 2>/dev/null | grep -q dns-cloudflare || fail 'Certbot dns-cloudflare plugin is missing'
confirm "issue the exact Grafana certificate"
"$SUDO_BIN" "$CERTBOT_BIN" certonly --dns-cloudflare \
--dns-cloudflare-credentials "$CREDENTIALS" --dns-cloudflare-propagation-seconds 60 \
--cert-name "$CERT_NAME" --domains "$GRAFANA_HOST" --non-interactive --agree-tos \
--email "$certbot_email"
validate_exact_certificate
printf 'HOST_NGINX_GRAFANA_CERTIFICATE=READY\n'
}
status_code() {
local code
if code="$(curl --disable --noproxy '*' --silent --show-error --output /dev/null \
--connect-timeout 2 --max-time 5 --write-out '%{http_code}' "$@" 2>/dev/null)"; then
printf '%s' "$code"
else
printf 000
fi
}
external_preflight() {
local stage=$1 code host
"$SYSTEMCTL_BIN" is-active --quiet nginx || fail 'Nginx is not active'
"$SUDO_BIN" -n "$NGINX_BIN" -t
if [[ "$TEST_MODE" == 1 ]]; then
"$PREFLIGHT_COMMAND" "$stage"
return
fi
case "$stage" in
metrics) return 0 ;;
deny)
validate_exact_certificate
[[ -z "$(ss -H -ltn '( sport = :1 )')" ]] || fail 'TCP 127.0.0.1:1 guard listener is not absent'
return 0
;;
full) ;;
*) fail "unknown transition stage: $stage" ;;
esac
validate_exact_certificate
[[ "$(dig +time=2 +tries=1 +short "@$LAN_IP" A "$GRAFANA_HOST" | tail -n1)" == "$LAN_IP" ]] || fail 'LAN private DNS answer mismatch'
[[ "$(dig +time=2 +tries=1 +short "@$TAIL_IP" A "$GRAFANA_HOST" | tail -n1)" == "$TAIL_IP" ]] || fail 'Tailscale private DNS answer mismatch'
[[ -z "$(dig +time=2 +tries=1 +short @1.1.1.1 A "$GRAFANA_HOST" | tr -d '[:space:]')" ]] || fail 'Grafana public A exists'
[[ -z "$(dig +time=2 +tries=1 +short @1.1.1.1 AAAA "$GRAFANA_HOST" | tr -d '[:space:]')" ]] || fail 'Grafana public AAAA exists'
kubectl -n observability exec deployment/grafana -c grafana --request-timeout=10s -- \
getent ahostsv4 "$GRAFANA_HOST" | awk -v ip="$LAN_IP" '$1 == ip { found=1 } END { exit !found }' ||
fail 'Grafana Pod private DNS answer mismatch'
kubectl -n observability get ingress grafana -o json --request-timeout=10s | jq -e --arg host "$GRAFANA_HOST" '
.spec.ingressClassName == "traefik" and (.spec.tls // [] | length == 0) and
([.spec.rules[] | select(.host == $host) | .http.paths[] |
select(.path == "/" and .pathType == "Prefix" and
.backend.service.name == "grafana" and .backend.service.port.number == 80)] | length == 1)
' >/dev/null || fail 'Grafana Ingress contract is not ready'
kubectl -n observability get endpointslice -l kubernetes.io/service-name=grafana -o json --request-timeout=10s |
jq -e '[.items[].endpoints[] | select(.conditions.ready == true) | .addresses[]] | length > 0' >/dev/null ||
fail 'Grafana EndpointSlice has no ready address'
[[ "$(status_code --header "Host: $GRAFANA_HOST" http://127.0.0.1:30080/api/health)" == 200 ]] || fail 'Grafana internal health is not 200'
[[ "$(status_code --resolve "$GITEA_HOST:443:127.0.0.1" "https://$GITEA_HOST/api/healthz")" == 200 ]] || fail 'Gitea regression failed'
[[ "$(status_code --resolve "$KEYCLOAK_HOST:443:127.0.0.1" "https://$KEYCLOAK_HOST/realms/hyeonworks/.well-known/openid-configuration")" == 200 ]] || fail 'Keycloak regression failed'
for host in "$STORAGE_HOST" "$DB_HOST"; do
code="$(status_code --interface "$LAN_IP" --resolve "$host:443:$LAN_IP" "https://$host/")"
[[ "$code" == 200 || "$code" == 302 || "$code" == 303 ]] || fail "admin endpoint regression failed: $host=$code"
done
[[ "$(status_code http://$LAN_IP:30080/)" == 000 ]] || fail 'LAN NodePort 30080 is reachable'
[[ "$(status_code --insecure https://$LAN_IP:30443/)" == 000 ]] || fail 'LAN NodePort 30443 is reachable'
}
stage_postcheck_once() {
local stage=$1
local code host
[[ "$(status_code --resolve "$GITEA_HOST:443:127.0.0.1" "https://$GITEA_HOST/metrics")" == 404 ]] || return 1
[[ "$(status_code --resolve "$GITEA_HOST:443:127.0.0.1" "https://$GITEA_HOST/api/healthz")" == 200 ]] || return 1
[[ "$(status_code --resolve "$KEYCLOAK_HOST:443:127.0.0.1" "https://$KEYCLOAK_HOST/realms/hyeonworks/.well-known/openid-configuration")" == 200 ]] || return 1
for host in "$STORAGE_HOST" "$DB_HOST"; do
code="$(status_code --interface "$LAN_IP" --resolve "$host:443:$LAN_IP" "https://$host/")"
[[ "$code" == 200 || "$code" == 302 || "$code" == 303 ]] || return 1
[[ "$(status_code --resolve "$host:443:127.0.0.1" "https://$host/")" == 403 ]] || return 1
done
[[ "$(status_code http://$LAN_IP:30080/)" == 000 ]] || return 1
[[ "$(status_code --insecure https://$LAN_IP:30443/)" == 000 ]] || return 1
[[ "$stage" == metrics ]] && return 0
[[ "$(status_code --resolve "$GRAFANA_HOST:443:127.0.0.1" "https://$GRAFANA_HOST/")" == 403 ]] || return 1
[[ "$(status_code --resolve "$GRAFANA_HOST:443:127.0.0.1" "https://$GRAFANA_HOST/metrics")" == 404 ]] || return 1
code="$(status_code --interface "$LAN_IP" --resolve "$GRAFANA_HOST:443:$LAN_IP" "https://$GRAFANA_HOST/")"
if [[ "$stage" == deny ]]; then
[[ "$code" == 502 ]] || return 1
else
[[ "$stage" == full && ( "$code" == 200 || "$code" == 302 ) ]] || return 1
fi
}
external_postcheck() {
local stage=$1 stable=0 attempt
"$SYSTEMCTL_BIN" is-active --quiet nginx || return 1
for ((attempt=1; attempt<=15; attempt++)); do
if { [[ "$TEST_MODE" == 1 ]] && "$POSTCHECK_COMMAND" "$stage"; } ||
{ [[ "$TEST_MODE" == 0 ]] && stage_postcheck_once "$stage"; }; then
stable=$((stable + 1))
(( stable >= 3 )) && break
else
stable=0
fi
[[ "$TEST_MODE" == 0 && attempt -lt 15 ]] && sleep 1
done
(( stable >= 3 )) || return 1
[[ "$TEST_MODE" == 1 ]] && return 0
if curl --disable --noproxy '*' --insecure --silent --output /dev/null --connect-timeout 2 --max-time 5 \
--resolve unconfigured.invalid:443:127.0.0.1 https://unconfigured.invalid/ 2>/dev/null; then
return 1
fi
}
confirm() {
local description=$1 answer
printf 'Type APPLY to %s: ' "$description"
if [[ "$TEST_MODE" == 1 ]]; then
answer=${PLATFORM_HOST_NGINX_CONFIRMATION:-}
printf '%s\n' "$answer"
else
[[ -t 0 ]] || fail '--execute requires an interactive terminal'
read -r answer
fi
[[ "$answer" == APPLY ]] || fail 'cancelled'
}
validate_full_first_cutover_residue_absent() {
local host_root child
host_root="$ROLLBACK_ROOT/host-nginx"
if ! "$SUDO_BIN" -n "$TEST_BIN" ! -e "$host_root" ||
! "$SUDO_BIN" -n "$TEST_BIN" ! -L "$host_root"; then
fail 'full first-cutover Host Nginx evidence root already exists or is unsafe'
return 1
fi
for child in \
"$host_root/stages.tsv" \
"$host_root/payloads" \
"$host_root/payloads/full-prior-0001.conf"; do
if ! "$SUDO_BIN" -n "$TEST_BIN" ! -e "$child" ||
! "$SUDO_BIN" -n "$TEST_BIN" ! -L "$child"; then
fail 'full first-cutover Host Nginx evidence residue exists or is unsafe'
return 1
fi
done
if ! "$SUDO_BIN" -n "$TEST_BIN" ! -e "$host_root" ||
! "$SUDO_BIN" -n "$TEST_BIN" ! -L "$host_root"; then
fail 'full first-cutover Host Nginx evidence root changed during validation'
return 1
fi
}
prepare_stage_ledger() {
local stage=$1 previous_sha=$2 candidate_sha=$3 host_root payload_root ledger attempt payload_rel payload tmp
host_root="$ROLLBACK_ROOT/host-nginx"
payload_root="$host_root/payloads"
ledger="$host_root/stages.tsv"
if ! ensure_root_directory_no_follow "$host_root"; then
return 1
fi
if ! ensure_root_directory_no_follow "$payload_root"; then
return 1
fi
if "$SUDO_BIN" -n "$TEST_BIN" -L "$ledger"; then
fail 'Host Nginx ledger must not be a symlink'
return 1
elif "$SUDO_BIN" -n "$TEST_BIN" -e "$ledger"; then
if ! validate_root_file_no_follow "$ledger" 600; then
return 1
fi
if [[ "$(root_cat "$ledger" | head -n1)" != "$STAGE_LEDGER_HEADER" ]]; then
fail 'Host Nginx ledger header mismatch'
return 1
fi
else
tmp="$(mktemp /tmp/platform-host-nginx-ledger.XXXXXX)"
printf '%s\n' "$STAGE_LEDGER_HEADER" >"$tmp"
if ! "$SUDO_BIN" install -o root -g root -m 0600 "$tmp" "$ledger"; then
rm -f -- "$tmp"
return 1
fi
rm -f -- "$tmp" || return 1
if ! validate_root_file_no_follow "$ledger" 600; then
return 1
fi
fi
attempt=$(( $(root_cat "$ledger" | awk -F '\t' -v stage="$stage" 'NR > 1 && $1 == stage { count++ } END { print count+1 }') ))
printf -v payload_rel 'payloads/%s-prior-%04d.conf' "$stage" "$attempt"
payload="$host_root/$payload_rel"
if "$SUDO_BIN" -n "$TEST_BIN" -L "$payload" || "$SUDO_BIN" -n "$TEST_BIN" -e "$payload"; then
fail "ledger payload already exists or is a symlink: $payload_rel"
return 1
fi
"$SUDO_BIN" install -o root -g root -m 0600 "$ACTIVE" "$payload"
if ! validate_root_file_no_follow "$payload" 600; then
return 1
fi
payload_sha="$(root_cat "$payload" | sha256sum | awk '{print $1}')"
[[ "$payload_sha" == "$previous_sha" ]] || fail 'ledger payload hash mismatch'
tmp="$(mktemp /tmp/platform-host-nginx-ledger.XXXXXX)"
root_cat "$ledger" >"$tmp"
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$stage" "$previous_sha" "$candidate_sha" \
"$payload_rel" "$payload_sha" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >>"$tmp"
if ! "$SUDO_BIN" install -o root -g root -m 0600 "$tmp" "$ledger"; then
rm -f -- "$tmp"
return 1
fi
rm -f -- "$tmp" || return 1
rollback_payload=$payload
rollback_previous_sha=$previous_sha
rollback_stage=$stage
return 0
}
rollback() {
local rc=0 actual
set +e
"$SUDO_BIN" -n "$TEST_BIN" -f "$rollback_payload" || rc=1
"$SUDO_BIN" -n "$TEST_BIN" -L "$rollback_payload" && rc=1
expected_owner_mode "$rollback_payload" 600 || rc=1
actual="$(root_cat "$rollback_payload" 2>/dev/null | sha256sum | awk '{print $1}')"
[[ "$actual" == "$rollback_previous_sha" ]] || rc=1
if (( rc == 0 )); then
"$SUDO_BIN" install -o root -g root -m 0644 "$rollback_payload" "$ACTIVE" || rc=1
[[ "$(sha_of "$ACTIVE")" == "$rollback_previous_sha" ]] || rc=1
"$SUDO_BIN" "$NGINX_BIN" -t || rc=1
"$SUDO_BIN" "$SYSTEMCTL_BIN" reload nginx || rc=1
"$SUDO_BIN" "$SYSTEMCTL_BIN" is-active --quiet nginx || rc=1
fi
if (( rc == 0 )); then
rollback_armed=false
printf 'HOST_NGINX_OBSERVABILITY_ROLLBACK=PASS\n' >&2
else
printf 'HOST_NGINX_OBSERVABILITY_ROLLBACK=FAIL\nMANUAL_RECOVERY_REQUIRED=YES\n' >&2
fi
set -e
return "$rc"
}
on_exit() {
local rc=$?
trap - EXIT INT TERM
if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then
rollback || true
fi
exit "$rc"
}
transition() {
local stage=$1 expected=$2 candidate=$3 description=$4 previous_sha candidate_sha active_sha
previous_sha="$(sha_of "$expected")"
candidate_sha="$(sha_of "$candidate")"
active_sha="$(sha_of "$ACTIVE")"
if [[ "$stage" == full ]]; then
if ! validate_full_evidence "$(sha_of "$DENY_CANDIDATE")"; then
return 1
fi
fi
if [[ "$stage" == full && "$active_sha" == "$candidate_sha" ]]; then
fail 'full cutover is already active without first-cutover provenance'
return 1
fi
if [[ "$active_sha" == "$candidate_sha" ]]; then
external_preflight "$stage"
external_postcheck "$stage" || fail "already-active $stage stage is unhealthy"
printf 'HOST_NGINX_%s_STAGE=ALREADY_ACTIVE\n' "${stage^^}"
return 0
fi
[[ "$active_sha" == "$previous_sha" ]] || fail "$stage active hash gate failed: $active_sha"
if [[ "$stage" == full ]]; then
validate_full_first_cutover_residue_absent || return 1
fi
external_preflight "$stage"
[[ "$(sha_of "$ACTIVE")" == "$previous_sha" ]] || fail "$stage active config changed during preflight"
printf 'HOST_NGINX_STAGE=%s\nHOST_NGINX_PREVIOUS_SHA256=%s\nHOST_NGINX_CANDIDATE_SHA256=%s\n' \
"$stage" "$previous_sha" "$candidate_sha"
confirm "$description"
[[ "$(sha_of "$ACTIVE")" == "$previous_sha" ]] || fail "$stage active config changed before commit"
[[ "$(sha_of "$candidate")" == "$candidate_sha" ]] || fail "$stage candidate changed before commit"
if ! prepare_stage_ledger "$stage" "$previous_sha" "$candidate_sha"; then
fail 'fresh rollback ID + source proof + complete Task 6 required'
return 1
fi
timestamp_backup="${ACTIVE}.before-observability-${stage}-$(date -u +%Y%m%dT%H%M%SZ)"
if "$SUDO_BIN" -n "$TEST_BIN" -e "$timestamp_backup" || "$SUDO_BIN" -n "$TEST_BIN" -L "$timestamp_backup"; then
fail "timestamp backup already exists or is unsafe: $timestamp_backup"
fail 'fresh rollback ID + source proof + complete Task 6 required'
return 1
fi
if ! "$SUDO_BIN" install -o root -g root -m 0644 "$ACTIVE" "$timestamp_backup"; then
fail 'fresh rollback ID + source proof + complete Task 6 required'
return 1
fi
if [[ "$(sha_of "$timestamp_backup")" != "$previous_sha" ]]; then
fail 'timestamp backup hash mismatch'
fail 'fresh rollback ID + source proof + complete Task 6 required'
return 1
fi
rollback_armed=true
"$SUDO_BIN" install -o root -g root -m 0644 "$candidate" "$ACTIVE"
[[ "$(sha_of "$ACTIVE")" == "$candidate_sha" ]] || fail 'installed active hash mismatch'
"$SUDO_BIN" "$SYSTEMCTL_BIN" daemon-reload
"$SUDO_BIN" "$NGINX_BIN" -t
"$SUDO_BIN" "$SYSTEMCTL_BIN" reload nginx
"$SUDO_BIN" "$SYSTEMCTL_BIN" is-active --quiet nginx
external_postcheck "$stage" || fail "$stage bounded postcheck failed"
rollback_armed=false
printf 'HOST_NGINX_%s_STAGE=PASS\nHOST_NGINX_TIMESTAMP_BACKUP=%s\n' "${stage^^}" "$timestamp_backup"
}
main() {
parse_args "$@"
validate_test_mode
if [[ "$TEST_MODE" == 0 ]]; then
validate_production_command_boundaries
fi
source_contract
if [[ "$execute" == false ]]; then
print_plan
return 0
fi
for cmd in awk cat curl date dig grep head install jq kubectl mktemp openssl python3 readlink sed sha256sum sort ss stat; do
command -v "$cmd" >/dev/null 2>&1 || fail "required command is missing: $cmd"
done
[[ -x "$NGINX_BIN" && -x "$SYSTEMCTL_BIN" && -x "$SUDO_BIN" ]] || fail 'required host command boundary is missing'
"$SUDO_BIN" -v
validate_host_files
validate_rollback_root
trap on_exit EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
case "$mode" in
metrics) transition metrics "$ADMIN_CANDIDATE" "$METRICS_CANDIDATE" 'install only the Gitea metrics guard' ;;
certificate) certificate_mode ;;
deny) transition deny "$METRICS_CANDIDATE" "$DENY_CANDIDATE" 'install only the Grafana deny guard' ;;
full) transition full "$DENY_CANDIDATE" "$FULL_CANDIDATE" 'replace only the Grafana deny upstream with Traefik' ;;
*) fail "invalid mode: $mode" ;;
esac
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
main "$@"
fi
Reference in New Issue View Git Blame Copy Permalink