300 lines
10 KiB
Bash
Executable File
300 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -Eeuo pipefail
|
|
set +x
|
|
umask 077
|
|
|
|
readonly EXPECTED_CANDIDATE_SHA256="5c5cd74b4992f537fd27c50cf2209573a80a9904e0b19b58c3154717be6ff4a5"
|
|
readonly EXPECTED_PRE_CUTOVER_SHA256="de7ebd4f69cd7d2204ee633e074bf6a4370a6f5e3f3fac9067b099d5d75269b5"
|
|
readonly REPOSITORY_ROOT="/home/donghyeon/workspace/platform"
|
|
readonly CANDIDATE="${REPOSITORY_ROOT}/infrastructure/networking/host-nginx/learn-services-keycloak.conf"
|
|
readonly ACTIVE="/etc/nginx/sites-available/learn-services"
|
|
readonly ENABLED="/etc/nginx/sites-enabled/learn-services"
|
|
readonly GITEA_HOST="git.learn.hyeonworks.com"
|
|
readonly KEYCLOAK_HOST="id.learn.hyeonworks.com"
|
|
readonly KEYCLOAK_ISSUER="https://${KEYCLOAK_HOST}/realms/hyeonworks"
|
|
readonly CURL_BIN="/usr/bin/curl"
|
|
readonly JQ_BIN="/usr/bin/jq"
|
|
readonly NGINX_BIN="/usr/sbin/nginx"
|
|
readonly SYSTEMCTL_BIN="/usr/bin/systemctl"
|
|
readonly INSTALL_BIN="/usr/bin/install"
|
|
readonly SHA256SUM_BIN="/usr/bin/sha256sum"
|
|
readonly STAT_BIN="/usr/bin/stat"
|
|
readonly READLINK_BIN="/usr/bin/readlink"
|
|
readonly MKTEMP_BIN="/usr/bin/mktemp"
|
|
readonly RM_BIN="/usr/bin/rm"
|
|
readonly DATE_BIN="/usr/bin/date"
|
|
readonly AWK_BIN="/usr/bin/awk"
|
|
readonly SLEEP_BIN="/usr/bin/sleep"
|
|
|
|
fail() {
|
|
printf 'ERROR: %s\n' "$*" >&2
|
|
return 1
|
|
}
|
|
|
|
usage() {
|
|
printf '%s\n' \
|
|
'Usage: sudo bash scripts/bootstrap/apply-host-nginx-keycloak.sh --execute' \
|
|
'' \
|
|
'Replaces the static id.learn response with the reviewed Keycloak proxy.' \
|
|
'The active site is backed up first; a failed test or health check restores' \
|
|
'the prior Gitea-only configuration and reloads Nginx automatically.'
|
|
}
|
|
|
|
[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || {
|
|
usage
|
|
exit 2
|
|
}
|
|
[[ "$EUID" -eq 0 ]] || fail "run this script through sudo"
|
|
[[ -t 0 ]] || fail "an interactive terminal is required"
|
|
|
|
for required_binary in \
|
|
"$CURL_BIN" "$JQ_BIN" "$NGINX_BIN" "$SYSTEMCTL_BIN" "$INSTALL_BIN" \
|
|
"$SHA256SUM_BIN" "$STAT_BIN" "$READLINK_BIN" "$MKTEMP_BIN" \
|
|
"$RM_BIN" "$DATE_BIN" "$AWK_BIN" "$SLEEP_BIN"; do
|
|
[[ -x "$required_binary" ]] || fail "required executable is missing: ${required_binary}"
|
|
done
|
|
|
|
[[ -f "$CANDIDATE" && ! -L "$CANDIDATE" ]] || fail "unsafe candidate: ${CANDIDATE}"
|
|
[[ -f "$ACTIVE" && ! -L "$ACTIVE" ]] || fail "unsafe active file: ${ACTIVE}"
|
|
[[ -L "$ENABLED" ]] || fail "enabled path is not a symlink: ${ENABLED}"
|
|
[[ "$("$READLINK_BIN" -f "$ENABLED")" == "$ACTIVE" ]] || \
|
|
fail "enabled symlink target changed"
|
|
[[ "$("$STAT_BIN" --format='%U:%G %a' "$ACTIVE")" == "root:root 644" ]] || \
|
|
fail "active file owner or mode changed"
|
|
"$SYSTEMCTL_BIN" is-active --quiet nginx || fail "nginx is not active"
|
|
|
|
readonly TEMP_DIR="$("$MKTEMP_BIN" -d /tmp/nginx-keycloak-cutover.XXXXXX)"
|
|
readonly CANDIDATE_SNAPSHOT="${TEMP_DIR}/learn-services.candidate"
|
|
readonly GITEA_HEALTH="${TEMP_DIR}/gitea-health.json"
|
|
readonly KEYCLOAK_DISCOVERY="${TEMP_DIR}/keycloak-discovery.json"
|
|
readonly LOGIN_HEADERS="${TEMP_DIR}/gitea-login-headers"
|
|
rollback_armed=0
|
|
backup=""
|
|
|
|
cleanup() {
|
|
case "$TEMP_DIR" in
|
|
/tmp/nginx-keycloak-cutover.*)
|
|
"$RM_BIN" -rf -- "$TEMP_DIR"
|
|
;;
|
|
*)
|
|
printf 'WARNING: refusing to remove unexpected temp path: %s\n' \
|
|
"$TEMP_DIR" >&2
|
|
;;
|
|
esac
|
|
}
|
|
|
|
check_gitea_health() {
|
|
"$JQ_BIN" -e '
|
|
.status == "pass" and
|
|
([.checks["database:ping"][], .checks["cache:ping"][]] |
|
|
all(.status == "pass"))
|
|
' "$GITEA_HEALTH" >/dev/null
|
|
}
|
|
|
|
check_keycloak_discovery() {
|
|
"$JQ_BIN" -e \
|
|
--arg issuer "$KEYCLOAK_ISSUER" '
|
|
.issuer == $issuer and
|
|
(.authorization_endpoint | startswith($issuer)) and
|
|
(.token_endpoint | startswith($issuer)) and
|
|
(.userinfo_endpoint | startswith($issuer)) and
|
|
(.jwks_uri | startswith($issuer))
|
|
' "$KEYCLOAK_DISCOVERY" >/dev/null
|
|
}
|
|
|
|
pre_cutover_checks() {
|
|
"$CURL_BIN" --disable --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \
|
|
--header "Host: ${GITEA_HOST}" \
|
|
--header 'X-Forwarded-Proto: https' \
|
|
http://127.0.0.1:30080/api/healthz >"$GITEA_HEALTH"
|
|
check_gitea_health
|
|
|
|
"$CURL_BIN" --disable --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \
|
|
--header "Host: ${KEYCLOAK_HOST}" \
|
|
--header 'X-Forwarded-Host: id.learn.hyeonworks.com' \
|
|
--header 'X-Forwarded-Proto: https' \
|
|
--header 'X-Forwarded-Port: 443' \
|
|
"http://127.0.0.1:30080/realms/hyeonworks/.well-known/openid-configuration" \
|
|
>"$KEYCLOAK_DISCOVERY"
|
|
check_keycloak_discovery
|
|
}
|
|
|
|
wait_for_proxy_state() {
|
|
local attempt
|
|
|
|
for ((attempt = 1; attempt <= 12; attempt++)); do
|
|
if "$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \
|
|
--connect-timeout 1 --max-time 3 \
|
|
--resolve "${GITEA_HOST}:443:127.0.0.1" \
|
|
"https://${GITEA_HOST}/api/healthz" >"$GITEA_HEALTH" 2>/dev/null &&
|
|
check_gitea_health 2>/dev/null &&
|
|
"$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \
|
|
--connect-timeout 1 --max-time 3 \
|
|
--resolve "${KEYCLOAK_HOST}:443:127.0.0.1" \
|
|
"https://${KEYCLOAK_HOST}/realms/hyeonworks/.well-known/openid-configuration" \
|
|
>"$KEYCLOAK_DISCOVERY" 2>/dev/null &&
|
|
check_keycloak_discovery 2>/dev/null; then
|
|
if (( attempt > 1 )); then
|
|
printf 'Nginx proxy state stabilized after %d probes.\n' "$attempt"
|
|
fi
|
|
return 0
|
|
fi
|
|
(( attempt < 12 )) && "$SLEEP_BIN" 1
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
wait_for_rollback_state() {
|
|
local attempt
|
|
local id_body
|
|
|
|
for ((attempt = 1; attempt <= 12; attempt++)); do
|
|
id_body="$(
|
|
"$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \
|
|
--connect-timeout 1 --max-time 3 \
|
|
--resolve "${KEYCLOAK_HOST}:443:127.0.0.1" \
|
|
"https://${KEYCLOAK_HOST}/" 2>/dev/null
|
|
)" || id_body=""
|
|
if [[ "$id_body" == "Keycloak domain reached Nginx successfully" ]] &&
|
|
"$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \
|
|
--connect-timeout 1 --max-time 3 \
|
|
--resolve "${GITEA_HOST}:443:127.0.0.1" \
|
|
"https://${GITEA_HOST}/api/healthz" >"$GITEA_HEALTH" 2>/dev/null &&
|
|
check_gitea_health 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
(( attempt < 12 )) && "$SLEEP_BIN" 1
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
rollback() {
|
|
local restore_rc=0
|
|
|
|
set +e
|
|
printf '\nROLLBACK: restoring %s\n' "$backup" >&2
|
|
"$INSTALL_BIN" -o root -g root -m 0644 "$backup" "$ACTIVE" || restore_rc=1
|
|
"$NGINX_BIN" -t || restore_rc=1
|
|
if (( restore_rc == 0 )); then
|
|
"$SYSTEMCTL_BIN" reload nginx || restore_rc=1
|
|
"$SYSTEMCTL_BIN" is-active --quiet nginx || restore_rc=1
|
|
wait_for_rollback_state || restore_rc=1
|
|
fi
|
|
|
|
if (( restore_rc == 0 )); then
|
|
rollback_armed=0
|
|
printf 'ROLLBACK complete. The Gitea proxy and static Keycloak hold were restored.\n' >&2
|
|
else
|
|
printf 'CRITICAL: automatic rollback failed; backup remains at %s\n' \
|
|
"$backup" >&2
|
|
fi
|
|
}
|
|
|
|
on_exit() {
|
|
local rc=$?
|
|
|
|
trap - EXIT INT TERM
|
|
if (( rc != 0 && rollback_armed == 1 )); then
|
|
rollback
|
|
fi
|
|
cleanup
|
|
exit "$rc"
|
|
}
|
|
|
|
trap on_exit EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
"$INSTALL_BIN" -o root -g root -m 0600 "$CANDIDATE" "$CANDIDATE_SNAPSHOT"
|
|
readonly CANDIDATE_SHA256="$(
|
|
"$SHA256SUM_BIN" "$CANDIDATE_SNAPSHOT" | "$AWK_BIN" '{print $1}'
|
|
)"
|
|
readonly ACTIVE_SHA256="$(
|
|
"$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}'
|
|
)"
|
|
[[ "$CANDIDATE_SHA256" == "$EXPECTED_CANDIDATE_SHA256" ]] || \
|
|
fail "candidate digest changed: ${CANDIDATE_SHA256}"
|
|
if [[ "$ACTIVE_SHA256" != "$EXPECTED_PRE_CUTOVER_SHA256" &&
|
|
"$ACTIVE_SHA256" != "$CANDIDATE_SHA256" ]]; then
|
|
fail "active config has an unexpected digest: ${ACTIVE_SHA256}"
|
|
fi
|
|
|
|
pre_cutover_checks
|
|
"$NGINX_BIN" -t
|
|
|
|
post_cutover_checks() {
|
|
local host
|
|
local redirect_result
|
|
|
|
wait_for_proxy_state || fail "Gitea and Keycloak HTTPS proxies did not converge"
|
|
|
|
for host in "$GITEA_HOST" "$KEYCLOAK_HOST"; do
|
|
redirect_result="$(
|
|
"$CURL_BIN" --disable --noproxy '*' --silent --show-error --max-time 10 \
|
|
--resolve "${host}:80:127.0.0.1" \
|
|
--output /dev/null \
|
|
--write-out $'%{http_code}\n%{redirect_url}' \
|
|
"http://${host}/"
|
|
)"
|
|
[[ "$redirect_result" == $'301\nhttps://'"${host}/" ]] || \
|
|
fail "${host} HTTP redirect check failed: ${redirect_result}"
|
|
done
|
|
|
|
"$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error --max-time 10 \
|
|
--resolve "${GITEA_HOST}:443:127.0.0.1" \
|
|
--dump-header "$LOGIN_HEADERS" --output /dev/null \
|
|
"https://${GITEA_HOST}/user/login"
|
|
"$AWK_BIN" '
|
|
BEGIN { found = 0; insecure = 0 }
|
|
tolower($0) ~ /^set-cookie:/ {
|
|
found++
|
|
if (tolower($0) !~ /; secure([;[:space:]]|$)/) insecure = 1
|
|
}
|
|
END { exit(found == 0 || insecure) }
|
|
' "$LOGIN_HEADERS" || fail "Gitea login cookie Secure check failed"
|
|
|
|
if "$CURL_BIN" --disable --noproxy '*' --insecure --silent --output /dev/null --max-time 5 \
|
|
--resolve unconfigured.invalid:443:127.0.0.1 \
|
|
https://unconfigured.invalid/ 2>/dev/null; then
|
|
fail "unknown TLS hostname was not rejected"
|
|
fi
|
|
}
|
|
|
|
if [[ "$ACTIVE_SHA256" == "$CANDIDATE_SHA256" ]]; then
|
|
post_cutover_checks
|
|
printf 'The reviewed Keycloak Host Nginx configuration is already active and healthy.\n'
|
|
exit 0
|
|
fi
|
|
|
|
backup="${ACTIVE}.before-keycloak-$("$DATE_BIN" +%Y%m%d%H%M%S)"
|
|
[[ ! -e "$backup" && ! -L "$backup" ]] || fail "backup path already exists: ${backup}"
|
|
|
|
printf '\nActive SHA-256: %s\n' "$ACTIVE_SHA256"
|
|
printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256"
|
|
printf 'Planned backup: %s\n' "$backup"
|
|
printf 'Type APPLY to expose Keycloak through Host Nginx: '
|
|
read -r confirmation
|
|
[[ "$confirmation" == "APPLY" ]] || fail "cancelled"
|
|
|
|
"$INSTALL_BIN" -o root -g root -m 0644 "$ACTIVE" "$backup"
|
|
[[ "$("$SHA256SUM_BIN" "$backup" | "$AWK_BIN" '{print $1}')" == \
|
|
"$ACTIVE_SHA256" ]] || fail "backup digest mismatch"
|
|
|
|
rollback_armed=1
|
|
"$INSTALL_BIN" -o root -g root -m 0644 "$CANDIDATE_SNAPSHOT" "$ACTIVE"
|
|
[[ "$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" == \
|
|
"$CANDIDATE_SHA256" ]] || fail "installed digest mismatch"
|
|
"$NGINX_BIN" -t
|
|
"$SYSTEMCTL_BIN" reload nginx
|
|
"$SYSTEMCTL_BIN" is-active --quiet nginx
|
|
post_cutover_checks
|
|
|
|
rollback_armed=0
|
|
printf '\nKEYCLOAK CUTOVER SUCCESS\n'
|
|
printf 'Backup: %s\n' "$backup"
|
|
printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256"
|
|
printf 'Gitea health, Keycloak discovery issuer, HTTPS cookies, redirects, and unknown-host rejection: PASS\n'
|