Add platform infrastructure configuration
This commit is contained in:
donghyeon-ka
Executable
+351
@@ -0,0 +1,351 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
# Do not expose headers, cookies, or future sensitive values through caller xtrace.
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
readonly EXPECTED_CANDIDATE_SHA256="de7ebd4f69cd7d2204ee633e074bf6a4370a6f5e3f3fac9067b099d5d75269b5"
|
||||
readonly EXPECTED_PRE_CUTOVER_SHA256="5b5941519ab677f751568827aa4f315193dfbd9eeac5e8a4ad85d724aaefe16f"
|
||||
readonly EXPECTED_PRE_CUTOVER_HEALTH_SHA256="6b683cb16987ff1f5ded22e9847ac0a45995947927c459b21f427523a41c7484"
|
||||
readonly REPOSITORY_ROOT="/home/donghyeon/workspace/platform"
|
||||
readonly CANDIDATE="${REPOSITORY_ROOT}/infrastructure/networking/host-nginx/learn-services.conf"
|
||||
readonly ACTIVE="/etc/nginx/sites-available/learn-services"
|
||||
readonly ENABLED="/etc/nginx/sites-enabled/learn-services"
|
||||
readonly CURL_BIN="/usr/bin/curl"
|
||||
readonly JQ_BIN="/usr/bin/jq"
|
||||
readonly NGINX_BIN="/usr/sbin/nginx"
|
||||
readonly SYSTEMCTL_BIN="/usr/bin/systemctl"
|
||||
readonly INSTALL_BIN="/usr/bin/install"
|
||||
readonly SHA256SUM_BIN="/usr/bin/sha256sum"
|
||||
readonly STAT_BIN="/usr/bin/stat"
|
||||
readonly READLINK_BIN="/usr/bin/readlink"
|
||||
readonly MKTEMP_BIN="/usr/bin/mktemp"
|
||||
readonly RM_BIN="/usr/bin/rm"
|
||||
readonly DATE_BIN="/usr/bin/date"
|
||||
readonly AWK_BIN="/usr/bin/awk"
|
||||
readonly SLEEP_BIN="/usr/bin/sleep"
|
||||
|
||||
fail() {
|
||||
printf 'ERROR: %s\n' "$*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
usage() {
|
||||
printf '%s\n' \
|
||||
'Usage: sudo bash scripts/bootstrap/apply-host-nginx-gitea.sh --execute' \
|
||||
'' \
|
||||
'Backs up the active learn-services site, installs the reviewed Gitea proxy,' \
|
||||
'tests and reloads Nginx, then runs local acceptance checks. Any failure after' \
|
||||
'the active file changes triggers an automatic restore and reload.'
|
||||
}
|
||||
|
||||
[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || {
|
||||
usage
|
||||
exit 2
|
||||
}
|
||||
[[ "$EUID" -eq 0 ]] || fail "run this script through sudo"
|
||||
[[ -t 0 ]] || fail "an interactive terminal is required"
|
||||
|
||||
for required_binary in \
|
||||
"$CURL_BIN" "$JQ_BIN" "$NGINX_BIN" "$SYSTEMCTL_BIN" "$INSTALL_BIN" \
|
||||
"$SHA256SUM_BIN" "$STAT_BIN" "$READLINK_BIN" "$MKTEMP_BIN" \
|
||||
"$RM_BIN" "$DATE_BIN" "$AWK_BIN" "$SLEEP_BIN"; do
|
||||
[[ -x "$required_binary" ]] || fail "required executable is missing: ${required_binary}"
|
||||
done
|
||||
|
||||
[[ -f "$CANDIDATE" && ! -L "$CANDIDATE" ]] || fail "unsafe candidate: ${CANDIDATE}"
|
||||
[[ -f "$ACTIVE" && ! -L "$ACTIVE" ]] || fail "unsafe active file: ${ACTIVE}"
|
||||
[[ -L "$ENABLED" ]] || fail "enabled path is not a symlink: ${ENABLED}"
|
||||
[[ "$("$READLINK_BIN" -f "$ENABLED")" == "$ACTIVE" ]] || fail "enabled symlink target changed"
|
||||
[[ "$("$STAT_BIN" --format='%U:%G %a' "$ACTIVE")" == "root:root 644" ]] || \
|
||||
fail "active file owner or mode changed"
|
||||
"$SYSTEMCTL_BIN" is-active --quiet nginx || fail "nginx is not active"
|
||||
|
||||
readonly TEMP_DIR="$("$MKTEMP_BIN" -d /tmp/nginx-gitea-cutover.XXXXXX)"
|
||||
readonly CANDIDATE_SNAPSHOT="${TEMP_DIR}/learn-services.candidate"
|
||||
readonly DIRECT_HEALTH="${TEMP_DIR}/direct-health.json"
|
||||
readonly NGINX_HEALTH="${TEMP_DIR}/nginx-health.json"
|
||||
readonly LOGIN_HEADERS="${TEMP_DIR}/login-headers"
|
||||
rollback_armed=0
|
||||
backup=""
|
||||
|
||||
cleanup() {
|
||||
case "$TEMP_DIR" in
|
||||
/tmp/nginx-gitea-cutover.*)
|
||||
"$RM_BIN" -rf -- "$TEMP_DIR"
|
||||
;;
|
||||
*)
|
||||
printf 'WARNING: refusing to remove unexpected temp path: %s\n' "$TEMP_DIR" >&2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
rollback() {
|
||||
local restore_install_rc
|
||||
local restore_test_rc
|
||||
local restore_reload_rc
|
||||
|
||||
set +e
|
||||
printf '\nROLLBACK: restoring %s\n' "$backup" >&2
|
||||
if "$INSTALL_BIN" -o root -g root -m 0644 "$backup" "$ACTIVE" &&
|
||||
[[ "$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" == "$("$SHA256SUM_BIN" "$backup" | "$AWK_BIN" '{print $1}')" ]]; then
|
||||
restore_install_rc=0
|
||||
else
|
||||
restore_install_rc=1
|
||||
fi
|
||||
"$NGINX_BIN" -t
|
||||
restore_test_rc=$?
|
||||
if (( restore_install_rc == 0 && restore_test_rc == 0 )); then
|
||||
if "$SYSTEMCTL_BIN" reload nginx &&
|
||||
"$SYSTEMCTL_BIN" is-active --quiet nginx &&
|
||||
wait_for_rollback_state; then
|
||||
restore_reload_rc=0
|
||||
else
|
||||
restore_reload_rc=1
|
||||
fi
|
||||
else
|
||||
restore_reload_rc=1
|
||||
fi
|
||||
|
||||
if (( restore_install_rc == 0 && restore_test_rc == 0 && restore_reload_rc == 0 )); then
|
||||
rollback_armed=0
|
||||
printf 'ROLLBACK complete. Active config and response were restored.\n' >&2
|
||||
else
|
||||
printf 'CRITICAL: automatic rollback failed; backup remains at %s\n' "$backup" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
local rc=$?
|
||||
|
||||
trap - EXIT INT TERM
|
||||
if (( rc != 0 && rollback_armed == 1 )); then
|
||||
rollback
|
||||
fi
|
||||
cleanup
|
||||
exit "$rc"
|
||||
}
|
||||
|
||||
trap on_exit EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
"$INSTALL_BIN" -o root -g root -m 0600 "$CANDIDATE" "$CANDIDATE_SNAPSHOT"
|
||||
readonly CANDIDATE_SHA256="$("$SHA256SUM_BIN" "$CANDIDATE_SNAPSHOT" | "$AWK_BIN" '{print $1}')"
|
||||
readonly ACTIVE_SHA256="$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')"
|
||||
[[ "$CANDIDATE_SHA256" == "$EXPECTED_CANDIDATE_SHA256" ]] || \
|
||||
fail "candidate digest changed: ${CANDIDATE_SHA256}"
|
||||
if [[ "$ACTIVE_SHA256" != "$EXPECTED_PRE_CUTOVER_SHA256" && "$ACTIVE_SHA256" != "$CANDIDATE_SHA256" ]]; then
|
||||
fail "active config has an unexpected digest: ${ACTIVE_SHA256}"
|
||||
fi
|
||||
|
||||
check_health_json() {
|
||||
local path="$1"
|
||||
|
||||
"$JQ_BIN" -e '
|
||||
.status == "pass" and
|
||||
([.checks["database:ping"][], .checks["cache:ping"][]] |
|
||||
all(.status == "pass"))
|
||||
' "$path" >/dev/null
|
||||
}
|
||||
|
||||
pre_cutover_check() {
|
||||
"$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \
|
||||
--header 'Host: git.learn.hyeonworks.com' \
|
||||
http://127.0.0.1:30080/api/healthz >"$DIRECT_HEALTH"
|
||||
check_health_json "$DIRECT_HEALTH"
|
||||
}
|
||||
|
||||
wait_for_nginx_health() {
|
||||
local attempt
|
||||
local probe_metadata=""
|
||||
local http_code="curl-error"
|
||||
local content_type="unavailable"
|
||||
local size_download="0"
|
||||
local body_sha256=""
|
||||
local classification="transport-error"
|
||||
local consecutive_passes=0
|
||||
|
||||
for ((attempt = 1; attempt <= 10; attempt++)); do
|
||||
if probe_metadata="$(
|
||||
"$CURL_BIN" --noproxy '*' --silent --connect-timeout 1 --max-time 2 \
|
||||
--resolve git.learn.hyeonworks.com:443:127.0.0.1 \
|
||||
--output "$NGINX_HEALTH" \
|
||||
--write-out $'%{http_code}\t%{content_type}\t%{size_download}' \
|
||||
https://git.learn.hyeonworks.com/api/healthz
|
||||
)"; then
|
||||
IFS=$'\t' read -r http_code content_type size_download <<<"$probe_metadata"
|
||||
body_sha256="$(
|
||||
"$SHA256SUM_BIN" "$NGINX_HEALTH" | "$AWK_BIN" '{print $1}'
|
||||
)"
|
||||
|
||||
if [[ "$http_code" == "200" ]] &&
|
||||
check_health_json "$NGINX_HEALTH" 2>/dev/null; then
|
||||
classification="healthy-json"
|
||||
consecutive_passes=$((consecutive_passes + 1))
|
||||
if (( consecutive_passes >= 2 )); then
|
||||
printf 'Nginx proxy health stabilized after %d probes.\n' "$attempt"
|
||||
return 0
|
||||
fi
|
||||
else
|
||||
consecutive_passes=0
|
||||
if [[ "$body_sha256" == "$EXPECTED_PRE_CUTOVER_HEALTH_SHA256" ]]; then
|
||||
classification="stale-old-generation"
|
||||
elif [[ "$http_code" =~ ^(502|503|504)$ ]]; then
|
||||
classification="transient-upstream"
|
||||
elif "$JQ_BIN" -e '
|
||||
type == "object" and
|
||||
has("status") and
|
||||
(.checks | type == "object")
|
||||
' "$NGINX_HEALTH" >/dev/null 2>&1; then
|
||||
classification="unhealthy-health-json"
|
||||
else
|
||||
printf 'ERROR: unexpected Nginx health response (status=%s, content-type=%s, bytes=%s, sha256=%s)\n' \
|
||||
"$http_code" "${content_type:-none}" "$size_download" "$body_sha256" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
http_code="curl-error"
|
||||
content_type="unavailable"
|
||||
size_download="0"
|
||||
body_sha256=""
|
||||
classification="transport-error"
|
||||
consecutive_passes=0
|
||||
fi
|
||||
|
||||
if (( attempt < 10 )); then
|
||||
"$SLEEP_BIN" 1
|
||||
fi
|
||||
done
|
||||
|
||||
fail "Nginx health did not converge during the bounded retry window (classification=${classification}, status=${http_code}, content-type=${content_type:-none}, bytes=${size_download})"
|
||||
}
|
||||
|
||||
wait_for_rollback_state() {
|
||||
local attempt
|
||||
local probe_metadata=""
|
||||
local http_code="curl-error"
|
||||
local content_type="unavailable"
|
||||
local size_download="0"
|
||||
local body_sha256=""
|
||||
|
||||
for ((attempt = 1; attempt <= 10; attempt++)); do
|
||||
if probe_metadata="$(
|
||||
"$CURL_BIN" --noproxy '*' --silent --connect-timeout 1 --max-time 2 \
|
||||
--resolve git.learn.hyeonworks.com:443:127.0.0.1 \
|
||||
--output "$NGINX_HEALTH" \
|
||||
--write-out $'%{http_code}\t%{content_type}\t%{size_download}' \
|
||||
https://git.learn.hyeonworks.com/api/healthz
|
||||
)"; then
|
||||
IFS=$'\t' read -r http_code content_type size_download <<<"$probe_metadata"
|
||||
body_sha256="$(
|
||||
"$SHA256SUM_BIN" "$NGINX_HEALTH" | "$AWK_BIN" '{print $1}'
|
||||
)"
|
||||
if [[ "$http_code" == "200" &&
|
||||
"$body_sha256" == "$EXPECTED_PRE_CUTOVER_HEALTH_SHA256" ]]; then
|
||||
if (( attempt > 1 )); then
|
||||
printf 'Rollback proxy state stabilized after %d probes.\n' "$attempt" >&2
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
else
|
||||
http_code="curl-error"
|
||||
content_type="unavailable"
|
||||
size_download="0"
|
||||
body_sha256=""
|
||||
fi
|
||||
|
||||
if (( attempt < 10 )); then
|
||||
"$SLEEP_BIN" 1
|
||||
fi
|
||||
done
|
||||
|
||||
printf 'CRITICAL: rollback response did not converge (status=%s, content-type=%s, bytes=%s, sha256=%s)\n' \
|
||||
"$http_code" "${content_type:-none}" "$size_download" "${body_sha256:-none}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
post_cutover_checks() {
|
||||
local id_body
|
||||
local redirect_result
|
||||
|
||||
wait_for_nginx_health
|
||||
|
||||
id_body="$(
|
||||
"$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \
|
||||
--resolve id.learn.hyeonworks.com:443:127.0.0.1 \
|
||||
https://id.learn.hyeonworks.com/
|
||||
)"
|
||||
[[ "$id_body" == "Keycloak domain reached Nginx successfully" ]] || \
|
||||
fail "Keycloak hold response changed"
|
||||
|
||||
redirect_result="$(
|
||||
"$CURL_BIN" --noproxy '*' --silent --show-error --max-time 10 \
|
||||
--resolve git.learn.hyeonworks.com:80:127.0.0.1 \
|
||||
--output /dev/null --write-out $'%{http_code}\n%{redirect_url}' \
|
||||
http://git.learn.hyeonworks.com/api/healthz
|
||||
)"
|
||||
[[ "$redirect_result" == $'301\nhttps://git.learn.hyeonworks.com/api/healthz' ]] || \
|
||||
fail "HTTP redirect check failed: ${redirect_result}"
|
||||
|
||||
if "$CURL_BIN" --noproxy '*' --insecure --silent --output /dev/null --max-time 5 \
|
||||
--resolve unconfigured.invalid:443:127.0.0.1 \
|
||||
https://unconfigured.invalid/ 2>/dev/null; then
|
||||
fail "unknown TLS hostname was not rejected"
|
||||
fi
|
||||
|
||||
"$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \
|
||||
--resolve git.learn.hyeonworks.com:443:127.0.0.1 \
|
||||
--dump-header "$LOGIN_HEADERS" --output /dev/null \
|
||||
https://git.learn.hyeonworks.com/user/login
|
||||
"$AWK_BIN" '
|
||||
BEGIN { found = 0; insecure = 0 }
|
||||
tolower($0) ~ /^set-cookie:/ {
|
||||
found++
|
||||
if (tolower($0) !~ /; secure([;[:space:]]|$)/) insecure = 1
|
||||
}
|
||||
END { exit(found == 0 || insecure) }
|
||||
' "$LOGIN_HEADERS" || fail "login cookie Secure check failed"
|
||||
}
|
||||
|
||||
pre_cutover_check
|
||||
"$NGINX_BIN" -t
|
||||
|
||||
if [[ "$ACTIVE_SHA256" == "$CANDIDATE_SHA256" ]]; then
|
||||
post_cutover_checks
|
||||
printf 'The reviewed Host Nginx configuration is already active and healthy.\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
backup="${ACTIVE}.before-gitea-$("$DATE_BIN" +%Y%m%d%H%M%S)"
|
||||
[[ ! -e "$backup" && ! -L "$backup" ]] || fail "backup path already exists: ${backup}"
|
||||
|
||||
printf '\nActive SHA-256: %s\n' "$ACTIVE_SHA256"
|
||||
printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256"
|
||||
printf 'Planned backup: %s\n' "$backup"
|
||||
printf 'Type APPLY to replace the Host Nginx site: '
|
||||
read -r confirmation
|
||||
[[ "$confirmation" == "APPLY" ]] || fail "cancelled"
|
||||
|
||||
"$INSTALL_BIN" -o root -g root -m 0644 "$ACTIVE" "$backup"
|
||||
[[ "$("$SHA256SUM_BIN" "$backup" | "$AWK_BIN" '{print $1}')" == "$ACTIVE_SHA256" ]] || \
|
||||
fail "backup digest mismatch"
|
||||
|
||||
rollback_armed=1
|
||||
"$INSTALL_BIN" -o root -g root -m 0644 "$CANDIDATE_SNAPSHOT" "$ACTIVE"
|
||||
[[ "$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" == "$CANDIDATE_SHA256" ]] || \
|
||||
fail "installed digest mismatch"
|
||||
"$NGINX_BIN" -t
|
||||
"$SYSTEMCTL_BIN" reload nginx
|
||||
"$SYSTEMCTL_BIN" is-active --quiet nginx
|
||||
post_cutover_checks
|
||||
|
||||
rollback_armed=0
|
||||
printf '\nCUTOVER SUCCESS\n'
|
||||
printf 'Backup: %s\n' "$backup"
|
||||
printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256"
|
||||
printf 'Local HTTPS health, redirect, Secure cookie, Keycloak hold, and unknown-host rejection: PASS\n'
|
||||
printf 'Run the public HTTPS and Git clone/push checks from a separate client next.\n'
|
||||
|
||||
Reference in New Issue
Block a user