#!/usr/bin/env bash set -Eeuo pipefail # Do not expose headers, cookies, or future sensitive values through caller xtrace. set +x umask 077 readonly EXPECTED_CANDIDATE_SHA256="de7ebd4f69cd7d2204ee633e074bf6a4370a6f5e3f3fac9067b099d5d75269b5" readonly EXPECTED_PRE_CUTOVER_SHA256="5b5941519ab677f751568827aa4f315193dfbd9eeac5e8a4ad85d724aaefe16f" readonly EXPECTED_PRE_CUTOVER_HEALTH_SHA256="6b683cb16987ff1f5ded22e9847ac0a45995947927c459b21f427523a41c7484" readonly REPOSITORY_ROOT="/home/donghyeon/workspace/platform" readonly CANDIDATE="${REPOSITORY_ROOT}/infrastructure/networking/host-nginx/learn-services.conf" readonly ACTIVE="/etc/nginx/sites-available/learn-services" readonly ENABLED="/etc/nginx/sites-enabled/learn-services" readonly CURL_BIN="/usr/bin/curl" readonly JQ_BIN="/usr/bin/jq" readonly NGINX_BIN="/usr/sbin/nginx" readonly SYSTEMCTL_BIN="/usr/bin/systemctl" readonly INSTALL_BIN="/usr/bin/install" readonly SHA256SUM_BIN="/usr/bin/sha256sum" readonly STAT_BIN="/usr/bin/stat" readonly READLINK_BIN="/usr/bin/readlink" readonly MKTEMP_BIN="/usr/bin/mktemp" readonly RM_BIN="/usr/bin/rm" readonly DATE_BIN="/usr/bin/date" readonly AWK_BIN="/usr/bin/awk" readonly SLEEP_BIN="/usr/bin/sleep" fail() { printf 'ERROR: %s\n' "$*" >&2 return 1 } usage() { printf '%s\n' \ 'Usage: sudo bash scripts/bootstrap/apply-host-nginx-gitea.sh --execute' \ '' \ 'Backs up the active learn-services site, installs the reviewed Gitea proxy,' \ 'tests and reloads Nginx, then runs local acceptance checks. Any failure after' \ 'the active file changes triggers an automatic restore and reload.' } [[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || { usage exit 2 } [[ "$EUID" -eq 0 ]] || fail "run this script through sudo" [[ -t 0 ]] || fail "an interactive terminal is required" for required_binary in \ "$CURL_BIN" "$JQ_BIN" "$NGINX_BIN" "$SYSTEMCTL_BIN" "$INSTALL_BIN" \ "$SHA256SUM_BIN" "$STAT_BIN" "$READLINK_BIN" "$MKTEMP_BIN" \ "$RM_BIN" "$DATE_BIN" "$AWK_BIN" "$SLEEP_BIN"; do [[ -x "$required_binary" ]] || fail "required executable is missing: ${required_binary}" done [[ -f "$CANDIDATE" && ! -L "$CANDIDATE" ]] || fail "unsafe candidate: ${CANDIDATE}" [[ -f "$ACTIVE" && ! -L "$ACTIVE" ]] || fail "unsafe active file: ${ACTIVE}" [[ -L "$ENABLED" ]] || fail "enabled path is not a symlink: ${ENABLED}" [[ "$("$READLINK_BIN" -f "$ENABLED")" == "$ACTIVE" ]] || fail "enabled symlink target changed" [[ "$("$STAT_BIN" --format='%U:%G %a' "$ACTIVE")" == "root:root 644" ]] || \ fail "active file owner or mode changed" "$SYSTEMCTL_BIN" is-active --quiet nginx || fail "nginx is not active" readonly TEMP_DIR="$("$MKTEMP_BIN" -d /tmp/nginx-gitea-cutover.XXXXXX)" readonly CANDIDATE_SNAPSHOT="${TEMP_DIR}/learn-services.candidate" readonly DIRECT_HEALTH="${TEMP_DIR}/direct-health.json" readonly NGINX_HEALTH="${TEMP_DIR}/nginx-health.json" readonly LOGIN_HEADERS="${TEMP_DIR}/login-headers" rollback_armed=0 backup="" cleanup() { case "$TEMP_DIR" in /tmp/nginx-gitea-cutover.*) "$RM_BIN" -rf -- "$TEMP_DIR" ;; *) printf 'WARNING: refusing to remove unexpected temp path: %s\n' "$TEMP_DIR" >&2 ;; esac } rollback() { local restore_install_rc local restore_test_rc local restore_reload_rc set +e printf '\nROLLBACK: restoring %s\n' "$backup" >&2 if "$INSTALL_BIN" -o root -g root -m 0644 "$backup" "$ACTIVE" && [[ "$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" == "$("$SHA256SUM_BIN" "$backup" | "$AWK_BIN" '{print $1}')" ]]; then restore_install_rc=0 else restore_install_rc=1 fi "$NGINX_BIN" -t restore_test_rc=$? if (( restore_install_rc == 0 && restore_test_rc == 0 )); then if "$SYSTEMCTL_BIN" reload nginx && "$SYSTEMCTL_BIN" is-active --quiet nginx && wait_for_rollback_state; then restore_reload_rc=0 else restore_reload_rc=1 fi else restore_reload_rc=1 fi if (( restore_install_rc == 0 && restore_test_rc == 0 && restore_reload_rc == 0 )); then rollback_armed=0 printf 'ROLLBACK complete. Active config and response were restored.\n' >&2 else printf 'CRITICAL: automatic rollback failed; backup remains at %s\n' "$backup" >&2 fi } on_exit() { local rc=$? trap - EXIT INT TERM if (( rc != 0 && rollback_armed == 1 )); then rollback fi cleanup exit "$rc" } trap on_exit EXIT trap 'exit 130' INT trap 'exit 143' TERM "$INSTALL_BIN" -o root -g root -m 0600 "$CANDIDATE" "$CANDIDATE_SNAPSHOT" readonly CANDIDATE_SHA256="$("$SHA256SUM_BIN" "$CANDIDATE_SNAPSHOT" | "$AWK_BIN" '{print $1}')" readonly ACTIVE_SHA256="$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" [[ "$CANDIDATE_SHA256" == "$EXPECTED_CANDIDATE_SHA256" ]] || \ fail "candidate digest changed: ${CANDIDATE_SHA256}" if [[ "$ACTIVE_SHA256" != "$EXPECTED_PRE_CUTOVER_SHA256" && "$ACTIVE_SHA256" != "$CANDIDATE_SHA256" ]]; then fail "active config has an unexpected digest: ${ACTIVE_SHA256}" fi check_health_json() { local path="$1" "$JQ_BIN" -e ' .status == "pass" and ([.checks["database:ping"][], .checks["cache:ping"][]] | all(.status == "pass")) ' "$path" >/dev/null } pre_cutover_check() { "$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \ --header 'Host: git.learn.hyeonworks.com' \ http://127.0.0.1:30080/api/healthz >"$DIRECT_HEALTH" check_health_json "$DIRECT_HEALTH" } wait_for_nginx_health() { local attempt local probe_metadata="" local http_code="curl-error" local content_type="unavailable" local size_download="0" local body_sha256="" local classification="transport-error" local consecutive_passes=0 for ((attempt = 1; attempt <= 10; attempt++)); do if probe_metadata="$( "$CURL_BIN" --noproxy '*' --silent --connect-timeout 1 --max-time 2 \ --resolve git.learn.hyeonworks.com:443:127.0.0.1 \ --output "$NGINX_HEALTH" \ --write-out $'%{http_code}\t%{content_type}\t%{size_download}' \ https://git.learn.hyeonworks.com/api/healthz )"; then IFS=$'\t' read -r http_code content_type size_download <<<"$probe_metadata" body_sha256="$( "$SHA256SUM_BIN" "$NGINX_HEALTH" | "$AWK_BIN" '{print $1}' )" if [[ "$http_code" == "200" ]] && check_health_json "$NGINX_HEALTH" 2>/dev/null; then classification="healthy-json" consecutive_passes=$((consecutive_passes + 1)) if (( consecutive_passes >= 2 )); then printf 'Nginx proxy health stabilized after %d probes.\n' "$attempt" return 0 fi else consecutive_passes=0 if [[ "$body_sha256" == "$EXPECTED_PRE_CUTOVER_HEALTH_SHA256" ]]; then classification="stale-old-generation" elif [[ "$http_code" =~ ^(502|503|504)$ ]]; then classification="transient-upstream" elif "$JQ_BIN" -e ' type == "object" and has("status") and (.checks | type == "object") ' "$NGINX_HEALTH" >/dev/null 2>&1; then classification="unhealthy-health-json" else printf 'ERROR: unexpected Nginx health response (status=%s, content-type=%s, bytes=%s, sha256=%s)\n' \ "$http_code" "${content_type:-none}" "$size_download" "$body_sha256" >&2 return 1 fi fi else http_code="curl-error" content_type="unavailable" size_download="0" body_sha256="" classification="transport-error" consecutive_passes=0 fi if (( attempt < 10 )); then "$SLEEP_BIN" 1 fi done fail "Nginx health did not converge during the bounded retry window (classification=${classification}, status=${http_code}, content-type=${content_type:-none}, bytes=${size_download})" } wait_for_rollback_state() { local attempt local probe_metadata="" local http_code="curl-error" local content_type="unavailable" local size_download="0" local body_sha256="" for ((attempt = 1; attempt <= 10; attempt++)); do if probe_metadata="$( "$CURL_BIN" --noproxy '*' --silent --connect-timeout 1 --max-time 2 \ --resolve git.learn.hyeonworks.com:443:127.0.0.1 \ --output "$NGINX_HEALTH" \ --write-out $'%{http_code}\t%{content_type}\t%{size_download}' \ https://git.learn.hyeonworks.com/api/healthz )"; then IFS=$'\t' read -r http_code content_type size_download <<<"$probe_metadata" body_sha256="$( "$SHA256SUM_BIN" "$NGINX_HEALTH" | "$AWK_BIN" '{print $1}' )" if [[ "$http_code" == "200" && "$body_sha256" == "$EXPECTED_PRE_CUTOVER_HEALTH_SHA256" ]]; then if (( attempt > 1 )); then printf 'Rollback proxy state stabilized after %d probes.\n' "$attempt" >&2 fi return 0 fi else http_code="curl-error" content_type="unavailable" size_download="0" body_sha256="" fi if (( attempt < 10 )); then "$SLEEP_BIN" 1 fi done printf 'CRITICAL: rollback response did not converge (status=%s, content-type=%s, bytes=%s, sha256=%s)\n' \ "$http_code" "${content_type:-none}" "$size_download" "${body_sha256:-none}" >&2 return 1 } post_cutover_checks() { local id_body local redirect_result wait_for_nginx_health id_body="$( "$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \ --resolve id.learn.hyeonworks.com:443:127.0.0.1 \ https://id.learn.hyeonworks.com/ )" [[ "$id_body" == "Keycloak domain reached Nginx successfully" ]] || \ fail "Keycloak hold response changed" redirect_result="$( "$CURL_BIN" --noproxy '*' --silent --show-error --max-time 10 \ --resolve git.learn.hyeonworks.com:80:127.0.0.1 \ --output /dev/null --write-out $'%{http_code}\n%{redirect_url}' \ http://git.learn.hyeonworks.com/api/healthz )" [[ "$redirect_result" == $'301\nhttps://git.learn.hyeonworks.com/api/healthz' ]] || \ fail "HTTP redirect check failed: ${redirect_result}" if "$CURL_BIN" --noproxy '*' --insecure --silent --output /dev/null --max-time 5 \ --resolve unconfigured.invalid:443:127.0.0.1 \ https://unconfigured.invalid/ 2>/dev/null; then fail "unknown TLS hostname was not rejected" fi "$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \ --resolve git.learn.hyeonworks.com:443:127.0.0.1 \ --dump-header "$LOGIN_HEADERS" --output /dev/null \ https://git.learn.hyeonworks.com/user/login "$AWK_BIN" ' BEGIN { found = 0; insecure = 0 } tolower($0) ~ /^set-cookie:/ { found++ if (tolower($0) !~ /; secure([;[:space:]]|$)/) insecure = 1 } END { exit(found == 0 || insecure) } ' "$LOGIN_HEADERS" || fail "login cookie Secure check failed" } pre_cutover_check "$NGINX_BIN" -t if [[ "$ACTIVE_SHA256" == "$CANDIDATE_SHA256" ]]; then post_cutover_checks printf 'The reviewed Host Nginx configuration is already active and healthy.\n' exit 0 fi backup="${ACTIVE}.before-gitea-$("$DATE_BIN" +%Y%m%d%H%M%S)" [[ ! -e "$backup" && ! -L "$backup" ]] || fail "backup path already exists: ${backup}" printf '\nActive SHA-256: %s\n' "$ACTIVE_SHA256" printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256" printf 'Planned backup: %s\n' "$backup" printf 'Type APPLY to replace the Host Nginx site: ' read -r confirmation [[ "$confirmation" == "APPLY" ]] || fail "cancelled" "$INSTALL_BIN" -o root -g root -m 0644 "$ACTIVE" "$backup" [[ "$("$SHA256SUM_BIN" "$backup" | "$AWK_BIN" '{print $1}')" == "$ACTIVE_SHA256" ]] || \ fail "backup digest mismatch" rollback_armed=1 "$INSTALL_BIN" -o root -g root -m 0644 "$CANDIDATE_SNAPSHOT" "$ACTIVE" [[ "$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" == "$CANDIDATE_SHA256" ]] || \ fail "installed digest mismatch" "$NGINX_BIN" -t "$SYSTEMCTL_BIN" reload nginx "$SYSTEMCTL_BIN" is-active --quiet nginx post_cutover_checks rollback_armed=0 printf '\nCUTOVER SUCCESS\n' printf 'Backup: %s\n' "$backup" printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256" printf 'Local HTTPS health, redirect, Secure cookie, Keycloak hold, and unknown-host rejection: PASS\n' printf 'Run the public HTTPS and Git clone/push checks from a separate client next.\n'