platform-core/scripts/bootstrap/apply-host-nginx-admin.sh

#!/usr/bin/env bash

set -Eeuo pipefail

readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)"
readonly CANDIDATE="$ROOT/infrastructure/networking/host-nginx/learn-services-admin.conf"
readonly ACTIVE="/etc/nginx/sites-available/learn-services"
readonly CERT_NAME="storage-admin.learn.hyeonworks.com"
readonly CERT_DIR="/etc/letsencrypt/live/$CERT_NAME"
readonly CREDENTIALS="/home/donghyeon/.secrets/certbot/cloudflare.ini"
readonly STORAGE_HOST="storage-admin.learn.hyeonworks.com"
readonly DB_HOST="db-admin.learn.hyeonworks.com"
readonly LAN_IP="192.168.0.107"
readonly TAIL_IP="100.92.240.34"
execute=false
certificate_only=false
certbot_email=""
rollback_armed=false
backup=""

fail() {
  printf 'ERROR: %s\n' "$*" >&2
  exit 1
}

usage() {
  cat <<'USAGE'
사용법:
  bash scripts/bootstrap/apply-host-nginx-admin.sh
  bash scripts/bootstrap/apply-host-nginx-admin.sh \
    --execute --certificate-only --certbot-email you@example.com
  bash scripts/bootstrap/apply-host-nginx-admin.sh --execute

dry-run은 공개 DNS, 후보 hash, Ingress endpoint, Cloudflare credential 상태를
검사합니다. --execute는 필요하면 공식 Cloudflare snap plugin과 SAN 인증서를
발급하고 Nginx를 timestamp backup 뒤 교체합니다.
USAGE
}

while (( $# > 0 )); do
  case "$1" in
    --certificate-only)
      certificate_only=true
      shift
      ;;
    --execute)
      execute=true
      shift
      ;;
    --certbot-email)
      (( $# >= 2 )) || fail "--certbot-email 값이 필요합니다"
      certbot_email="$2"
      shift 2
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      usage >&2
      fail "지원하지 않는 인자: $1"
      ;;
  esac
done

for cmd in awk curl dig install jq kubectl openssl rg sha256sum sleep stat sudo; do
  command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다"
done
[[ -f "$CANDIDATE" && ! -L "$CANDIDATE" ]] || fail "Nginx 후보가 없습니다"
[[ -f "$ACTIVE" && ! -L "$ACTIVE" ]] || fail "활성 Nginx site가 없습니다"
[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요"

for name in "$STORAGE_HOST" "$DB_HOST"; do
  [[ -z "$(dig +short @1.1.1.1 A "$name" | tr -d '[:space:]')" ]] || fail "$name 공개 A가 있습니다"
  [[ -z "$(dig +short @1.1.1.1 AAAA "$name" | tr -d '[:space:]')" ]] || fail "$name 공개 AAAA가 있습니다"
done

kubectl -n object-storage get ingress minio-aistor-console >/dev/null 2>&1 || \
  printf 'WARNING: AIStor Console Ingress가 아직 적용되지 않았습니다.\n' >&2
kubectl -n platform-admin get ingress pgadmin >/dev/null 2>&1 || \
  printf 'WARNING: pgAdmin Ingress가 아직 적용되지 않았습니다.\n' >&2

candidate_sha="$(sha256sum "$CANDIDATE" | awk '{print $1}')"
active_sha="$(sha256sum "$ACTIVE" | awk '{print $1}')"
printf 'Active SHA-256:    %s\n' "$active_sha"
printf 'Candidate SHA-256: %s\n' "$candidate_sha"
printf 'Certificate SAN:   %s, %s\n' "$STORAGE_HOST" "$DB_HOST"

# Ubuntu의 sites-enabled/default는 미등록 HTTP Host에 welcome page(200)를
# 반환할 수 있다. HTTP status만으로 관리 도메인의 부분 적용을 판정하지 않고,
# 실제 활성 site에 관리 server_name이 들어갔는지를 먼저 확인한다.
if [[ "$active_sha" != "$candidate_sha" ]] && \
  { rg -q -F "$STORAGE_HOST" "$ACTIVE" || rg -q -F "$DB_HOST" "$ACTIVE"; }; then
  fail "활성 Nginx site에 후보와 다른 admin 도메인 설정이 부분 적용돼 있습니다"
fi

check_cloudflare_credentials() {
  sudo test -f "$CREDENTIALS" || \
    fail "Cloudflare 제한 토큰 파일이 없습니다: $CREDENTIALS"

  owner="$(sudo stat -c '%U:%G' "$CREDENTIALS")"
  mode="$(sudo stat -c '%a' "$CREDENTIALS")"
  size="$(sudo stat -c '%s' "$CREDENTIALS")"
  [[ "$owner" == root:root && "$mode" == 600 ]] || \
    fail "$CREDENTIALS 소유권/권한은 root:root 0600이어야 합니다"
  (( size > 0 )) || fail "$CREDENTIALS 파일이 비어 있습니다"

  contract="$(
    sudo awk -F= '
      BEGIN { count=0; nonempty=0; unexpected=0 }
      /^[[:space:]]*($|#)/ { next }
      /^[[:space:]]*dns_cloudflare_api_token[[:space:]]*=/ {
        count++
        value=$0
        sub(/^[^=]*=[[:space:]]*/, "", value)
        sub(/[[:space:]]+$/, "", value)
        if (length(value) >= 20) nonempty++
        next
      }
      { unexpected++ }
      END { printf "%d:%d:%d", count, nonempty, unexpected }
    ' "$CREDENTIALS"
  )"
  [[ "$contract" == "1:1:0" ]] || \
    fail "$CREDENTIALS에는 비어 있지 않은 dns_cloudflare_api_token 한 개만 있어야 합니다"

  printf 'Cloudflare credential: root:root 0600 및 key 형식 확인\n'
}

if [[ "$execute" == false ]]; then
  if sudo -n test -f "$CREDENTIALS" 2>/dev/null; then
    check_cloudflare_credentials
  else
    printf 'Cloudflare credential: root 전용이므로 --execute에서 sudo로 확인\n'
  fi
  printf 'DRY RUN PASS: --execute를 지정하지 않아 인증서와 Nginx를 변경하지 않았습니다.\n'
  exit 0
fi

[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다"
sudo -v
check_cloudflare_credentials
context="$(kubectl config current-context)"
printf 'Type APPLY to issue/renew the admin certificate and replace Host Nginx: '
read -r answer
[[ "$answer" == APPLY ]] || fail "취소했습니다"

if [[ ! -x /snap/bin/certbot ]]; then
  fail "지원 대상으로 고정한 snap certbot이 없습니다"
fi
if ! /snap/bin/certbot plugins 2>/dev/null | rg -q 'dns-cloudflare'; then
  sudo snap set certbot trust-plugin-with-root=ok
  sudo snap install certbot-dns-cloudflare
fi

if ! sudo test -s "$CERT_DIR/fullchain.pem" || ! sudo test -s "$CERT_DIR/privkey.pem"; then
  [[ "$certbot_email" == *@* ]] || fail "최초 인증서 발급에는 --certbot-email이 필요합니다"
  sudo /snap/bin/certbot certonly \
    --dns-cloudflare \
    --dns-cloudflare-credentials "$CREDENTIALS" \
    --dns-cloudflare-propagation-seconds 60 \
    --cert-name "$CERT_NAME" \
    --domains "$STORAGE_HOST" \
    --domains "$DB_HOST" \
    --non-interactive \
    --agree-tos \
    --email "$certbot_email"
fi

cert_text="$(sudo openssl x509 -in "$CERT_DIR/fullchain.pem" -noout -text)"
printf '%s\n' "$cert_text" | rg -q "DNS:$STORAGE_HOST" || fail "인증서에 storage-admin SAN이 없습니다"
printf '%s\n' "$cert_text" | rg -q "DNS:$DB_HOST" || fail "인증서에 db-admin SAN이 없습니다"
san_count="$(printf '%s\n' "$cert_text" | rg -o 'DNS:[^,[:space:]]+' | sort -u | wc -l | tr -d '[:space:]')"
[[ "$san_count" == 2 ]] || fail "admin 인증서 SAN은 정확히 두 개여야 합니다"

sudo install -d -o root -g root -m 0755 /etc/letsencrypt/renewal-hooks/deploy
sudo install -o root -g root -m 0755 \
  "$ROOT/infrastructure/networking/host-nginx/reload-nginx.sh" \
  /etc/letsencrypt/renewal-hooks/deploy/reload-nginx

if [[ "$certificate_only" == true ]]; then
  printf 'ADMIN CERTIFICATE READY\n'
  printf 'Nginx cutover는 서비스 적용 뒤 --certificate-only 없이 다시 실행하세요.\n'
  exit 0
fi

for host in "$STORAGE_HOST" "$DB_HOST"; do
  code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
    --noproxy '*' --resolve "$host:80:127.0.0.1" -H "Host: $host" "http://$host/")"
  [[ "$code" == 200 || "$code" == 404 || "$code" == 301 ]] || \
    fail "기존 HTTP 경계가 예상과 다릅니다: $host=$code"
  printf '기존 HTTP fallback: %s=%s\n' "$host" "$code"
  route_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
    --noproxy '*' -H "Host: $host" "http://127.0.0.1:30080/")"
  [[ "$route_code" == 200 || "$route_code" == 302 || "$route_code" == 303 ]] || \
    fail "Traefik route가 준비되지 않았습니다: $host=$route_code"
done

probe_https_code() {
  host="$1"
  target_ip="$2"
  source_ip="${3:-}"

  interface_args=()
  if [[ -n "$source_ip" ]]; then
    interface_args=(--interface "$source_ip")
  fi

  curl --disable --silent --output /dev/null --write-out '%{http_code}' \
    --connect-timeout 1 --max-time 3 --noproxy '*' \
    "${interface_args[@]}" --resolve "$host:443:$target_ip" "https://$host/" \
    2>/dev/null || true
}

wait_for_admin_proxy_state() {
  consecutive_passes=0
  storage_lan=000
  db_lan=000
  storage_denied=000
  db_denied=000

  for ((attempt = 1; attempt <= 15; attempt++)); do
    storage_lan="$(probe_https_code "$STORAGE_HOST" "$LAN_IP" "$LAN_IP")"
    db_lan="$(probe_https_code "$DB_HOST" "$LAN_IP" "$LAN_IP")"
    storage_denied="$(probe_https_code "$STORAGE_HOST" 127.0.0.1)"
    db_denied="$(probe_https_code "$DB_HOST" 127.0.0.1)"

    if [[ "$storage_lan" == 200 || "$storage_lan" == 302 || "$storage_lan" == 303 ]] &&
      [[ "$db_lan" == 200 || "$db_lan" == 302 || "$db_lan" == 303 ]] &&
      [[ "$storage_denied" == 403 && "$db_denied" == 403 ]]; then
      consecutive_passes=$((consecutive_passes + 1))
      if (( consecutive_passes >= 2 )); then
        printf 'Nginx admin proxy state stabilized after %d probes.\n' "$attempt"
        return 0
      fi
    else
      consecutive_passes=0
    fi

    if (( attempt == 1 || attempt % 5 == 0 )); then
      printf 'Nginx admin proxy 대기: probe=%d LAN=%s/%s denied=%s/%s\n' \
        "$attempt" "$storage_lan" "$db_lan" "$storage_denied" "$db_denied"
    fi
    (( attempt < 15 )) && sleep 1
  done

  printf 'ERROR: Nginx admin proxy가 안정화되지 않았습니다: LAN=%s/%s denied=%s/%s\n' \
    "$storage_lan" "$db_lan" "$storage_denied" "$db_denied" >&2
  return 1
}

wait_for_admin_tls_rejection() {
  consecutive_passes=0

  for ((attempt = 1; attempt <= 15; attempt++)); do
    if ! curl --disable --insecure --silent --output /dev/null \
      --connect-timeout 1 --max-time 3 --noproxy '*' \
      --resolve "$STORAGE_HOST:443:127.0.0.1" "https://$STORAGE_HOST/" 2>/dev/null &&
      ! curl --disable --insecure --silent --output /dev/null \
        --connect-timeout 1 --max-time 3 --noproxy '*' \
        --resolve "$DB_HOST:443:127.0.0.1" "https://$DB_HOST/" 2>/dev/null; then
      consecutive_passes=$((consecutive_passes + 1))
      if (( consecutive_passes >= 2 )); then
        return 0
      fi
    else
      consecutive_passes=0
    fi
    (( attempt < 15 )) && sleep 1
  done

  return 1
}

backup="$ACTIVE.before-admin-$(date +%Y%m%d%H%M%S)"
printf 'Planned backup: %s\n' "$backup"
sudo install -o root -g root -m 0644 "$ACTIVE" "$backup"

rollback() {
  set +e
  printf '\nROLLBACK: %s 복원\n' "$backup" >&2
  sudo install -o root -g root -m 0644 "$backup" "$ACTIVE"
  sudo /usr/sbin/nginx -t
  sudo systemctl reload nginx
  if ! wait_for_admin_tls_rejection; then
    printf 'WARNING: rollback 뒤 admin SNI 거부 안정화를 확인하지 못했습니다.\n' >&2
  fi
  rollback_armed=false
  printf 'ROLLBACK complete.\n' >&2
}

finish() {
  rc=$?
  trap - EXIT INT TERM
  if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then
    rollback
  fi
  exit "$rc"
}
trap finish EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

rollback_armed=true
sudo install -o root -g root -m 0644 "$CANDIDATE" "$ACTIVE"
sudo systemctl daemon-reload
sudo /usr/sbin/nginx -t
sudo systemctl reload nginx
sudo systemctl is-active --quiet nginx

wait_for_admin_proxy_state || fail "LAN HTTPS와 allowlist 상태가 수렴하지 않았습니다"

for host in "$STORAGE_HOST" "$DB_HOST"; do
  redirect_result="$(
    curl --disable --silent --show-error --output /dev/null \
      --write-out $'%{http_code}\n%{redirect_url}' --max-time 5 --noproxy '*' \
      --resolve "$host:80:$LAN_IP" "http://$host/"
  )"
  [[ "$redirect_result" == $'301\nhttps://'"$host/" ]] || \
    fail "HTTP redirect 검증 실패: $host=$redirect_result"
done

if curl --insecure --silent --output /dev/null --noproxy '*' \
  --resolve unconfigured.invalid:443:127.0.0.1 https://unconfigured.invalid/ 2>/dev/null; then
  fail "unknown SNI가 거부되지 않았습니다"
fi

sudo /snap/bin/certbot renew --dry-run
rollback_armed=false
printf 'ADMIN NGINX CUTOVER SUCCESS\n'
printf 'Backup: %s\n' "$backup"
printf 'Candidate SHA-256: %s\n' "$candidate_sha"