Add platform infrastructure configuration
This commit is contained in:
donghyeon-ka
Executable
+266
@@ -0,0 +1,266 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)"
|
||||
readonly TARGET_NODE="donghyeon-system-product-name"
|
||||
readonly PG_PATH="/srv/k3s/ssd/pgadmin"
|
||||
readonly PG_SECRET="pgadmin-bootstrap"
|
||||
readonly PG_OIDC_SECRET="pgadmin-keycloak-oidc"
|
||||
readonly AI_OIDC_SECRET="aistor-keycloak-oidc"
|
||||
readonly NAMES="admin-namespace pgadmin-local-pv coredns-custom aistor-admin-oidc pgadmin"
|
||||
execute=false
|
||||
password_file=""
|
||||
work=""
|
||||
rollback_armed=false
|
||||
bootstrap_existed=false
|
||||
|
||||
fail() {
|
||||
printf 'ERROR: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
사용법:
|
||||
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
|
||||
bash scripts/bootstrap/apply-admin-services.sh
|
||||
|
||||
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
|
||||
bash scripts/bootstrap/apply-admin-services.sh \
|
||||
--execute \
|
||||
--pgadmin-password-file /home/donghyeon/.secrets/pgadmin/bootstrap-password
|
||||
|
||||
인자 없이 실행하면 공식 차트 hash와 manifest만 검증합니다.
|
||||
--execute는 Local PV, pgAdmin bootstrap Secret, pgAdmin, AIStor OIDC profile을
|
||||
적용합니다. 비밀번호와 OIDC Secret 값은 출력하지 않습니다.
|
||||
USAGE
|
||||
}
|
||||
|
||||
while (( $# > 0 )); do
|
||||
case "$1" in
|
||||
--execute)
|
||||
execute=true
|
||||
shift
|
||||
;;
|
||||
--pgadmin-password-file)
|
||||
(( $# >= 2 )) || fail "--pgadmin-password-file 값이 필요합니다"
|
||||
password_file="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
usage >&2
|
||||
fail "지원하지 않는 인자: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요"
|
||||
for cmd in awk curl find findmnt install jq kubectl mktemp od rg seq sha256sum sleep sort stat tail tr wc; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다"
|
||||
done
|
||||
|
||||
work="$(mktemp -d /tmp/platform-admin-apply.XXXXXX)"
|
||||
|
||||
rollback() {
|
||||
set +e
|
||||
printf '\nROLLBACK: 관리 UI 트래픽을 내리고 AIStor 이전 spec을 복원합니다.\n' >&2
|
||||
kubectl -n platform-admin scale deployment/pgadmin --replicas=0 >/dev/null 2>&1 || true
|
||||
kubectl -n platform-admin delete ingress pgadmin --ignore-not-found >/dev/null 2>&1
|
||||
kubectl -n object-storage delete ingress minio-aistor-console --ignore-not-found >/dev/null 2>&1
|
||||
kubectl apply -f "$work/objectstore-before.json" >/dev/null 2>&1 || true
|
||||
for statefulset_name in $(
|
||||
kubectl -n object-storage get objectstore minio-aistor \
|
||||
-o jsonpath='{range .status.pools[*]}{.ssName}{"\n"}{end}' 2>/dev/null
|
||||
); do
|
||||
kubectl -n object-storage rollout restart "statefulset/$statefulset_name" >/dev/null 2>&1 || true
|
||||
kubectl -n object-storage rollout status "statefulset/$statefulset_name" \
|
||||
--timeout=300s >/dev/null 2>&1 || true
|
||||
done
|
||||
if [[ "$bootstrap_existed" == true ]]; then
|
||||
kubectl apply -f "$work/pgadmin-bootstrap-before.yaml" >/dev/null 2>&1 || true
|
||||
else
|
||||
kubectl -n platform-admin delete secret "$PG_SECRET" --ignore-not-found >/dev/null 2>&1
|
||||
fi
|
||||
rollback_armed=false
|
||||
printf 'ROLLBACK complete. pgAdmin PVC/PV와 AIStor 데이터는 삭제하지 않았습니다.\n' >&2
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
rc=$?
|
||||
trap - EXIT INT TERM
|
||||
if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then
|
||||
rollback
|
||||
fi
|
||||
case "$work" in
|
||||
/tmp/platform-admin-apply.*) rm -rf -- "$work" ;;
|
||||
esac
|
||||
exit "$rc"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
unexpected_error() {
|
||||
rc=$?
|
||||
line="$1"
|
||||
trap - ERR
|
||||
printf 'ERROR: 예상하지 못한 명령 실패(line=%s, exit=%s)\n' "$line" "$rc" >&2
|
||||
exit "$rc"
|
||||
}
|
||||
trap 'unexpected_error "$LINENO"' ERR
|
||||
|
||||
if [[ ! -v PLATFORM_HELM_BIN ]]; then
|
||||
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm
|
||||
fi
|
||||
PLATFORM_HELM_BIN="$PLATFORM_HELM_BIN" \
|
||||
bash "$ROOT/scripts/validate/render-admin-services.sh" --verified-output-dir "$work"
|
||||
|
||||
for name in $NAMES; do
|
||||
file="$work/$name.yaml"
|
||||
[[ -f "$file" && ! -L "$file" && -O "$file" && -s "$file" ]] || fail "안전하지 않은 handoff: $file"
|
||||
[[ "$(stat -c '%a' "$file")" == 600 ]] || fail "handoff 권한이 0600이 아닙니다"
|
||||
sha256sum "$file" | awk '{print $1}' >"$work/$name.sha256"
|
||||
done
|
||||
|
||||
context="$(kubectl config current-context)"
|
||||
api="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')"
|
||||
ready="$(kubectl get node "$TARGET_NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')"
|
||||
[[ "$ready" == True ]] || fail "대상 노드가 Ready가 아닙니다"
|
||||
printf 'Current context: %s\n' "$context"
|
||||
printf 'API server: %s\n' "$api"
|
||||
for name in $NAMES; do
|
||||
printf 'SHA-256 %-20s %s\n' "$name" "$(cat "$work/$name.sha256")"
|
||||
done
|
||||
|
||||
if [[ "$execute" == false ]]; then
|
||||
printf 'DRY RUN PASS: --execute를 지정하지 않아 클러스터를 변경하지 않았습니다.\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다"
|
||||
[[ "$password_file" == /* && -f "$password_file" && ! -L "$password_file" ]] || \
|
||||
fail "pgAdmin 비밀번호 파일은 일반 파일인 절대 경로여야 합니다"
|
||||
[[ "$(stat -c '%a' "$password_file")" == 600 ]] || fail "pgAdmin 비밀번호 파일 권한은 0600이어야 합니다"
|
||||
password_bytes="$(wc -c <"$password_file" | tr -d '[:space:]')"
|
||||
(( password_bytes >= 16 && password_bytes <= 256 )) || fail "pgAdmin 비밀번호는 16~256 bytes여야 합니다"
|
||||
last_byte="$(tail -c 1 "$password_file" | od -An -t x1 | tr -d '[:space:]')"
|
||||
[[ "$last_byte" != 0a && "$last_byte" != 0d ]] || \
|
||||
fail "pgAdmin 비밀번호 파일 끝에 개행이 없어야 합니다"
|
||||
|
||||
for contract in "object-storage $AI_OIDC_SECRET client-id client-secret" \
|
||||
"platform-admin $PG_OIDC_SECRET client-id client-secret"; do
|
||||
set -- $contract
|
||||
namespace="$1"
|
||||
secret="$2"
|
||||
key_a="$3"
|
||||
key_b="$4"
|
||||
kubectl -n "$namespace" get "secret/$secret" >/dev/null 2>&1 || \
|
||||
fail "$namespace/$secret Secret이 없습니다. Keycloak OIDC 구성을 먼저 실행하세요"
|
||||
keys="$(kubectl -n "$namespace" get "secret/$secret" \
|
||||
-o go-template='{{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}' | LC_ALL=C sort)"
|
||||
[[ "$keys" == "$key_a"$'\n'"$key_b" ]] || fail "$namespace/$secret key 계약이 다릅니다"
|
||||
done
|
||||
|
||||
core_data="$(kubectl -n kube-system get configmap coredns-custom \
|
||||
-o jsonpath='{.data.learn-hyeonworks\.server}' 2>/dev/null)" || \
|
||||
fail "coredns-custom이 없습니다. apply-private-dns.sh를 먼저 실행하세요"
|
||||
for host in git.learn.hyeonworks.com id.learn.hyeonworks.com \
|
||||
storage-admin.learn.hyeonworks.com db-admin.learn.hyeonworks.com; do
|
||||
printf '%s\n' "$core_data" | rg -q -F "$host" || fail "CoreDNS에 $host가 없습니다"
|
||||
done
|
||||
|
||||
printf 'Type APPLY %s to deploy private admin services: ' "$context"
|
||||
read -r answer
|
||||
[[ "$answer" == "APPLY $context" ]] || fail "취소했습니다"
|
||||
[[ "$(kubectl config current-context)" == "$context" ]] || fail "context가 바뀌었습니다"
|
||||
[[ "$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" == "$api" ]] || \
|
||||
fail "API server가 바뀌었습니다"
|
||||
for name in $NAMES; do
|
||||
current_sha="$(sha256sum "$work/$name.yaml" | awk '{print $1}')"
|
||||
[[ "$current_sha" == "$(cat "$work/$name.sha256")" ]] || fail "$name handoff가 확인 뒤 바뀌었습니다"
|
||||
done
|
||||
|
||||
printf '[1/6] sudo 인증 확인\n'
|
||||
sudo -v
|
||||
printf '[2/6] pgAdmin SSD filesystem 경계와 Local PV 경로 확인\n'
|
||||
root_source="$(sudo findmnt --kernel --first-only --noheadings --output SOURCE --target /)"
|
||||
srv_source="$(sudo findmnt --kernel --first-only --noheadings --output SOURCE --target /srv)"
|
||||
srv_target="$(sudo findmnt --kernel --first-only --noheadings --output TARGET --target /srv)"
|
||||
[[ "$srv_source" == "$root_source" && "$srv_target" == / ]] || \
|
||||
fail "/srv가 root SSD 파일시스템 경계가 아닙니다"
|
||||
if sudo test -L "$PG_PATH"; then
|
||||
fail "$PG_PATH가 심볼릭 링크입니다"
|
||||
fi
|
||||
if sudo test -e "$PG_PATH"; then
|
||||
sudo test -d "$PG_PATH" || fail "$PG_PATH가 디렉터리가 아닙니다"
|
||||
else
|
||||
sudo install -d -o root -g root -m 0750 "$PG_PATH"
|
||||
fi
|
||||
[[ "$(sudo findmnt --kernel --first-only --noheadings --output SOURCE --target "$PG_PATH")" == "$root_source" ]] || \
|
||||
fail "$PG_PATH가 root SSD에 있지 않습니다"
|
||||
[[ "$(sudo findmnt --kernel --first-only --noheadings --output TARGET --target "$PG_PATH")" == / ]] || \
|
||||
fail "$PG_PATH 아래에 다른 mount가 있습니다"
|
||||
|
||||
printf '[3/6] rollback용 AIStor spec과 기존 pgAdmin Secret 상태 저장\n'
|
||||
kubectl -n object-storage get objectstore minio-aistor -o json | \
|
||||
jq '{apiVersion,kind,metadata:{name:.metadata.name,namespace:.metadata.namespace},spec}' \
|
||||
>"$work/objectstore-before.json"
|
||||
if kubectl -n platform-admin get secret "$PG_SECRET" -o yaml >"$work/pgadmin-bootstrap-before.yaml" 2>/dev/null; then
|
||||
bootstrap_existed=true
|
||||
fi
|
||||
|
||||
rollback_armed=true
|
||||
printf '[4/6] namespace, Local PV, Secret, AIStor OIDC와 pgAdmin 적용\n'
|
||||
kubectl apply -f "$work/admin-namespace.yaml"
|
||||
kubectl apply -f "$work/pgadmin-local-pv.yaml"
|
||||
|
||||
kubectl -n platform-admin create secret generic "$PG_SECRET" \
|
||||
--from-file="password=$password_file" \
|
||||
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
kubectl -n platform-admin label secret "$PG_SECRET" \
|
||||
app.kubernetes.io/name=pgadmin4 \
|
||||
app.kubernetes.io/component=bootstrap-credential \
|
||||
app.kubernetes.io/part-of=platform \
|
||||
app.kubernetes.io/managed-by=bootstrap-script \
|
||||
--overwrite >/dev/null
|
||||
|
||||
kubectl apply -f "$work/aistor-admin-oidc.yaml"
|
||||
kubectl apply -f "$work/pgadmin.yaml"
|
||||
|
||||
printf '[5/6] pgAdmin rollout, EndpointSlice와 AIStor health 대기\n'
|
||||
kubectl -n platform-admin rollout status deployment/pgadmin --timeout=300s
|
||||
kubectl -n platform-admin wait --for=jsonpath='{.endpoints[0].conditions.ready}'=true \
|
||||
endpointslice --selector=kubernetes.io/service-name=pgadmin --timeout=120s
|
||||
|
||||
healthy=false
|
||||
for attempt in $(seq 1 60); do
|
||||
health="$(kubectl -n object-storage get objectstore minio-aistor -o jsonpath='{.status.healthStatus}' 2>/dev/null)"
|
||||
if [[ "$health" == green ]]; then
|
||||
healthy=true
|
||||
break
|
||||
fi
|
||||
if (( attempt == 1 || attempt % 3 == 0 )); then
|
||||
printf 'AIStor health 대기: status=%s elapsed=%ss/300s\n' \
|
||||
"${health:-unknown}" "$((attempt * 5))"
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
[[ "$healthy" == true ]] || fail "AIStor ObjectStore가 green으로 복귀하지 않았습니다"
|
||||
|
||||
printf '[6/6] Traefik Host routing 확인\n'
|
||||
pg_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
|
||||
--noproxy '*' -H 'Host: db-admin.learn.hyeonworks.com' http://127.0.0.1:30080/)"
|
||||
ai_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
|
||||
--noproxy '*' -H 'Host: storage-admin.learn.hyeonworks.com' http://127.0.0.1:30080/)"
|
||||
[[ "$pg_code" == 200 || "$pg_code" == 302 || "$pg_code" == 303 ]] || fail "pgAdmin Traefik 응답 실패: $pg_code"
|
||||
[[ "$ai_code" == 200 || "$ai_code" == 302 || "$ai_code" == 303 ]] || fail "AIStor Console Traefik 응답 실패: $ai_code"
|
||||
|
||||
rollback_armed=false
|
||||
printf 'ADMIN SERVICES APPLY SUCCESS\n'
|
||||
printf 'pgAdmin PVC/PV: Retain, AIStor S3 API: cluster-internal only\n'
|
||||
Reference in New Issue
Block a user