Files

471 lines
18 KiB
Bash
Executable File

#!/usr/bin/env bash
set -Eeuo pipefail
# 호출자가 bash -x로 실행해도 Secret 경로와 향후 입력이 추적되지 않도록 한다.
set +x
readonly EXPECTED_HELM_VERSION="v3.19.4"
readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1"
readonly TARGET_NODE="donghyeon-system-product-name"
readonly EXPECTED_API_SERVICE_IP="10.43.0.1"
readonly EXPECTED_API_ENDPOINT_IP="192.168.0.107"
readonly EXPECTED_AISTOR_DEVICE="/dev/sdb3"
readonly EXPECTED_AISTOR_MOUNT="/srv/k3s/aistor"
readonly EXPECTED_STORAGE_CLASS="aistor-local-xfs-retain"
readonly EXPECTED_PV="aistor-data-local-pv"
readonly EXPECTED_OBJECTSTORE="minio-aistor"
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly -a VERIFIED_MANIFEST_NAMES=(
phase2-namespaces
aistor-local-pv
keycloak-operator
platform-postgres-keycloak
keycloak
aistor-operator
minio-aistor
aistor-network-policies
)
readonly -a AISTOR_CRDS=(
customresourcedefinition/adminjobs.aistor.min.io
customresourcedefinition/objectstores.aistor.min.io
customresourcedefinition/policybindings.sts.min.io
)
license_file=""
root_config_file=""
generate_root_config=false
execute_requested=false
mutation_started=false
current_step="preflight"
report_retained_state() {
if [[ "$mutation_started" == true ]]; then
printf '%s\n' \
"SAFE STOP during ${current_step}." \
'No Namespace, Secret, PV, PVC, Operator, ObjectStore, or XFS data was deleted.' \
'The PV reclaim policy remains Retain and PVC protection remains enabled.' \
'Diagnose the failed wait or policy, then rerun this script.' >&2
fi
}
fail() {
printf 'ERROR: %s\n' "$*" >&2
report_retained_state
exit 1
}
on_error() {
local status="$1"
local line="$2"
trap - ERR
set +e
printf 'ERROR: command failed at line %s (exit %s).\n' "$line" "$status" >&2
report_retained_state
exit "$status"
}
on_signal() {
local status="$1"
trap - INT TERM
set +e
printf 'INTERRUPTED: stopping without deleting cluster or XFS state.\n' >&2
report_retained_state
exit "$status"
}
usage() {
cat <<'USAGE'
Usage:
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
bash scripts/bootstrap/apply-aistor.sh \
--license-file /home/donghyeon/.secrets/aistor/minio.license \
--root-config-file /home/donghyeon/.secrets/aistor/root.env \
--generate-root-config \
--execute
Renders, verifies, and applies the internal-only MinIO AIStor path:
aistor and object-storage namespaces
two out-of-Git Secret contracts
one 900Gi Retain Local PV on /srv/k3s/aistor
AIStor Operator 5.10.0 and CRDs
one-server, one-drive ObjectStore 1.0.16
default-deny NetworkPolicies with only required internal paths
It does not configure Host Nginx, Traefik Ingress, NodePort, LoadBalancer,
public DNS, credential rotation, or deletion.
USAGE
}
while (( $# > 0 )); do
case "$1" in
--license-file)
(( $# >= 2 )) || fail "--license-file requires a path"
license_file="$2"
shift 2
;;
--root-config-file)
(( $# >= 2 )) || fail "--root-config-file requires a path"
root_config_file="$2"
shift 2
;;
--generate-root-config)
generate_root_config=true
shift
;;
--execute)
execute_requested=true
shift
;;
-h|--help)
usage
exit 0
;;
*)
usage >&2
fail "unsupported argument: $1"
;;
esac
done
[[ "$execute_requested" == true ]] || {
usage >&2
exit 2
}
[[ "$license_file" == /* ]] || fail "--license-file must be an absolute path"
[[ "$root_config_file" == /* ]] || \
fail "--root-config-file must be an absolute path"
for command_name in cmp curl df find findmnt jq kubectl mktemp mountpoint \
rg sed seq sha256sum sleep stat tail tr wc; do
command -v "$command_name" >/dev/null 2>&1 || \
fail "${command_name} is required"
done
if [[ -n "${PLATFORM_HELM_BIN:-}" ]]; then
[[ "$PLATFORM_HELM_BIN" == /* ]] || \
fail "PLATFORM_HELM_BIN must be an absolute path"
[[ -f "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \
fail "PLATFORM_HELM_BIN is not an executable file: ${PLATFORM_HELM_BIN}"
readonly HELM_BIN="$PLATFORM_HELM_BIN"
else
HELM_BIN="$(command -v helm 2>/dev/null)" || \
fail "Helm ${EXPECTED_HELM_VERSION} is required"
readonly HELM_BIN
fi
[[ "$("$HELM_BIN" version --template '{{.Version}}')" == "$EXPECTED_HELM_VERSION" ]] || \
fail "Helm must be exactly ${EXPECTED_HELM_VERSION}"
kustomize_version="$(
kubectl version --client --output=yaml |
sed -n 's/^kustomizeVersion: //p'
)"
[[ "$kustomize_version" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \
fail "expected Kustomize ${EXPECTED_KUSTOMIZE_VERSION}, found ${kustomize_version:-unknown}"
umask 077
render_temp_dir="$(mktemp -d /tmp/platform-phase2-apply.XXXXXX)"
cleanup() {
case "$render_temp_dir" in
/tmp/platform-phase2-apply.*)
rm -rf -- "$render_temp_dir"
;;
*)
printf 'WARNING: refusing to remove unexpected render directory: %s\n' \
"$render_temp_dir" >&2
;;
esac
}
trap cleanup EXIT
trap 'on_error "$?" "$LINENO"' ERR
trap 'on_signal 130' INT
trap 'on_signal 143' TERM
cd -- "$REPOSITORY_ROOT"
PLATFORM_HELM_BIN="$HELM_BIN" \
bash scripts/validate/render-phase2.sh \
--verified-output-dir "$render_temp_dir"
declare -A verified_manifest_sha256=()
for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do
manifest_path="${render_temp_dir}/${manifest_name}.yaml"
[[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \
fail "verified manifest is missing or unsafe: ${manifest_path}"
[[ "$(stat --format='%a' -- "$manifest_path")" == "600" ]] || \
fail "verified manifest must have mode 0600: ${manifest_path}"
checksum_output="$(sha256sum -- "$manifest_path")"
verified_manifest_sha256["$manifest_name"]="${checksum_output%% *}"
done
verified_entry_count="$(
find "$render_temp_dir" -mindepth 1 -maxdepth 1 |
wc -l | tr -d '[:space:]'
)"
[[ "$verified_entry_count" == "${#VERIFIED_MANIFEST_NAMES[@]}" ]] || \
fail "verified handoff must contain exactly eight manifest files"
verify_manifest_unchanged() {
local manifest_name="$1"
local manifest_path="${render_temp_dir}/${manifest_name}.yaml"
local checksum_output
local actual_sha256
[[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \
fail "verified manifest became missing or unsafe: ${manifest_path}"
checksum_output="$(sha256sum -- "$manifest_path")"
actual_sha256="${checksum_output%% *}"
[[ "$actual_sha256" == "${verified_manifest_sha256[$manifest_name]}" ]] || \
fail "verified manifest changed before apply: ${manifest_name}.yaml"
}
current_context="$(kubectl config current-context)"
api_server="$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')"
node_ready="$(
kubectl get node "$TARGET_NODE" \
--output='go-template={{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}true{{end}}{{end}}'
)"
[[ "$node_ready" == "true" ]] || fail "target node is not Ready: ${TARGET_NODE}"
cluster_api_ip="$(
kubectl --namespace default get service kubernetes \
--output=jsonpath='{.spec.clusterIP}'
)"
[[ "$cluster_api_ip" == "$EXPECTED_API_SERVICE_IP" ]] || \
fail "Kubernetes API Service IP changed: expected ${EXPECTED_API_SERVICE_IP}, found ${cluster_api_ip}"
cluster_api_endpoint="$(
kubectl --namespace default get endpointslice \
--selector=kubernetes.io/service-name=kubernetes \
--output=jsonpath='{.items[0].endpoints[0].addresses[0]}'
)"
[[ "$cluster_api_endpoint" == "$EXPECTED_API_ENDPOINT_IP" ]] || \
fail "Kubernetes API endpoint changed: expected ${EXPECTED_API_ENDPOINT_IP}, found ${cluster_api_endpoint}"
mountpoint --quiet "$EXPECTED_AISTOR_MOUNT" || \
fail "${EXPECTED_AISTOR_MOUNT} is not a mountpoint"
mount_source="$(findmnt --noheadings --output SOURCE --target "$EXPECTED_AISTOR_MOUNT" | tr -d '[:space:]')"
mount_fstype="$(findmnt --noheadings --output FSTYPE --target "$EXPECTED_AISTOR_MOUNT" | tr -d '[:space:]')"
mount_options="$(findmnt --noheadings --output OPTIONS --target "$EXPECTED_AISTOR_MOUNT")"
[[ "$mount_source" == "$EXPECTED_AISTOR_DEVICE" ]] || \
fail "AIStor mount source changed: expected ${EXPECTED_AISTOR_DEVICE}, found ${mount_source}"
[[ "$mount_fstype" == "xfs" ]] || \
fail "AIStor mount must be XFS, found ${mount_fstype}"
[[ ",${mount_options}," == *,rw,* ]] || fail "AIStor XFS mount is not writable"
[[ -d "$EXPECTED_AISTOR_MOUNT" && ! -L "$EXPECTED_AISTOR_MOUNT" ]] || \
fail "AIStor mount path must be a non-symlink directory"
available_bytes="$(
df --block-size=1 --output=avail "$EXPECTED_AISTOR_MOUNT" |
tail -n 1 | tr -d '[:space:]'
)"
minimum_bytes=$((900 * 1024 * 1024 * 1024))
(( available_bytes >= minimum_bytes )) || \
fail "AIStor XFS has less than 900Gi available"
if ! kubectl get persistentvolume "$EXPECTED_PV" >/dev/null 2>&1; then
[[ -z "$(find "$EXPECTED_AISTOR_MOUNT" -mindepth 1 -maxdepth 1 -print -quit)" ]] || \
fail "initial AIStor XFS root is not empty; refusing to bind an unknown data directory"
fi
unexpected_consumers="$(
kubectl get persistentvolumeclaim --all-namespaces --output=json |
jq --arg storage_class "$EXPECTED_STORAGE_CLASS" \
'[.items[] | select(.spec.storageClassName == $storage_class)] | length'
)"
if (( unexpected_consumers > 0 )); then
existing_expected_claim="$(
kubectl --namespace object-storage get persistentvolumeclaim \
--ignore-not-found --output=json |
jq --arg storage_class "$EXPECTED_STORAGE_CLASS" \
'[.items[] | select(.spec.storageClassName == $storage_class)] | length'
)"
[[ "$unexpected_consumers" == "1" && "$existing_expected_claim" == "1" ]] || \
fail "the AIStor StorageClass has an unexpected PVC consumer"
fi
if kubectl get customresourcedefinition objectstores.aistor.min.io >/dev/null 2>&1; then
unexpected_objectstores="$(
kubectl get objectstores.aistor.min.io --all-namespaces --output=json |
jq --arg name "$EXPECTED_OBJECTSTORE" \
'[.items[] | select(.metadata.namespace != "object-storage" or .metadata.name != $name)] | length'
)"
[[ "$unexpected_objectstores" == "0" ]] || \
fail "an unexpected AIStor ObjectStore already exists"
fi
[[ -f "$license_file" && ! -L "$license_file" && -O "$license_file" && -s "$license_file" ]] || \
fail "license file must be a non-empty, current-user-owned regular file"
[[ "$(stat --format='%a' -- "$license_file")" == "600" ]] || \
fail "license file must have mode 0600"
printf '\nKubernetes context: %s\nAPI server: %s\nTarget node: %s\n' \
"$current_context" "$api_server" "$TARGET_NODE"
printf 'XFS: %s -> %s (%s, at least 900Gi available)\n' \
"$mount_source" "$EXPECTED_AISTOR_MOUNT" "$mount_fstype"
printf '%s\n' \
'Scope: internal-only AIStor Operator, 900Gi Retain Local PV, one ObjectStore, and NetworkPolicies.' \
'Excluded: Host Nginx, Traefik, NodePort, LoadBalancer, public DNS, rotation, and deletion.' \
'Failure boundary: all applied state and XFS data are retained; rerunning is the recovery path.'
[[ -t 0 ]] || fail "an interactive terminal is required"
printf 'Type APPLY AISTOR %s to start the cluster mutation: ' "$current_context"
read -r confirmation
[[ "$confirmation" == "APPLY AISTOR ${current_context}" ]] || fail "cancelled"
assert_cluster_identity() {
[[ "$(kubectl config current-context)" == "$current_context" ]] || \
fail "kubectl context changed after confirmation"
[[ "$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" == "$api_server" ]] || \
fail "Kubernetes API server changed after confirmation"
kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \
fail "target node disappeared after confirmation: ${TARGET_NODE}"
mountpoint --quiet "$EXPECTED_AISTOR_MOUNT" || \
fail "AIStor XFS mount disappeared after confirmation"
}
assert_cluster_identity
for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do
verify_manifest_unchanged "$manifest_name"
done
mutation_started=true
current_step="[1/7] AIStor namespaces"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged phase2-namespaces
kubectl apply --dry-run=server \
--filename="${render_temp_dir}/phase2-namespaces.yaml" >/dev/null
kubectl apply --filename="${render_temp_dir}/phase2-namespaces.yaml"
current_step="[2/7] AIStor Secret contracts"
printf '\n%s\n' "$current_step"
assert_cluster_identity
secret_args=(
--license-file "$license_file"
--root-config-file "$root_config_file"
--execute
)
if [[ "$generate_root_config" == true ]]; then
secret_args+=(--generate-root-config)
fi
bash scripts/bootstrap/create-aistor-secrets.sh "${secret_args[@]}"
current_step="[3/7] 900Gi Retain Local PV"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged aistor-local-pv
kubectl apply --dry-run=server \
--filename="${render_temp_dir}/aistor-local-pv.yaml" >/dev/null
kubectl apply --filename="${render_temp_dir}/aistor-local-pv.yaml"
current_step="[4/7] AIStor Operator and CRDs"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged aistor-operator
kubectl apply --server-side \
--filename="${render_temp_dir}/aistor-operator.yaml"
kubectl wait --for=condition=Established "${AISTOR_CRDS[@]}" --timeout=5m
kubectl --namespace aistor rollout status \
deployment/adminjob-operator --timeout=10m
kubectl --namespace aistor rollout status \
deployment/object-store-operator --timeout=10m
kubectl --namespace aistor rollout status \
deployment/object-store-webhook --timeout=10m
current_step="[5/7] One-node, one-drive AIStor ObjectStore"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged minio-aistor
kubectl apply --server-side --dry-run=server \
--filename="${render_temp_dir}/minio-aistor.yaml" >/dev/null
kubectl apply --server-side \
--filename="${render_temp_dir}/minio-aistor.yaml"
statefulset_name=""
for _ in $(seq 1 180); do
statefulset_names="$(
kubectl --namespace object-storage get statefulset \
--selector="aistor.min.io/objectStore=${EXPECTED_OBJECTSTORE}" \
--output=name
)"
statefulset_count="$(printf '%s\n' "$statefulset_names" | sed '/^$/d' | wc -l | tr -d '[:space:]')"
if [[ "$statefulset_count" == "1" ]]; then
statefulset_name="$statefulset_names"
break
fi
sleep 2
done
[[ -n "$statefulset_name" ]] || \
fail "the ObjectStore Operator did not create exactly one StatefulSet"
kubectl --namespace object-storage rollout status "$statefulset_name" --timeout=15m
kubectl --namespace object-storage wait \
--for=condition=Ready pod \
--selector="aistor.min.io/objectStore=${EXPECTED_OBJECTSTORE}" \
--timeout=10m
current_step="[6/7] AIStor default-deny NetworkPolicies"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged aistor-network-policies
kubectl apply --dry-run=server \
--filename="${render_temp_dir}/aistor-network-policies.yaml" >/dev/null
kubectl apply \
--filename="${render_temp_dir}/aistor-network-policies.yaml"
kubectl --namespace object-storage wait \
--for=condition=Ready pod \
--selector="aistor.min.io/objectStore=${EXPECTED_OBJECTSTORE}" \
--timeout=5m
current_step="[7/7] Storage, service, and exposure acceptance"
printf '\n%s\n' "$current_step"
assert_cluster_identity
pvc_json="$(
kubectl --namespace object-storage get persistentvolumeclaim --output=json |
jq --arg storage_class "$EXPECTED_STORAGE_CLASS" \
'{apiVersion, kind, items: [.items[] | select(.spec.storageClassName == $storage_class)]}'
)"
[[ "$(jq '.items | length' <<<"$pvc_json")" == "1" ]] || \
fail "expected exactly one AIStor PVC"
[[ "$(jq -r '.items[0].status.phase' <<<"$pvc_json")" == "Bound" ]] || \
fail "AIStor PVC is not Bound"
[[ "$(jq -r '.items[0].spec.volumeName' <<<"$pvc_json")" == "$EXPECTED_PV" ]] || \
fail "AIStor PVC did not bind the expected Local PV"
[[ "$(jq -r '.items[0].spec.resources.requests.storage' <<<"$pvc_json")" == "900Gi" ]] || \
fail "AIStor PVC request is not 900Gi"
for service_name in minio minio-aistor-console minio-aistor-hl; do
service_type="$(
kubectl --namespace object-storage get service "$service_name" \
--output=jsonpath='{.spec.type}'
)"
[[ "$service_type" == "ClusterIP" ]] || \
fail "${service_name} must remain ClusterIP"
done
[[ "$(
kubectl --namespace object-storage get service minio \
--output=jsonpath='{.spec.ports[0].port}:{.spec.ports[0].targetPort}'
)" == "80:9000" ]] || fail "S3 Service must map 80/TCP to 9000/TCP"
[[ "$(
kubectl --namespace object-storage get service minio-aistor-console \
--output=jsonpath='{.spec.ports[0].port}:{.spec.ports[0].targetPort}'
)" == "9090:9090" ]] || fail "Console Service must map 9090/TCP to 9090/TCP"
[[ "$(
kubectl --namespace object-storage get service minio-aistor-hl \
--output=jsonpath='{.spec.clusterIP}'
)" == "None" ]] || fail "AIStor headless Service must remain headless"
[[ -z "$(
kubectl --namespace object-storage get service --output=json |
jq -r '.items[].spec.ports[]? | select(.nodePort != null) | .nodePort'
)" ]] || fail "an AIStor service unexpectedly has a NodePort"
[[ -z "$(kubectl --namespace object-storage get ingress --output=name)" ]] || \
fail "AIStor must not have an Ingress"
printf '\nAISTOR APPLY SUCCESS\n'
printf 'ObjectStore: object-storage/%s\n' "$EXPECTED_OBJECTSTORE"
printf 'Storage: %s -> %s (900Gi PVC, Retain)\n' \
"$EXPECTED_AISTOR_DEVICE" "$EXPECTED_AISTOR_MOUNT"
printf '%s\n' \
'Exposure: ClusterIP only; no Host Nginx, Traefik, NodePort, or public DNS.' \
"Root credential file: ${root_config_file}" \
'Next: run the authenticated S3 write/read smoke test.'