#!/usr/bin/env bash set -Eeuo pipefail # Never inherit xtrace across credential, bearer-token, or client-secret handling. set +x umask 077 readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" readonly EXPECTED_CONTEXT='default' readonly EXPECTED_API_SERVER='https://127.0.0.1:6443' readonly EXPECTED_NODE='donghyeon-system-product-name' readonly KEYCLOAK_NAMESPACE='keycloak' readonly KEYCLOAK_NAME='keycloak' readonly KEYCLOAK_SERVICE='keycloak-service' readonly KEYCLOAK_ADMIN_SECRET='keycloak-initial-admin' readonly KEYCLOAK_HOST='id.learn.hyeonworks.com' readonly KEYCLOAK_REALM='hyeonworks' readonly OBSERVABILITY_NAMESPACE='observability' readonly OIDC_SECRET='grafana-keycloak-oidc' readonly CLIENT_ID='grafana' readonly GRAFANA_URL='https://grafana.learn.hyeonworks.com' readonly REDIRECT_URI='https://grafana.learn.hyeonworks.com/login/generic_oauth' readonly POST_LOGOUT_URI='https://grafana.learn.hyeonworks.com/*' readonly ADMIN_GROUP_NAME='platform-observability-admins' readonly VIEWER_GROUP_NAME='platform-observability-viewers' readonly ADMIN_GROUP_PATH='/platform-observability-admins' readonly VIEWER_GROUP_PATH='/platform-observability-viewers' readonly MAPPER_NAME='grafana-groups' readonly EVIDENCE_SCHEMA='platform-observability-recovery-evidence-v1' readonly EVIDENCE_RESOURCE='keycloak/hyeonworks/client/grafana' readonly EVIDENCE_MAX_AGE_SECONDS=2592000 readonly PRODUCTION_EVIDENCE_DIR='/etc/hyeonworks/platform/recovery-evidence' readonly PRODUCTION_ENCRYPTION_SCRIPT="$ROOT/scripts/validate/k3s-secret-encryption.sh" readonly PRODUCTION_RESTORE_SCRIPT="$ROOT/scripts/validate/k3s-secret-encryption-restore-evidence.sh" readonly VALIDATOR_PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' readonly VALIDATOR_HOME='/home/donghyeon' readonly KUBECTL_REQUEST_TIMEOUT='5s' TEST_MODE=false KUBECTL_BIN='/usr/local/bin/kubectl' SUDO_BIN='/usr/bin/sudo' ENCRYPTION_SCRIPT="$PRODUCTION_ENCRYPTION_SCRIPT" RESTORE_SCRIPT="$PRODUCTION_RESTORE_SCRIPT" EVIDENCE_DIR="$PRODUCTION_EVIDENCE_DIR" KUBECTL_PROCESS_TIMEOUT='10s' PORT_FORWARD_PROCESS_TIMEOUT='120s' PROXY_PROCESS_TIMEOUT='120s' CURL_PROCESS_TIMEOUT='12s' CURL_MAX_TIME='8' CURL_CONNECT_TIMEOUT='2' EVIDENCE_EXPECTED_UID=0 EVIDENCE_EXPECTED_GID=0 EVIDENCE_DIR_MODE=700 TEST_CONFIRMATIONS='' TEST_NOW_UTC='' execute_requested=false check_requested=false admin_username='' viewer_username='' temporary_dir='' port_forward_pid='' port_forward_start='' proxy_pid='' proxy_start='' local_port='' local_base='' kube_socket='' confirmation_index=0 declare -a confirmation_answers=() transaction_active=false rollback_in_progress=false exit_handler_running=false manual_recovery_required=false pending_kind='' pending_before='' pending_desired='' pending_id='' pending_token='' pending_user_id='' pending_group_id='' pending_status_file='' transaction_token='' client_state='absent' client_id='' client_before='' client_desired='' client_after='' client_restore='' client_secret_file='' client_mutation='none' admin_group_state='absent' admin_group_id='' admin_group_before='' admin_group_after='' admin_group_mutation='none' viewer_group_state='absent' viewer_group_id='' viewer_group_before='' viewer_group_after='' viewer_group_mutation='none' mapper_state='absent' mapper_id='' mapper_before='' mapper_desired='' mapper_after='' mapper_mutation='none' secret_state='absent' secret_before='' secret_uid='' secret_resource_version='' secret_created_uid='' secret_created_resource_version='' secret_mutation='none' admin_user_id='' viewer_user_id='' admin_membership_before=false viewer_membership_before=false admin_membership_added=false viewer_membership_added=false marker_had_prior=false marker_prior='' marker_desired='' marker_written=false last_http_status='' last_body='' last_status_file='' last_curl_rc=0 request_counter=0 usage() { cat <<'USAGE' Usage: bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh --execute bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh \ --execute --admin USER --viewer USER bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh \ --check-recovery-evidence No arguments prints the fixed, payload-free plan and performs no external reads. Execute reconciles only the exact Grafana client, two groups, full-path groups mapper, OIDC Secret, and explicitly requested existing-user memberships. USAGE } emit_manual_recovery() { [[ "$manual_recovery_required" == false ]] || return 0 manual_recovery_required=true printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2 } fail() { local message=$1 if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then transaction_fail "$message" fi printf 'ERROR: %s\n' "$message" >&2 exit 1 } reject_production_overrides() { local variable for variable in ${!PLATFORM_GRAFANA_OIDC_@}; do printf 'ERROR: production boundary override is forbidden: %s\n' "$variable" >&2 exit 1 done } parse_arguments() { local seen_execute=false seen_check=false seen_admin=false seen_viewer=false while (( $# > 0 )); do case "$1" in --execute) [[ "$seen_execute" == false ]] || fail 'duplicate --execute' seen_execute=true; execute_requested=true; shift ;; --check-recovery-evidence) [[ "$seen_check" == false ]] || fail 'duplicate --check-recovery-evidence' seen_check=true; check_requested=true; shift ;; --admin) [[ "$seen_admin" == false && $# -ge 2 ]] || fail 'invalid --admin' seen_admin=true; admin_username=$2; shift 2 ;; --viewer) [[ "$seen_viewer" == false && $# -ge 2 ]] || fail 'invalid --viewer' seen_viewer=true; viewer_username=$2; shift 2 ;; -h|--help) usage; exit 0 ;; *) fail "unsupported argument: $1" ;; esac done if [[ "$check_requested" == true ]]; then [[ "$execute_requested" == false && "$seen_admin" == false && "$seen_viewer" == false ]] || fail 'recovery evidence mode cannot be combined with execute or memberships' return fi if [[ "$execute_requested" == false ]]; then [[ "$seen_admin" == false && "$seen_viewer" == false ]] || fail 'membership selection requires --execute' return fi local username for username in "$admin_username" "$viewer_username"; do [[ -z "$username" || "$username" =~ ^[A-Za-z0-9][A-Za-z0-9._@+-]{0,127}$ ]] || fail 'membership usernames must use the bounded printable Keycloak username form' done } print_dry_run() { printf '%s\n' \ 'GRAFANA_OIDC_DRY_RUN=PASS' \ 'CONTEXT=default' \ 'KEYCLOAK_CLIENT=grafana' \ 'KEYCLOAK_GROUPS=/platform-observability-admins,/platform-observability-viewers' \ 'KEYCLOAK_MAPPER=grafana-groups:groups:full-path:id-token,access-token,userinfo' \ 'KUBERNETES_SECRET=observability/grafana-keycloak-oidc:Opaque:client-id,client-secret' \ 'MEMBERSHIPS=EXPLICIT_ONLY' \ 'MUTATION=NOT_REQUESTED' } attest_test_executable() { local path=$1 metadata [[ -f "$path" && ! -L "$path" && -O "$path" && -x "$path" ]] || return 1 metadata="$(stat -c '%u:%a:%h:%F' -- "$path")" || return 1 [[ "$metadata" == "${EUID}:755:1:regular file" ]] } configure_test_boundaries() { local fixture=$1 resolved metadata [[ "$fixture" =~ ^/tmp/platform-grafana-oidc-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+$ ]] || fail 'test fixture root has an invalid shape' [[ -d "$fixture" && ! -L "$fixture" && -O "$fixture" ]] || fail 'test fixture root is unsafe' resolved="$(cd -- "$fixture" && pwd -P)" || fail 'cannot resolve test fixture root' [[ "$resolved" == "$fixture" ]] || fail 'test fixture contains a symlink boundary' metadata="$(stat -c '%u:%a:%F' -- "$fixture")" || fail 'cannot stat test fixture root' [[ "$metadata" == "${EUID}:700:directory" ]] || fail 'test fixture metadata is unsafe' [[ "$(cd -- "$fixture/bin" && pwd -P)" == "$fixture/bin" && "$(cd -- "$fixture/evidence-parent" && pwd -P)" == "$fixture/evidence-parent" ]] || fail 'test fixture child has a symlink boundary' KUBECTL_BIN="$fixture/bin/kubectl" SUDO_BIN="$fixture/bin/sudo" ENCRYPTION_SCRIPT="$fixture/bin/encryption" RESTORE_SCRIPT="$fixture/bin/restore" EVIDENCE_DIR="$fixture/evidence-parent/recovery-evidence" EVIDENCE_EXPECTED_UID=$EUID EVIDENCE_EXPECTED_GID="$(id -g)" EVIDENCE_DIR_MODE=700 KUBECTL_PROCESS_TIMEOUT='3s' PORT_FORWARD_PROCESS_TIMEOUT='30s' PROXY_PROCESS_TIMEOUT='30s' CURL_PROCESS_TIMEOUT='4s' CURL_MAX_TIME='2' CURL_CONNECT_TIMEOUT='1' TEST_CONFIRMATIONS="${PLATFORM_GRAFANA_OIDC_CONFIRMATIONS:-}" TEST_NOW_UTC="${PLATFORM_GRAFANA_OIDC_NOW_UTC:-}" mapfile -t confirmation_answers <<<"$TEST_CONFIRMATIONS" [[ "$TEST_NOW_UTC" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || fail 'test clock is invalid' attest_test_executable "$KUBECTL_BIN" || fail 'test kubectl boundary is unsafe' attest_test_executable "$SUDO_BIN" || fail 'test sudo boundary is unsafe' attest_test_executable "$ENCRYPTION_SCRIPT" || fail 'test encryption boundary is unsafe' attest_test_executable "$RESTORE_SCRIPT" || fail 'test restore boundary is unsafe' [[ "$SUDO_BIN" != /usr/bin/sudo ]] || fail 'real sudo is forbidden in the fixture entrypoint' export PLATFORM_TEST_TARGET_PID=$BASHPID } require_commands() { local command_name for command_name in base64 chmod cmp cp curl date grep id jq kill mktemp python3 rm sort stat timeout tr wc; do command -v "$command_name" >/dev/null 2>&1 || fail "$command_name is required" done [[ -x "$KUBECTL_BIN" && ! -L "$KUBECTL_BIN" ]] || fail 'kubectl boundary is unavailable or unsafe' [[ -x "$SUDO_BIN" && ! -L "$SUDO_BIN" ]] || fail 'sudo boundary is unavailable or unsafe' [[ -f "$ENCRYPTION_SCRIPT" && ! -L "$ENCRYPTION_SCRIPT" && -r "$ENCRYPTION_SCRIPT" ]] || fail 'k3s encryption validator is unavailable' [[ -f "$RESTORE_SCRIPT" && ! -L "$RESTORE_SCRIPT" && -r "$RESTORE_SCRIPT" ]] || fail 'k3s restore validator is unavailable' } make_temporary_dir() { temporary_dir="$(mktemp -d /tmp/platform-grafana-oidc.XXXXXX)" || fail 'cannot create private temporary directory' [[ "$temporary_dir" == /tmp/platform-grafana-oidc.?????? && ! -L "$temporary_dir" ]] || fail 'private temporary directory shape is unsafe' chmod 0700 -- "$temporary_dir" [[ "$(stat -c '%u:%a:%h:%F' -- "$temporary_dir")" == "${EUID}:700:2:directory" ]] || fail 'private temporary directory metadata is unsafe' } private_file_ok() { local path=$1 allow_empty=${2:-false} metadata [[ "$path" == "$temporary_dir"/* && -f "$path" && ! -L "$path" && -O "$path" ]] || return 1 metadata="$(stat -c '%u:%a:%h' -- "$path")" || return 1 [[ "$metadata" == "${EUID}:600:1" ]] || return 1 [[ "$allow_empty" == true || -s "$path" ]] } process_start_time() { local pid=$1 python3 -I -S - "$pid" <<'PY' import pathlib,sys raw=pathlib.Path('/proc',sys.argv[1],'stat').read_text() end=raw.rfind(') ') if end < 0: raise SystemExit(1) fields=raw[end+2:].split() if len(fields)<20 or not fields[19].isdigit(): raise SystemExit(1) print(fields[19],end='') PY } stop_owned_process() { local pid=$1 expected_start=$2 attempt current [[ -n "$pid" && -n "$expected_start" && "$pid" =~ ^[0-9]+$ ]] || return 0 current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; } [[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; } kill -TERM "$pid" 2>/dev/null || true for ((attempt=0; attempt<30; attempt++)); do if ! kill -0 "$pid" 2>/dev/null; then wait "$pid" 2>/dev/null || true; return 0; fi /usr/bin/sleep 0.05 done current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; } [[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; } kill -KILL "$pid" 2>/dev/null || true wait "$pid" 2>/dev/null || true } safe_temporary_path() { [[ -n "$temporary_dir" && "$temporary_dir" == /tmp/platform-grafana-oidc.?????? && ( "$1" == "$temporary_dir" || "$1" == "$temporary_dir"/* ) ]] } cleanup() { stop_owned_process "$proxy_pid" "$proxy_start" || true stop_owned_process "$port_forward_pid" "$port_forward_start" || true if [[ -n "$temporary_dir" ]] && safe_temporary_path "$temporary_dir"; then rm -rf -- "$temporary_dir"; fi } kubectl_bounded() { /usr/bin/timeout --signal=TERM --kill-after=1s "$KUBECTL_PROCESS_TIMEOUT" \ "$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" "$@" } run_common_gates() { local -a safe_environment=(/usr/bin/env -i "PATH=$VALIDATOR_PATH" "HOME=$VALIDATOR_HOME") if [[ "$TEST_MODE" == true ]]; then safe_environment+=( "PLATFORM_TEST_COMMAND_LOG=${PLATFORM_TEST_COMMAND_LOG:-}" "PLATFORM_TEST_STATE=${PLATFORM_TEST_STATE:-}" "PLATFORM_TEST_ENCRYPTION_COUNT=${PLATFORM_TEST_ENCRYPTION_COUNT:-}" "PLATFORM_TEST_RESTORE_COUNT=${PLATFORM_TEST_RESTORE_COUNT:-}" "PLATFORM_TEST_ENCRYPTION_FAIL_AT=${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" "PLATFORM_TEST_RESTORE_FAIL_AT=${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" "PLATFORM_TEST_VALIDATOR_ENV_LOG=${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" ) fi "${safe_environment[@]}" /usr/bin/bash "$ENCRYPTION_SCRIPT" --expect-reencrypted >/dev/null 2>&1 || fail 'k3s Secret encryption is not fully re-encrypted' "${safe_environment[@]}" /usr/bin/bash "$RESTORE_SCRIPT" --check >/dev/null 2>&1 || fail 'current k3s restore evidence is unavailable' } read_confirmation() { local prompt=$1 expected=$2 answer='' printf '%s' "$prompt" >&2 if [[ "$TEST_MODE" == true ]]; then (( confirmation_index < ${#confirmation_answers[@]} )) || fail 'confirmation input is missing' answer="${confirmation_answers[$confirmation_index]}"; confirmation_index=$((confirmation_index + 1)) else [[ -t 0 ]] || fail 'execute mode requires an interactive terminal' IFS= read -r answer fi [[ "$answer" == "$expected" ]] || fail 'confirmation was not exact' } now_utc() { if [[ "$TEST_MODE" == true ]]; then printf '%s\n' "$TEST_NOW_UTC"; else date -u +%Y-%m-%dT%H:%M:%SZ; fi } timestamp_epoch() { local timestamp=$1 normalized normalized="$(date -u -d "$timestamp" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" || return 1 [[ "$normalized" == "$timestamp" ]] || return 1 date -u -d "$timestamp" +%s } now_epoch() { timestamp_epoch "$(now_utc)"; } json_equal() { python3 -I -S - "$1" "$2" <<'PY' import json,pathlib,sys try: a=json.loads(pathlib.Path(sys.argv[1]).read_text()); b=json.loads(pathlib.Path(sys.argv[2]).read_text()) except Exception: raise SystemExit(1) raise SystemExit(0 if a==b else 1) PY } new_transaction_token() { python3 -I -S -c 'import secrets; print(secrets.token_hex(16))'; } validate_context_and_authority() { local context api node_file namespace result context="$(kubectl_bounded config current-context)" || fail 'cannot read Kubernetes context' [[ "$context" == "$EXPECTED_CONTEXT" ]] || fail 'current Kubernetes context is not default' api="$(kubectl_bounded config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" || fail 'cannot read Kubernetes API server' [[ "$api" == "$EXPECTED_API_SERVER" ]] || fail 'Kubernetes API server is not the exact local endpoint' node_file="$temporary_dir/node.json" kubectl_bounded get node "$EXPECTED_NODE" --output=json >"$node_file" || fail 'cannot read exact target node' chmod 0600 -- "$node_file" private_file_ok "$node_file" || fail 'target node response is unsafe' jq -e --arg name "$EXPECTED_NODE" ' .apiVersion == "v1" and .kind == "Node" and .metadata.name == $name and (.metadata.uid | type == "string" and length > 0) and any(.status.conditions[]?; .type == "Ready" and .status == "True") ' "$node_file" >/dev/null || fail 'target node identity or readiness changed' for namespace in "$KEYCLOAK_NAMESPACE" "$OBSERVABILITY_NAMESPACE"; do result="$(kubectl_bounded get namespace "$namespace" --output=name)" || fail "cannot read namespace $namespace" [[ "$result" == "namespace/$namespace" ]] || fail "namespace identity changed: $namespace" done for permission in \ 'get keycloak.k8s.keycloak.org/keycloak keycloak' \ 'get service/keycloak-service keycloak' \ 'get secret/keycloak-initial-admin keycloak' \ 'get secret/grafana-keycloak-oidc observability' \ 'create secret observability' \ 'delete secret/grafana-keycloak-oidc observability' \ 'get nodes -' \ 'get namespaces -' \ 'get pods keycloak' \ 'list pods keycloak' \ 'create pods/portforward keycloak'; do set -- $permission if [[ "$3" == - ]]; then result="$(kubectl_bounded auth can-i "$1" "$2")" || fail 'cannot verify exact Kubernetes authority' else result="$(kubectl_bounded auth can-i "$1" "$2" --namespace="$3")" || fail 'cannot verify exact Kubernetes authority' fi [[ "$result" == yes ]] || fail "missing exact Kubernetes authority: $permission" done } snapshot_kubernetes_objects() { local keycloak_file service_file admin_file keycloak_file="$temporary_dir/keycloak-resource.json" service_file="$temporary_dir/keycloak-service.json" admin_file="$temporary_dir/keycloak-admin-secret.json" kubectl_bounded --namespace "$KEYCLOAK_NAMESPACE" get "keycloak.k8s.keycloak.org/$KEYCLOAK_NAME" --output=json >"$keycloak_file" || fail 'cannot read exact Keycloak resource' kubectl_bounded --namespace "$KEYCLOAK_NAMESPACE" get "service/$KEYCLOAK_SERVICE" --output=json >"$service_file" || fail 'cannot read exact Keycloak Service' kubectl_bounded --namespace "$KEYCLOAK_NAMESPACE" get "secret/$KEYCLOAK_ADMIN_SECRET" --output=json >"$admin_file" || fail 'cannot read exact initial-admin Secret' chmod 0600 -- "$keycloak_file" "$service_file" "$admin_file" private_file_ok "$keycloak_file" && private_file_ok "$service_file" && private_file_ok "$admin_file" || fail 'Kubernetes bootstrap snapshots are unsafe' jq -e --arg name "$KEYCLOAK_NAME" --arg namespace "$KEYCLOAK_NAMESPACE" ' .apiVersion == "k8s.keycloak.org/v2beta1" and .kind == "Keycloak" and .metadata.name == $name and .metadata.namespace == $namespace and (.metadata.uid | type == "string" and length > 0) and any(.status.conditions[]?; .type == "Ready" and .status == "True") ' "$keycloak_file" >/dev/null || fail 'Keycloak resource identity or readiness changed' jq -e --arg name "$KEYCLOAK_SERVICE" --arg namespace "$KEYCLOAK_NAMESPACE" ' .apiVersion == "v1" and .kind == "Service" and .metadata.name == $name and .metadata.namespace == $namespace and (.metadata.uid | type == "string" and length > 0) and ([.spec.ports[]? | select(.port == 8080)] | length) == 1 ' "$service_file" >/dev/null || fail 'Keycloak Service contract changed' local admin_username_file admin_password_file admin_username_file="$temporary_dir/admin-username" admin_password_file="$temporary_dir/admin-password" python3 -I -S - "$admin_file" "$admin_username_file" "$admin_password_file" <<'PY' || import base64,binascii,json,os,pathlib,sys source,user_path,password_path=sys.argv[1:] try: item=json.loads(pathlib.Path(source).read_text()) except Exception: raise SystemExit(1) meta=item.get('metadata') or {}; data=item.get('data') if item.get('apiVersion')!='v1' or item.get('kind')!='Secret' or item.get('type')!='kubernetes.io/basic-auth' or meta.get('namespace')!='keycloak' or meta.get('name')!='keycloak-initial-admin' or not meta.get('uid') or sorted(data or {})!=['password','username']: raise SystemExit(1) for key,destination in [('username',user_path),('password',password_path)]: try: payload=base64.b64decode(data[key],validate=True) except (binascii.Error,ValueError): raise SystemExit(1) if not payload or b'\x00' in payload or b'\r' in payload or b'\n' in payload: raise SystemExit(1) fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600) try: os.write(fd,payload) finally: os.close(fd) PY fail 'initial-admin Secret contract is invalid' private_file_ok "$admin_username_file" && private_file_ok "$admin_password_file" || fail 'private initial-admin credential snapshot is unsafe' rm -f -- "$admin_file" } allocate_loopback_port() { local reservation_file="$temporary_dir/port-reservation" python3 -I -S - "$reservation_file" <<'PY' || return 1 import os,socket,sys s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) s.bind(('127.0.0.1',0)); port=s.getsockname()[1] fd=os.open(sys.argv[1],os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600) try: os.write(fd,str(port).encode()) finally: os.close(fd); s.close() PY private_file_ok "$reservation_file" || return 1 IFS= read -r local_port <"$reservation_file" [[ "$local_port" =~ ^[0-9]+$ ]] || return 1 (( local_port >= 1024 && local_port <= 65535 )) || return 1 local_base="http://127.0.0.1:$local_port" } loopback_port_open() { python3 -I -S - "$local_port" <<'PY' import socket,sys port=int(sys.argv[1]) with socket.create_connection(('127.0.0.1',port),timeout=0.2): pass PY } start_port_forward() { local log="$temporary_dir/port-forward.log" ready=false attempt allocate_loopback_port || fail 'cannot allocate a validated loopback port' /usr/bin/timeout --signal=TERM --kill-after=1s "$PORT_FORWARD_PROCESS_TIMEOUT" \ "$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" \ port-forward --address=127.0.0.1 --namespace="$KEYCLOAK_NAMESPACE" \ "service/$KEYCLOAK_SERVICE" "$local_port:8080" >"$log" 2>&1 & port_forward_pid=$! port_forward_start="$(process_start_time "$port_forward_pid")" || fail 'cannot pin port-forward process identity' for ((attempt=0; attempt<60; attempt++)); do kill -0 "$port_forward_pid" 2>/dev/null || fail 'Keycloak port-forward exited before readiness' if grep -Fqx "Forwarding from 127.0.0.1:$local_port -> 8080" "$log" && loopback_port_open >/dev/null 2>&1; then ready=true; break; fi /usr/bin/sleep 0.05 done [[ "$ready" == true ]] || fail 'Keycloak port-forward did not become ready in time' [[ "$(process_start_time "$port_forward_pid")" == "$port_forward_start" ]] || fail 'Keycloak port-forward process identity changed' } write_curl_config() { local destination=$1 token_file=${2:-} { printf '%s\n' \ 'silent' \ 'show-error' \ 'noproxy = "*"' \ "connect-timeout = \"$CURL_CONNECT_TIMEOUT\"" \ "max-time = \"$CURL_MAX_TIME\"" \ "header = \"Host: $KEYCLOAK_HOST\"" \ 'header = "X-Forwarded-Proto: https"' \ 'header = "X-Forwarded-Port: 443"' if [[ -n "$token_file" ]]; then printf 'header = "Authorization: Bearer ' tr -d '\r\n' <"$token_file" printf '"\n' fi } >"$destination" chmod 0600 -- "$destination" private_file_ok "$destination" || fail 'curl configuration is unsafe' } authenticate_keycloak() { local token_response token_file public_config auth_config status token_response="$temporary_dir/token-response.json" token_file="$temporary_dir/access-token" public_config="$temporary_dir/curl-public.conf" auth_config="$temporary_dir/curl-auth.conf" write_curl_config "$public_config" status="$(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable \ --config "$public_config" --request POST \ --output "$token_response" --write-out '%{http_code}' \ --data-urlencode 'grant_type=password' \ --data-urlencode 'client_id=admin-cli' \ --data-urlencode "username@$temporary_dir/admin-username" \ --data-urlencode "password@$temporary_dir/admin-password" \ "$local_base/realms/master/protocol/openid-connect/token" 2>/dev/null)" || fail 'temporary Keycloak administrator authentication failed' chmod 0600 -- "$token_response" [[ "$status" == 200 ]] || fail 'temporary Keycloak administrator authentication returned an unexpected status' jq -erj '.access_token | select(type == "string" and length >= 16)' "$token_response" >"$token_file" || fail 'Keycloak token response is invalid' chmod 0600 -- "$token_file" private_file_ok "$token_file" || fail 'temporary Keycloak token file is unsafe' write_curl_config "$auth_config" "$token_file" rm -f -- "$token_response" "$temporary_dir/admin-username" "$temporary_dir/admin-password" } admin_request() { local method=$1 url=$2 body=${3:-} output status_file rc=0 request_counter=$((request_counter + 1)) output="$temporary_dir/admin-response-$request_counter.json" status_file="$temporary_dir/admin-status-$request_counter" : >"$output"; : >"$status_file"; chmod 0600 -- "$output" "$status_file" local -a command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable --config "$temporary_dir/curl-auth.conf" --request "$method" --output "$output" --write-out '%{http_code}' "$url") if [[ -n "$body" ]]; then private_file_ok "$body" || fail 'request body is not a private snapshot' command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable --config "$temporary_dir/curl-auth.conf" --request "$method" --header 'Content-Type: application/json' --data-binary "@$body" --output "$output" --write-out '%{http_code}' "$url") fi "${command[@]}" >"$status_file" 2>/dev/null || rc=$? private_file_ok "$output" true && private_file_ok "$status_file" true || fail 'Admin API response files are unsafe' last_body=$output last_status_file=$status_file last_curl_rc=$rc last_http_status='' if (( rc == 0 )); then IFS= read -r last_http_status <"$status_file" || true [[ "$last_http_status" =~ ^[0-9]{3}$ ]] || { last_curl_rc=90; last_http_status=''; } fi } admin_get_query() { local url=$1 query_name=$2 query_value=$3 max=$4 output status_file rc=0 request_counter=$((request_counter + 1)) output="$temporary_dir/admin-response-$request_counter.json" status_file="$temporary_dir/admin-status-$request_counter" : >"$output"; : >"$status_file"; chmod 0600 -- "$output" "$status_file" /usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable \ --config "$temporary_dir/curl-auth.conf" --get \ --data-urlencode "$query_name=$query_value" --data-urlencode "max=$max" \ --output "$output" --write-out '%{http_code}' "$url" >"$status_file" 2>/dev/null || rc=$? private_file_ok "$output" true && private_file_ok "$status_file" true || fail 'Admin API query response files are unsafe' last_body=$output; last_status_file=$status_file; last_curl_rc=$rc; last_http_status='' if (( rc == 0 )); then IFS= read -r last_http_status <"$status_file" || true; fi } require_http() { local expected=$1 operation=$2 (( last_curl_rc == 0 )) || fail "$operation transport failed" [[ "$last_http_status" == "$expected" ]] || fail "$operation returned HTTP ${last_http_status:-none}" } build_desired_client() { client_desired="$temporary_dir/client-desired.json" jq -n --arg client_id "$CLIENT_ID" --arg root "$GRAFANA_URL" --arg redirect "$REDIRECT_URI" \ --arg logout "$POST_LOGOUT_URI" '{ clientId:$client_id, name:"Grafana", description:"Grafana confidential OIDC client managed by the platform bootstrap", enabled:true, protocol:"openid-connect", clientAuthenticatorType:"client-secret", publicClient:false, standardFlowEnabled:true, implicitFlowEnabled:false, directAccessGrantsEnabled:false, serviceAccountsEnabled:false, authorizationServicesEnabled:false, consentRequired:false, fullScopeAllowed:false, rootUrl:$root, baseUrl:$root, redirectUris:[$redirect], webOrigins:[$root], attributes:{ "post.logout.redirect.uris":$logout, "oauth2.device.authorization.grant.enabled":"false", "oidc.ciba.grant.enabled":"false" } }' >"$client_desired" chmod 0600 -- "$client_desired" } build_desired_mapper() { mapper_desired="$temporary_dir/mapper-desired.json" jq -n --arg name "$MAPPER_NAME" '{ name:$name, protocol:"openid-connect", protocolMapper:"oidc-group-membership-mapper", consentRequired:false, config:{ "claim.name":"groups", "full.path":"true", "id.token.claim":"true", "access.token.claim":"true", "userinfo.token.claim":"true" } }' >"$mapper_desired" chmod 0600 -- "$mapper_desired" } lookup_client() { admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2 require_http 200 'Grafana client lookup' [[ "$(jq 'length' "$last_body")" == 0 || "$(jq 'length' "$last_body")" == 1 ]] || fail 'Grafana client lookup is ambiguous' if [[ "$(jq 'length' "$last_body")" == 0 ]]; then client_state=absent; client_id=''; client_before='' else client_state=existing client_id="$(jq -er '.[0].id | select(type == "string" and length > 0)' "$last_body")" || fail 'existing Grafana client has no exact internal id' client_before="$temporary_dir/client-before.json" admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" require_http 200 'existing Grafana client snapshot' jq -e --arg id "$client_id" --arg client_id "$CLIENT_ID" \ '.id == $id and .clientId == $client_id' "$last_body" >/dev/null || fail 'existing client identity changed' cp -- "$last_body" "$client_before"; chmod 0600 -- "$client_before" fi } lookup_group() { local label=$1 name=$2 path=$3 state_var=$4 id_var=$5 before_var=$6 admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2 require_http 200 "$label group lookup" local count count="$(jq 'length' "$last_body")" || fail "$label group response is invalid" [[ "$count" == 0 || "$count" == 1 ]] || fail "$label group lookup is ambiguous" if [[ "$count" == 0 ]]; then printf -v "$state_var" '%s' absent; printf -v "$id_var" '%s' ''; printf -v "$before_var" '%s' '' else local gid before gid="$(jq -er --arg path "$path" '.[0] | select(.path == $path) | .id | select(type == "string" and length > 0)' "$last_body")" || fail "$label group has no exact internal id" before="$temporary_dir/${label}-group-before.json" admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid" require_http 200 "$label group snapshot" jq -e --arg id "$gid" --arg name "$name" --arg path "$path" \ '.id == $id and .name == $name and .path == $path' "$last_body" >/dev/null || fail "$label group identity changed" cp -- "$last_body" "$before"; chmod 0600 -- "$before" printf -v "$state_var" '%s' existing; printf -v "$id_var" '%s' "$gid"; printf -v "$before_var" '%s' "$before" fi } lookup_mapper() { [[ "$client_state" == existing ]] || { mapper_state=absent; mapper_id=''; mapper_before=''; return; } admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" require_http 200 'Grafana mapper lookup' local count_by_name count_by_claim count_by_name="$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)] | length' "$last_body")" || fail 'mapper response is invalid' count_by_claim="$(jq '[.[] | select(.config["claim.name"] == "groups")] | length' "$last_body")" || fail 'mapper response is invalid' [[ "$count_by_name" == 0 || "$count_by_name" == 1 ]] || fail 'Grafana mapper name is ambiguous' [[ "$count_by_claim" == 0 || "$count_by_claim" == 1 ]] || fail 'Grafana groups claim mapper is ambiguous' if [[ "$count_by_name" == 1 ]]; then mapper_state=existing mapper_id="$(jq -er --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)][0].id | select(type == "string" and length > 0)' "$last_body")" || fail 'existing Grafana mapper has no exact id' [[ "$count_by_claim" == 0 || "$(jq -r --arg id "$mapper_id" '[.[] | select(.config["claim.name"] == "groups")][0].id // empty' "$last_body")" == "$mapper_id" ]] || fail 'a different mapper already owns the groups claim' mapper_before="$temporary_dir/mapper-before.json" jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_before" || fail 'cannot snapshot exact existing mapper' chmod 0600 -- "$mapper_before" else [[ "$count_by_claim" == 0 ]] || fail 'groups claim is owned by a differently named mapper' mapper_state=absent; mapper_id=''; mapper_before='' fi } lookup_user() { local label=$1 username=$2 id_var=$3 [[ -n "$username" ]] || return 0 admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/users?exact=true" username "$username" 2 require_http 200 "$label user lookup" [[ "$(jq 'length' "$last_body")" == 1 ]] || fail "$label username lookup must return exactly one user" local uid uid="$(jq -er --arg username "$username" '.[0] | select(.username == $username) | .id | select(type == "string" and length > 0)' "$last_body")" || fail "$label user identity is invalid" printf -v "$id_var" '%s' "$uid" } snapshot_membership() { local label=$1 user_id=$2 group_id=$3 state_var=$4 [[ -n "$user_id" ]] || return 0 admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups" briefRepresentation false 101 require_http 200 "$label membership lookup" [[ "$(jq 'length' "$last_body")" -le 100 ]] || fail "$label membership list exceeds the bounded cardinality" if jq -e --arg id "$group_id" 'any(.[]; .id == $id)' "$last_body" >/dev/null; then printf -v "$state_var" '%s' true else printf -v "$state_var" '%s' false fi } retrieve_client_secret() { client_secret_file="$temporary_dir/client-secret" admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret" require_http 200 'Grafana client-secret retrieval' jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$client_secret_file" || fail 'Grafana client secret response is invalid' chmod 0600 -- "$client_secret_file" private_file_ok "$client_secret_file" || fail 'Grafana client secret file is unsafe' } start_kubernetes_proxy() { local log="$temporary_dir/kubernetes-proxy.log" attempt kube_socket="$temporary_dir/kube-api.sock" /usr/bin/timeout --signal=TERM --kill-after=1s "$PROXY_PROCESS_TIMEOUT" \ "$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" proxy \ --unix-socket="$kube_socket" --api-prefix=/ \ --accept-paths="^/api/v1/namespaces/${OBSERVABILITY_NAMESPACE}/secrets(/${OIDC_SECRET})?$" \ >"$log" 2>&1 & proxy_pid=$! proxy_start="$(process_start_time "$proxy_pid")" || fail 'cannot pin Kubernetes proxy identity' for ((attempt=0; attempt<60; attempt++)); do [[ -S "$kube_socket" ]] && break kill -0 "$proxy_pid" 2>/dev/null || fail 'bounded Kubernetes proxy exited' /usr/bin/sleep 0.05 done [[ -S "$kube_socket" && "$(process_start_time "$proxy_pid")" == "$proxy_start" ]] || fail 'bounded Kubernetes proxy did not become ready safely' } kube_request() { local method=$1 path=$2 body=${3:-} output status_file rc=0 request_counter=$((request_counter + 1)) output="$temporary_dir/kube-response-$request_counter.json" status_file="$temporary_dir/kube-status-$request_counter" : >"$output"; : >"$status_file"; chmod 0600 -- "$output" "$status_file" local -a command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable --silent --show-error --noproxy '*' --connect-timeout "$CURL_CONNECT_TIMEOUT" --max-time "$CURL_MAX_TIME" --unix-socket "$kube_socket" --request "$method" --output "$output" --write-out '%{http_code}' "http://localhost$path") if [[ -n "$body" ]]; then private_file_ok "$body" || fail 'Kubernetes request body is unsafe' command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable --silent --show-error --noproxy '*' --connect-timeout "$CURL_CONNECT_TIMEOUT" --max-time "$CURL_MAX_TIME" --unix-socket "$kube_socket" --request "$method" --header 'Content-Type: application/json' --data-binary "@$body" --output "$output" --write-out '%{http_code}' "http://localhost$path") fi "${command[@]}" >"$status_file" 2>/dev/null || rc=$? private_file_ok "$output" true && private_file_ok "$status_file" true || fail 'Kubernetes API response files are unsafe' last_body=$output; last_status_file=$status_file; last_curl_rc=$rc; last_http_status='' if (( rc == 0 )); then IFS= read -r last_http_status <"$status_file" || true; fi } validate_secret_json() { local source=$1 prefix=$2 python3 -I -S - "$source" "$prefix" <<'PY' import base64,binascii,json,os,pathlib,re,sys source,prefix=sys.argv[1:] try: item=json.loads(pathlib.Path(source).read_text()) except Exception: raise SystemExit(1) meta=item.get('metadata') or {}; data=item.get('data') if item.get('apiVersion')!='v1' or item.get('kind')!='Secret' or item.get('type')!='Opaque' or meta.get('namespace')!='observability' or meta.get('name')!='grafana-keycloak-oidc' or sorted(data or {})!=['client-id','client-secret']: raise SystemExit(1) uid=meta.get('uid'); rv=meta.get('resourceVersion') if not isinstance(uid,str) or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*',uid) or not isinstance(rv,str) or not re.fullmatch(r'[0-9]+',rv): raise SystemExit(1) for key in ('client-id','client-secret'): try: payload=base64.b64decode(data[key],validate=True) except (binascii.Error,ValueError): raise SystemExit(1) if (key=='client-id' and payload!=b'grafana') or (key=='client-secret' and not payload): raise SystemExit(1) fd=os.open(prefix+'.'+key,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600) try: os.write(fd,payload) finally: os.close(fd) print(uid+'\t'+rv,end='') PY } snapshot_oidc_secret() { local prefix values kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" (( last_curl_rc == 0 )) || fail 'Grafana OIDC Secret lookup transport failed' case "$last_http_status" in 200) secret_state=existing; secret_before="$temporary_dir/secret-before.json" cp -- "$last_body" "$secret_before"; chmod 0600 -- "$secret_before" prefix="$temporary_dir/secret-before" values="$(validate_secret_json "$secret_before" "$prefix")" || fail 'existing Grafana OIDC Secret contract is invalid' IFS=$'\t' read -r secret_uid secret_resource_version <<<"$values" [[ "$client_state" == existing && -n "$client_secret_file" ]] || fail 'Grafana OIDC Secret exists without one exact existing Keycloak client' [[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] || fail 'existing Grafana OIDC Secret client-id differs' cmp --silent -- "$prefix.client-secret" "$client_secret_file" || fail 'existing Grafana OIDC Secret does not match the exact existing Keycloak credential; rotation refused' ;; 404) secret_state=absent; secret_before=''; secret_uid=''; secret_resource_version='' ;; *) fail "Grafana OIDC Secret lookup returned HTTP ${last_http_status:-none}" ;; esac } sudo_refresh() { "$SUDO_BIN" -v || fail 'sudo credential refresh failed' "$SUDO_BIN" -n /usr/bin/true || fail 'narrow non-interactive sudo is unavailable' } validate_evidence_stream() { local file=$1 timestamp_var=$2 parsed local -a lines=() mapfile -t lines <"$file" [[ "$(wc -l <"$file" | tr -d '[:space:]')" == 4 && ${#lines[@]} -eq 4 ]] || return 1 [[ "${lines[0]}" == "schema=$EVIDENCE_SCHEMA" ]] || return 1 [[ "${lines[1]}" == "context=$EXPECTED_CONTEXT" ]] || return 1 [[ "${lines[2]}" == "resource=$EVIDENCE_RESOURCE" ]] || return 1 [[ "${lines[3]}" =~ ^checked_at_utc=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)$ ]] || return 1 parsed="${BASH_REMATCH[1]}" timestamp_epoch "$parsed" >/dev/null || return 1 printf -v "$timestamp_var" '%s' "$parsed" } root_marker_helper() { local output=$1 shift "$SUDO_BIN" -n /usr/bin/python3 -I -S - platform-grafana-oidc-marker-helper "$@" \ >"$output" 2>/dev/null <<'PY' import errno,os,pathlib,stat,sys tag,action,directory,basename,uid_raw,gid_raw,dir_mode_raw,source,expected_kind,expected_source,source_uid_raw=sys.argv[1:] if tag!='platform-grafana-oidc-marker-helper' or action not in {'check-parent','check-dir','ensure-dir','read','install','unlink-exact'} or basename!='keycloak.env': raise SystemExit(2) if not directory.startswith('/') or '\x00' in directory: raise SystemExit(2) uid=int(uid_raw); gid=int(gid_raw); directory_mode=int(dir_mode_raw,8); source_uid=int(source_uid_raw) flags=os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW|os.O_CLOEXEC def open_dir(path,mode): fd=os.open('/',flags) try: for part in pathlib.PurePosixPath(path).parts[1:]: if part in {'','.','..'}: raise OSError(errno.EINVAL,'component') nxt=os.open(part,flags,dir_fd=fd); os.close(fd); fd=nxt st=os.fstat(fd) if not stat.S_ISDIR(st.st_mode) or stat.S_IMODE(st.st_mode)!=mode or st.st_uid!=uid or st.st_gid!=gid: raise OSError(errno.EPERM,'directory metadata') return fd except BaseException: os.close(fd); raise def read_file(directory_fd,name): fd=os.open(name,os.O_RDONLY|os.O_NOFOLLOW|os.O_CLOEXEC,dir_fd=directory_fd) try: before=os.fstat(fd) if not stat.S_ISREG(before.st_mode) or stat.S_IMODE(before.st_mode)!=0o600 or before.st_uid!=uid or before.st_gid!=gid or before.st_nlink!=1: raise OSError(errno.EPERM,'file metadata') chunks=[] while True: chunk=os.read(fd,65536) if not chunk: break chunks.append(chunk) after=os.fstat(fd) a=(before.st_dev,before.st_ino,before.st_size,before.st_mtime_ns,before.st_ctime_ns,before.st_nlink) b=(after.st_dev,after.st_ino,after.st_size,after.st_mtime_ns,after.st_ctime_ns,after.st_nlink) if a!=b: raise OSError(errno.ESTALE,'changed') payload=b''.join(chunks) if len(payload)!=before.st_size: raise OSError(errno.EIO,'short read') return payload finally: os.close(fd) def read_source(path): fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW|os.O_CLOEXEC) try: before=os.fstat(fd) if not stat.S_ISREG(before.st_mode) or stat.S_IMODE(before.st_mode)!=0o600 or before.st_uid!=source_uid or before.st_nlink!=1: raise OSError(errno.EPERM,'source metadata') chunks=[] while True: chunk=os.read(fd,65536) if not chunk: break chunks.append(chunk) after=os.fstat(fd) if (before.st_dev,before.st_ino,before.st_size,before.st_mtime_ns,before.st_ctime_ns,before.st_nlink)!=(after.st_dev,after.st_ino,after.st_size,after.st_mtime_ns,after.st_ctime_ns,after.st_nlink): raise OSError(errno.ESTALE,'source changed') payload=b''.join(chunks) if len(payload)!=before.st_size: raise OSError(errno.EIO,'short source') return payload finally: os.close(fd) if action in {'check-parent','check-dir','ensure-dir'}: parent,leaf=os.path.split(directory.rstrip('/')) if leaf!='recovery-evidence' or not parent: raise OSError(errno.EINVAL,'directory') parent_fd=open_dir(parent,directory_mode) if action=='check-parent': os.close(parent_fd); raise SystemExit(0) try: try: directory_fd=os.open(leaf,flags,dir_fd=parent_fd) except FileNotFoundError: if action=='check-dir': raise SystemExit(4) os.mkdir(leaf,directory_mode,dir_fd=parent_fd); os.fsync(parent_fd); directory_fd=os.open(leaf,flags,dir_fd=parent_fd) try: st=os.fstat(directory_fd) if not stat.S_ISDIR(st.st_mode) or stat.S_IMODE(st.st_mode)!=directory_mode or st.st_uid!=uid or st.st_gid!=gid: raise OSError(errno.EPERM,'evidence directory') finally: os.close(directory_fd) finally: os.close(parent_fd) raise SystemExit(0) try: directory_fd=open_dir(directory,directory_mode) except FileNotFoundError: if action=='read': raise SystemExit(4) raise try: if action=='read': try: payload=read_file(directory_fd,basename) except FileNotFoundError: raise SystemExit(4) os.write(1,payload); raise SystemExit(0) desired=read_source(source) if expected_kind=='absent': try: read_file(directory_fd,basename) except FileNotFoundError: pass else: raise OSError(errno.EEXIST,'expected absent') elif expected_kind=='file': if read_file(directory_fd,basename)!=read_source(expected_source): raise OSError(errno.ESTALE,'precondition') else: raise OSError(errno.EINVAL,'kind') if action=='unlink-exact': if read_file(directory_fd,basename)!=desired: raise OSError(errno.ESTALE,'unlink precondition') os.unlink(basename,dir_fd=directory_fd); os.fsync(directory_fd) try: os.stat(basename,dir_fd=directory_fd,follow_symlinks=False) except FileNotFoundError: raise SystemExit(0) raise OSError(errno.EEXIST,'remained') temporary=f'.{basename}.new.{os.getpid():x}' fd=os.open(temporary,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW|os.O_CLOEXEC,0o600,dir_fd=directory_fd); exists=True try: os.fchown(fd,uid,gid); os.fchmod(fd,0o600); view=memoryview(desired) while view: n=os.write(fd,view) if n<=0: raise OSError(errno.EIO,'short write') view=view[n:] os.fsync(fd) if os.fstat(fd).st_nlink!=1: raise OSError(errno.EPERM,'links') finally: os.close(fd) try: os.replace(temporary,basename,src_dir_fd=directory_fd,dst_dir_fd=directory_fd); exists=False; os.fsync(directory_fd) if read_file(directory_fd,basename)!=desired: raise OSError(errno.EIO,'verify') finally: if exists: try: os.unlink(temporary,dir_fd=directory_fd) except FileNotFoundError: pass finally: os.close(directory_fd) PY } root_marker_read() { local destination=$1 rc if root_marker_helper "$destination" read "$EVIDENCE_DIR" keycloak.env \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" '' '' '' "$EUID"; then chmod 0600 -- "$destination"; return 0 else rc=$?; rm -f -- "$destination"; return "$rc"; fi } prepare_marker_snapshot() { local rc timestamp marker_prior="$temporary_dir/marker-prior.env" marker_desired="$temporary_dir/marker-desired.env" root_marker_helper /dev/null check-parent "$EVIDENCE_DIR" keycloak.env \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" '' '' '' "$EUID" || fail 'recovery evidence parent boundary is missing or unsafe' if root_marker_read "$marker_prior"; then validate_evidence_stream "$marker_prior" timestamp || fail 'existing Keycloak recovery marker is malformed' marker_had_prior=true else rc=$?; [[ "$rc" == 4 ]] || fail 'cannot safely inspect Keycloak recovery marker' marker_had_prior=false fi printf '%s\n' "schema=$EVIDENCE_SCHEMA" "context=$EXPECTED_CONTEXT" \ "resource=$EVIDENCE_RESOURCE" "checked_at_utc=$(now_utc)" >"$marker_desired" chmod 0600 -- "$marker_desired" validate_evidence_stream "$marker_desired" timestamp || fail 'generated recovery marker is invalid' } ensure_evidence_directory() { root_marker_helper /dev/null ensure-dir "$EVIDENCE_DIR" keycloak.env \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" '' '' '' "$EUID" || fail 'cannot safely ensure recovery evidence directory' } install_marker() { pending_kind=marker if [[ "$marker_had_prior" == true ]]; then root_marker_helper /dev/null install "$EVIDENCE_DIR" keycloak.env \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_desired" file "$marker_prior" "$EUID" || fail 'cannot atomically replace Keycloak recovery marker' else root_marker_helper /dev/null install "$EVIDENCE_DIR" keycloak.env \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_desired" absent '' "$EUID" || fail 'cannot atomically install Keycloak recovery marker' fi local current="$temporary_dir/marker-installed.env" root_marker_read "$current" && cmp --silent -- "$marker_desired" "$current" || fail 'installed Keycloak recovery marker cannot be verified' marker_written=true pending_kind='' } restore_marker() { local current="$temporary_dir/marker-rollback-current.env" [[ "$marker_written" == true ]] || return 0 root_marker_read "$current" && cmp --silent -- "$marker_desired" "$current" || return 1 if [[ "$marker_had_prior" == true ]]; then root_marker_helper /dev/null install "$EVIDENCE_DIR" keycloak.env \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_prior" file "$marker_desired" "$EUID" else root_marker_helper /dev/null unlink-exact "$EVIDENCE_DIR" keycloak.env \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_desired" file "$marker_desired" "$EUID" fi } check_recovery_evidence() { local marker timestamp marker_epoch current_epoch age validate_context_and_authority sudo_refresh marker="$temporary_dir/check-keycloak.env" root_marker_read "$marker" || fail 'Keycloak recovery evidence is missing or unsafe' validate_evidence_stream "$marker" timestamp || fail 'Keycloak recovery evidence is malformed' marker_epoch="$(timestamp_epoch "$timestamp")" || fail 'recovery evidence timestamp is invalid' current_epoch="$(now_epoch)" || fail 'current UTC clock is invalid' age=$((current_epoch - marker_epoch)) (( age >= 0 && age <= EVIDENCE_MAX_AGE_SECONDS )) || fail 'Keycloak recovery evidence is outside the 30-day window' printf 'KEYCLOAK_RECOVERY_EVIDENCE=PASS\n' } build_group_body() { local name=$1 destination=$2 token=${3:-} if [[ -n "$token" ]]; then jq -n --arg name "$name" --arg token "$token" \ '{name:$name,attributes:{"platform.observability.transaction":[$token]}}' >"$destination" else jq -n --arg name "$name" '{name:$name}' >"$destination" fi chmod 0600 -- "$destination" } client_body_with_secret() { local source=$1 destination=$2 private_file_ok "$source" && private_file_ok "$client_secret_file" || return 1 jq --rawfile secret "$client_secret_file" '. + {secret:$secret}' "$source" >"$destination" || return 1 chmod 0600 -- "$destination" private_file_ok "$destination" } client_contract_matches() { local source=$1 expected_id=$2 jq -e --arg id "$expected_id" --arg root "$GRAFANA_URL" --arg redirect "$REDIRECT_URI" \ --arg logout "$POST_LOGOUT_URI" ' .id == $id and .clientId == "grafana" and .name == "Grafana" and .description == "Grafana confidential OIDC client managed by the platform bootstrap" and .enabled == true and .protocol == "openid-connect" and .clientAuthenticatorType == "client-secret" and .publicClient == false and .standardFlowEnabled == true and .implicitFlowEnabled == false and .directAccessGrantsEnabled == false and .serviceAccountsEnabled == false and .authorizationServicesEnabled == false and .consentRequired == false and .fullScopeAllowed == false and .rootUrl == $root and .baseUrl == $root and .redirectUris == [$redirect] and .webOrigins == [$root] and .attributes["post.logout.redirect.uris"] == $logout and .attributes["oauth2.device.authorization.grant.enabled"] == "false" and .attributes["oidc.ciba.grant.enabled"] == "false" and (.attributes["platform.observability.transaction"] // null) == null ' "$source" >/dev/null } mapper_contract_matches() { local source=$1 expected_id=$2 jq -e --arg id "$expected_id" --arg name "$MAPPER_NAME" ' .id == $id and .name == $name and .protocol == "openid-connect" and .protocolMapper == "oidc-group-membership-mapper" and .consentRequired == false and .config == {"claim.name":"groups","full.path":"true","id.token.claim":"true", "access.token.claim":"true","userinfo.token.claim":"true"} ' "$source" >/dev/null } capture_client_after() { client_after="$temporary_dir/client-after.json" admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" require_http 200 'post-mutation Grafana client snapshot' client_contract_matches "$last_body" "$client_id" || fail 'post-mutation Grafana client contract differs' cp -- "$last_body" "$client_after"; chmod 0600 -- "$client_after" admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret" require_http 200 'post-mutation Grafana credential verification' local verification="$temporary_dir/client-secret-verification" jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$verification" || fail 'post-mutation Grafana credential response is invalid' chmod 0600 -- "$verification" cmp --silent -- "$client_secret_file" "$verification" || fail 'Grafana client PUT changed the existing credential unexpectedly' } classify_client_create() { admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2 (( last_curl_rc == 0 )) || return 2 [[ "$last_http_status" == 200 ]] || return 2 local count count="$(jq 'length' "$last_body" 2>/dev/null)" || return 2 if [[ "$count" == 0 ]]; then return 1; fi [[ "$count" == 1 ]] || return 2 local owned_id owned_id="$(jq -er --arg token "$transaction_token" '.[0] | select(.attributes["platform.observability.transaction"] == $token) | .id | select(type == "string" and length > 0)' "$last_body" 2>/dev/null)" || return 2 client_id=$owned_id client_mutation=created return 0 } create_client() { local create_body="$temporary_dir/client-create.json" status jq --arg token "$transaction_token" \ '.attributes["platform.observability.transaction"]=$token' "$client_desired" >"$create_body" chmod 0600 -- "$create_body" pending_kind=client-create; pending_desired=$create_body; pending_token=$transaction_token admin_request POST "$local_base/admin/realms/$KEYCLOAK_REALM/clients" "$create_body" status=$last_http_status if (( last_curl_rc != 0 )) || [[ "$status" != 201 ]]; then if classify_client_create; then : elif [[ "$?" != 1 ]]; then emit_manual_recovery fi fail "Grafana client create was not cleanly confirmed" fi classify_client_create || { emit_manual_recovery; fail 'Grafana client create response could not be owned exactly'; } client_after="$temporary_dir/client-created-tagged.json" admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" require_http 200 'created Grafana client ownership snapshot' cp -- "$last_body" "$client_after"; chmod 0600 -- "$client_after" retrieve_client_secret local update_body="$temporary_dir/client-created-final.json" client_body_with_secret "$client_desired" "$update_body" || fail 'cannot build credential-preserving client finalization' pending_kind=client-created-finalize; pending_before=$client_after; pending_desired=$client_desired; pending_id=$client_id admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" "$update_body" if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then fail 'created Grafana client finalization was not cleanly confirmed' fi capture_client_after pending_kind='' } update_client() { local update_body="$temporary_dir/client-update.json" client_body_with_secret "$client_desired" "$update_body" || fail 'cannot build credential-preserving client update' client_restore="$temporary_dir/client-restore.json" client_body_with_secret "$client_before" "$client_restore" || fail 'cannot build exact credential-preserving client rollback snapshot' pending_kind=client-update; pending_before=$client_before; pending_desired=$client_desired; pending_id=$client_id admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" "$update_body" if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then fail 'Grafana client update was not cleanly confirmed' fi client_mutation=updated; pending_kind='' capture_client_after } classify_group_create() { local name=$1 path=$2 id_var=$3 mutation_var=$4 admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2 (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 local count gid count="$(jq 'length' "$last_body" 2>/dev/null)" || return 2 if [[ "$count" == 0 ]]; then return 1; fi [[ "$count" == 1 ]] || return 2 gid="$(jq -er --arg path "$path" --arg token "$transaction_token" ' .[0] | select(.path == $path and .attributes["platform.observability.transaction"] == [$token]) | .id | select(type == "string" and length > 0)' "$last_body" 2>/dev/null)" || return 2 printf -v "$id_var" '%s' "$gid"; printf -v "$mutation_var" '%s' created return 0 } create_group() { local label=$1 name=$2 path=$3 id_var=$4 mutation_var=$5 after_var=$6 local create_body="$temporary_dir/${label}-group-create.json" final_body="$temporary_dir/${label}-group-final.json" build_group_body "$name" "$create_body" "$transaction_token" pending_kind="${label}-group-create"; pending_desired=$create_body; pending_token=$transaction_token admin_request POST "$local_base/admin/realms/$KEYCLOAK_REALM/groups" "$create_body" if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 201 ]]; then if classify_group_create "$name" "$path" "$id_var" "$mutation_var"; then : elif [[ "$?" != 1 ]]; then emit_manual_recovery; fi fail "$label group create was not cleanly confirmed" fi classify_group_create "$name" "$path" "$id_var" "$mutation_var" || { emit_manual_recovery; fail "$label group create response could not be owned exactly"; } local gid=${!id_var} admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid" require_http 200 "$label created group ownership snapshot" local after="$temporary_dir/${label}-group-after.json" cp -- "$last_body" "$after"; chmod 0600 -- "$after"; printf -v "$after_var" '%s' "$after" pending_kind="${label}-group-finalize"; pending_before=$after; pending_id=$gid build_group_body "$name" "$final_body" pending_desired=$final_body admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid" "$final_body" if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then if classify_pending; then fail "$label group finalization response was lost after an exact state transition" else [[ "$?" != 2 ]] || emit_manual_recovery fail "$label group finalization was not cleanly confirmed" fi fi admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid" require_http 200 "$label group post-mutation snapshot" jq -e --arg id "$gid" --arg name "$name" --arg path "$path" ' .id == $id and .name == $name and .path == $path and ((.attributes // {}) | has("platform.observability.transaction") | not) ' "$last_body" >/dev/null || fail "$label group final contract differs" cp -- "$last_body" "$after"; chmod 0600 -- "$after"; printf -v "$after_var" '%s' "$after" pending_kind='' } create_mapper() { local create_body="$temporary_dir/mapper-create.json" jq --arg token "$transaction_token" '.config["platform.observability.transaction"]=$token' \ "$mapper_desired" >"$create_body"; chmod 0600 -- "$create_body" pending_kind=mapper-create; pending_desired=$create_body; pending_token=$transaction_token admin_request POST "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" "$create_body" if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 201 ]]; then fail 'Grafana groups mapper create was not cleanly confirmed' fi admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" require_http 200 'post-create Grafana mapper lookup' [[ "$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)] | length' "$last_body")" == 1 ]] || { emit_manual_recovery; fail 'created Grafana mapper is ambiguous'; } mapper_id="$(jq -er --arg name "$MAPPER_NAME" --arg token "$transaction_token" ' [.[] | select(.name == $name and .config["platform.observability.transaction"] == $token)][0].id | select(type == "string" and length > 0)' "$last_body")" || { emit_manual_recovery; fail 'created Grafana mapper ownership is not exact'; } mapper_mutation=created mapper_after="$temporary_dir/mapper-created-tagged.json" jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_after" || fail 'cannot snapshot created Grafana mapper ownership' chmod 0600 -- "$mapper_after" pending_kind=mapper-created-finalize; pending_before=$mapper_after; pending_desired=$mapper_desired; pending_id=$mapper_id admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id" "$mapper_desired" if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then fail 'created Grafana mapper finalization was not cleanly confirmed' fi capture_mapper_after pending_kind='' } capture_mapper_after() { mapper_after="$temporary_dir/mapper-after.json" admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" require_http 200 'post-mutation Grafana mapper snapshot' jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_after" || fail 'cannot capture post-mutation Grafana mapper' chmod 0600 -- "$mapper_after" mapper_contract_matches "$mapper_after" "$mapper_id" || fail 'post-mutation Grafana mapper contract differs' } update_mapper() { pending_kind=mapper-update; pending_before=$mapper_before; pending_desired=$mapper_desired; pending_id=$mapper_id admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id" "$mapper_desired" if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then fail 'Grafana mapper update was not cleanly confirmed'; fi mapper_mutation=updated; pending_kind=''; capture_mapper_after } build_secret_manifest() { local destination=$1 python3 -I -S - "$client_secret_file" "$transaction_token" "$destination" <<'PY' || return 1 import base64,json,os,pathlib,re,sys source,token,destination=sys.argv[1:] if re.fullmatch(r'[0-9a-f]{32}',token) is None: raise SystemExit(1) payload=pathlib.Path(source).read_bytes() if not payload: raise SystemExit(1) item={'apiVersion':'v1','kind':'Secret','type':'Opaque','metadata':{'namespace':'observability','name':'grafana-keycloak-oidc','annotations':{'observability.hyeonworks.com/create-transaction':token}},'data':{'client-id':base64.b64encode(b'grafana').decode(),'client-secret':base64.b64encode(payload).decode()}} fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600) try: os.write(fd,(json.dumps(item,sort_keys=True,separators=(',',':'))+'\n').encode()) finally: os.close(fd) PY } classify_secret_create() { kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" (( last_curl_rc == 0 )) || return 2 if [[ "$last_http_status" == 404 ]]; then return 1; fi [[ "$last_http_status" == 200 ]] || return 2 local prefix="$temporary_dir/secret-classify" values values="$(validate_secret_json "$last_body" "$prefix")" || return 2 jq -e --arg token "$transaction_token" '.metadata.annotations["observability.hyeonworks.com/create-transaction"] == $token' "$last_body" >/dev/null || return 2 [[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] || return 2 cmp --silent -- "$prefix.client-secret" "$client_secret_file" || return 2 IFS=$'\t' read -r secret_created_uid secret_created_resource_version <<<"$values" secret_mutation=created return 0 } create_oidc_secret() { local manifest="$temporary_dir/secret-create.json" build_secret_manifest "$manifest" || fail 'cannot build private Grafana OIDC Secret manifest' pending_kind=secret-create; pending_desired=$manifest; pending_token=$transaction_token kube_request POST "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets" "$manifest" if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 201 ]]; then if classify_secret_create; then :; elif [[ "$?" == 2 ]]; then emit_manual_recovery; fi fail 'Grafana OIDC Secret create was not cleanly confirmed' fi classify_secret_create || { emit_manual_recovery; fail 'Grafana OIDC Secret create response could not be owned exactly'; } pending_kind='' } membership_present() { local user_id=$1 group_id=$2 admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups" briefRepresentation false 101 (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 [[ "$(jq 'length' "$last_body")" -le 100 ]] || return 2 jq -e --arg id "$group_id" 'any(.[]; .id == $id)' "$last_body" >/dev/null } add_membership() { local label=$1 user_id=$2 group_id=$3 before=$4 added_var=$5 [[ -n "$user_id" ]] || return 0 [[ "$before" == false ]] || return 0 pending_kind="${label}-membership"; pending_user_id=$user_id; pending_group_id=$group_id admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups/$group_id" if (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]]; then membership_present "$user_id" "$group_id" || fail "$label membership was not observable after PUT" printf -v "$added_var" '%s' true; pending_kind=''; return 0 fi if membership_present "$user_id" "$group_id"; then printf -v "$added_var" '%s' true else [[ "$?" != 2 ]] || emit_manual_recovery fi fail "$label membership PUT was not cleanly confirmed" } classify_mapper_create() { admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 local count count="$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)] | length' "$last_body" 2>/dev/null)" || return 2 if [[ "$count" == 0 ]]; then return 1; fi [[ "$count" == 1 ]] || return 2 mapper_id="$(jq -er --arg name "$MAPPER_NAME" --arg token "$transaction_token" ' [.[] | select(.name == $name and .config["platform.observability.transaction"] == $token)][0].id | select(type == "string" and length > 0)' "$last_body" 2>/dev/null)" || return 2 mapper_after="$temporary_dir/mapper-after-pending.json" jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_after" || return 2 chmod 0600 -- "$mapper_after"; mapper_mutation=created; return 0 } capture_pending_group_after() { local label=$1 id=$2 after_var=$3 admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 local after="$temporary_dir/${label}-group-after-pending.json" cp -- "$last_body" "$after"; chmod 0600 -- "$after"; printf -v "$after_var" '%s' "$after" } classify_pending() { local result current target case "$pending_kind" in '') return 0 ;; client-create) if classify_client_create; then client_after="$temporary_dir/client-after-pending.json" admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" if (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]]; then cp -- "$last_body" "$client_after"; chmod 0600 -- "$client_after"; pending_kind=''; return 0 fi return 2 else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi ;; client-created-finalize) current="$temporary_dir/client-created-finalize-current.json" admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 cp -- "$last_body" "$current"; chmod 0600 -- "$current" if client_contract_matches "$current" "$client_id" || jq -e --arg id "$client_id" --arg token "$transaction_token" ' .id == $id and .clientId == "grafana" and .attributes["platform.observability.transaction"] == $token ' "$current" >/dev/null; then client_after=$current; client_mutation=created; pending_kind=''; return 0 fi return 2 ;; admin-group-create) if classify_group_create "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" admin_group_id admin_group_mutation; then capture_pending_group_after admin "$admin_group_id" admin_group_after || return 2 pending_kind=''; return 0 else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi ;; viewer-group-create) if classify_group_create "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" viewer_group_id viewer_group_mutation; then capture_pending_group_after viewer "$viewer_group_id" viewer_group_after || return 2 pending_kind=''; return 0 else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi ;; admin-group-finalize|viewer-group-finalize) if [[ "$pending_kind" == admin-group-finalize ]]; then target=admin; pending_id=$admin_group_id else target=viewer; pending_id=$viewer_group_id fi admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$pending_id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 current="$temporary_dir/${target}-group-finalize-current.json" cp -- "$last_body" "$current"; chmod 0600 -- "$current" if [[ "$target" == admin ]]; then jq -e --arg id "$admin_group_id" --arg name "$ADMIN_GROUP_NAME" --arg path "$ADMIN_GROUP_PATH" --arg token "$transaction_token" ' .id == $id and .name == $name and .path == $path and ((.attributes // {}) == {} or .attributes["platform.observability.transaction"] == [$token]) ' "$current" >/dev/null || return 2 admin_group_after=$current else jq -e --arg id "$viewer_group_id" --arg name "$VIEWER_GROUP_NAME" --arg path "$VIEWER_GROUP_PATH" --arg token "$transaction_token" ' .id == $id and .name == $name and .path == $path and ((.attributes // {}) == {} or .attributes["platform.observability.transaction"] == [$token]) ' "$current" >/dev/null || return 2 viewer_group_after=$current fi pending_kind=''; return 0 ;; mapper-create) if classify_mapper_create; then pending_kind=''; return 0; else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi ;; mapper-created-finalize) admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 current="$temporary_dir/mapper-created-finalize-current.json" jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 2 chmod 0600 -- "$current" if mapper_contract_matches "$current" "$mapper_id" || jq -e --arg id "$mapper_id" --arg token "$transaction_token" ' .id == $id and .name == "grafana-groups" and .config["platform.observability.transaction"] == $token ' "$current" >/dev/null; then mapper_after=$current; mapper_mutation=created; pending_kind=''; return 0 fi return 2 ;; secret-create) if classify_secret_create; then pending_kind=''; return 0; else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi ;; admin-membership) if membership_present "$pending_user_id" "$pending_group_id"; then admin_membership_added=true; pending_kind=''; return 0 else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi ;; viewer-membership) if membership_present "$pending_user_id" "$pending_group_id"; then viewer_membership_added=true; pending_kind=''; return 0 else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi ;; marker) current="$temporary_dir/marker-pending-current.env" if root_marker_read "$current"; then if cmp --silent -- "$marker_desired" "$current"; then marker_written=true; pending_kind=''; return 0; fi if [[ "$marker_had_prior" == true ]] && cmp --silent -- "$marker_prior" "$current"; then pending_kind=''; return 1; fi return 2 else result=$? [[ "$result" == 4 && "$marker_had_prior" == false ]] && { pending_kind=''; return 1; } return 2 fi ;; client-update) current="$temporary_dir/client-pending-current.json" admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 cp -- "$last_body" "$current"; chmod 0600 -- "$current" if client_contract_matches "$current" "$client_id"; then client_after=$current; client_mutation=updated; pending_kind=''; return 0 fi if json_equal "$current" "$client_before"; then pending_kind=''; return 1; fi return 2 ;; mapper-update) admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 current="$temporary_dir/mapper-pending-current.json" jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 2 chmod 0600 -- "$current" if mapper_contract_matches "$current" "$mapper_id"; then mapper_after=$current; mapper_mutation=updated; pending_kind=''; return 0; fi if json_equal "$current" "$mapper_before"; then pending_kind=''; return 1; fi return 2 ;; *) return 2 ;; esac } delete_created_secret() { [[ "$secret_mutation" == created ]] || return 0 local current="$temporary_dir/rollback-secret-current.json" prefix="$temporary_dir/rollback-secret" values options kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 cp -- "$last_body" "$current"; chmod 0600 -- "$current" values="$(validate_secret_json "$current" "$prefix")" || return 1 local uid rv IFS=$'\t' read -r uid rv <<<"$values" [[ "$uid" == "$secret_created_uid" && "$rv" == "$secret_created_resource_version" ]] || return 1 jq -e --arg token "$transaction_token" '.metadata.annotations["observability.hyeonworks.com/create-transaction"] == $token' "$current" >/dev/null || return 1 [[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] && cmp --silent -- "$prefix.client-secret" "$client_secret_file" || return 1 options="$temporary_dir/secret-delete-options.json" jq -n --arg uid "$uid" --arg rv "$rv" '{apiVersion:"meta.k8s.io/v1",kind:"DeleteOptions",propagationPolicy:"Background",preconditions:{uid:$uid,resourceVersion:$rv}}' >"$options" chmod 0600 -- "$options" kube_request DELETE "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" "$options" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 || "$last_http_status" == 202 ]] || return 1 kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 404 ]] } rollback_membership() { local user_id=$1 group_id=$2 added=$3 [[ "$added" == true ]] || return 0 admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups/$group_id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 if membership_present "$user_id" "$group_id"; then return 1; else [[ "$?" == 1 ]]; fi } rollback_mapper() { [[ "$mapper_mutation" != none ]] || return 0 admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 local current="$temporary_dir/rollback-mapper-current.json" jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 1 chmod 0600 -- "$current" json_equal "$current" "$mapper_after" || return 1 if [[ "$mapper_mutation" == created ]]; then admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 jq -e --arg id "$mapper_id" \ 'type == "array" and ([.[] | select(.id == $id)] | length) == 0' "$last_body" >/dev/null else admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id" "$mapper_before" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 1 json_equal "$current" "$mapper_before" fi } rollback_client() { [[ "$client_mutation" != none ]] || return 0 admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 local current="$temporary_dir/rollback-client-current.json" cp -- "$last_body" "$current"; chmod 0600 -- "$current" json_equal "$current" "$client_after" || return 1 if [[ "$client_mutation" == created ]]; then if jq -e --arg token "$transaction_token" '.attributes["platform.observability.transaction"] == $token' "$current" >/dev/null; then : elif client_contract_matches "$current" "$client_id"; then : else return 1; fi admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2 (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 && "$(jq 'length' "$last_body")" == 0 ]] else admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 local current_secret="$temporary_dir/rollback-client-current-secret" jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$current_secret" || return 1 chmod 0600 -- "$current_secret"; cmp --silent -- "$client_secret_file" "$current_secret" || return 1 admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" "$client_restore" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 json_equal "$last_body" "$client_before" fi } rollback_group() { local id=$1 mutation=$2 after=$3 [[ "$mutation" == created ]] || return 0 admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 json_equal "$last_body" "$after" || return 1 admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 404 ]] } rollback_transaction() { local rc=0 [[ "$rollback_in_progress" == false ]] || return 1 rollback_in_progress=true restore_marker || rc=1 rollback_membership "$viewer_user_id" "$viewer_group_id" "$viewer_membership_added" || rc=1 rollback_membership "$admin_user_id" "$admin_group_id" "$admin_membership_added" || rc=1 delete_created_secret || rc=1 rollback_mapper || rc=1 rollback_client || rc=1 rollback_group "$viewer_group_id" "$viewer_group_mutation" "$viewer_group_after" || rc=1 rollback_group "$admin_group_id" "$admin_group_mutation" "$admin_group_after" || rc=1 if (( rc == 0 )); then printf 'GRAFANA_OIDC_ROLLBACK=PASS\n' >&2 else printf 'GRAFANA_OIDC_ROLLBACK=FAIL\n' >&2; emit_manual_recovery; fi rollback_in_progress=false return "$rc" } transaction_fail() { local message=$1 status=${2:-1} classification trap - ERR HUP INT TERM set +e if [[ -n "$pending_kind" ]]; then classify_pending; classification=$? [[ "$classification" != 2 ]] || emit_manual_recovery fi rollback_transaction || true transaction_active=false printf 'ERROR: %s\n' "$message" >&2 exit "$status" } on_signal() { transaction_fail "interrupted by $1" "$2"; } on_process_exit() { local rc=$1 classification [[ "$exit_handler_running" == false ]] || exit "$rc" exit_handler_running=true trap - EXIT ERR HUP INT TERM set +e if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then if [[ -n "$pending_kind" ]]; then classify_pending; classification=$?; [[ "$classification" != 2 ]] || emit_manual_recovery; fi rollback_transaction || true transaction_active=false (( rc != 0 )) || rc=1 fi cleanup exit "$rc" } verify_group_prestate() { local label=$1 name=$2 path=$3 state=$4 id=$5 before=$6 count admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2 require_http 200 "$label group prestate reinspection" count="$(jq 'length' "$last_body")" || return 1 if [[ "$state" == absent ]]; then [[ "$count" == 0 ]]; return; fi [[ "$count" == 1 && "$(jq -r --arg path "$path" '.[0] | select(.path == $path) | .id' "$last_body")" == "$id" ]] || return 1 admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id" require_http 200 "$label group prestate snapshot reinspection" json_equal "$last_body" "$before" } verify_prestate() { local count current_secret="$temporary_dir/prestate-current-secret" current_mapper="$temporary_dir/prestate-current-mapper.json" validate_context_and_authority admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2 require_http 200 'Grafana client prestate reinspection' count="$(jq 'length' "$last_body")" if [[ "$client_state" == absent ]]; then [[ "$count" == 0 ]] || fail 'Grafana client appeared after snapshot' else [[ "$count" == 1 && "$(jq -r '.[0].id' "$last_body")" == "$client_id" ]] || fail 'Grafana client cardinality or identity changed' admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" require_http 200 'Grafana client exact prestate reinspection' json_equal "$last_body" "$client_before" || fail 'Grafana client changed after snapshot' admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret" require_http 200 'Grafana credential prestate reinspection' jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$current_secret" || fail 'Grafana credential reinspection response is invalid' chmod 0600 -- "$current_secret" cmp --silent -- "$client_secret_file" "$current_secret" || fail 'Grafana credential changed after snapshot' fi verify_group_prestate admin "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" "$admin_group_state" "$admin_group_id" "$admin_group_before" || fail 'admin group changed after snapshot' verify_group_prestate viewer "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" "$viewer_group_state" "$viewer_group_id" "$viewer_group_before" || fail 'viewer group changed after snapshot' if [[ "$client_state" == existing ]]; then admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" require_http 200 'Grafana mapper prestate reinspection' if [[ "$mapper_state" == absent ]]; then [[ "$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name or .config["claim.name"] == "groups")] | length' "$last_body")" == 0 ]] || fail 'Grafana mapper appeared after snapshot' else [[ "$(jq --arg id "$mapper_id" '[.[] | select(.id == $id)] | length' "$last_body")" == 1 ]] || fail 'Grafana mapper identity changed' jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current_mapper" chmod 0600 -- "$current_mapper" json_equal "$current_mapper" "$mapper_before" || fail 'Grafana mapper changed after snapshot' fi fi if [[ -n "$admin_username" ]]; then local prior_id=$admin_user_id; lookup_user admin "$admin_username" admin_user_id [[ "$admin_user_id" == "$prior_id" ]] || fail 'admin user identity changed' if [[ "$admin_group_state" == existing ]]; then local membership_rc if membership_present "$admin_user_id" "$admin_group_id"; then membership_rc=0; else membership_rc=$?; fi case "$membership_rc" in 0) [[ "$admin_membership_before" == true ]] || fail 'admin membership appeared after snapshot' ;; 1) [[ "$admin_membership_before" == false ]] || fail 'admin membership disappeared after snapshot' ;; *) fail 'admin membership reinspection failed' ;; esac fi fi if [[ -n "$viewer_username" ]]; then local prior_viewer_id=$viewer_user_id; lookup_user viewer "$viewer_username" viewer_user_id [[ "$viewer_user_id" == "$prior_viewer_id" ]] || fail 'viewer user identity changed' if [[ "$viewer_group_state" == existing ]]; then local viewer_membership_rc if membership_present "$viewer_user_id" "$viewer_group_id"; then viewer_membership_rc=0; else viewer_membership_rc=$?; fi case "$viewer_membership_rc" in 0) [[ "$viewer_membership_before" == true ]] || fail 'viewer membership appeared after snapshot' ;; 1) [[ "$viewer_membership_before" == false ]] || fail 'viewer membership disappeared after snapshot' ;; *) fail 'viewer membership reinspection failed' ;; esac fi fi kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" (( last_curl_rc == 0 )) || fail 'Grafana OIDC Secret prestate reinspection transport failed' if [[ "$secret_state" == absent ]]; then [[ "$last_http_status" == 404 ]] || fail 'Grafana OIDC Secret appeared after snapshot' else [[ "$last_http_status" == 200 ]] || fail 'Grafana OIDC Secret disappeared after snapshot' json_equal "$last_body" "$secret_before" || fail 'Grafana OIDC Secret changed after snapshot' fi } snapshot_transaction_prestate() { transaction_token="$(new_transaction_token)" || fail 'cannot create a transaction ownership token' [[ "$transaction_token" =~ ^[0-9a-f]{32}$ ]] || fail 'transaction ownership token is invalid' build_desired_client build_desired_mapper lookup_client lookup_group admin "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" admin_group_state admin_group_id admin_group_before lookup_group viewer "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" viewer_group_state viewer_group_id viewer_group_before lookup_mapper lookup_user admin "$admin_username" admin_user_id lookup_user viewer "$viewer_username" viewer_user_id if [[ -n "$admin_user_id" && "$admin_group_state" == existing ]]; then snapshot_membership admin "$admin_user_id" "$admin_group_id" admin_membership_before fi if [[ -n "$viewer_user_id" && "$viewer_group_state" == existing ]]; then snapshot_membership viewer "$viewer_user_id" "$viewer_group_id" viewer_membership_before fi if [[ "$client_state" == existing ]]; then retrieve_client_secret; fi start_kubernetes_proxy snapshot_oidc_secret sudo_refresh prepare_marker_snapshot } verify_final_contract() { local count current_mapper="$temporary_dir/final-mapper.json" prefix values admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2 require_http 200 'final Grafana client lookup' [[ "$(jq 'length' "$last_body")" == 1 && "$(jq -r '.[0].id' "$last_body")" == "$client_id" ]] || fail 'final Grafana client cardinality differs' admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" require_http 200 'final Grafana client read' client_contract_matches "$last_body" "$client_id" || fail 'final Grafana client contract differs' local label name path id for label in admin viewer; do if [[ "$label" == admin ]]; then name=$ADMIN_GROUP_NAME; path=$ADMIN_GROUP_PATH; id=$admin_group_id else name=$VIEWER_GROUP_NAME; path=$VIEWER_GROUP_PATH; id=$viewer_group_id; fi admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2 require_http 200 "final $label group lookup" [[ "$(jq 'length' "$last_body")" == 1 && "$(jq -r --arg path "$path" '.[0] | select(.path == $path) | .id' "$last_body")" == "$id" ]] || fail "final $label group contract differs" done admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" require_http 200 'final Grafana mapper lookup' [[ "$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name or .config["claim.name"] == "groups")] | length' "$last_body")" == 1 ]] || fail 'final Grafana mapper cardinality differs' jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current_mapper" || fail 'final Grafana mapper identity differs' chmod 0600 -- "$current_mapper"; mapper_contract_matches "$current_mapper" "$mapper_id" || fail 'final Grafana mapper contract differs' kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || fail 'final Grafana OIDC Secret is absent' prefix="$temporary_dir/final-secret"; values="$(validate_secret_json "$last_body" "$prefix")" || fail 'final Grafana OIDC Secret contract differs' [[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] && cmp --silent -- "$prefix.client-secret" "$client_secret_file" || fail 'final Grafana OIDC Secret relationship differs' if [[ -n "$admin_user_id" ]]; then membership_present "$admin_user_id" "$admin_group_id" || fail 'requested admin membership is absent'; fi if [[ -n "$viewer_user_id" ]]; then membership_present "$viewer_user_id" "$viewer_group_id" || fail 'requested viewer membership is absent'; fi admin_request GET "$local_base/realms/$KEYCLOAK_REALM/.well-known/openid-configuration" require_http 200 'Keycloak OIDC discovery verification' jq -e --arg issuer "https://$KEYCLOAK_HOST/realms/$KEYCLOAK_REALM" '.issuer == $issuer' "$last_body" >/dev/null || fail 'Keycloak public issuer differs' } reconcile_transaction() { transaction_active=true trap 'transaction_fail "unexpected command failure at line $LINENO"' ERR trap 'on_signal HUP 129' HUP trap 'on_signal INT 130' INT trap 'on_signal TERM 143' TERM if [[ "$admin_group_state" == absent ]]; then create_group admin "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" admin_group_id admin_group_mutation admin_group_after fi if [[ "$viewer_group_state" == absent ]]; then create_group viewer "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" viewer_group_id viewer_group_mutation viewer_group_after fi if [[ "$client_state" == absent ]]; then create_client elif client_contract_matches "$client_before" "$client_id"; then client_mutation=none else update_client fi if [[ "$mapper_state" == absent ]]; then create_mapper elif mapper_contract_matches "$mapper_before" "$mapper_id"; then mapper_mutation=none else update_mapper fi if [[ "$secret_state" == absent ]]; then create_oidc_secret; fi add_membership admin "$admin_user_id" "$admin_group_id" "$admin_membership_before" admin_membership_added add_membership viewer "$viewer_user_id" "$viewer_group_id" "$viewer_membership_before" viewer_membership_added verify_final_contract install_marker trap - ERR HUP INT TERM transaction_active=false printf '%s\n' \ 'GRAFANA_OIDC_CLIENT=RECONCILED' \ 'GRAFANA_OIDC_GROUPS=RECONCILED' \ 'GRAFANA_OIDC_MAPPER=RECONCILED' \ "GRAFANA_OIDC_SECRET=$(if [[ "$secret_state" == absent ]]; then printf CREATE_CONFIRMED; else printf REUSED_UNCHANGED; fi)" \ 'GRAFANA_OIDC_TRANSACTION=PASS' } execute_transaction() { run_common_gates validate_context_and_authority snapshot_kubernetes_objects start_port_forward authenticate_keycloak snapshot_transaction_prestate read_confirmation 'Type APPLY default: ' 'APPLY default' read_confirmation 'Type RECOVERY KEYCLOAK default: ' 'RECOVERY KEYCLOAK default' verify_prestate run_common_gates ensure_evidence_directory reconcile_transaction } grafana_oidc_main() { parse_arguments "$@" (( EUID != 0 )) || fail 'whole-script root execution is forbidden; use narrow sudo from the invoking user' if [[ "$execute_requested" == false && "$check_requested" == false ]]; then print_dry_run; return 0; fi require_commands make_temporary_dir trap 'on_process_exit "$?"' EXIT if [[ "$check_requested" == true ]]; then check_recovery_evidence; else execute_transaction; fi } platform_grafana_oidc_fixture_main() { local fixture=${1:-} [[ "${BASH_SOURCE[0]}" != "$0" ]] || fail 'fixture entrypoint must be sourced' [[ -n "$fixture" ]] || fail 'fixture root is required' shift TEST_MODE=true configure_test_boundaries "$fixture" grafana_oidc_main "$@" } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then reject_production_overrides grafana_oidc_main "$@" fi