1929 lines
92 KiB
Bash
Executable File
1929 lines
92 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -Eeuo pipefail
|
|
# Never inherit xtrace across credential, bearer-token, or client-secret handling.
|
|
set +x
|
|
umask 077
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly EXPECTED_CONTEXT='default'
|
|
readonly EXPECTED_API_SERVER='https://127.0.0.1:6443'
|
|
readonly EXPECTED_NODE='donghyeon-system-product-name'
|
|
readonly KEYCLOAK_NAMESPACE='keycloak'
|
|
readonly KEYCLOAK_NAME='keycloak'
|
|
readonly KEYCLOAK_SERVICE='keycloak-service'
|
|
readonly KEYCLOAK_ADMIN_SECRET='keycloak-initial-admin'
|
|
readonly KEYCLOAK_HOST='id.learn.hyeonworks.com'
|
|
readonly KEYCLOAK_REALM='hyeonworks'
|
|
readonly OBSERVABILITY_NAMESPACE='observability'
|
|
readonly OIDC_SECRET='grafana-keycloak-oidc'
|
|
readonly CLIENT_ID='grafana'
|
|
readonly GRAFANA_URL='https://grafana.learn.hyeonworks.com'
|
|
readonly REDIRECT_URI='https://grafana.learn.hyeonworks.com/login/generic_oauth'
|
|
readonly POST_LOGOUT_URI='https://grafana.learn.hyeonworks.com/*'
|
|
readonly ADMIN_GROUP_NAME='platform-observability-admins'
|
|
readonly VIEWER_GROUP_NAME='platform-observability-viewers'
|
|
readonly ADMIN_GROUP_PATH='/platform-observability-admins'
|
|
readonly VIEWER_GROUP_PATH='/platform-observability-viewers'
|
|
readonly MAPPER_NAME='grafana-groups'
|
|
readonly EVIDENCE_SCHEMA='platform-observability-recovery-evidence-v1'
|
|
readonly EVIDENCE_RESOURCE='keycloak/hyeonworks/client/grafana'
|
|
readonly EVIDENCE_MAX_AGE_SECONDS=2592000
|
|
readonly PRODUCTION_EVIDENCE_DIR='/etc/hyeonworks/platform/recovery-evidence'
|
|
readonly PRODUCTION_ENCRYPTION_SCRIPT="$ROOT/scripts/validate/k3s-secret-encryption.sh"
|
|
readonly PRODUCTION_RESTORE_SCRIPT="$ROOT/scripts/validate/k3s-secret-encryption-restore-evidence.sh"
|
|
readonly VALIDATOR_PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
|
|
readonly VALIDATOR_HOME='/home/donghyeon'
|
|
readonly KUBECTL_REQUEST_TIMEOUT='5s'
|
|
|
|
TEST_MODE=false
|
|
KUBECTL_BIN='/usr/local/bin/kubectl'
|
|
SUDO_BIN='/usr/bin/sudo'
|
|
ENCRYPTION_SCRIPT="$PRODUCTION_ENCRYPTION_SCRIPT"
|
|
RESTORE_SCRIPT="$PRODUCTION_RESTORE_SCRIPT"
|
|
EVIDENCE_DIR="$PRODUCTION_EVIDENCE_DIR"
|
|
KUBECTL_PROCESS_TIMEOUT='10s'
|
|
PORT_FORWARD_PROCESS_TIMEOUT='120s'
|
|
PROXY_PROCESS_TIMEOUT='120s'
|
|
CURL_PROCESS_TIMEOUT='12s'
|
|
CURL_MAX_TIME='8'
|
|
CURL_CONNECT_TIMEOUT='2'
|
|
EVIDENCE_EXPECTED_UID=0
|
|
EVIDENCE_EXPECTED_GID=0
|
|
EVIDENCE_DIR_MODE=700
|
|
TEST_CONFIRMATIONS=''
|
|
TEST_NOW_UTC=''
|
|
|
|
execute_requested=false
|
|
check_requested=false
|
|
admin_username=''
|
|
viewer_username=''
|
|
temporary_dir=''
|
|
port_forward_pid=''
|
|
port_forward_start=''
|
|
proxy_pid=''
|
|
proxy_start=''
|
|
local_port=''
|
|
local_base=''
|
|
kube_socket=''
|
|
confirmation_index=0
|
|
declare -a confirmation_answers=()
|
|
|
|
transaction_active=false
|
|
rollback_in_progress=false
|
|
exit_handler_running=false
|
|
manual_recovery_required=false
|
|
pending_kind=''
|
|
pending_before=''
|
|
pending_desired=''
|
|
pending_id=''
|
|
pending_token=''
|
|
pending_user_id=''
|
|
pending_group_id=''
|
|
pending_status_file=''
|
|
transaction_token=''
|
|
|
|
client_state='absent'
|
|
client_id=''
|
|
client_before=''
|
|
client_desired=''
|
|
client_after=''
|
|
client_restore=''
|
|
client_secret_file=''
|
|
client_mutation='none'
|
|
admin_group_state='absent'
|
|
admin_group_id=''
|
|
admin_group_before=''
|
|
admin_group_after=''
|
|
admin_group_mutation='none'
|
|
viewer_group_state='absent'
|
|
viewer_group_id=''
|
|
viewer_group_before=''
|
|
viewer_group_after=''
|
|
viewer_group_mutation='none'
|
|
mapper_state='absent'
|
|
mapper_id=''
|
|
mapper_before=''
|
|
mapper_desired=''
|
|
mapper_after=''
|
|
mapper_mutation='none'
|
|
secret_state='absent'
|
|
secret_before=''
|
|
secret_uid=''
|
|
secret_resource_version=''
|
|
secret_created_uid=''
|
|
secret_created_resource_version=''
|
|
secret_mutation='none'
|
|
admin_user_id=''
|
|
viewer_user_id=''
|
|
admin_membership_before=false
|
|
viewer_membership_before=false
|
|
admin_membership_added=false
|
|
viewer_membership_added=false
|
|
marker_had_prior=false
|
|
marker_prior=''
|
|
marker_desired=''
|
|
marker_written=false
|
|
|
|
last_http_status=''
|
|
last_body=''
|
|
last_status_file=''
|
|
last_curl_rc=0
|
|
request_counter=0
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage:
|
|
bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh
|
|
bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh --execute
|
|
bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh \
|
|
--execute --admin USER --viewer USER
|
|
bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh \
|
|
--check-recovery-evidence
|
|
|
|
No arguments prints the fixed, payload-free plan and performs no external reads.
|
|
Execute reconciles only the exact Grafana client, two groups, full-path groups
|
|
mapper, OIDC Secret, and explicitly requested existing-user memberships.
|
|
USAGE
|
|
}
|
|
|
|
emit_manual_recovery() {
|
|
[[ "$manual_recovery_required" == false ]] || return 0
|
|
manual_recovery_required=true
|
|
printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2
|
|
}
|
|
|
|
fail() {
|
|
local message=$1
|
|
if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then
|
|
transaction_fail "$message"
|
|
fi
|
|
printf 'ERROR: %s\n' "$message" >&2
|
|
exit 1
|
|
}
|
|
|
|
reject_production_overrides() {
|
|
local variable
|
|
for variable in ${!PLATFORM_GRAFANA_OIDC_@}; do
|
|
printf 'ERROR: production boundary override is forbidden: %s\n' "$variable" >&2
|
|
exit 1
|
|
done
|
|
}
|
|
|
|
parse_arguments() {
|
|
local seen_execute=false seen_check=false seen_admin=false seen_viewer=false
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
--execute)
|
|
[[ "$seen_execute" == false ]] || fail 'duplicate --execute'
|
|
seen_execute=true; execute_requested=true; shift
|
|
;;
|
|
--check-recovery-evidence)
|
|
[[ "$seen_check" == false ]] || fail 'duplicate --check-recovery-evidence'
|
|
seen_check=true; check_requested=true; shift
|
|
;;
|
|
--admin)
|
|
[[ "$seen_admin" == false && $# -ge 2 ]] || fail 'invalid --admin'
|
|
seen_admin=true; admin_username=$2; shift 2
|
|
;;
|
|
--viewer)
|
|
[[ "$seen_viewer" == false && $# -ge 2 ]] || fail 'invalid --viewer'
|
|
seen_viewer=true; viewer_username=$2; shift 2
|
|
;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) fail "unsupported argument: $1" ;;
|
|
esac
|
|
done
|
|
if [[ "$check_requested" == true ]]; then
|
|
[[ "$execute_requested" == false && "$seen_admin" == false && "$seen_viewer" == false ]] ||
|
|
fail 'recovery evidence mode cannot be combined with execute or memberships'
|
|
return
|
|
fi
|
|
if [[ "$execute_requested" == false ]]; then
|
|
[[ "$seen_admin" == false && "$seen_viewer" == false ]] ||
|
|
fail 'membership selection requires --execute'
|
|
return
|
|
fi
|
|
local username
|
|
for username in "$admin_username" "$viewer_username"; do
|
|
[[ -z "$username" || "$username" =~ ^[A-Za-z0-9][A-Za-z0-9._@+-]{0,127}$ ]] ||
|
|
fail 'membership usernames must use the bounded printable Keycloak username form'
|
|
done
|
|
}
|
|
|
|
print_dry_run() {
|
|
printf '%s\n' \
|
|
'GRAFANA_OIDC_DRY_RUN=PASS' \
|
|
'CONTEXT=default' \
|
|
'KEYCLOAK_CLIENT=grafana' \
|
|
'KEYCLOAK_GROUPS=/platform-observability-admins,/platform-observability-viewers' \
|
|
'KEYCLOAK_MAPPER=grafana-groups:groups:full-path:id-token,access-token,userinfo' \
|
|
'KUBERNETES_SECRET=observability/grafana-keycloak-oidc:Opaque:client-id,client-secret' \
|
|
'MEMBERSHIPS=EXPLICIT_ONLY' \
|
|
'MUTATION=NOT_REQUESTED'
|
|
}
|
|
|
|
attest_test_executable() {
|
|
local path=$1 metadata
|
|
[[ -f "$path" && ! -L "$path" && -O "$path" && -x "$path" ]] || return 1
|
|
metadata="$(stat -c '%u:%a:%h:%F' -- "$path")" || return 1
|
|
[[ "$metadata" == "${EUID}:755:1:regular file" ]]
|
|
}
|
|
|
|
configure_test_boundaries() {
|
|
local fixture=$1 resolved metadata
|
|
[[ "$fixture" =~ ^/tmp/platform-grafana-oidc-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+$ ]] ||
|
|
fail 'test fixture root has an invalid shape'
|
|
[[ -d "$fixture" && ! -L "$fixture" && -O "$fixture" ]] || fail 'test fixture root is unsafe'
|
|
resolved="$(cd -- "$fixture" && pwd -P)" || fail 'cannot resolve test fixture root'
|
|
[[ "$resolved" == "$fixture" ]] || fail 'test fixture contains a symlink boundary'
|
|
metadata="$(stat -c '%u:%a:%F' -- "$fixture")" || fail 'cannot stat test fixture root'
|
|
[[ "$metadata" == "${EUID}:700:directory" ]] || fail 'test fixture metadata is unsafe'
|
|
[[ "$(cd -- "$fixture/bin" && pwd -P)" == "$fixture/bin" &&
|
|
"$(cd -- "$fixture/evidence-parent" && pwd -P)" == "$fixture/evidence-parent" ]] ||
|
|
fail 'test fixture child has a symlink boundary'
|
|
KUBECTL_BIN="$fixture/bin/kubectl"
|
|
SUDO_BIN="$fixture/bin/sudo"
|
|
ENCRYPTION_SCRIPT="$fixture/bin/encryption"
|
|
RESTORE_SCRIPT="$fixture/bin/restore"
|
|
EVIDENCE_DIR="$fixture/evidence-parent/recovery-evidence"
|
|
EVIDENCE_EXPECTED_UID=$EUID
|
|
EVIDENCE_EXPECTED_GID="$(id -g)"
|
|
EVIDENCE_DIR_MODE=700
|
|
KUBECTL_PROCESS_TIMEOUT='3s'
|
|
PORT_FORWARD_PROCESS_TIMEOUT='30s'
|
|
PROXY_PROCESS_TIMEOUT='30s'
|
|
CURL_PROCESS_TIMEOUT='4s'
|
|
CURL_MAX_TIME='2'
|
|
CURL_CONNECT_TIMEOUT='1'
|
|
TEST_CONFIRMATIONS="${PLATFORM_GRAFANA_OIDC_CONFIRMATIONS:-}"
|
|
TEST_NOW_UTC="${PLATFORM_GRAFANA_OIDC_NOW_UTC:-}"
|
|
mapfile -t confirmation_answers <<<"$TEST_CONFIRMATIONS"
|
|
[[ "$TEST_NOW_UTC" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] ||
|
|
fail 'test clock is invalid'
|
|
attest_test_executable "$KUBECTL_BIN" || fail 'test kubectl boundary is unsafe'
|
|
attest_test_executable "$SUDO_BIN" || fail 'test sudo boundary is unsafe'
|
|
attest_test_executable "$ENCRYPTION_SCRIPT" || fail 'test encryption boundary is unsafe'
|
|
attest_test_executable "$RESTORE_SCRIPT" || fail 'test restore boundary is unsafe'
|
|
[[ "$SUDO_BIN" != /usr/bin/sudo ]] || fail 'real sudo is forbidden in the fixture entrypoint'
|
|
export PLATFORM_TEST_TARGET_PID=$BASHPID
|
|
}
|
|
|
|
require_commands() {
|
|
local command_name
|
|
for command_name in base64 chmod cmp cp curl date grep id jq kill mktemp python3 rm sort stat timeout tr wc; do
|
|
command -v "$command_name" >/dev/null 2>&1 || fail "$command_name is required"
|
|
done
|
|
[[ -x "$KUBECTL_BIN" && ! -L "$KUBECTL_BIN" ]] || fail 'kubectl boundary is unavailable or unsafe'
|
|
[[ -x "$SUDO_BIN" && ! -L "$SUDO_BIN" ]] || fail 'sudo boundary is unavailable or unsafe'
|
|
[[ -f "$ENCRYPTION_SCRIPT" && ! -L "$ENCRYPTION_SCRIPT" && -r "$ENCRYPTION_SCRIPT" ]] ||
|
|
fail 'k3s encryption validator is unavailable'
|
|
[[ -f "$RESTORE_SCRIPT" && ! -L "$RESTORE_SCRIPT" && -r "$RESTORE_SCRIPT" ]] ||
|
|
fail 'k3s restore validator is unavailable'
|
|
}
|
|
|
|
make_temporary_dir() {
|
|
temporary_dir="$(mktemp -d /tmp/platform-grafana-oidc.XXXXXX)" || fail 'cannot create private temporary directory'
|
|
[[ "$temporary_dir" == /tmp/platform-grafana-oidc.?????? && ! -L "$temporary_dir" ]] ||
|
|
fail 'private temporary directory shape is unsafe'
|
|
chmod 0700 -- "$temporary_dir"
|
|
[[ "$(stat -c '%u:%a:%h:%F' -- "$temporary_dir")" == "${EUID}:700:2:directory" ]] ||
|
|
fail 'private temporary directory metadata is unsafe'
|
|
}
|
|
|
|
private_file_ok() {
|
|
local path=$1 allow_empty=${2:-false} metadata
|
|
[[ "$path" == "$temporary_dir"/* && -f "$path" && ! -L "$path" && -O "$path" ]] || return 1
|
|
metadata="$(stat -c '%u:%a:%h' -- "$path")" || return 1
|
|
[[ "$metadata" == "${EUID}:600:1" ]] || return 1
|
|
[[ "$allow_empty" == true || -s "$path" ]]
|
|
}
|
|
|
|
process_start_time() {
|
|
local pid=$1
|
|
python3 -I -S - "$pid" <<'PY'
|
|
import pathlib,sys
|
|
raw=pathlib.Path('/proc',sys.argv[1],'stat').read_text()
|
|
end=raw.rfind(') ')
|
|
if end < 0: raise SystemExit(1)
|
|
fields=raw[end+2:].split()
|
|
if len(fields)<20 or not fields[19].isdigit(): raise SystemExit(1)
|
|
print(fields[19],end='')
|
|
PY
|
|
}
|
|
|
|
stop_owned_process() {
|
|
local pid=$1 expected_start=$2 attempt current
|
|
[[ -n "$pid" && -n "$expected_start" && "$pid" =~ ^[0-9]+$ ]] || return 0
|
|
current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; }
|
|
[[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; }
|
|
kill -TERM "$pid" 2>/dev/null || true
|
|
for ((attempt=0; attempt<30; attempt++)); do
|
|
if ! kill -0 "$pid" 2>/dev/null; then wait "$pid" 2>/dev/null || true; return 0; fi
|
|
/usr/bin/sleep 0.05
|
|
done
|
|
current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; }
|
|
[[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; }
|
|
kill -KILL "$pid" 2>/dev/null || true
|
|
wait "$pid" 2>/dev/null || true
|
|
}
|
|
|
|
safe_temporary_path() {
|
|
[[ -n "$temporary_dir" && "$temporary_dir" == /tmp/platform-grafana-oidc.?????? &&
|
|
( "$1" == "$temporary_dir" || "$1" == "$temporary_dir"/* ) ]]
|
|
}
|
|
|
|
cleanup() {
|
|
stop_owned_process "$proxy_pid" "$proxy_start" || true
|
|
stop_owned_process "$port_forward_pid" "$port_forward_start" || true
|
|
if [[ -n "$temporary_dir" ]] && safe_temporary_path "$temporary_dir"; then rm -rf -- "$temporary_dir"; fi
|
|
}
|
|
|
|
kubectl_bounded() {
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s "$KUBECTL_PROCESS_TIMEOUT" \
|
|
"$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" "$@"
|
|
}
|
|
|
|
run_common_gates() {
|
|
local -a safe_environment=(/usr/bin/env -i "PATH=$VALIDATOR_PATH" "HOME=$VALIDATOR_HOME")
|
|
if [[ "$TEST_MODE" == true ]]; then
|
|
safe_environment+=(
|
|
"PLATFORM_TEST_COMMAND_LOG=${PLATFORM_TEST_COMMAND_LOG:-}"
|
|
"PLATFORM_TEST_STATE=${PLATFORM_TEST_STATE:-}"
|
|
"PLATFORM_TEST_ENCRYPTION_COUNT=${PLATFORM_TEST_ENCRYPTION_COUNT:-}"
|
|
"PLATFORM_TEST_RESTORE_COUNT=${PLATFORM_TEST_RESTORE_COUNT:-}"
|
|
"PLATFORM_TEST_ENCRYPTION_FAIL_AT=${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}"
|
|
"PLATFORM_TEST_RESTORE_FAIL_AT=${PLATFORM_TEST_RESTORE_FAIL_AT:-0}"
|
|
"PLATFORM_TEST_VALIDATOR_ENV_LOG=${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}"
|
|
)
|
|
fi
|
|
"${safe_environment[@]}" /usr/bin/bash "$ENCRYPTION_SCRIPT" --expect-reencrypted >/dev/null 2>&1 ||
|
|
fail 'k3s Secret encryption is not fully re-encrypted'
|
|
"${safe_environment[@]}" /usr/bin/bash "$RESTORE_SCRIPT" --check >/dev/null 2>&1 ||
|
|
fail 'current k3s restore evidence is unavailable'
|
|
}
|
|
|
|
read_confirmation() {
|
|
local prompt=$1 expected=$2 answer=''
|
|
printf '%s' "$prompt" >&2
|
|
if [[ "$TEST_MODE" == true ]]; then
|
|
(( confirmation_index < ${#confirmation_answers[@]} )) || fail 'confirmation input is missing'
|
|
answer="${confirmation_answers[$confirmation_index]}"; confirmation_index=$((confirmation_index + 1))
|
|
else
|
|
[[ -t 0 ]] || fail 'execute mode requires an interactive terminal'
|
|
IFS= read -r answer
|
|
fi
|
|
[[ "$answer" == "$expected" ]] || fail 'confirmation was not exact'
|
|
}
|
|
|
|
now_utc() {
|
|
if [[ "$TEST_MODE" == true ]]; then printf '%s\n' "$TEST_NOW_UTC"; else date -u +%Y-%m-%dT%H:%M:%SZ; fi
|
|
}
|
|
|
|
timestamp_epoch() {
|
|
local timestamp=$1 normalized
|
|
normalized="$(date -u -d "$timestamp" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" || return 1
|
|
[[ "$normalized" == "$timestamp" ]] || return 1
|
|
date -u -d "$timestamp" +%s
|
|
}
|
|
|
|
now_epoch() { timestamp_epoch "$(now_utc)"; }
|
|
|
|
json_equal() {
|
|
python3 -I -S - "$1" "$2" <<'PY'
|
|
import json,pathlib,sys
|
|
try: a=json.loads(pathlib.Path(sys.argv[1]).read_text()); b=json.loads(pathlib.Path(sys.argv[2]).read_text())
|
|
except Exception: raise SystemExit(1)
|
|
raise SystemExit(0 if a==b else 1)
|
|
PY
|
|
}
|
|
|
|
new_transaction_token() { python3 -I -S -c 'import secrets; print(secrets.token_hex(16))'; }
|
|
|
|
validate_context_and_authority() {
|
|
local context api node_file namespace result
|
|
context="$(kubectl_bounded config current-context)" || fail 'cannot read Kubernetes context'
|
|
[[ "$context" == "$EXPECTED_CONTEXT" ]] || fail 'current Kubernetes context is not default'
|
|
api="$(kubectl_bounded config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" ||
|
|
fail 'cannot read Kubernetes API server'
|
|
[[ "$api" == "$EXPECTED_API_SERVER" ]] || fail 'Kubernetes API server is not the exact local endpoint'
|
|
node_file="$temporary_dir/node.json"
|
|
kubectl_bounded get node "$EXPECTED_NODE" --output=json >"$node_file" || fail 'cannot read exact target node'
|
|
chmod 0600 -- "$node_file"
|
|
private_file_ok "$node_file" || fail 'target node response is unsafe'
|
|
jq -e --arg name "$EXPECTED_NODE" '
|
|
.apiVersion == "v1" and .kind == "Node" and .metadata.name == $name and
|
|
(.metadata.uid | type == "string" and length > 0) and
|
|
any(.status.conditions[]?; .type == "Ready" and .status == "True")
|
|
' "$node_file" >/dev/null || fail 'target node identity or readiness changed'
|
|
|
|
for namespace in "$KEYCLOAK_NAMESPACE" "$OBSERVABILITY_NAMESPACE"; do
|
|
result="$(kubectl_bounded get namespace "$namespace" --output=name)" || fail "cannot read namespace $namespace"
|
|
[[ "$result" == "namespace/$namespace" ]] || fail "namespace identity changed: $namespace"
|
|
done
|
|
for permission in \
|
|
'get keycloak.k8s.keycloak.org/keycloak keycloak' \
|
|
'get service/keycloak-service keycloak' \
|
|
'get secret/keycloak-initial-admin keycloak' \
|
|
'get secret/grafana-keycloak-oidc observability' \
|
|
'create secret observability' \
|
|
'delete secret/grafana-keycloak-oidc observability' \
|
|
'get nodes -' \
|
|
'get namespaces -' \
|
|
'get pods keycloak' \
|
|
'list pods keycloak' \
|
|
'create pods/portforward keycloak'; do
|
|
set -- $permission
|
|
if [[ "$3" == - ]]; then
|
|
result="$(kubectl_bounded auth can-i "$1" "$2")" || fail 'cannot verify exact Kubernetes authority'
|
|
else
|
|
result="$(kubectl_bounded auth can-i "$1" "$2" --namespace="$3")" || fail 'cannot verify exact Kubernetes authority'
|
|
fi
|
|
[[ "$result" == yes ]] || fail "missing exact Kubernetes authority: $permission"
|
|
done
|
|
}
|
|
|
|
snapshot_kubernetes_objects() {
|
|
local keycloak_file service_file admin_file
|
|
keycloak_file="$temporary_dir/keycloak-resource.json"
|
|
service_file="$temporary_dir/keycloak-service.json"
|
|
admin_file="$temporary_dir/keycloak-admin-secret.json"
|
|
kubectl_bounded --namespace "$KEYCLOAK_NAMESPACE" get "keycloak.k8s.keycloak.org/$KEYCLOAK_NAME" --output=json >"$keycloak_file" ||
|
|
fail 'cannot read exact Keycloak resource'
|
|
kubectl_bounded --namespace "$KEYCLOAK_NAMESPACE" get "service/$KEYCLOAK_SERVICE" --output=json >"$service_file" ||
|
|
fail 'cannot read exact Keycloak Service'
|
|
kubectl_bounded --namespace "$KEYCLOAK_NAMESPACE" get "secret/$KEYCLOAK_ADMIN_SECRET" --output=json >"$admin_file" ||
|
|
fail 'cannot read exact initial-admin Secret'
|
|
chmod 0600 -- "$keycloak_file" "$service_file" "$admin_file"
|
|
private_file_ok "$keycloak_file" && private_file_ok "$service_file" && private_file_ok "$admin_file" ||
|
|
fail 'Kubernetes bootstrap snapshots are unsafe'
|
|
jq -e --arg name "$KEYCLOAK_NAME" --arg namespace "$KEYCLOAK_NAMESPACE" '
|
|
.apiVersion == "k8s.keycloak.org/v2beta1" and .kind == "Keycloak" and
|
|
.metadata.name == $name and .metadata.namespace == $namespace and
|
|
(.metadata.uid | type == "string" and length > 0) and
|
|
any(.status.conditions[]?; .type == "Ready" and .status == "True")
|
|
' "$keycloak_file" >/dev/null || fail 'Keycloak resource identity or readiness changed'
|
|
jq -e --arg name "$KEYCLOAK_SERVICE" --arg namespace "$KEYCLOAK_NAMESPACE" '
|
|
.apiVersion == "v1" and .kind == "Service" and .metadata.name == $name and
|
|
.metadata.namespace == $namespace and (.metadata.uid | type == "string" and length > 0) and
|
|
([.spec.ports[]? | select(.port == 8080)] | length) == 1
|
|
' "$service_file" >/dev/null || fail 'Keycloak Service contract changed'
|
|
|
|
local admin_username_file admin_password_file
|
|
admin_username_file="$temporary_dir/admin-username"
|
|
admin_password_file="$temporary_dir/admin-password"
|
|
python3 -I -S - "$admin_file" "$admin_username_file" "$admin_password_file" <<'PY' ||
|
|
import base64,binascii,json,os,pathlib,sys
|
|
source,user_path,password_path=sys.argv[1:]
|
|
try: item=json.loads(pathlib.Path(source).read_text())
|
|
except Exception: raise SystemExit(1)
|
|
meta=item.get('metadata') or {}; data=item.get('data')
|
|
if item.get('apiVersion')!='v1' or item.get('kind')!='Secret' or item.get('type')!='kubernetes.io/basic-auth' or meta.get('namespace')!='keycloak' or meta.get('name')!='keycloak-initial-admin' or not meta.get('uid') or sorted(data or {})!=['password','username']:
|
|
raise SystemExit(1)
|
|
for key,destination in [('username',user_path),('password',password_path)]:
|
|
try: payload=base64.b64decode(data[key],validate=True)
|
|
except (binascii.Error,ValueError): raise SystemExit(1)
|
|
if not payload or b'\x00' in payload or b'\r' in payload or b'\n' in payload: raise SystemExit(1)
|
|
fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600)
|
|
try: os.write(fd,payload)
|
|
finally: os.close(fd)
|
|
PY
|
|
fail 'initial-admin Secret contract is invalid'
|
|
private_file_ok "$admin_username_file" && private_file_ok "$admin_password_file" ||
|
|
fail 'private initial-admin credential snapshot is unsafe'
|
|
rm -f -- "$admin_file"
|
|
}
|
|
|
|
allocate_loopback_port() {
|
|
local reservation_file="$temporary_dir/port-reservation"
|
|
python3 -I -S - "$reservation_file" <<'PY' || return 1
|
|
import os,socket,sys
|
|
s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
|
|
s.bind(('127.0.0.1',0)); port=s.getsockname()[1]
|
|
fd=os.open(sys.argv[1],os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600)
|
|
try: os.write(fd,str(port).encode())
|
|
finally: os.close(fd); s.close()
|
|
PY
|
|
private_file_ok "$reservation_file" || return 1
|
|
IFS= read -r local_port <"$reservation_file"
|
|
[[ "$local_port" =~ ^[0-9]+$ ]] || return 1
|
|
(( local_port >= 1024 && local_port <= 65535 )) || return 1
|
|
local_base="http://127.0.0.1:$local_port"
|
|
}
|
|
|
|
loopback_port_open() {
|
|
python3 -I -S - "$local_port" <<'PY'
|
|
import socket,sys
|
|
port=int(sys.argv[1])
|
|
with socket.create_connection(('127.0.0.1',port),timeout=0.2): pass
|
|
PY
|
|
}
|
|
|
|
start_port_forward() {
|
|
local log="$temporary_dir/port-forward.log" ready=false attempt
|
|
allocate_loopback_port || fail 'cannot allocate a validated loopback port'
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s "$PORT_FORWARD_PROCESS_TIMEOUT" \
|
|
"$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" \
|
|
port-forward --address=127.0.0.1 --namespace="$KEYCLOAK_NAMESPACE" \
|
|
"service/$KEYCLOAK_SERVICE" "$local_port:8080" >"$log" 2>&1 &
|
|
port_forward_pid=$!
|
|
port_forward_start="$(process_start_time "$port_forward_pid")" || fail 'cannot pin port-forward process identity'
|
|
for ((attempt=0; attempt<60; attempt++)); do
|
|
kill -0 "$port_forward_pid" 2>/dev/null || fail 'Keycloak port-forward exited before readiness'
|
|
if grep -Fqx "Forwarding from 127.0.0.1:$local_port -> 8080" "$log" &&
|
|
loopback_port_open >/dev/null 2>&1; then ready=true; break; fi
|
|
/usr/bin/sleep 0.05
|
|
done
|
|
[[ "$ready" == true ]] || fail 'Keycloak port-forward did not become ready in time'
|
|
[[ "$(process_start_time "$port_forward_pid")" == "$port_forward_start" ]] ||
|
|
fail 'Keycloak port-forward process identity changed'
|
|
}
|
|
|
|
write_curl_config() {
|
|
local destination=$1 token_file=${2:-}
|
|
{
|
|
printf '%s\n' \
|
|
'silent' \
|
|
'show-error' \
|
|
'noproxy = "*"' \
|
|
"connect-timeout = \"$CURL_CONNECT_TIMEOUT\"" \
|
|
"max-time = \"$CURL_MAX_TIME\"" \
|
|
"header = \"Host: $KEYCLOAK_HOST\"" \
|
|
'header = "X-Forwarded-Proto: https"' \
|
|
'header = "X-Forwarded-Port: 443"'
|
|
if [[ -n "$token_file" ]]; then
|
|
printf 'header = "Authorization: Bearer '
|
|
tr -d '\r\n' <"$token_file"
|
|
printf '"\n'
|
|
fi
|
|
} >"$destination"
|
|
chmod 0600 -- "$destination"
|
|
private_file_ok "$destination" || fail 'curl configuration is unsafe'
|
|
}
|
|
|
|
authenticate_keycloak() {
|
|
local token_response token_file public_config auth_config status
|
|
token_response="$temporary_dir/token-response.json"
|
|
token_file="$temporary_dir/access-token"
|
|
public_config="$temporary_dir/curl-public.conf"
|
|
auth_config="$temporary_dir/curl-auth.conf"
|
|
write_curl_config "$public_config"
|
|
status="$(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable \
|
|
--config "$public_config" --request POST \
|
|
--output "$token_response" --write-out '%{http_code}' \
|
|
--data-urlencode 'grant_type=password' \
|
|
--data-urlencode 'client_id=admin-cli' \
|
|
--data-urlencode "username@$temporary_dir/admin-username" \
|
|
--data-urlencode "password@$temporary_dir/admin-password" \
|
|
"$local_base/realms/master/protocol/openid-connect/token" 2>/dev/null)" ||
|
|
fail 'temporary Keycloak administrator authentication failed'
|
|
chmod 0600 -- "$token_response"
|
|
[[ "$status" == 200 ]] || fail 'temporary Keycloak administrator authentication returned an unexpected status'
|
|
jq -erj '.access_token | select(type == "string" and length >= 16)' "$token_response" >"$token_file" ||
|
|
fail 'Keycloak token response is invalid'
|
|
chmod 0600 -- "$token_file"
|
|
private_file_ok "$token_file" || fail 'temporary Keycloak token file is unsafe'
|
|
write_curl_config "$auth_config" "$token_file"
|
|
rm -f -- "$token_response" "$temporary_dir/admin-username" "$temporary_dir/admin-password"
|
|
}
|
|
|
|
admin_request() {
|
|
local method=$1 url=$2 body=${3:-} output status_file rc=0
|
|
request_counter=$((request_counter + 1))
|
|
output="$temporary_dir/admin-response-$request_counter.json"
|
|
status_file="$temporary_dir/admin-status-$request_counter"
|
|
: >"$output"; : >"$status_file"; chmod 0600 -- "$output" "$status_file"
|
|
local -a command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable
|
|
--config "$temporary_dir/curl-auth.conf" --request "$method" --output "$output"
|
|
--write-out '%{http_code}' "$url")
|
|
if [[ -n "$body" ]]; then
|
|
private_file_ok "$body" || fail 'request body is not a private snapshot'
|
|
command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable
|
|
--config "$temporary_dir/curl-auth.conf" --request "$method" --header 'Content-Type: application/json'
|
|
--data-binary "@$body" --output "$output" --write-out '%{http_code}' "$url")
|
|
fi
|
|
"${command[@]}" >"$status_file" 2>/dev/null || rc=$?
|
|
private_file_ok "$output" true && private_file_ok "$status_file" true ||
|
|
fail 'Admin API response files are unsafe'
|
|
last_body=$output
|
|
last_status_file=$status_file
|
|
last_curl_rc=$rc
|
|
last_http_status=''
|
|
if (( rc == 0 )); then
|
|
IFS= read -r last_http_status <"$status_file" || true
|
|
[[ "$last_http_status" =~ ^[0-9]{3}$ ]] || { last_curl_rc=90; last_http_status=''; }
|
|
fi
|
|
}
|
|
|
|
admin_get_query() {
|
|
local url=$1 query_name=$2 query_value=$3 max=$4 output status_file rc=0
|
|
request_counter=$((request_counter + 1))
|
|
output="$temporary_dir/admin-response-$request_counter.json"
|
|
status_file="$temporary_dir/admin-status-$request_counter"
|
|
: >"$output"; : >"$status_file"; chmod 0600 -- "$output" "$status_file"
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable \
|
|
--config "$temporary_dir/curl-auth.conf" --get \
|
|
--data-urlencode "$query_name=$query_value" --data-urlencode "max=$max" \
|
|
--output "$output" --write-out '%{http_code}' "$url" >"$status_file" 2>/dev/null || rc=$?
|
|
private_file_ok "$output" true && private_file_ok "$status_file" true ||
|
|
fail 'Admin API query response files are unsafe'
|
|
last_body=$output; last_status_file=$status_file; last_curl_rc=$rc; last_http_status=''
|
|
if (( rc == 0 )); then IFS= read -r last_http_status <"$status_file" || true; fi
|
|
}
|
|
|
|
require_http() {
|
|
local expected=$1 operation=$2
|
|
(( last_curl_rc == 0 )) || fail "$operation transport failed"
|
|
[[ "$last_http_status" == "$expected" ]] || fail "$operation returned HTTP ${last_http_status:-none}"
|
|
}
|
|
|
|
build_desired_client() {
|
|
client_desired="$temporary_dir/client-desired.json"
|
|
jq -n --arg client_id "$CLIENT_ID" --arg root "$GRAFANA_URL" --arg redirect "$REDIRECT_URI" \
|
|
--arg logout "$POST_LOGOUT_URI" '{
|
|
clientId:$client_id,
|
|
name:"Grafana",
|
|
description:"Grafana confidential OIDC client managed by the platform bootstrap",
|
|
enabled:true,
|
|
protocol:"openid-connect",
|
|
clientAuthenticatorType:"client-secret",
|
|
publicClient:false,
|
|
standardFlowEnabled:true,
|
|
implicitFlowEnabled:false,
|
|
directAccessGrantsEnabled:false,
|
|
serviceAccountsEnabled:false,
|
|
authorizationServicesEnabled:false,
|
|
consentRequired:false,
|
|
fullScopeAllowed:false,
|
|
rootUrl:$root,
|
|
baseUrl:$root,
|
|
redirectUris:[$redirect],
|
|
webOrigins:[$root],
|
|
attributes:{
|
|
"post.logout.redirect.uris":$logout,
|
|
"oauth2.device.authorization.grant.enabled":"false",
|
|
"oidc.ciba.grant.enabled":"false"
|
|
}
|
|
}' >"$client_desired"
|
|
chmod 0600 -- "$client_desired"
|
|
}
|
|
|
|
build_desired_mapper() {
|
|
mapper_desired="$temporary_dir/mapper-desired.json"
|
|
jq -n --arg name "$MAPPER_NAME" '{
|
|
name:$name,
|
|
protocol:"openid-connect",
|
|
protocolMapper:"oidc-group-membership-mapper",
|
|
consentRequired:false,
|
|
config:{
|
|
"claim.name":"groups",
|
|
"full.path":"true",
|
|
"id.token.claim":"true",
|
|
"access.token.claim":"true",
|
|
"userinfo.token.claim":"true"
|
|
}
|
|
}' >"$mapper_desired"
|
|
chmod 0600 -- "$mapper_desired"
|
|
}
|
|
|
|
lookup_client() {
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2
|
|
require_http 200 'Grafana client lookup'
|
|
[[ "$(jq 'length' "$last_body")" == 0 || "$(jq 'length' "$last_body")" == 1 ]] ||
|
|
fail 'Grafana client lookup is ambiguous'
|
|
if [[ "$(jq 'length' "$last_body")" == 0 ]]; then
|
|
client_state=absent; client_id=''; client_before=''
|
|
else
|
|
client_state=existing
|
|
client_id="$(jq -er '.[0].id | select(type == "string" and length > 0)' "$last_body")" ||
|
|
fail 'existing Grafana client has no exact internal id'
|
|
client_before="$temporary_dir/client-before.json"
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id"
|
|
require_http 200 'existing Grafana client snapshot'
|
|
jq -e --arg id "$client_id" --arg client_id "$CLIENT_ID" \
|
|
'.id == $id and .clientId == $client_id' "$last_body" >/dev/null || fail 'existing client identity changed'
|
|
cp -- "$last_body" "$client_before"; chmod 0600 -- "$client_before"
|
|
fi
|
|
}
|
|
|
|
lookup_group() {
|
|
local label=$1 name=$2 path=$3 state_var=$4 id_var=$5 before_var=$6
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2
|
|
require_http 200 "$label group lookup"
|
|
local count
|
|
count="$(jq 'length' "$last_body")" ||
|
|
fail "$label group response is invalid"
|
|
[[ "$count" == 0 || "$count" == 1 ]] || fail "$label group lookup is ambiguous"
|
|
if [[ "$count" == 0 ]]; then
|
|
printf -v "$state_var" '%s' absent; printf -v "$id_var" '%s' ''; printf -v "$before_var" '%s' ''
|
|
else
|
|
local gid before
|
|
gid="$(jq -er --arg path "$path" '.[0] | select(.path == $path) | .id | select(type == "string" and length > 0)' "$last_body")" ||
|
|
fail "$label group has no exact internal id"
|
|
before="$temporary_dir/${label}-group-before.json"
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid"
|
|
require_http 200 "$label group snapshot"
|
|
jq -e --arg id "$gid" --arg name "$name" --arg path "$path" \
|
|
'.id == $id and .name == $name and .path == $path' "$last_body" >/dev/null ||
|
|
fail "$label group identity changed"
|
|
cp -- "$last_body" "$before"; chmod 0600 -- "$before"
|
|
printf -v "$state_var" '%s' existing; printf -v "$id_var" '%s' "$gid"; printf -v "$before_var" '%s' "$before"
|
|
fi
|
|
}
|
|
|
|
lookup_mapper() {
|
|
[[ "$client_state" == existing ]] || { mapper_state=absent; mapper_id=''; mapper_before=''; return; }
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models"
|
|
require_http 200 'Grafana mapper lookup'
|
|
local count_by_name count_by_claim
|
|
count_by_name="$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)] | length' "$last_body")" ||
|
|
fail 'mapper response is invalid'
|
|
count_by_claim="$(jq '[.[] | select(.config["claim.name"] == "groups")] | length' "$last_body")" ||
|
|
fail 'mapper response is invalid'
|
|
[[ "$count_by_name" == 0 || "$count_by_name" == 1 ]] || fail 'Grafana mapper name is ambiguous'
|
|
[[ "$count_by_claim" == 0 || "$count_by_claim" == 1 ]] || fail 'Grafana groups claim mapper is ambiguous'
|
|
if [[ "$count_by_name" == 1 ]]; then
|
|
mapper_state=existing
|
|
mapper_id="$(jq -er --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)][0].id | select(type == "string" and length > 0)' "$last_body")" ||
|
|
fail 'existing Grafana mapper has no exact id'
|
|
[[ "$count_by_claim" == 0 ||
|
|
"$(jq -r --arg id "$mapper_id" '[.[] | select(.config["claim.name"] == "groups")][0].id // empty' "$last_body")" == "$mapper_id" ]] ||
|
|
fail 'a different mapper already owns the groups claim'
|
|
mapper_before="$temporary_dir/mapper-before.json"
|
|
jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_before" ||
|
|
fail 'cannot snapshot exact existing mapper'
|
|
chmod 0600 -- "$mapper_before"
|
|
else
|
|
[[ "$count_by_claim" == 0 ]] || fail 'groups claim is owned by a differently named mapper'
|
|
mapper_state=absent; mapper_id=''; mapper_before=''
|
|
fi
|
|
}
|
|
|
|
lookup_user() {
|
|
local label=$1 username=$2 id_var=$3
|
|
[[ -n "$username" ]] || return 0
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/users?exact=true" username "$username" 2
|
|
require_http 200 "$label user lookup"
|
|
[[ "$(jq 'length' "$last_body")" == 1 ]] || fail "$label username lookup must return exactly one user"
|
|
local uid
|
|
uid="$(jq -er --arg username "$username" '.[0] | select(.username == $username) | .id | select(type == "string" and length > 0)' "$last_body")" ||
|
|
fail "$label user identity is invalid"
|
|
printf -v "$id_var" '%s' "$uid"
|
|
}
|
|
|
|
snapshot_membership() {
|
|
local label=$1 user_id=$2 group_id=$3 state_var=$4
|
|
[[ -n "$user_id" ]] || return 0
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups" briefRepresentation false 101
|
|
require_http 200 "$label membership lookup"
|
|
[[ "$(jq 'length' "$last_body")" -le 100 ]] || fail "$label membership list exceeds the bounded cardinality"
|
|
if jq -e --arg id "$group_id" 'any(.[]; .id == $id)' "$last_body" >/dev/null; then
|
|
printf -v "$state_var" '%s' true
|
|
else
|
|
printf -v "$state_var" '%s' false
|
|
fi
|
|
}
|
|
|
|
retrieve_client_secret() {
|
|
client_secret_file="$temporary_dir/client-secret"
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret"
|
|
require_http 200 'Grafana client-secret retrieval'
|
|
jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$client_secret_file" ||
|
|
fail 'Grafana client secret response is invalid'
|
|
chmod 0600 -- "$client_secret_file"
|
|
private_file_ok "$client_secret_file" || fail 'Grafana client secret file is unsafe'
|
|
}
|
|
|
|
start_kubernetes_proxy() {
|
|
local log="$temporary_dir/kubernetes-proxy.log" attempt
|
|
kube_socket="$temporary_dir/kube-api.sock"
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s "$PROXY_PROCESS_TIMEOUT" \
|
|
"$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" proxy \
|
|
--unix-socket="$kube_socket" --api-prefix=/ \
|
|
--accept-paths="^/api/v1/namespaces/${OBSERVABILITY_NAMESPACE}/secrets(/${OIDC_SECRET})?$" \
|
|
>"$log" 2>&1 &
|
|
proxy_pid=$!
|
|
proxy_start="$(process_start_time "$proxy_pid")" || fail 'cannot pin Kubernetes proxy identity'
|
|
for ((attempt=0; attempt<60; attempt++)); do
|
|
[[ -S "$kube_socket" ]] && break
|
|
kill -0 "$proxy_pid" 2>/dev/null || fail 'bounded Kubernetes proxy exited'
|
|
/usr/bin/sleep 0.05
|
|
done
|
|
[[ -S "$kube_socket" && "$(process_start_time "$proxy_pid")" == "$proxy_start" ]] ||
|
|
fail 'bounded Kubernetes proxy did not become ready safely'
|
|
}
|
|
|
|
kube_request() {
|
|
local method=$1 path=$2 body=${3:-} output status_file rc=0
|
|
request_counter=$((request_counter + 1))
|
|
output="$temporary_dir/kube-response-$request_counter.json"
|
|
status_file="$temporary_dir/kube-status-$request_counter"
|
|
: >"$output"; : >"$status_file"; chmod 0600 -- "$output" "$status_file"
|
|
local -a command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable
|
|
--silent --show-error --noproxy '*' --connect-timeout "$CURL_CONNECT_TIMEOUT" --max-time "$CURL_MAX_TIME"
|
|
--unix-socket "$kube_socket" --request "$method" --output "$output" --write-out '%{http_code}'
|
|
"http://localhost$path")
|
|
if [[ -n "$body" ]]; then
|
|
private_file_ok "$body" || fail 'Kubernetes request body is unsafe'
|
|
command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable
|
|
--silent --show-error --noproxy '*' --connect-timeout "$CURL_CONNECT_TIMEOUT" --max-time "$CURL_MAX_TIME"
|
|
--unix-socket "$kube_socket" --request "$method" --header 'Content-Type: application/json'
|
|
--data-binary "@$body" --output "$output" --write-out '%{http_code}' "http://localhost$path")
|
|
fi
|
|
"${command[@]}" >"$status_file" 2>/dev/null || rc=$?
|
|
private_file_ok "$output" true && private_file_ok "$status_file" true ||
|
|
fail 'Kubernetes API response files are unsafe'
|
|
last_body=$output; last_status_file=$status_file; last_curl_rc=$rc; last_http_status=''
|
|
if (( rc == 0 )); then IFS= read -r last_http_status <"$status_file" || true; fi
|
|
}
|
|
|
|
validate_secret_json() {
|
|
local source=$1 prefix=$2
|
|
python3 -I -S - "$source" "$prefix" <<'PY'
|
|
import base64,binascii,json,os,pathlib,re,sys
|
|
source,prefix=sys.argv[1:]
|
|
try: item=json.loads(pathlib.Path(source).read_text())
|
|
except Exception: raise SystemExit(1)
|
|
meta=item.get('metadata') or {}; data=item.get('data')
|
|
if item.get('apiVersion')!='v1' or item.get('kind')!='Secret' or item.get('type')!='Opaque' or meta.get('namespace')!='observability' or meta.get('name')!='grafana-keycloak-oidc' or sorted(data or {})!=['client-id','client-secret']:
|
|
raise SystemExit(1)
|
|
uid=meta.get('uid'); rv=meta.get('resourceVersion')
|
|
if not isinstance(uid,str) or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*',uid) or not isinstance(rv,str) or not re.fullmatch(r'[0-9]+',rv): raise SystemExit(1)
|
|
for key in ('client-id','client-secret'):
|
|
try: payload=base64.b64decode(data[key],validate=True)
|
|
except (binascii.Error,ValueError): raise SystemExit(1)
|
|
if (key=='client-id' and payload!=b'grafana') or (key=='client-secret' and not payload): raise SystemExit(1)
|
|
fd=os.open(prefix+'.'+key,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600)
|
|
try: os.write(fd,payload)
|
|
finally: os.close(fd)
|
|
print(uid+'\t'+rv,end='')
|
|
PY
|
|
}
|
|
|
|
snapshot_oidc_secret() {
|
|
local prefix values
|
|
kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET"
|
|
(( last_curl_rc == 0 )) || fail 'Grafana OIDC Secret lookup transport failed'
|
|
case "$last_http_status" in
|
|
200)
|
|
secret_state=existing; secret_before="$temporary_dir/secret-before.json"
|
|
cp -- "$last_body" "$secret_before"; chmod 0600 -- "$secret_before"
|
|
prefix="$temporary_dir/secret-before"
|
|
values="$(validate_secret_json "$secret_before" "$prefix")" || fail 'existing Grafana OIDC Secret contract is invalid'
|
|
IFS=$'\t' read -r secret_uid secret_resource_version <<<"$values"
|
|
[[ "$client_state" == existing && -n "$client_secret_file" ]] ||
|
|
fail 'Grafana OIDC Secret exists without one exact existing Keycloak client'
|
|
[[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] || fail 'existing Grafana OIDC Secret client-id differs'
|
|
cmp --silent -- "$prefix.client-secret" "$client_secret_file" ||
|
|
fail 'existing Grafana OIDC Secret does not match the exact existing Keycloak credential; rotation refused'
|
|
;;
|
|
404) secret_state=absent; secret_before=''; secret_uid=''; secret_resource_version='' ;;
|
|
*) fail "Grafana OIDC Secret lookup returned HTTP ${last_http_status:-none}" ;;
|
|
esac
|
|
}
|
|
|
|
sudo_refresh() {
|
|
"$SUDO_BIN" -v || fail 'sudo credential refresh failed'
|
|
"$SUDO_BIN" -n /usr/bin/true || fail 'narrow non-interactive sudo is unavailable'
|
|
}
|
|
|
|
validate_evidence_stream() {
|
|
local file=$1 timestamp_var=$2 parsed
|
|
local -a lines=()
|
|
mapfile -t lines <"$file"
|
|
[[ "$(wc -l <"$file" | tr -d '[:space:]')" == 4 && ${#lines[@]} -eq 4 ]] || return 1
|
|
[[ "${lines[0]}" == "schema=$EVIDENCE_SCHEMA" ]] || return 1
|
|
[[ "${lines[1]}" == "context=$EXPECTED_CONTEXT" ]] || return 1
|
|
[[ "${lines[2]}" == "resource=$EVIDENCE_RESOURCE" ]] || return 1
|
|
[[ "${lines[3]}" =~ ^checked_at_utc=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)$ ]] || return 1
|
|
parsed="${BASH_REMATCH[1]}"
|
|
timestamp_epoch "$parsed" >/dev/null || return 1
|
|
printf -v "$timestamp_var" '%s' "$parsed"
|
|
}
|
|
|
|
root_marker_helper() {
|
|
local output=$1
|
|
shift
|
|
"$SUDO_BIN" -n /usr/bin/python3 -I -S - platform-grafana-oidc-marker-helper "$@" \
|
|
>"$output" 2>/dev/null <<'PY'
|
|
import errno,os,pathlib,stat,sys
|
|
tag,action,directory,basename,uid_raw,gid_raw,dir_mode_raw,source,expected_kind,expected_source,source_uid_raw=sys.argv[1:]
|
|
if tag!='platform-grafana-oidc-marker-helper' or action not in {'check-parent','check-dir','ensure-dir','read','install','unlink-exact'} or basename!='keycloak.env': raise SystemExit(2)
|
|
if not directory.startswith('/') or '\x00' in directory: raise SystemExit(2)
|
|
uid=int(uid_raw); gid=int(gid_raw); directory_mode=int(dir_mode_raw,8); source_uid=int(source_uid_raw)
|
|
flags=os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW|os.O_CLOEXEC
|
|
def open_dir(path,mode):
|
|
fd=os.open('/',flags)
|
|
try:
|
|
for part in pathlib.PurePosixPath(path).parts[1:]:
|
|
if part in {'','.','..'}: raise OSError(errno.EINVAL,'component')
|
|
nxt=os.open(part,flags,dir_fd=fd); os.close(fd); fd=nxt
|
|
st=os.fstat(fd)
|
|
if not stat.S_ISDIR(st.st_mode) or stat.S_IMODE(st.st_mode)!=mode or st.st_uid!=uid or st.st_gid!=gid: raise OSError(errno.EPERM,'directory metadata')
|
|
return fd
|
|
except BaseException:
|
|
os.close(fd); raise
|
|
def read_file(directory_fd,name):
|
|
fd=os.open(name,os.O_RDONLY|os.O_NOFOLLOW|os.O_CLOEXEC,dir_fd=directory_fd)
|
|
try:
|
|
before=os.fstat(fd)
|
|
if not stat.S_ISREG(before.st_mode) or stat.S_IMODE(before.st_mode)!=0o600 or before.st_uid!=uid or before.st_gid!=gid or before.st_nlink!=1: raise OSError(errno.EPERM,'file metadata')
|
|
chunks=[]
|
|
while True:
|
|
chunk=os.read(fd,65536)
|
|
if not chunk: break
|
|
chunks.append(chunk)
|
|
after=os.fstat(fd)
|
|
a=(before.st_dev,before.st_ino,before.st_size,before.st_mtime_ns,before.st_ctime_ns,before.st_nlink)
|
|
b=(after.st_dev,after.st_ino,after.st_size,after.st_mtime_ns,after.st_ctime_ns,after.st_nlink)
|
|
if a!=b: raise OSError(errno.ESTALE,'changed')
|
|
payload=b''.join(chunks)
|
|
if len(payload)!=before.st_size: raise OSError(errno.EIO,'short read')
|
|
return payload
|
|
finally: os.close(fd)
|
|
def read_source(path):
|
|
fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW|os.O_CLOEXEC)
|
|
try:
|
|
before=os.fstat(fd)
|
|
if not stat.S_ISREG(before.st_mode) or stat.S_IMODE(before.st_mode)!=0o600 or before.st_uid!=source_uid or before.st_nlink!=1: raise OSError(errno.EPERM,'source metadata')
|
|
chunks=[]
|
|
while True:
|
|
chunk=os.read(fd,65536)
|
|
if not chunk: break
|
|
chunks.append(chunk)
|
|
after=os.fstat(fd)
|
|
if (before.st_dev,before.st_ino,before.st_size,before.st_mtime_ns,before.st_ctime_ns,before.st_nlink)!=(after.st_dev,after.st_ino,after.st_size,after.st_mtime_ns,after.st_ctime_ns,after.st_nlink): raise OSError(errno.ESTALE,'source changed')
|
|
payload=b''.join(chunks)
|
|
if len(payload)!=before.st_size: raise OSError(errno.EIO,'short source')
|
|
return payload
|
|
finally: os.close(fd)
|
|
if action in {'check-parent','check-dir','ensure-dir'}:
|
|
parent,leaf=os.path.split(directory.rstrip('/'))
|
|
if leaf!='recovery-evidence' or not parent: raise OSError(errno.EINVAL,'directory')
|
|
parent_fd=open_dir(parent,directory_mode)
|
|
if action=='check-parent':
|
|
os.close(parent_fd); raise SystemExit(0)
|
|
try:
|
|
try: directory_fd=os.open(leaf,flags,dir_fd=parent_fd)
|
|
except FileNotFoundError:
|
|
if action=='check-dir': raise SystemExit(4)
|
|
os.mkdir(leaf,directory_mode,dir_fd=parent_fd); os.fsync(parent_fd); directory_fd=os.open(leaf,flags,dir_fd=parent_fd)
|
|
try:
|
|
st=os.fstat(directory_fd)
|
|
if not stat.S_ISDIR(st.st_mode) or stat.S_IMODE(st.st_mode)!=directory_mode or st.st_uid!=uid or st.st_gid!=gid: raise OSError(errno.EPERM,'evidence directory')
|
|
finally: os.close(directory_fd)
|
|
finally: os.close(parent_fd)
|
|
raise SystemExit(0)
|
|
try: directory_fd=open_dir(directory,directory_mode)
|
|
except FileNotFoundError:
|
|
if action=='read': raise SystemExit(4)
|
|
raise
|
|
try:
|
|
if action=='read':
|
|
try: payload=read_file(directory_fd,basename)
|
|
except FileNotFoundError: raise SystemExit(4)
|
|
os.write(1,payload); raise SystemExit(0)
|
|
desired=read_source(source)
|
|
if expected_kind=='absent':
|
|
try: read_file(directory_fd,basename)
|
|
except FileNotFoundError: pass
|
|
else: raise OSError(errno.EEXIST,'expected absent')
|
|
elif expected_kind=='file':
|
|
if read_file(directory_fd,basename)!=read_source(expected_source): raise OSError(errno.ESTALE,'precondition')
|
|
else: raise OSError(errno.EINVAL,'kind')
|
|
if action=='unlink-exact':
|
|
if read_file(directory_fd,basename)!=desired: raise OSError(errno.ESTALE,'unlink precondition')
|
|
os.unlink(basename,dir_fd=directory_fd); os.fsync(directory_fd)
|
|
try: os.stat(basename,dir_fd=directory_fd,follow_symlinks=False)
|
|
except FileNotFoundError: raise SystemExit(0)
|
|
raise OSError(errno.EEXIST,'remained')
|
|
temporary=f'.{basename}.new.{os.getpid():x}'
|
|
fd=os.open(temporary,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW|os.O_CLOEXEC,0o600,dir_fd=directory_fd); exists=True
|
|
try:
|
|
os.fchown(fd,uid,gid); os.fchmod(fd,0o600); view=memoryview(desired)
|
|
while view:
|
|
n=os.write(fd,view)
|
|
if n<=0: raise OSError(errno.EIO,'short write')
|
|
view=view[n:]
|
|
os.fsync(fd)
|
|
if os.fstat(fd).st_nlink!=1: raise OSError(errno.EPERM,'links')
|
|
finally: os.close(fd)
|
|
try:
|
|
os.replace(temporary,basename,src_dir_fd=directory_fd,dst_dir_fd=directory_fd); exists=False; os.fsync(directory_fd)
|
|
if read_file(directory_fd,basename)!=desired: raise OSError(errno.EIO,'verify')
|
|
finally:
|
|
if exists:
|
|
try: os.unlink(temporary,dir_fd=directory_fd)
|
|
except FileNotFoundError: pass
|
|
finally: os.close(directory_fd)
|
|
PY
|
|
}
|
|
|
|
root_marker_read() {
|
|
local destination=$1 rc
|
|
if root_marker_helper "$destination" read "$EVIDENCE_DIR" keycloak.env \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" '' '' '' "$EUID"; then
|
|
chmod 0600 -- "$destination"; return 0
|
|
else rc=$?; rm -f -- "$destination"; return "$rc"; fi
|
|
}
|
|
|
|
prepare_marker_snapshot() {
|
|
local rc timestamp
|
|
marker_prior="$temporary_dir/marker-prior.env"
|
|
marker_desired="$temporary_dir/marker-desired.env"
|
|
root_marker_helper /dev/null check-parent "$EVIDENCE_DIR" keycloak.env \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" '' '' '' "$EUID" ||
|
|
fail 'recovery evidence parent boundary is missing or unsafe'
|
|
if root_marker_read "$marker_prior"; then
|
|
validate_evidence_stream "$marker_prior" timestamp || fail 'existing Keycloak recovery marker is malformed'
|
|
marker_had_prior=true
|
|
else
|
|
rc=$?; [[ "$rc" == 4 ]] || fail 'cannot safely inspect Keycloak recovery marker'
|
|
marker_had_prior=false
|
|
fi
|
|
printf '%s\n' "schema=$EVIDENCE_SCHEMA" "context=$EXPECTED_CONTEXT" \
|
|
"resource=$EVIDENCE_RESOURCE" "checked_at_utc=$(now_utc)" >"$marker_desired"
|
|
chmod 0600 -- "$marker_desired"
|
|
validate_evidence_stream "$marker_desired" timestamp || fail 'generated recovery marker is invalid'
|
|
}
|
|
|
|
ensure_evidence_directory() {
|
|
root_marker_helper /dev/null ensure-dir "$EVIDENCE_DIR" keycloak.env \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" '' '' '' "$EUID" ||
|
|
fail 'cannot safely ensure recovery evidence directory'
|
|
}
|
|
|
|
install_marker() {
|
|
pending_kind=marker
|
|
if [[ "$marker_had_prior" == true ]]; then
|
|
root_marker_helper /dev/null install "$EVIDENCE_DIR" keycloak.env \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_desired" file "$marker_prior" "$EUID" ||
|
|
fail 'cannot atomically replace Keycloak recovery marker'
|
|
else
|
|
root_marker_helper /dev/null install "$EVIDENCE_DIR" keycloak.env \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_desired" absent '' "$EUID" ||
|
|
fail 'cannot atomically install Keycloak recovery marker'
|
|
fi
|
|
local current="$temporary_dir/marker-installed.env"
|
|
root_marker_read "$current" && cmp --silent -- "$marker_desired" "$current" ||
|
|
fail 'installed Keycloak recovery marker cannot be verified'
|
|
marker_written=true
|
|
pending_kind=''
|
|
}
|
|
|
|
restore_marker() {
|
|
local current="$temporary_dir/marker-rollback-current.env"
|
|
[[ "$marker_written" == true ]] || return 0
|
|
root_marker_read "$current" && cmp --silent -- "$marker_desired" "$current" || return 1
|
|
if [[ "$marker_had_prior" == true ]]; then
|
|
root_marker_helper /dev/null install "$EVIDENCE_DIR" keycloak.env \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_prior" file "$marker_desired" "$EUID"
|
|
else
|
|
root_marker_helper /dev/null unlink-exact "$EVIDENCE_DIR" keycloak.env \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_desired" file "$marker_desired" "$EUID"
|
|
fi
|
|
}
|
|
|
|
check_recovery_evidence() {
|
|
local marker timestamp marker_epoch current_epoch age
|
|
validate_context_and_authority
|
|
sudo_refresh
|
|
marker="$temporary_dir/check-keycloak.env"
|
|
root_marker_read "$marker" || fail 'Keycloak recovery evidence is missing or unsafe'
|
|
validate_evidence_stream "$marker" timestamp || fail 'Keycloak recovery evidence is malformed'
|
|
marker_epoch="$(timestamp_epoch "$timestamp")" || fail 'recovery evidence timestamp is invalid'
|
|
current_epoch="$(now_epoch)" || fail 'current UTC clock is invalid'
|
|
age=$((current_epoch - marker_epoch))
|
|
(( age >= 0 && age <= EVIDENCE_MAX_AGE_SECONDS )) || fail 'Keycloak recovery evidence is outside the 30-day window'
|
|
printf 'KEYCLOAK_RECOVERY_EVIDENCE=PASS\n'
|
|
}
|
|
|
|
build_group_body() {
|
|
local name=$1 destination=$2 token=${3:-}
|
|
if [[ -n "$token" ]]; then
|
|
jq -n --arg name "$name" --arg token "$token" \
|
|
'{name:$name,attributes:{"platform.observability.transaction":[$token]}}' >"$destination"
|
|
else
|
|
jq -n --arg name "$name" '{name:$name}' >"$destination"
|
|
fi
|
|
chmod 0600 -- "$destination"
|
|
}
|
|
|
|
client_body_with_secret() {
|
|
local source=$1 destination=$2
|
|
private_file_ok "$source" && private_file_ok "$client_secret_file" || return 1
|
|
jq --rawfile secret "$client_secret_file" '. + {secret:$secret}' "$source" >"$destination" || return 1
|
|
chmod 0600 -- "$destination"
|
|
private_file_ok "$destination"
|
|
}
|
|
|
|
client_contract_matches() {
|
|
local source=$1 expected_id=$2
|
|
jq -e --arg id "$expected_id" --arg root "$GRAFANA_URL" --arg redirect "$REDIRECT_URI" \
|
|
--arg logout "$POST_LOGOUT_URI" '
|
|
.id == $id and .clientId == "grafana" and .name == "Grafana" and
|
|
.description == "Grafana confidential OIDC client managed by the platform bootstrap" and
|
|
.enabled == true and .protocol == "openid-connect" and
|
|
.clientAuthenticatorType == "client-secret" and .publicClient == false and
|
|
.standardFlowEnabled == true and .implicitFlowEnabled == false and
|
|
.directAccessGrantsEnabled == false and .serviceAccountsEnabled == false and
|
|
.authorizationServicesEnabled == false and .consentRequired == false and
|
|
.fullScopeAllowed == false and .rootUrl == $root and .baseUrl == $root and
|
|
.redirectUris == [$redirect] and .webOrigins == [$root] and
|
|
.attributes["post.logout.redirect.uris"] == $logout and
|
|
.attributes["oauth2.device.authorization.grant.enabled"] == "false" and
|
|
.attributes["oidc.ciba.grant.enabled"] == "false" and
|
|
(.attributes["platform.observability.transaction"] // null) == null
|
|
' "$source" >/dev/null
|
|
}
|
|
|
|
mapper_contract_matches() {
|
|
local source=$1 expected_id=$2
|
|
jq -e --arg id "$expected_id" --arg name "$MAPPER_NAME" '
|
|
.id == $id and .name == $name and .protocol == "openid-connect" and
|
|
.protocolMapper == "oidc-group-membership-mapper" and .consentRequired == false and
|
|
.config == {"claim.name":"groups","full.path":"true","id.token.claim":"true",
|
|
"access.token.claim":"true","userinfo.token.claim":"true"}
|
|
' "$source" >/dev/null
|
|
}
|
|
|
|
capture_client_after() {
|
|
client_after="$temporary_dir/client-after.json"
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id"
|
|
require_http 200 'post-mutation Grafana client snapshot'
|
|
client_contract_matches "$last_body" "$client_id" || fail 'post-mutation Grafana client contract differs'
|
|
cp -- "$last_body" "$client_after"; chmod 0600 -- "$client_after"
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret"
|
|
require_http 200 'post-mutation Grafana credential verification'
|
|
local verification="$temporary_dir/client-secret-verification"
|
|
jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$verification" ||
|
|
fail 'post-mutation Grafana credential response is invalid'
|
|
chmod 0600 -- "$verification"
|
|
cmp --silent -- "$client_secret_file" "$verification" ||
|
|
fail 'Grafana client PUT changed the existing credential unexpectedly'
|
|
}
|
|
|
|
classify_client_create() {
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2
|
|
(( last_curl_rc == 0 )) || return 2
|
|
[[ "$last_http_status" == 200 ]] || return 2
|
|
local count
|
|
count="$(jq 'length' "$last_body" 2>/dev/null)" || return 2
|
|
if [[ "$count" == 0 ]]; then return 1; fi
|
|
[[ "$count" == 1 ]] || return 2
|
|
local owned_id
|
|
owned_id="$(jq -er --arg token "$transaction_token" '.[0] |
|
|
select(.attributes["platform.observability.transaction"] == $token) |
|
|
.id | select(type == "string" and length > 0)' "$last_body" 2>/dev/null)" || return 2
|
|
client_id=$owned_id
|
|
client_mutation=created
|
|
return 0
|
|
}
|
|
|
|
create_client() {
|
|
local create_body="$temporary_dir/client-create.json" status
|
|
jq --arg token "$transaction_token" \
|
|
'.attributes["platform.observability.transaction"]=$token' "$client_desired" >"$create_body"
|
|
chmod 0600 -- "$create_body"
|
|
pending_kind=client-create; pending_desired=$create_body; pending_token=$transaction_token
|
|
admin_request POST "$local_base/admin/realms/$KEYCLOAK_REALM/clients" "$create_body"
|
|
status=$last_http_status
|
|
if (( last_curl_rc != 0 )) || [[ "$status" != 201 ]]; then
|
|
if classify_client_create; then
|
|
:
|
|
elif [[ "$?" != 1 ]]; then
|
|
emit_manual_recovery
|
|
fi
|
|
fail "Grafana client create was not cleanly confirmed"
|
|
fi
|
|
classify_client_create || { emit_manual_recovery; fail 'Grafana client create response could not be owned exactly'; }
|
|
client_after="$temporary_dir/client-created-tagged.json"
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id"
|
|
require_http 200 'created Grafana client ownership snapshot'
|
|
cp -- "$last_body" "$client_after"; chmod 0600 -- "$client_after"
|
|
retrieve_client_secret
|
|
local update_body="$temporary_dir/client-created-final.json"
|
|
client_body_with_secret "$client_desired" "$update_body" || fail 'cannot build credential-preserving client finalization'
|
|
pending_kind=client-created-finalize; pending_before=$client_after; pending_desired=$client_desired; pending_id=$client_id
|
|
admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" "$update_body"
|
|
if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then
|
|
fail 'created Grafana client finalization was not cleanly confirmed'
|
|
fi
|
|
capture_client_after
|
|
pending_kind=''
|
|
}
|
|
|
|
update_client() {
|
|
local update_body="$temporary_dir/client-update.json"
|
|
client_body_with_secret "$client_desired" "$update_body" || fail 'cannot build credential-preserving client update'
|
|
client_restore="$temporary_dir/client-restore.json"
|
|
client_body_with_secret "$client_before" "$client_restore" || fail 'cannot build exact credential-preserving client rollback snapshot'
|
|
pending_kind=client-update; pending_before=$client_before; pending_desired=$client_desired; pending_id=$client_id
|
|
admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" "$update_body"
|
|
if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then
|
|
fail 'Grafana client update was not cleanly confirmed'
|
|
fi
|
|
client_mutation=updated; pending_kind=''
|
|
capture_client_after
|
|
}
|
|
|
|
classify_group_create() {
|
|
local name=$1 path=$2 id_var=$3 mutation_var=$4
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2
|
|
local count gid
|
|
count="$(jq 'length' "$last_body" 2>/dev/null)" || return 2
|
|
if [[ "$count" == 0 ]]; then return 1; fi
|
|
[[ "$count" == 1 ]] || return 2
|
|
gid="$(jq -er --arg path "$path" --arg token "$transaction_token" '
|
|
.[0] | select(.path == $path and .attributes["platform.observability.transaction"] == [$token]) | .id |
|
|
select(type == "string" and length > 0)' "$last_body" 2>/dev/null)" || return 2
|
|
printf -v "$id_var" '%s' "$gid"; printf -v "$mutation_var" '%s' created
|
|
return 0
|
|
}
|
|
|
|
create_group() {
|
|
local label=$1 name=$2 path=$3 id_var=$4 mutation_var=$5 after_var=$6
|
|
local create_body="$temporary_dir/${label}-group-create.json" final_body="$temporary_dir/${label}-group-final.json"
|
|
build_group_body "$name" "$create_body" "$transaction_token"
|
|
pending_kind="${label}-group-create"; pending_desired=$create_body; pending_token=$transaction_token
|
|
admin_request POST "$local_base/admin/realms/$KEYCLOAK_REALM/groups" "$create_body"
|
|
if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 201 ]]; then
|
|
if classify_group_create "$name" "$path" "$id_var" "$mutation_var"; then :
|
|
elif [[ "$?" != 1 ]]; then emit_manual_recovery; fi
|
|
fail "$label group create was not cleanly confirmed"
|
|
fi
|
|
classify_group_create "$name" "$path" "$id_var" "$mutation_var" ||
|
|
{ emit_manual_recovery; fail "$label group create response could not be owned exactly"; }
|
|
local gid=${!id_var}
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid"
|
|
require_http 200 "$label created group ownership snapshot"
|
|
local after="$temporary_dir/${label}-group-after.json"
|
|
cp -- "$last_body" "$after"; chmod 0600 -- "$after"; printf -v "$after_var" '%s' "$after"
|
|
pending_kind="${label}-group-finalize"; pending_before=$after; pending_id=$gid
|
|
build_group_body "$name" "$final_body"
|
|
pending_desired=$final_body
|
|
admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid" "$final_body"
|
|
if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then
|
|
if classify_pending; then
|
|
fail "$label group finalization response was lost after an exact state transition"
|
|
else
|
|
[[ "$?" != 2 ]] || emit_manual_recovery
|
|
fail "$label group finalization was not cleanly confirmed"
|
|
fi
|
|
fi
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid"
|
|
require_http 200 "$label group post-mutation snapshot"
|
|
jq -e --arg id "$gid" --arg name "$name" --arg path "$path" '
|
|
.id == $id and .name == $name and .path == $path and
|
|
((.attributes // {}) | has("platform.observability.transaction") | not)
|
|
' "$last_body" >/dev/null || fail "$label group final contract differs"
|
|
cp -- "$last_body" "$after"; chmod 0600 -- "$after"; printf -v "$after_var" '%s' "$after"
|
|
pending_kind=''
|
|
}
|
|
|
|
create_mapper() {
|
|
local create_body="$temporary_dir/mapper-create.json"
|
|
jq --arg token "$transaction_token" '.config["platform.observability.transaction"]=$token' \
|
|
"$mapper_desired" >"$create_body"; chmod 0600 -- "$create_body"
|
|
pending_kind=mapper-create; pending_desired=$create_body; pending_token=$transaction_token
|
|
admin_request POST "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" "$create_body"
|
|
if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 201 ]]; then
|
|
fail 'Grafana groups mapper create was not cleanly confirmed'
|
|
fi
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models"
|
|
require_http 200 'post-create Grafana mapper lookup'
|
|
[[ "$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)] | length' "$last_body")" == 1 ]] ||
|
|
{ emit_manual_recovery; fail 'created Grafana mapper is ambiguous'; }
|
|
mapper_id="$(jq -er --arg name "$MAPPER_NAME" --arg token "$transaction_token" '
|
|
[.[] | select(.name == $name and .config["platform.observability.transaction"] == $token)][0].id |
|
|
select(type == "string" and length > 0)' "$last_body")" ||
|
|
{ emit_manual_recovery; fail 'created Grafana mapper ownership is not exact'; }
|
|
mapper_mutation=created
|
|
mapper_after="$temporary_dir/mapper-created-tagged.json"
|
|
jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_after" ||
|
|
fail 'cannot snapshot created Grafana mapper ownership'
|
|
chmod 0600 -- "$mapper_after"
|
|
pending_kind=mapper-created-finalize; pending_before=$mapper_after; pending_desired=$mapper_desired; pending_id=$mapper_id
|
|
admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id" "$mapper_desired"
|
|
if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then
|
|
fail 'created Grafana mapper finalization was not cleanly confirmed'
|
|
fi
|
|
capture_mapper_after
|
|
pending_kind=''
|
|
}
|
|
|
|
capture_mapper_after() {
|
|
mapper_after="$temporary_dir/mapper-after.json"
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models"
|
|
require_http 200 'post-mutation Grafana mapper snapshot'
|
|
jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_after" ||
|
|
fail 'cannot capture post-mutation Grafana mapper'
|
|
chmod 0600 -- "$mapper_after"
|
|
mapper_contract_matches "$mapper_after" "$mapper_id" || fail 'post-mutation Grafana mapper contract differs'
|
|
}
|
|
|
|
update_mapper() {
|
|
pending_kind=mapper-update; pending_before=$mapper_before; pending_desired=$mapper_desired; pending_id=$mapper_id
|
|
admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id" "$mapper_desired"
|
|
if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then fail 'Grafana mapper update was not cleanly confirmed'; fi
|
|
mapper_mutation=updated; pending_kind=''; capture_mapper_after
|
|
}
|
|
|
|
build_secret_manifest() {
|
|
local destination=$1
|
|
python3 -I -S - "$client_secret_file" "$transaction_token" "$destination" <<'PY' || return 1
|
|
import base64,json,os,pathlib,re,sys
|
|
source,token,destination=sys.argv[1:]
|
|
if re.fullmatch(r'[0-9a-f]{32}',token) is None: raise SystemExit(1)
|
|
payload=pathlib.Path(source).read_bytes()
|
|
if not payload: raise SystemExit(1)
|
|
item={'apiVersion':'v1','kind':'Secret','type':'Opaque','metadata':{'namespace':'observability','name':'grafana-keycloak-oidc','annotations':{'observability.hyeonworks.com/create-transaction':token}},'data':{'client-id':base64.b64encode(b'grafana').decode(),'client-secret':base64.b64encode(payload).decode()}}
|
|
fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600)
|
|
try: os.write(fd,(json.dumps(item,sort_keys=True,separators=(',',':'))+'\n').encode())
|
|
finally: os.close(fd)
|
|
PY
|
|
}
|
|
|
|
classify_secret_create() {
|
|
kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET"
|
|
(( last_curl_rc == 0 )) || return 2
|
|
if [[ "$last_http_status" == 404 ]]; then return 1; fi
|
|
[[ "$last_http_status" == 200 ]] || return 2
|
|
local prefix="$temporary_dir/secret-classify" values
|
|
values="$(validate_secret_json "$last_body" "$prefix")" || return 2
|
|
jq -e --arg token "$transaction_token" '.metadata.annotations["observability.hyeonworks.com/create-transaction"] == $token' "$last_body" >/dev/null || return 2
|
|
[[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] || return 2
|
|
cmp --silent -- "$prefix.client-secret" "$client_secret_file" || return 2
|
|
IFS=$'\t' read -r secret_created_uid secret_created_resource_version <<<"$values"
|
|
secret_mutation=created
|
|
return 0
|
|
}
|
|
|
|
create_oidc_secret() {
|
|
local manifest="$temporary_dir/secret-create.json"
|
|
build_secret_manifest "$manifest" || fail 'cannot build private Grafana OIDC Secret manifest'
|
|
pending_kind=secret-create; pending_desired=$manifest; pending_token=$transaction_token
|
|
kube_request POST "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets" "$manifest"
|
|
if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 201 ]]; then
|
|
if classify_secret_create; then :; elif [[ "$?" == 2 ]]; then emit_manual_recovery; fi
|
|
fail 'Grafana OIDC Secret create was not cleanly confirmed'
|
|
fi
|
|
classify_secret_create || { emit_manual_recovery; fail 'Grafana OIDC Secret create response could not be owned exactly'; }
|
|
pending_kind=''
|
|
}
|
|
|
|
membership_present() {
|
|
local user_id=$1 group_id=$2
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups" briefRepresentation false 101
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2
|
|
[[ "$(jq 'length' "$last_body")" -le 100 ]] || return 2
|
|
jq -e --arg id "$group_id" 'any(.[]; .id == $id)' "$last_body" >/dev/null
|
|
}
|
|
|
|
add_membership() {
|
|
local label=$1 user_id=$2 group_id=$3 before=$4 added_var=$5
|
|
[[ -n "$user_id" ]] || return 0
|
|
[[ "$before" == false ]] || return 0
|
|
pending_kind="${label}-membership"; pending_user_id=$user_id; pending_group_id=$group_id
|
|
admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups/$group_id"
|
|
if (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]]; then
|
|
membership_present "$user_id" "$group_id" || fail "$label membership was not observable after PUT"
|
|
printf -v "$added_var" '%s' true; pending_kind=''; return 0
|
|
fi
|
|
if membership_present "$user_id" "$group_id"; then
|
|
printf -v "$added_var" '%s' true
|
|
else
|
|
[[ "$?" != 2 ]] || emit_manual_recovery
|
|
fi
|
|
fail "$label membership PUT was not cleanly confirmed"
|
|
}
|
|
|
|
classify_mapper_create() {
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2
|
|
local count
|
|
count="$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)] | length' "$last_body" 2>/dev/null)" || return 2
|
|
if [[ "$count" == 0 ]]; then return 1; fi
|
|
[[ "$count" == 1 ]] || return 2
|
|
mapper_id="$(jq -er --arg name "$MAPPER_NAME" --arg token "$transaction_token" '
|
|
[.[] | select(.name == $name and .config["platform.observability.transaction"] == $token)][0].id |
|
|
select(type == "string" and length > 0)' "$last_body" 2>/dev/null)" || return 2
|
|
mapper_after="$temporary_dir/mapper-after-pending.json"
|
|
jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_after" || return 2
|
|
chmod 0600 -- "$mapper_after"; mapper_mutation=created; return 0
|
|
}
|
|
|
|
capture_pending_group_after() {
|
|
local label=$1 id=$2 after_var=$3
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1
|
|
local after="$temporary_dir/${label}-group-after-pending.json"
|
|
cp -- "$last_body" "$after"; chmod 0600 -- "$after"; printf -v "$after_var" '%s' "$after"
|
|
}
|
|
|
|
classify_pending() {
|
|
local result current target
|
|
case "$pending_kind" in
|
|
'') return 0 ;;
|
|
client-create)
|
|
if classify_client_create; then
|
|
client_after="$temporary_dir/client-after-pending.json"
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id"
|
|
if (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]]; then
|
|
cp -- "$last_body" "$client_after"; chmod 0600 -- "$client_after"; pending_kind=''; return 0
|
|
fi
|
|
return 2
|
|
else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi
|
|
;;
|
|
client-created-finalize)
|
|
current="$temporary_dir/client-created-finalize-current.json"
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2
|
|
cp -- "$last_body" "$current"; chmod 0600 -- "$current"
|
|
if client_contract_matches "$current" "$client_id" ||
|
|
jq -e --arg id "$client_id" --arg token "$transaction_token" '
|
|
.id == $id and .clientId == "grafana" and
|
|
.attributes["platform.observability.transaction"] == $token
|
|
' "$current" >/dev/null; then
|
|
client_after=$current; client_mutation=created; pending_kind=''; return 0
|
|
fi
|
|
return 2
|
|
;;
|
|
admin-group-create)
|
|
if classify_group_create "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" admin_group_id admin_group_mutation; then
|
|
capture_pending_group_after admin "$admin_group_id" admin_group_after || return 2
|
|
pending_kind=''; return 0
|
|
else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi
|
|
;;
|
|
viewer-group-create)
|
|
if classify_group_create "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" viewer_group_id viewer_group_mutation; then
|
|
capture_pending_group_after viewer "$viewer_group_id" viewer_group_after || return 2
|
|
pending_kind=''; return 0
|
|
else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi
|
|
;;
|
|
admin-group-finalize|viewer-group-finalize)
|
|
if [[ "$pending_kind" == admin-group-finalize ]]; then
|
|
target=admin; pending_id=$admin_group_id
|
|
else
|
|
target=viewer; pending_id=$viewer_group_id
|
|
fi
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$pending_id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2
|
|
current="$temporary_dir/${target}-group-finalize-current.json"
|
|
cp -- "$last_body" "$current"; chmod 0600 -- "$current"
|
|
if [[ "$target" == admin ]]; then
|
|
jq -e --arg id "$admin_group_id" --arg name "$ADMIN_GROUP_NAME" --arg path "$ADMIN_GROUP_PATH" --arg token "$transaction_token" '
|
|
.id == $id and .name == $name and .path == $path and
|
|
((.attributes // {}) == {} or .attributes["platform.observability.transaction"] == [$token])
|
|
' "$current" >/dev/null || return 2
|
|
admin_group_after=$current
|
|
else
|
|
jq -e --arg id "$viewer_group_id" --arg name "$VIEWER_GROUP_NAME" --arg path "$VIEWER_GROUP_PATH" --arg token "$transaction_token" '
|
|
.id == $id and .name == $name and .path == $path and
|
|
((.attributes // {}) == {} or .attributes["platform.observability.transaction"] == [$token])
|
|
' "$current" >/dev/null || return 2
|
|
viewer_group_after=$current
|
|
fi
|
|
pending_kind=''; return 0
|
|
;;
|
|
mapper-create)
|
|
if classify_mapper_create; then pending_kind=''; return 0; else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi
|
|
;;
|
|
mapper-created-finalize)
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2
|
|
current="$temporary_dir/mapper-created-finalize-current.json"
|
|
jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 2
|
|
chmod 0600 -- "$current"
|
|
if mapper_contract_matches "$current" "$mapper_id" ||
|
|
jq -e --arg id "$mapper_id" --arg token "$transaction_token" '
|
|
.id == $id and .name == "grafana-groups" and
|
|
.config["platform.observability.transaction"] == $token
|
|
' "$current" >/dev/null; then
|
|
mapper_after=$current; mapper_mutation=created; pending_kind=''; return 0
|
|
fi
|
|
return 2
|
|
;;
|
|
secret-create)
|
|
if classify_secret_create; then pending_kind=''; return 0; else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi
|
|
;;
|
|
admin-membership)
|
|
if membership_present "$pending_user_id" "$pending_group_id"; then admin_membership_added=true; pending_kind=''; return 0
|
|
else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi
|
|
;;
|
|
viewer-membership)
|
|
if membership_present "$pending_user_id" "$pending_group_id"; then viewer_membership_added=true; pending_kind=''; return 0
|
|
else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi
|
|
;;
|
|
marker)
|
|
current="$temporary_dir/marker-pending-current.env"
|
|
if root_marker_read "$current"; then
|
|
if cmp --silent -- "$marker_desired" "$current"; then marker_written=true; pending_kind=''; return 0; fi
|
|
if [[ "$marker_had_prior" == true ]] && cmp --silent -- "$marker_prior" "$current"; then pending_kind=''; return 1; fi
|
|
return 2
|
|
else
|
|
result=$?
|
|
[[ "$result" == 4 && "$marker_had_prior" == false ]] && { pending_kind=''; return 1; }
|
|
return 2
|
|
fi
|
|
;;
|
|
client-update)
|
|
current="$temporary_dir/client-pending-current.json"
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2
|
|
cp -- "$last_body" "$current"; chmod 0600 -- "$current"
|
|
if client_contract_matches "$current" "$client_id"; then
|
|
client_after=$current; client_mutation=updated; pending_kind=''; return 0
|
|
fi
|
|
if json_equal "$current" "$client_before"; then pending_kind=''; return 1; fi
|
|
return 2
|
|
;;
|
|
mapper-update)
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2
|
|
current="$temporary_dir/mapper-pending-current.json"
|
|
jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 2
|
|
chmod 0600 -- "$current"
|
|
if mapper_contract_matches "$current" "$mapper_id"; then mapper_after=$current; mapper_mutation=updated; pending_kind=''; return 0; fi
|
|
if json_equal "$current" "$mapper_before"; then pending_kind=''; return 1; fi
|
|
return 2
|
|
;;
|
|
*) return 2 ;;
|
|
esac
|
|
}
|
|
|
|
delete_created_secret() {
|
|
[[ "$secret_mutation" == created ]] || return 0
|
|
local current="$temporary_dir/rollback-secret-current.json" prefix="$temporary_dir/rollback-secret" values options
|
|
kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1
|
|
cp -- "$last_body" "$current"; chmod 0600 -- "$current"
|
|
values="$(validate_secret_json "$current" "$prefix")" || return 1
|
|
local uid rv
|
|
IFS=$'\t' read -r uid rv <<<"$values"
|
|
[[ "$uid" == "$secret_created_uid" && "$rv" == "$secret_created_resource_version" ]] || return 1
|
|
jq -e --arg token "$transaction_token" '.metadata.annotations["observability.hyeonworks.com/create-transaction"] == $token' "$current" >/dev/null || return 1
|
|
[[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] && cmp --silent -- "$prefix.client-secret" "$client_secret_file" || return 1
|
|
options="$temporary_dir/secret-delete-options.json"
|
|
jq -n --arg uid "$uid" --arg rv "$rv" '{apiVersion:"meta.k8s.io/v1",kind:"DeleteOptions",propagationPolicy:"Background",preconditions:{uid:$uid,resourceVersion:$rv}}' >"$options"
|
|
chmod 0600 -- "$options"
|
|
kube_request DELETE "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" "$options"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 || "$last_http_status" == 202 ]] || return 1
|
|
kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 404 ]]
|
|
}
|
|
|
|
rollback_membership() {
|
|
local user_id=$1 group_id=$2 added=$3
|
|
[[ "$added" == true ]] || return 0
|
|
admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups/$group_id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1
|
|
if membership_present "$user_id" "$group_id"; then return 1; else [[ "$?" == 1 ]]; fi
|
|
}
|
|
|
|
rollback_mapper() {
|
|
[[ "$mapper_mutation" != none ]] || return 0
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1
|
|
local current="$temporary_dir/rollback-mapper-current.json"
|
|
jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 1
|
|
chmod 0600 -- "$current"
|
|
json_equal "$current" "$mapper_after" || return 1
|
|
if [[ "$mapper_mutation" == created ]]; then
|
|
admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1
|
|
jq -e --arg id "$mapper_id" \
|
|
'type == "array" and ([.[] | select(.id == $id)] | length) == 0' "$last_body" >/dev/null
|
|
else
|
|
admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id" "$mapper_before"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1
|
|
jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 1
|
|
json_equal "$current" "$mapper_before"
|
|
fi
|
|
}
|
|
|
|
rollback_client() {
|
|
[[ "$client_mutation" != none ]] || return 0
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1
|
|
local current="$temporary_dir/rollback-client-current.json"
|
|
cp -- "$last_body" "$current"; chmod 0600 -- "$current"
|
|
json_equal "$current" "$client_after" || return 1
|
|
if [[ "$client_mutation" == created ]]; then
|
|
if jq -e --arg token "$transaction_token" '.attributes["platform.observability.transaction"] == $token' "$current" >/dev/null; then :
|
|
elif client_contract_matches "$current" "$client_id"; then :
|
|
else return 1; fi
|
|
admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 && "$(jq 'length' "$last_body")" == 0 ]]
|
|
else
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1
|
|
local current_secret="$temporary_dir/rollback-client-current-secret"
|
|
jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$current_secret" || return 1
|
|
chmod 0600 -- "$current_secret"; cmp --silent -- "$client_secret_file" "$current_secret" || return 1
|
|
admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" "$client_restore"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1
|
|
json_equal "$last_body" "$client_before"
|
|
fi
|
|
}
|
|
|
|
rollback_group() {
|
|
local id=$1 mutation=$2 after=$3
|
|
[[ "$mutation" == created ]] || return 0
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1
|
|
json_equal "$last_body" "$after" || return 1
|
|
admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 404 ]]
|
|
}
|
|
|
|
rollback_transaction() {
|
|
local rc=0
|
|
[[ "$rollback_in_progress" == false ]] || return 1
|
|
rollback_in_progress=true
|
|
restore_marker || rc=1
|
|
rollback_membership "$viewer_user_id" "$viewer_group_id" "$viewer_membership_added" || rc=1
|
|
rollback_membership "$admin_user_id" "$admin_group_id" "$admin_membership_added" || rc=1
|
|
delete_created_secret || rc=1
|
|
rollback_mapper || rc=1
|
|
rollback_client || rc=1
|
|
rollback_group "$viewer_group_id" "$viewer_group_mutation" "$viewer_group_after" || rc=1
|
|
rollback_group "$admin_group_id" "$admin_group_mutation" "$admin_group_after" || rc=1
|
|
if (( rc == 0 )); then printf 'GRAFANA_OIDC_ROLLBACK=PASS\n' >&2
|
|
else printf 'GRAFANA_OIDC_ROLLBACK=FAIL\n' >&2; emit_manual_recovery; fi
|
|
rollback_in_progress=false
|
|
return "$rc"
|
|
}
|
|
|
|
transaction_fail() {
|
|
local message=$1 status=${2:-1} classification
|
|
trap - ERR HUP INT TERM
|
|
set +e
|
|
if [[ -n "$pending_kind" ]]; then
|
|
classify_pending; classification=$?
|
|
[[ "$classification" != 2 ]] || emit_manual_recovery
|
|
fi
|
|
rollback_transaction || true
|
|
transaction_active=false
|
|
printf 'ERROR: %s\n' "$message" >&2
|
|
exit "$status"
|
|
}
|
|
|
|
on_signal() { transaction_fail "interrupted by $1" "$2"; }
|
|
|
|
on_process_exit() {
|
|
local rc=$1 classification
|
|
[[ "$exit_handler_running" == false ]] || exit "$rc"
|
|
exit_handler_running=true
|
|
trap - EXIT ERR HUP INT TERM
|
|
set +e
|
|
if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then
|
|
if [[ -n "$pending_kind" ]]; then classify_pending; classification=$?; [[ "$classification" != 2 ]] || emit_manual_recovery; fi
|
|
rollback_transaction || true
|
|
transaction_active=false
|
|
(( rc != 0 )) || rc=1
|
|
fi
|
|
cleanup
|
|
exit "$rc"
|
|
}
|
|
|
|
verify_group_prestate() {
|
|
local label=$1 name=$2 path=$3 state=$4 id=$5 before=$6 count
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2
|
|
require_http 200 "$label group prestate reinspection"
|
|
count="$(jq 'length' "$last_body")" || return 1
|
|
if [[ "$state" == absent ]]; then [[ "$count" == 0 ]]; return; fi
|
|
[[ "$count" == 1 && "$(jq -r --arg path "$path" '.[0] | select(.path == $path) | .id' "$last_body")" == "$id" ]] || return 1
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id"
|
|
require_http 200 "$label group prestate snapshot reinspection"
|
|
json_equal "$last_body" "$before"
|
|
}
|
|
|
|
verify_prestate() {
|
|
local count current_secret="$temporary_dir/prestate-current-secret" current_mapper="$temporary_dir/prestate-current-mapper.json"
|
|
validate_context_and_authority
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2
|
|
require_http 200 'Grafana client prestate reinspection'
|
|
count="$(jq 'length' "$last_body")"
|
|
if [[ "$client_state" == absent ]]; then
|
|
[[ "$count" == 0 ]] || fail 'Grafana client appeared after snapshot'
|
|
else
|
|
[[ "$count" == 1 && "$(jq -r '.[0].id' "$last_body")" == "$client_id" ]] || fail 'Grafana client cardinality or identity changed'
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id"
|
|
require_http 200 'Grafana client exact prestate reinspection'
|
|
json_equal "$last_body" "$client_before" || fail 'Grafana client changed after snapshot'
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret"
|
|
require_http 200 'Grafana credential prestate reinspection'
|
|
jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$current_secret" ||
|
|
fail 'Grafana credential reinspection response is invalid'
|
|
chmod 0600 -- "$current_secret"
|
|
cmp --silent -- "$client_secret_file" "$current_secret" || fail 'Grafana credential changed after snapshot'
|
|
fi
|
|
verify_group_prestate admin "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" "$admin_group_state" "$admin_group_id" "$admin_group_before" ||
|
|
fail 'admin group changed after snapshot'
|
|
verify_group_prestate viewer "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" "$viewer_group_state" "$viewer_group_id" "$viewer_group_before" ||
|
|
fail 'viewer group changed after snapshot'
|
|
if [[ "$client_state" == existing ]]; then
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models"
|
|
require_http 200 'Grafana mapper prestate reinspection'
|
|
if [[ "$mapper_state" == absent ]]; then
|
|
[[ "$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name or .config["claim.name"] == "groups")] | length' "$last_body")" == 0 ]] ||
|
|
fail 'Grafana mapper appeared after snapshot'
|
|
else
|
|
[[ "$(jq --arg id "$mapper_id" '[.[] | select(.id == $id)] | length' "$last_body")" == 1 ]] ||
|
|
fail 'Grafana mapper identity changed'
|
|
jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current_mapper"
|
|
chmod 0600 -- "$current_mapper"
|
|
json_equal "$current_mapper" "$mapper_before" || fail 'Grafana mapper changed after snapshot'
|
|
fi
|
|
fi
|
|
if [[ -n "$admin_username" ]]; then
|
|
local prior_id=$admin_user_id; lookup_user admin "$admin_username" admin_user_id
|
|
[[ "$admin_user_id" == "$prior_id" ]] || fail 'admin user identity changed'
|
|
if [[ "$admin_group_state" == existing ]]; then
|
|
local membership_rc
|
|
if membership_present "$admin_user_id" "$admin_group_id"; then membership_rc=0; else membership_rc=$?; fi
|
|
case "$membership_rc" in
|
|
0) [[ "$admin_membership_before" == true ]] || fail 'admin membership appeared after snapshot' ;;
|
|
1) [[ "$admin_membership_before" == false ]] || fail 'admin membership disappeared after snapshot' ;;
|
|
*) fail 'admin membership reinspection failed' ;;
|
|
esac
|
|
fi
|
|
fi
|
|
if [[ -n "$viewer_username" ]]; then
|
|
local prior_viewer_id=$viewer_user_id; lookup_user viewer "$viewer_username" viewer_user_id
|
|
[[ "$viewer_user_id" == "$prior_viewer_id" ]] || fail 'viewer user identity changed'
|
|
if [[ "$viewer_group_state" == existing ]]; then
|
|
local viewer_membership_rc
|
|
if membership_present "$viewer_user_id" "$viewer_group_id"; then viewer_membership_rc=0; else viewer_membership_rc=$?; fi
|
|
case "$viewer_membership_rc" in
|
|
0) [[ "$viewer_membership_before" == true ]] || fail 'viewer membership appeared after snapshot' ;;
|
|
1) [[ "$viewer_membership_before" == false ]] || fail 'viewer membership disappeared after snapshot' ;;
|
|
*) fail 'viewer membership reinspection failed' ;;
|
|
esac
|
|
fi
|
|
fi
|
|
kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET"
|
|
(( last_curl_rc == 0 )) || fail 'Grafana OIDC Secret prestate reinspection transport failed'
|
|
if [[ "$secret_state" == absent ]]; then
|
|
[[ "$last_http_status" == 404 ]] || fail 'Grafana OIDC Secret appeared after snapshot'
|
|
else
|
|
[[ "$last_http_status" == 200 ]] || fail 'Grafana OIDC Secret disappeared after snapshot'
|
|
json_equal "$last_body" "$secret_before" || fail 'Grafana OIDC Secret changed after snapshot'
|
|
fi
|
|
}
|
|
|
|
snapshot_transaction_prestate() {
|
|
transaction_token="$(new_transaction_token)" || fail 'cannot create a transaction ownership token'
|
|
[[ "$transaction_token" =~ ^[0-9a-f]{32}$ ]] || fail 'transaction ownership token is invalid'
|
|
build_desired_client
|
|
build_desired_mapper
|
|
lookup_client
|
|
lookup_group admin "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" admin_group_state admin_group_id admin_group_before
|
|
lookup_group viewer "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" viewer_group_state viewer_group_id viewer_group_before
|
|
lookup_mapper
|
|
lookup_user admin "$admin_username" admin_user_id
|
|
lookup_user viewer "$viewer_username" viewer_user_id
|
|
if [[ -n "$admin_user_id" && "$admin_group_state" == existing ]]; then
|
|
snapshot_membership admin "$admin_user_id" "$admin_group_id" admin_membership_before
|
|
fi
|
|
if [[ -n "$viewer_user_id" && "$viewer_group_state" == existing ]]; then
|
|
snapshot_membership viewer "$viewer_user_id" "$viewer_group_id" viewer_membership_before
|
|
fi
|
|
if [[ "$client_state" == existing ]]; then retrieve_client_secret; fi
|
|
start_kubernetes_proxy
|
|
snapshot_oidc_secret
|
|
sudo_refresh
|
|
prepare_marker_snapshot
|
|
}
|
|
|
|
verify_final_contract() {
|
|
local count current_mapper="$temporary_dir/final-mapper.json" prefix values
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2
|
|
require_http 200 'final Grafana client lookup'
|
|
[[ "$(jq 'length' "$last_body")" == 1 && "$(jq -r '.[0].id' "$last_body")" == "$client_id" ]] ||
|
|
fail 'final Grafana client cardinality differs'
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id"
|
|
require_http 200 'final Grafana client read'
|
|
client_contract_matches "$last_body" "$client_id" || fail 'final Grafana client contract differs'
|
|
local label name path id
|
|
for label in admin viewer; do
|
|
if [[ "$label" == admin ]]; then name=$ADMIN_GROUP_NAME; path=$ADMIN_GROUP_PATH; id=$admin_group_id
|
|
else name=$VIEWER_GROUP_NAME; path=$VIEWER_GROUP_PATH; id=$viewer_group_id; fi
|
|
admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2
|
|
require_http 200 "final $label group lookup"
|
|
[[ "$(jq 'length' "$last_body")" == 1 &&
|
|
"$(jq -r --arg path "$path" '.[0] | select(.path == $path) | .id' "$last_body")" == "$id" ]] ||
|
|
fail "final $label group contract differs"
|
|
done
|
|
admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models"
|
|
require_http 200 'final Grafana mapper lookup'
|
|
[[ "$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name or .config["claim.name"] == "groups")] | length' "$last_body")" == 1 ]] ||
|
|
fail 'final Grafana mapper cardinality differs'
|
|
jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current_mapper" || fail 'final Grafana mapper identity differs'
|
|
chmod 0600 -- "$current_mapper"; mapper_contract_matches "$current_mapper" "$mapper_id" || fail 'final Grafana mapper contract differs'
|
|
kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET"
|
|
(( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || fail 'final Grafana OIDC Secret is absent'
|
|
prefix="$temporary_dir/final-secret"; values="$(validate_secret_json "$last_body" "$prefix")" || fail 'final Grafana OIDC Secret contract differs'
|
|
[[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] && cmp --silent -- "$prefix.client-secret" "$client_secret_file" ||
|
|
fail 'final Grafana OIDC Secret relationship differs'
|
|
if [[ -n "$admin_user_id" ]]; then membership_present "$admin_user_id" "$admin_group_id" || fail 'requested admin membership is absent'; fi
|
|
if [[ -n "$viewer_user_id" ]]; then membership_present "$viewer_user_id" "$viewer_group_id" || fail 'requested viewer membership is absent'; fi
|
|
admin_request GET "$local_base/realms/$KEYCLOAK_REALM/.well-known/openid-configuration"
|
|
require_http 200 'Keycloak OIDC discovery verification'
|
|
jq -e --arg issuer "https://$KEYCLOAK_HOST/realms/$KEYCLOAK_REALM" '.issuer == $issuer' "$last_body" >/dev/null ||
|
|
fail 'Keycloak public issuer differs'
|
|
}
|
|
|
|
reconcile_transaction() {
|
|
transaction_active=true
|
|
trap 'transaction_fail "unexpected command failure at line $LINENO"' ERR
|
|
trap 'on_signal HUP 129' HUP
|
|
trap 'on_signal INT 130' INT
|
|
trap 'on_signal TERM 143' TERM
|
|
if [[ "$admin_group_state" == absent ]]; then
|
|
create_group admin "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" admin_group_id admin_group_mutation admin_group_after
|
|
fi
|
|
if [[ "$viewer_group_state" == absent ]]; then
|
|
create_group viewer "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" viewer_group_id viewer_group_mutation viewer_group_after
|
|
fi
|
|
if [[ "$client_state" == absent ]]; then
|
|
create_client
|
|
elif client_contract_matches "$client_before" "$client_id"; then
|
|
client_mutation=none
|
|
else
|
|
update_client
|
|
fi
|
|
if [[ "$mapper_state" == absent ]]; then
|
|
create_mapper
|
|
elif mapper_contract_matches "$mapper_before" "$mapper_id"; then
|
|
mapper_mutation=none
|
|
else
|
|
update_mapper
|
|
fi
|
|
if [[ "$secret_state" == absent ]]; then create_oidc_secret; fi
|
|
add_membership admin "$admin_user_id" "$admin_group_id" "$admin_membership_before" admin_membership_added
|
|
add_membership viewer "$viewer_user_id" "$viewer_group_id" "$viewer_membership_before" viewer_membership_added
|
|
verify_final_contract
|
|
install_marker
|
|
trap - ERR HUP INT TERM
|
|
transaction_active=false
|
|
printf '%s\n' \
|
|
'GRAFANA_OIDC_CLIENT=RECONCILED' \
|
|
'GRAFANA_OIDC_GROUPS=RECONCILED' \
|
|
'GRAFANA_OIDC_MAPPER=RECONCILED' \
|
|
"GRAFANA_OIDC_SECRET=$(if [[ "$secret_state" == absent ]]; then printf CREATE_CONFIRMED; else printf REUSED_UNCHANGED; fi)" \
|
|
'GRAFANA_OIDC_TRANSACTION=PASS'
|
|
}
|
|
|
|
execute_transaction() {
|
|
run_common_gates
|
|
validate_context_and_authority
|
|
snapshot_kubernetes_objects
|
|
start_port_forward
|
|
authenticate_keycloak
|
|
snapshot_transaction_prestate
|
|
read_confirmation 'Type APPLY default: ' 'APPLY default'
|
|
read_confirmation 'Type RECOVERY KEYCLOAK default: ' 'RECOVERY KEYCLOAK default'
|
|
verify_prestate
|
|
run_common_gates
|
|
ensure_evidence_directory
|
|
reconcile_transaction
|
|
}
|
|
|
|
grafana_oidc_main() {
|
|
parse_arguments "$@"
|
|
(( EUID != 0 )) || fail 'whole-script root execution is forbidden; use narrow sudo from the invoking user'
|
|
if [[ "$execute_requested" == false && "$check_requested" == false ]]; then print_dry_run; return 0; fi
|
|
require_commands
|
|
make_temporary_dir
|
|
trap 'on_process_exit "$?"' EXIT
|
|
if [[ "$check_requested" == true ]]; then check_recovery_evidence; else execute_transaction; fi
|
|
}
|
|
|
|
platform_grafana_oidc_fixture_main() {
|
|
local fixture=${1:-}
|
|
[[ "${BASH_SOURCE[0]}" != "$0" ]] || fail 'fixture entrypoint must be sourced'
|
|
[[ -n "$fixture" ]] || fail 'fixture root is required'
|
|
shift
|
|
TEST_MODE=true
|
|
configure_test_boundaries "$fixture"
|
|
grafana_oidc_main "$@"
|
|
}
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
reject_production_overrides
|
|
grafana_oidc_main "$@"
|
|
fi
|