From 16c337bcc91b27ac0f074154b8337e4c8dc17ebb Mon Sep 17 00:00:00 2001 From: donghyeon-ka Date: Fri, 28 Aug 2026 17:35:41 +0900 Subject: [PATCH] Add platform infrastructure configuration --- .gitignore | 43 + README.md | 243 +- bootstrap/argocd/README.md | 25 + .../k3s-secret-encryption-restore-drill.md | 127 + bootstrap/manual/k3s-secret-encryption.md | 118 + bootstrap/manual/phase1-gitea.md | 50 + bootstrap/manual/phase2-keycloak-aistor.md | 139 + bootstrap/manual/phase3-admin-surfaces.md | 189 + .../manual/phase4-observability-access.md | 701 ++ clusters/home/applications/.gitkeep | 0 clusters/home/projects/.gitkeep | 0 components/mesh-ready/README.md | 17 + components/policies/.gitkeep | 0 ...-alertmanager-receiver-postcheck-design.md | 69 + ...ability-alertmanager-receiver-postcheck.md | 380 + ...vability-structured-slack-notifications.md | 373 + ...-observability-task7-cutover-correction.md | 1290 +++ .../controllers/aistor-operator/README.md | 56 + .../aistor-operator/base/README.md | 5 + .../aistor-operator/kustomization.yaml | 29 + .../aistor-operator/overlays/home/README.md | 4 + .../aistor-operator/values/home.yaml | 37 + .../controllers/cloudnative-pg/README.md | 32 + .../controllers/cloudnative-pg/base/README.md | 5 + .../cloudnative-pg/kustomization.yaml | 17 + .../cloudnative-pg/overlays/home/README.md | 5 + .../cloudnative-pg/values/home.yaml | 31 + .../controllers/keycloak-operator/README.md | 35 + .../keycloak-operator/base/README.md | 5 + .../keycloak-operator/kustomization.yaml | 12 + .../keycloak-operator/overlays/home/README.md | 5 + .../prometheus-operator-crds/README.md | 5 + .../kustomization.yaml | 15 + .../prometheus-operator-crds/values/home.yaml | 22 + .../namespaces/admin-tools/README.md | 7 + .../namespaces/admin-tools/kustomization.yaml | 5 + .../namespaces/admin-tools/namespace.yaml | 13 + infrastructure/namespaces/base/README.md | 17 + .../namespaces/base/cnpg-system.yaml | 13 + infrastructure/namespaces/base/gitea.yaml | 13 + .../namespaces/base/kustomization.yaml | 7 + .../namespaces/base/platform-data.yaml | 13 + .../namespaces/observability/README.md | 17 + .../observability/discovery-namespaces.txt | 5 + .../observability/kustomization.yaml | 6 + .../observability/observability-agent.yaml | 12 + .../observability/observability.yaml | 12 + .../overlays/home/kustomization.yaml | 5 + infrastructure/namespaces/phase2/README.md | 22 + infrastructure/namespaces/phase2/aistor.yaml | 13 + .../namespaces/phase2/keycloak.yaml | 13 + .../namespaces/phase2/kustomization.yaml | 7 + .../namespaces/phase2/object-storage.yaml | 13 + infrastructure/networking/aistor/README.md | 20 + .../networking/aistor/kustomization.yaml | 5 + .../networking/aistor/network-policies.yaml | 313 + .../networking/host-nginx/README.md | 309 + .../host-nginx/learn-services-admin.conf | 165 + .../learn-services-grafana-deny-guard.conf | 214 + .../host-nginx/learn-services-keycloak.conf | 104 + .../learn-services-metrics-guard.conf | 169 + .../learn-services-observability.conf | 214 + .../networking/host-nginx/learn-services.conf | 94 + .../networking/host-nginx/reload-nginx.sh | 6 + .../networking/private-dns/host/README.md | 53 + .../private-dns/host/dnsmasq-lan.conf | 16 + .../private-dns/host/dnsmasq-tailscale.conf | 16 + .../host/hyeonworks-dnsmasq-lan.service | 23 + .../host/hyeonworks-dnsmasq-tailscale.service | 23 + .../private-dns/kubernetes/README.md | 8 + .../kubernetes/coredns-custom.yaml | 24 + .../private-dns/kubernetes/kustomization.yaml | 5 + infrastructure/networking/traefik/README.md | 287 + .../traefik/base/helm-chart-config.yaml | 45 + .../traefik/base/kustomization.yaml | 5 + .../networking/traefik/kustomization.yaml | 5 + .../overlays/baseline/kustomization.yaml | 14 + .../baseline/service-boundary-only-patch.yaml | 28 + .../overlays/observe/kustomization.yaml | 5 + .../traefik/overlays/trust/kustomization.yaml | 14 + .../trust/trusted-proxy-cidr-patch.yaml | 41 + .../traefik/scripts/apply-observe.sh | 123 + .../networking/traefik/scripts/apply-trust.sh | 131 + .../networking/traefik/scripts/common.sh | 464 + .../traefik/scripts/observe-client-host.sh | 58 + .../traefik/scripts/rollback-to-observe.sh | 75 + .../networking/traefik/scripts/validate.sh | 161 + .../security/k3s/40-secrets-encryption.yaml | 2 + infrastructure/security/k3s/README.md | 34 + infrastructure/security/secrets/README.md | 92 + .../storage/aistor-local-pv/README.md | 38 + .../aistor-local-pv/aistor-data-pv.yaml | 27 + .../aistor-local-pv/kustomization.yaml | 6 + .../aistor-local-pv/storage-class.yaml | 12 + .../storage/observability-local-pv/README.md | 32 + .../observability-local-pv/kustomization.yaml | 6 + .../persistent-volumes.yaml | 155 + .../storage-classes.yaml | 53 + .../storage/pgadmin-local-pv/README.md | 12 + .../pgadmin-local-pv/kustomization.yaml | 6 + .../storage/pgadmin-local-pv/pgadmin-pv.yaml | 26 + .../pgadmin-local-pv/storage-class.yaml | 11 + infrastructure/storage/ssd-local-pv/README.md | 33 + .../storage/ssd-local-pv/gitea-pv.yaml | 26 + .../storage/ssd-local-pv/kustomization.yaml | 7 + .../ssd-local-pv/platform-postgres-pv.yaml | 26 + .../storage/ssd-local-pv/storage-classes.yaml | 23 + scripts/bootstrap/apply-admin-services.sh | 266 + scripts/bootstrap/apply-aistor.sh | 470 + scripts/bootstrap/apply-gitea-oidc.sh | 673 ++ scripts/bootstrap/apply-host-nginx-admin.sh | 332 + scripts/bootstrap/apply-host-nginx-gitea.sh | 351 + .../bootstrap/apply-host-nginx-keycloak.sh | 299 + .../apply-host-nginx-observability.sh | 1132 +++ .../bootstrap/apply-k3s-secret-encryption.sh | 1624 +++ scripts/bootstrap/apply-keycloak.sh | 407 + .../bootstrap/apply-observability-access.sh | 1419 +++ scripts/bootstrap/apply-phase1-gitea.sh | 195 + scripts/bootstrap/apply-private-dns.sh | 287 + .../backup-slack-webhook-recovery.sh | 22 + scripts/bootstrap/close-k3s-local-recovery.sh | 49 + .../configure-keycloak-admin-oidc.sh | 400 + .../configure-keycloak-gitea-oidc.sh | 445 + .../configure-keycloak-grafana-oidc.sh | 1928 ++++ scripts/bootstrap/create-aistor-secrets.sh | 324 + scripts/bootstrap/create-keycloak-secrets.sh | 298 + .../bootstrap/create-observability-secrets.sh | 1826 ++++ scripts/bootstrap/create-phase1-secrets.sh | 216 + scripts/bootstrap/create-phase2-secrets.sh | 323 + scripts/bootstrap/open-k3s-local-recovery.sh | 49 + .../bootstrap/prepare-k3s-local-recovery.sh | 51 + .../prepare-observability-local-paths.sh | 564 ++ scripts/bootstrap/prepare-ssd-local-paths.sh | 112 + .../prove-k3s-local-recovery-header.sh | 70 + scripts/lib/k3s-local-recovery-a1.sh | 1142 +++ .../lib/k3s-local-recovery-header-proof.sh | 344 + scripts/lib/k3s-local-recovery.sh | 2038 ++++ scripts/lib/k3s-secret-encryption.sh | 569 ++ scripts/lib/slack-webhook-recovery.sh | 1232 +++ .../libexec/k3s-local-recovery-a1-check.sh | 243 + .../libexec/slack-webhook-recovery-dirfd.py | 1095 +++ .../libexec/slack-webhook-recovery-vault.py | 947 ++ scripts/validate/README.md | 561 ++ scripts/validate/admin-ui-smoke.sh | 168 + scripts/validate/aistor-s3-smoke.sh | 193 + .../capture-observability-metric-inventory.sh | 812 ++ .../k3s-local-recovery-feasibility.sh | 431 + scripts/validate/k3s-local-recovery.sh | 718 ++ .../k3s-secret-encryption-restore-evidence.sh | 636 ++ scripts/validate/k3s-secret-encryption.sh | 344 + scripts/validate/observability-smoke.sh | 1808 ++++ scripts/validate/render-admin-services.sh | 243 + .../validate/render-observability-access.sh | 825 ++ scripts/validate/render-observability-core.sh | 1650 ++++ scripts/validate/render-phase1.sh | 671 ++ scripts/validate/render-phase2.sh | 439 + .../scan-platform-sensitive-source.sh | 1725 ++++ .../test-apply-host-nginx-observability.sh | 1181 +++ .../test-apply-observability-access.sh | 1651 ++++ .../test-backup-slack-webhook-recovery.sh | 2197 +++++ ...-capture-observability-metric-inventory.sh | 751 ++ .../test-configure-keycloak-grafana-oidc.sh | 651 ++ .../test-create-observability-secrets.sh | 1728 ++++ .../test-host-nginx-observability-source.sh | 56 + .../validate/test-k3s-local-recovery-a1.sh | 3057 ++++++ .../validate/test-k3s-local-recovery-fmask.sh | 22 + .../test-k3s-local-recovery-header-proof.sh | 797 ++ .../validate/test-k3s-local-recovery-lsblk.sh | 43 + scripts/validate/test-k3s-local-recovery.sh | 8668 +++++++++++++++++ ...-k3s-secret-encryption-restore-evidence.sh | 705 ++ .../test-k3s-secret-encryption-status.sh | 2360 +++++ .../validate/test-observability-alerting.sh | 273 + .../validate/test-observability-dashboards.sh | 612 ++ .../test-observability-slack-risk-policy.sh | 163 + scripts/validate/test-observability-smoke.sh | 1111 +++ .../test-platform-observability-rules.sh | 1070 ++ .../test-private-dns-observability.sh | 59 + .../test-render-observability-access.sh | 1066 ++ .../test-render-observability-core.sh | 1093 +++ .../test-scan-platform-sensitive-source.sh | 1485 +++ .../test-slack-webhook-recovery-dirfd.py | 1397 +++ .../test-slack-webhook-recovery-vault.py | 1179 +++ .../test-validate-blackbox-edge-source.sh | 562 ++ .../validate/validate-blackbox-edge-source.sh | 728 ++ services/gitea/README.md | 229 + services/gitea/base/ingress.yaml | 25 + services/gitea/base/kustomization.yaml | 7 + services/gitea/base/network-policies.yaml | 161 + services/gitea/base/pvc.yaml | 19 + services/gitea/branding/kustomization.yaml | 15 + .../branding/public/assets/css/hyeonworks.css | 59 + .../branding/public/assets/img/favicon.svg | 12 + .../gitea/branding/public/assets/img/logo.svg | 13 + .../templates/custom/extra_links.tmpl | 1 + .../branding/templates/custom/header.tmpl | 2 + services/gitea/kustomization.yaml | 40 + .../gitea/profiles/oidc/kustomization.yaml | 39 + .../gitea/profiles/oidc/network-policy.yaml | 24 + .../oidc/patches/delete-ssh-service.yaml | 6 + .../servicemonitor-job-label-patch.yaml | 7 + .../gitea/profiles/oidc/values/baseline.yaml | 189 + services/gitea/profiles/oidc/values/oidc.yaml | 49 + services/keycloak/README.md | 111 + services/keycloak/base/ingress.yaml | 23 + services/keycloak/base/keycloak.yaml | 94 + services/keycloak/base/kustomization.yaml | 6 + services/keycloak/kustomization.yaml | 13 + services/keycloak/overlays/home/README.md | 6 + services/minio-aistor/README.md | 81 + services/minio-aistor/base/README.md | 7 + .../minio-aistor/base/deployment/README.md | 6 + .../base/deployment/kustomization.yaml | 17 + .../base/deployment/values/home.yaml | 67 + services/minio-aistor/kustomization.yaml | 5 + services/minio-aistor/overlays/home/README.md | 8 + .../profiles/admin-oidc/README.md | 16 + .../profiles/admin-oidc/console-ingress.yaml | 24 + .../profiles/admin-oidc/kustomization.yaml | 15 + .../profiles/admin-oidc/network-policies.yaml | 101 + .../admin-oidc/objectstore-oidc-patch.yaml | 32 + services/minio-aistor/values/home.yaml | 67 + services/observability/README.md | 18 + .../alertmanager-public-https-egress.yaml | 24 + .../observability/alerting/kustomization.yaml | 8 + .../alerting/platform-alertmanager.yaml | 80 + services/observability/alloy/README.md | 12 + services/observability/alloy/config.alloy | 214 + .../observability/alloy/kustomization.yaml | 51 + services/observability/alloy/pvc.yaml | 17 + services/observability/alloy/values/home.yaml | 155 + .../observability/blackbox-exporter/README.md | 22 + .../blackbox-exporter/kustomization.yaml | 45 + .../blackbox-exporter/network-policy.yaml | 88 + .../probes-private-edge.yaml | 28 + .../probes-private-internal.yaml | 44 + .../blackbox-exporter/probes-public.yaml | 27 + .../blackbox-exporter/values/home.yaml | 128 + .../dashboards/platform/https-endpoints.json | 208 + .../dashboards/platform/kubernetes-node.json | 266 + .../dashboards/platform/kustomization.yaml | 28 + .../platform/observability-backends.json | 384 + .../platform/platform-services.json | 419 + .../dashboards/platform/workload-health.json | 218 + services/observability/grafana/README.md | 24 + .../grafana/dashboard-sidecar-rbac.yaml | 27 + .../dashboard-sidecar-token-patch.yaml | 37 + .../grafana/ingress-class-patch.yaml | 7 + .../observability/grafana/kustomization.yaml | 24 + .../observability/grafana/values/home.yaml | 262 + .../kube-prometheus-stack/README.md | 15 + .../kube-prometheus-stack/kustomization.yaml | 37 + .../kube-prometheus-stack/values/home.yaml | 282 + services/observability/kustomization.yaml | 16 + services/observability/loki/README.md | 16 + .../observability/loki/kustomization.yaml | 17 + services/observability/loki/values/home.yaml | 146 + .../network-policies/kustomization.yaml | 8 + .../object-storage-counterpart.yaml | 31 + .../network-policies/observability-agent.yaml | 140 + .../network-policies/observability.yaml | 548 ++ .../observability/node-exporter/README.md | 11 + .../node-exporter/kustomization.yaml | 23 + .../node-exporter/values/home.yaml | 78 + .../observability/object-storage/README.md | 10 + .../object-storage/policies/loki.json | 29 + .../object-storage/policies/tempo.json | 29 + .../rules/platform/certificate-rules.yaml | 87 + .../rules/platform/core-rules.yaml | 117 + .../rules/platform/kustomization.yaml | 10 + .../rules/platform/storage-quota-rules.yaml | 55 + .../platform/verified-service-rules.yaml | 88 + services/observability/targets/README.md | 24 + .../targets/aistor/service-monitor.yaml | 28 + .../targets/cloudnative-pg/pod-monitor.yaml | 24 + .../targets/counterpart-network-policies.yaml | 108 + .../observability/targets/kustomization.yaml | 8 + services/observability/tempo/README.md | 15 + .../observability/tempo/kustomization.yaml | 56 + services/observability/tempo/values/home.yaml | 125 + services/pgadmin/README.md | 19 + services/pgadmin/config.yaml | 42 + services/pgadmin/ingress-class-patch.yaml | 5 + services/pgadmin/kustomization.yaml | 38 + .../network-policies/kustomization.yaml | 6 + .../pgadmin/network-policies/pgadmin.yaml | 150 + .../network-policies/platform-postgres.yaml | 27 + services/pgadmin/values/home.yaml | 175 + services/platform-postgres-keycloak/README.md | 42 + .../base/keycloak-database-role.yaml | 22 + .../base/keycloak-database.yaml | 14 + .../base/kustomization.yaml | 15 + ...form-postgres-keycloak-network-policy.yaml | 21 + .../kustomization.yaml | 5 + services/platform-postgres/README.md | 73 + services/platform-postgres/base/cluster.yaml | 46 + .../base/gitea-database-role.yaml | 22 + .../base/gitea-database.yaml | 14 + .../platform-postgres/base/kustomization.yaml | 16 + .../platform-postgres-network-policy.yaml | 53 + services/platform-postgres/kustomization.yaml | 5 + .../overlays/home/kustomization.yaml | 8 + .../home/platform-postgres-storage-patch.yaml | 8 + 302 files changed, 83259 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 bootstrap/argocd/README.md create mode 100644 bootstrap/manual/k3s-secret-encryption-restore-drill.md create mode 100644 bootstrap/manual/k3s-secret-encryption.md create mode 100644 bootstrap/manual/phase1-gitea.md create mode 100644 bootstrap/manual/phase2-keycloak-aistor.md create mode 100644 bootstrap/manual/phase3-admin-surfaces.md create mode 100644 bootstrap/manual/phase4-observability-access.md create mode 100644 clusters/home/applications/.gitkeep create mode 100644 clusters/home/projects/.gitkeep create mode 100644 components/mesh-ready/README.md create mode 100644 components/policies/.gitkeep create mode 100644 docs/platform/plans/2026-08-14-observability-alertmanager-receiver-postcheck-design.md create mode 100644 docs/superpowers/plans/2026-08-14-observability-alertmanager-receiver-postcheck.md create mode 100644 docs/superpowers/plans/2026-08-15-observability-structured-slack-notifications.md create mode 100644 docs/superpowers/plans/2026-08-15-observability-task7-cutover-correction.md create mode 100644 infrastructure/controllers/aistor-operator/README.md create mode 100644 infrastructure/controllers/aistor-operator/base/README.md create mode 100644 infrastructure/controllers/aistor-operator/kustomization.yaml create mode 100644 infrastructure/controllers/aistor-operator/overlays/home/README.md create mode 100644 infrastructure/controllers/aistor-operator/values/home.yaml create mode 100644 infrastructure/controllers/cloudnative-pg/README.md create mode 100644 infrastructure/controllers/cloudnative-pg/base/README.md create mode 100644 infrastructure/controllers/cloudnative-pg/kustomization.yaml create mode 100644 infrastructure/controllers/cloudnative-pg/overlays/home/README.md create mode 100644 infrastructure/controllers/cloudnative-pg/values/home.yaml create mode 100644 infrastructure/controllers/keycloak-operator/README.md create mode 100644 infrastructure/controllers/keycloak-operator/base/README.md create mode 100644 infrastructure/controllers/keycloak-operator/kustomization.yaml create mode 100644 infrastructure/controllers/keycloak-operator/overlays/home/README.md create mode 100644 infrastructure/controllers/prometheus-operator-crds/README.md create mode 100644 infrastructure/controllers/prometheus-operator-crds/kustomization.yaml create mode 100644 infrastructure/controllers/prometheus-operator-crds/values/home.yaml create mode 100644 infrastructure/namespaces/admin-tools/README.md create mode 100644 infrastructure/namespaces/admin-tools/kustomization.yaml create mode 100644 infrastructure/namespaces/admin-tools/namespace.yaml create mode 100644 infrastructure/namespaces/base/README.md create mode 100644 infrastructure/namespaces/base/cnpg-system.yaml create mode 100644 infrastructure/namespaces/base/gitea.yaml create mode 100644 infrastructure/namespaces/base/kustomization.yaml create mode 100644 infrastructure/namespaces/base/platform-data.yaml create mode 100644 infrastructure/namespaces/observability/README.md create mode 100644 infrastructure/namespaces/observability/discovery-namespaces.txt create mode 100644 infrastructure/namespaces/observability/kustomization.yaml create mode 100644 infrastructure/namespaces/observability/observability-agent.yaml create mode 100644 infrastructure/namespaces/observability/observability.yaml create mode 100644 infrastructure/namespaces/overlays/home/kustomization.yaml create mode 100644 infrastructure/namespaces/phase2/README.md create mode 100644 infrastructure/namespaces/phase2/aistor.yaml create mode 100644 infrastructure/namespaces/phase2/keycloak.yaml create mode 100644 infrastructure/namespaces/phase2/kustomization.yaml create mode 100644 infrastructure/namespaces/phase2/object-storage.yaml create mode 100644 infrastructure/networking/aistor/README.md create mode 100644 infrastructure/networking/aistor/kustomization.yaml create mode 100644 infrastructure/networking/aistor/network-policies.yaml create mode 100644 infrastructure/networking/host-nginx/README.md create mode 100644 infrastructure/networking/host-nginx/learn-services-admin.conf create mode 100644 infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf create mode 100644 infrastructure/networking/host-nginx/learn-services-keycloak.conf create mode 100644 infrastructure/networking/host-nginx/learn-services-metrics-guard.conf create mode 100644 infrastructure/networking/host-nginx/learn-services-observability.conf create mode 100644 infrastructure/networking/host-nginx/learn-services.conf create mode 100755 infrastructure/networking/host-nginx/reload-nginx.sh create mode 100644 infrastructure/networking/private-dns/host/README.md create mode 100644 infrastructure/networking/private-dns/host/dnsmasq-lan.conf create mode 100644 infrastructure/networking/private-dns/host/dnsmasq-tailscale.conf create mode 100644 infrastructure/networking/private-dns/host/hyeonworks-dnsmasq-lan.service create mode 100644 infrastructure/networking/private-dns/host/hyeonworks-dnsmasq-tailscale.service create mode 100644 infrastructure/networking/private-dns/kubernetes/README.md create mode 100644 infrastructure/networking/private-dns/kubernetes/coredns-custom.yaml create mode 100644 infrastructure/networking/private-dns/kubernetes/kustomization.yaml create mode 100644 infrastructure/networking/traefik/README.md create mode 100644 infrastructure/networking/traefik/base/helm-chart-config.yaml create mode 100644 infrastructure/networking/traefik/base/kustomization.yaml create mode 100644 infrastructure/networking/traefik/kustomization.yaml create mode 100644 infrastructure/networking/traefik/overlays/baseline/kustomization.yaml create mode 100644 infrastructure/networking/traefik/overlays/baseline/service-boundary-only-patch.yaml create mode 100644 infrastructure/networking/traefik/overlays/observe/kustomization.yaml create mode 100644 infrastructure/networking/traefik/overlays/trust/kustomization.yaml create mode 100644 infrastructure/networking/traefik/overlays/trust/trusted-proxy-cidr-patch.yaml create mode 100755 infrastructure/networking/traefik/scripts/apply-observe.sh create mode 100755 infrastructure/networking/traefik/scripts/apply-trust.sh create mode 100755 infrastructure/networking/traefik/scripts/common.sh create mode 100755 infrastructure/networking/traefik/scripts/observe-client-host.sh create mode 100755 infrastructure/networking/traefik/scripts/rollback-to-observe.sh create mode 100755 infrastructure/networking/traefik/scripts/validate.sh create mode 100644 infrastructure/security/k3s/40-secrets-encryption.yaml create mode 100644 infrastructure/security/k3s/README.md create mode 100644 infrastructure/security/secrets/README.md create mode 100644 infrastructure/storage/aistor-local-pv/README.md create mode 100644 infrastructure/storage/aistor-local-pv/aistor-data-pv.yaml create mode 100644 infrastructure/storage/aistor-local-pv/kustomization.yaml create mode 100644 infrastructure/storage/aistor-local-pv/storage-class.yaml create mode 100644 infrastructure/storage/observability-local-pv/README.md create mode 100644 infrastructure/storage/observability-local-pv/kustomization.yaml create mode 100644 infrastructure/storage/observability-local-pv/persistent-volumes.yaml create mode 100644 infrastructure/storage/observability-local-pv/storage-classes.yaml create mode 100644 infrastructure/storage/pgadmin-local-pv/README.md create mode 100644 infrastructure/storage/pgadmin-local-pv/kustomization.yaml create mode 100644 infrastructure/storage/pgadmin-local-pv/pgadmin-pv.yaml create mode 100644 infrastructure/storage/pgadmin-local-pv/storage-class.yaml create mode 100644 infrastructure/storage/ssd-local-pv/README.md create mode 100644 infrastructure/storage/ssd-local-pv/gitea-pv.yaml create mode 100644 infrastructure/storage/ssd-local-pv/kustomization.yaml create mode 100644 infrastructure/storage/ssd-local-pv/platform-postgres-pv.yaml create mode 100644 infrastructure/storage/ssd-local-pv/storage-classes.yaml create mode 100755 scripts/bootstrap/apply-admin-services.sh create mode 100755 scripts/bootstrap/apply-aistor.sh create mode 100755 scripts/bootstrap/apply-gitea-oidc.sh create mode 100755 scripts/bootstrap/apply-host-nginx-admin.sh create mode 100755 scripts/bootstrap/apply-host-nginx-gitea.sh create mode 100755 scripts/bootstrap/apply-host-nginx-keycloak.sh create mode 100755 scripts/bootstrap/apply-host-nginx-observability.sh create mode 100644 scripts/bootstrap/apply-k3s-secret-encryption.sh create mode 100755 scripts/bootstrap/apply-keycloak.sh create mode 100644 scripts/bootstrap/apply-observability-access.sh create mode 100755 scripts/bootstrap/apply-phase1-gitea.sh create mode 100755 scripts/bootstrap/apply-private-dns.sh create mode 100755 scripts/bootstrap/backup-slack-webhook-recovery.sh create mode 100644 scripts/bootstrap/close-k3s-local-recovery.sh create mode 100755 scripts/bootstrap/configure-keycloak-admin-oidc.sh create mode 100755 scripts/bootstrap/configure-keycloak-gitea-oidc.sh create mode 100755 scripts/bootstrap/configure-keycloak-grafana-oidc.sh create mode 100755 scripts/bootstrap/create-aistor-secrets.sh create mode 100755 scripts/bootstrap/create-keycloak-secrets.sh create mode 100755 scripts/bootstrap/create-observability-secrets.sh create mode 100755 scripts/bootstrap/create-phase1-secrets.sh create mode 100755 scripts/bootstrap/create-phase2-secrets.sh create mode 100644 scripts/bootstrap/open-k3s-local-recovery.sh create mode 100644 scripts/bootstrap/prepare-k3s-local-recovery.sh create mode 100644 scripts/bootstrap/prepare-observability-local-paths.sh create mode 100755 scripts/bootstrap/prepare-ssd-local-paths.sh create mode 100644 scripts/bootstrap/prove-k3s-local-recovery-header.sh create mode 100644 scripts/lib/k3s-local-recovery-a1.sh create mode 100644 scripts/lib/k3s-local-recovery-header-proof.sh create mode 100644 scripts/lib/k3s-local-recovery.sh create mode 100644 scripts/lib/k3s-secret-encryption.sh create mode 100644 scripts/lib/slack-webhook-recovery.sh create mode 100644 scripts/libexec/k3s-local-recovery-a1-check.sh create mode 100644 scripts/libexec/slack-webhook-recovery-dirfd.py create mode 100644 scripts/libexec/slack-webhook-recovery-vault.py create mode 100644 scripts/validate/README.md create mode 100755 scripts/validate/admin-ui-smoke.sh create mode 100755 scripts/validate/aistor-s3-smoke.sh create mode 100644 scripts/validate/capture-observability-metric-inventory.sh create mode 100644 scripts/validate/k3s-local-recovery-feasibility.sh create mode 100644 scripts/validate/k3s-local-recovery.sh create mode 100755 scripts/validate/k3s-secret-encryption-restore-evidence.sh create mode 100644 scripts/validate/k3s-secret-encryption.sh create mode 100644 scripts/validate/observability-smoke.sh create mode 100755 scripts/validate/render-admin-services.sh create mode 100644 scripts/validate/render-observability-access.sh create mode 100755 scripts/validate/render-observability-core.sh create mode 100755 scripts/validate/render-phase1.sh create mode 100755 scripts/validate/render-phase2.sh create mode 100644 scripts/validate/scan-platform-sensitive-source.sh create mode 100755 scripts/validate/test-apply-host-nginx-observability.sh create mode 100644 scripts/validate/test-apply-observability-access.sh create mode 100755 scripts/validate/test-backup-slack-webhook-recovery.sh create mode 100644 scripts/validate/test-capture-observability-metric-inventory.sh create mode 100755 scripts/validate/test-configure-keycloak-grafana-oidc.sh create mode 100755 scripts/validate/test-create-observability-secrets.sh create mode 100644 scripts/validate/test-host-nginx-observability-source.sh create mode 100644 scripts/validate/test-k3s-local-recovery-a1.sh create mode 100644 scripts/validate/test-k3s-local-recovery-fmask.sh create mode 100644 scripts/validate/test-k3s-local-recovery-header-proof.sh create mode 100644 scripts/validate/test-k3s-local-recovery-lsblk.sh create mode 100644 scripts/validate/test-k3s-local-recovery.sh create mode 100755 scripts/validate/test-k3s-secret-encryption-restore-evidence.sh create mode 100644 scripts/validate/test-k3s-secret-encryption-status.sh create mode 100755 scripts/validate/test-observability-alerting.sh create mode 100755 scripts/validate/test-observability-dashboards.sh create mode 100644 scripts/validate/test-observability-slack-risk-policy.sh create mode 100644 scripts/validate/test-observability-smoke.sh create mode 100644 scripts/validate/test-platform-observability-rules.sh create mode 100644 scripts/validate/test-private-dns-observability.sh create mode 100755 scripts/validate/test-render-observability-access.sh create mode 100755 scripts/validate/test-render-observability-core.sh create mode 100644 scripts/validate/test-scan-platform-sensitive-source.sh create mode 100644 scripts/validate/test-slack-webhook-recovery-dirfd.py create mode 100644 scripts/validate/test-slack-webhook-recovery-vault.py create mode 100644 scripts/validate/test-validate-blackbox-edge-source.sh create mode 100644 scripts/validate/validate-blackbox-edge-source.sh create mode 100644 services/gitea/README.md create mode 100644 services/gitea/base/ingress.yaml create mode 100644 services/gitea/base/kustomization.yaml create mode 100644 services/gitea/base/network-policies.yaml create mode 100644 services/gitea/base/pvc.yaml create mode 100644 services/gitea/branding/kustomization.yaml create mode 100644 services/gitea/branding/public/assets/css/hyeonworks.css create mode 100644 services/gitea/branding/public/assets/img/favicon.svg create mode 100644 services/gitea/branding/public/assets/img/logo.svg create mode 100644 services/gitea/branding/templates/custom/extra_links.tmpl create mode 100644 services/gitea/branding/templates/custom/header.tmpl create mode 100644 services/gitea/kustomization.yaml create mode 100644 services/gitea/profiles/oidc/kustomization.yaml create mode 100644 services/gitea/profiles/oidc/network-policy.yaml create mode 100644 services/gitea/profiles/oidc/patches/delete-ssh-service.yaml create mode 100644 services/gitea/profiles/oidc/patches/servicemonitor-job-label-patch.yaml create mode 100644 services/gitea/profiles/oidc/values/baseline.yaml create mode 100644 services/gitea/profiles/oidc/values/oidc.yaml create mode 100644 services/keycloak/README.md create mode 100644 services/keycloak/base/ingress.yaml create mode 100644 services/keycloak/base/keycloak.yaml create mode 100644 services/keycloak/base/kustomization.yaml create mode 100644 services/keycloak/kustomization.yaml create mode 100644 services/keycloak/overlays/home/README.md create mode 100644 services/minio-aistor/README.md create mode 100644 services/minio-aistor/base/README.md create mode 100644 services/minio-aistor/base/deployment/README.md create mode 100644 services/minio-aistor/base/deployment/kustomization.yaml create mode 100644 services/minio-aistor/base/deployment/values/home.yaml create mode 100644 services/minio-aistor/kustomization.yaml create mode 100644 services/minio-aistor/overlays/home/README.md create mode 100644 services/minio-aistor/profiles/admin-oidc/README.md create mode 100644 services/minio-aistor/profiles/admin-oidc/console-ingress.yaml create mode 100644 services/minio-aistor/profiles/admin-oidc/kustomization.yaml create mode 100644 services/minio-aistor/profiles/admin-oidc/network-policies.yaml create mode 100644 services/minio-aistor/profiles/admin-oidc/objectstore-oidc-patch.yaml create mode 100644 services/minio-aistor/values/home.yaml create mode 100644 services/observability/README.md create mode 100644 services/observability/alerting/alertmanager-public-https-egress.yaml create mode 100644 services/observability/alerting/kustomization.yaml create mode 100644 services/observability/alerting/platform-alertmanager.yaml create mode 100644 services/observability/alloy/README.md create mode 100644 services/observability/alloy/config.alloy create mode 100644 services/observability/alloy/kustomization.yaml create mode 100644 services/observability/alloy/pvc.yaml create mode 100644 services/observability/alloy/values/home.yaml create mode 100644 services/observability/blackbox-exporter/README.md create mode 100644 services/observability/blackbox-exporter/kustomization.yaml create mode 100644 services/observability/blackbox-exporter/network-policy.yaml create mode 100644 services/observability/blackbox-exporter/probes-private-edge.yaml create mode 100644 services/observability/blackbox-exporter/probes-private-internal.yaml create mode 100644 services/observability/blackbox-exporter/probes-public.yaml create mode 100644 services/observability/blackbox-exporter/values/home.yaml create mode 100644 services/observability/dashboards/platform/https-endpoints.json create mode 100644 services/observability/dashboards/platform/kubernetes-node.json create mode 100644 services/observability/dashboards/platform/kustomization.yaml create mode 100644 services/observability/dashboards/platform/observability-backends.json create mode 100644 services/observability/dashboards/platform/platform-services.json create mode 100644 services/observability/dashboards/platform/workload-health.json create mode 100644 services/observability/grafana/README.md create mode 100644 services/observability/grafana/dashboard-sidecar-rbac.yaml create mode 100644 services/observability/grafana/dashboard-sidecar-token-patch.yaml create mode 100644 services/observability/grafana/ingress-class-patch.yaml create mode 100644 services/observability/grafana/kustomization.yaml create mode 100644 services/observability/grafana/values/home.yaml create mode 100644 services/observability/kube-prometheus-stack/README.md create mode 100644 services/observability/kube-prometheus-stack/kustomization.yaml create mode 100644 services/observability/kube-prometheus-stack/values/home.yaml create mode 100644 services/observability/kustomization.yaml create mode 100644 services/observability/loki/README.md create mode 100644 services/observability/loki/kustomization.yaml create mode 100644 services/observability/loki/values/home.yaml create mode 100644 services/observability/network-policies/kustomization.yaml create mode 100644 services/observability/network-policies/object-storage-counterpart.yaml create mode 100644 services/observability/network-policies/observability-agent.yaml create mode 100644 services/observability/network-policies/observability.yaml create mode 100644 services/observability/node-exporter/README.md create mode 100644 services/observability/node-exporter/kustomization.yaml create mode 100644 services/observability/node-exporter/values/home.yaml create mode 100644 services/observability/object-storage/README.md create mode 100644 services/observability/object-storage/policies/loki.json create mode 100644 services/observability/object-storage/policies/tempo.json create mode 100644 services/observability/rules/platform/certificate-rules.yaml create mode 100644 services/observability/rules/platform/core-rules.yaml create mode 100644 services/observability/rules/platform/kustomization.yaml create mode 100644 services/observability/rules/platform/storage-quota-rules.yaml create mode 100644 services/observability/rules/platform/verified-service-rules.yaml create mode 100644 services/observability/targets/README.md create mode 100644 services/observability/targets/aistor/service-monitor.yaml create mode 100644 services/observability/targets/cloudnative-pg/pod-monitor.yaml create mode 100644 services/observability/targets/counterpart-network-policies.yaml create mode 100644 services/observability/targets/kustomization.yaml create mode 100644 services/observability/tempo/README.md create mode 100644 services/observability/tempo/kustomization.yaml create mode 100644 services/observability/tempo/values/home.yaml create mode 100644 services/pgadmin/README.md create mode 100644 services/pgadmin/config.yaml create mode 100644 services/pgadmin/ingress-class-patch.yaml create mode 100644 services/pgadmin/kustomization.yaml create mode 100644 services/pgadmin/network-policies/kustomization.yaml create mode 100644 services/pgadmin/network-policies/pgadmin.yaml create mode 100644 services/pgadmin/network-policies/platform-postgres.yaml create mode 100644 services/pgadmin/values/home.yaml create mode 100644 services/platform-postgres-keycloak/README.md create mode 100644 services/platform-postgres-keycloak/base/keycloak-database-role.yaml create mode 100644 services/platform-postgres-keycloak/base/keycloak-database.yaml create mode 100644 services/platform-postgres-keycloak/base/kustomization.yaml create mode 100644 services/platform-postgres-keycloak/base/platform-postgres-keycloak-network-policy.yaml create mode 100644 services/platform-postgres-keycloak/kustomization.yaml create mode 100644 services/platform-postgres/README.md create mode 100644 services/platform-postgres/base/cluster.yaml create mode 100644 services/platform-postgres/base/gitea-database-role.yaml create mode 100644 services/platform-postgres/base/gitea-database.yaml create mode 100644 services/platform-postgres/base/kustomization.yaml create mode 100644 services/platform-postgres/base/platform-postgres-network-policy.yaml create mode 100644 services/platform-postgres/kustomization.yaml create mode 100644 services/platform-postgres/overlays/home/kustomization.yaml create mode 100644 services/platform-postgres/overlays/home/platform-postgres-storage-patch.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d72f655 --- /dev/null +++ b/.gitignore @@ -0,0 +1,43 @@ +# Local environment and editor files +.env +.env.* +!.env.example +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# Credentials and private keys +*.key +*.pem +*.p12 +*.pfx +*.jks +*.crt +*.kubeconfig +kubeconfig +**/secrets/*.yaml +**/secrets/*.yml +!**/secrets/.gitkeep + +# Host-specific recovery hardware identity (UUID, serial, WWN) +infrastructure/security/k3s/local-recovery.env + +# Generated and temporary files +__pycache__/ +*.py[cod] +.pytest_cache/ +*.tmp +*.bak +*.orig +*.rej +.DS_Store + +# Helm and validation artifacts +charts/ +*.tgz +.helm/ +rendered/ +dist/ +.superpowers/ diff --git a/README.md b/README.md index f0d1654..0a18d2e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,243 @@ -# platform-core +# 플랫폼 인프라 +단일 노드 k3s에서 사용하는 공통 플랫폼 인프라의 선언형 구성을 관리하는 +저장소입니다. 상세 아키텍처, 운영 절차, 의사결정 기록은 +[중앙 Platform 문서](/home/donghyeon/workspace/docs/platform)에서 관리합니다. + +## 현재 상태 + +Phase 1과 Phase 2의 선언 및 수동 초기 구축 진입점이 구현되어 있습니다. +Phase 1에는 namespace, SSD Local PV, CloudNativePG, 공용 PostgreSQL, Gitea, +Traefik Ingress와 NetworkPolicy가 포함됩니다. Phase 2 선언에는 Keycloak, 공용 +PostgreSQL의 Keycloak DB·Role, AIStor Operator, 단일 MinIO AIStor ObjectStore, +namespace와 Local PV가 포함됩니다. + +Phase 1은 실제 클러스터에 적용했습니다. CloudNativePG·PostgreSQL·Gitea, +PV/PVC와 내부 Traefik health가 정상이며 Gitea 네 컨테이너의 restricted +securityContext도 검증했습니다. Host Nginx의 Gitea proxy 전환과 로컬·public +HTTP/HTTPS 검증도 통과했습니다. + +Phase 2는 전체를 한 번에 적용하지 않았습니다. **Keycloak-only 범위**인 전용 DB와 +Role, NetworkPolicy, 공식 Operator, 단일 Keycloak 인스턴스, `hyeonworks` realm, +confidential `gitea` client와 `gitea/gitea-keycloak-oidc` Secret까지 실제 적용을 +완료했습니다. Traefik `web` entrypoint에는 실제 관측한 Host Nginx source +`10.42.0.1/32`만 forwarded-header trusted IP로 적용했습니다. + +`id.learn.hyeonworks.com`의 Host Nginx static hold를 Keycloak proxy로 바꾸는 +전환과 Gitea OIDC·브랜딩 rollout도 완료했습니다. Host Nginx 후보 SHA-256은 +`5c5cd74b4992f537fd27c50cf2209573a80a9904e0b19b58c3154717be6ff4a5`, 백업은 +`/etc/nginx/sites-available/learn-services.before-keycloak-20260723160519`이며, +적용한 Gitea manifest SHA-256은 `d25a757...a157`입니다. 활성 OAuth source, +외부 인증 전용 가입 정책, authorization-code redirect와 브랜딩 자동 검증을 모두 +통과했습니다. 실제 realm 사용자의 브라우저 login/callback/logout와 비상 관리자 +로그인은 별도 수용 시험으로 남아 있습니다. + +AIStor는 Keycloak 경로와 분리해 실제 적용했습니다. `aistor`와 `object-storage` +namespace, 900Gi Retain Local PV, 공식 Operator와 단일 ObjectStore, 기본 차단 +NetworkPolicy가 동작 중입니다. ObjectStore는 `Initialized/green`, PVC는 +`aistor-data-local-pv`에 Bound이며 S3와 Console은 `ClusterIP`로만 노출됩니다. +고정 다이제스트의 공식 AIStor Client로 버킷 생성, 객체 쓰기·읽기 checksum, +객체·버킷 삭제까지 인증된 스모크 테스트를 통과했습니다. + +| 구성요소 | 고정 버전 | 상태 | +| --- | --- | --- | +| CloudNativePG | Operator `1.30.0`, Chart `0.29.0` | Phase 1 적용, Ready/Available `1/1` | +| PostgreSQL | `17.9-standard-trixie` | Phase 1 적용, healthy `1/1`, PVC Bound | +| Gitea | 애플리케이션 `1.27.0`, Chart `12.7.0` | OIDC·브랜딩 rollout 완료; OAuth source·정책·redirect·자산 자동 검증 통과 | +| Keycloak | Operator·애플리케이션 `26.7.0` | Ready `1/1`; realm·client·OIDC Secret·Host Nginx 공개 전환 완료 | +| Traefik | Chart `40.1.3+up40.1.0`, 이미지 `3.7.4` | NodePort `30080/30443`, `web` trust `10.42.0.1/32` 적용 | +| AIStor Operator | Chart `5.10.0` | 적용, Operator·AdminJob·Webhook Ready `1/1` | +| AIStor ObjectStore | Chart `1.0.16` | 적용, `Initialized/green`, 900Gi PVC Bound, 인증 S3 스모크 통과 | +| 렌더 도구 | Kustomize `5.8.1`, Helm `3.19.4` | 정확히 일치해야 함 | + +Argo CD와 Istio는 현재 단계에서 구현하거나 배포하지 않습니다. + +## 실제 Kustomize 빌드 루트 + +Phase 1 최초 적용은 다음 경로만 빌드 루트로 사용합니다. + +| 순서 | 빌드 루트 | 렌더 방식 | +| ---: | --- | --- | +| 1 | `infrastructure/namespaces/overlays/home` | Kustomize | +| 2 | `infrastructure/storage/ssd-local-pv` | Kustomize | +| 3 | `infrastructure/controllers/cloudnative-pg` | Kustomize + Helm | +| 4 | `services/platform-postgres` | Kustomize | +| 5 | `services/gitea` | Kustomize + Helm | + +`services/gitea`는 신규 설치용 baseline 빌드 루트이며 렌더 결과는 +`gitea.yaml`입니다. Keycloak OIDC와 브랜딩은 최초 Phase 1 적용 대상이 아니며, +Host Nginx ID 전환 뒤 전용 빌드 루트 `services/gitea/profiles/oidc`에서 +`gitea-oidc.yaml`로 렌더링하고 수명주기 스크립트로만 적용합니다. + +Phase 2는 Phase 1 전체를 먼저 검증한 뒤 다음 빌드 루트를 추가로 렌더합니다. + +| 순서 | 빌드 루트 | 렌더 방식 | +| ---: | --- | --- | +| 1 | `infrastructure/namespaces/phase2` | Kustomize | +| 2 | `infrastructure/storage/aistor-local-pv` | Kustomize | +| 3 | `infrastructure/controllers/keycloak-operator` | Kustomize 원격 리소스 | +| 4 | `services/platform-postgres-keycloak` | Kustomize | +| 5 | `services/keycloak` | Kustomize | +| 6 | `infrastructure/controllers/aistor-operator` | Kustomize + Helm | +| 7 | `services/minio-aistor` | Kustomize + Helm | + +하위 `base`와 `overlays/home`는 직접 적용 대상이 아닐 수 있습니다. Helm values와 +환경 patch가 형제 디렉터리에 있는 구성은 Kustomize 기본 `RootOnly` 제한을 +유지하기 위해 서비스 또는 컨트롤러 루트에서 렌더합니다. + +## Secret 원칙 + +평문 Secret, 비밀번호, 라이선스, 개인 키와 소스 관리되는 `kind: Secret` YAML을 +저장소에 두지 않습니다. 각 단계의 초기 구축 스크립트가 계약 전체의 +존재·타입·키·교차 namespace 동일성을 검증하고, 전체가 없을 때만 명시 확인 후 +생성합니다. 일부만 존재하면 중단하며 자동 회전하지 않습니다. + +최초 생성은 여러 Kubernetes API 요청으로 수행되므로 중간 실패 시 일부 객체가 남을 +수 있습니다. 후속 실행은 이를 삭제하거나 덮어쓰지 않고 부분 상태로 감지해 +중단합니다. + +Phase 1 계약: + +- `platform-data/gitea-db-credentials` +- `gitea/gitea-db-credentials` +- `gitea/gitea-admin` + +Phase 2 계약: + +- `platform-data/keycloak-db-credentials` +- `keycloak/keycloak-db-credentials` +- `gitea/gitea-keycloak-oidc` +- `aistor/minio-license` +- `object-storage/aistor-root-configuration` + +`gitea/gitea-keycloak-oidc`는 `Opaque` 유형이며 정확히 `key`, `secret` 두 key만 +갖습니다. 현재 Keycloak-only 작업으로 생성됐지만 payload는 문서, values와 렌더 +결과에 기록하지 않습니다. + +자세한 계약은 [Secret 문서](infrastructure/security/secrets/README.md)를 따릅니다. +Helm 차트가 렌더하는 초기화 스크립트·비민감 설정용 내부 Secret은 애플리케이션 +자격 증명 계약과 구분합니다. + +## k3s Secret 저장 암호화와 복구 증거 + +k3s Secret encryption at rest의 상태 확인과 fail-stop 활성화 절차는 +[암호화 수동 운영 절차](bootstrap/manual/k3s-secret-encryption.md)를 따릅니다. datastore나 +snapshot 단독 탈취를 완화하지만 같은 host의 root 침해나 full-disk 탈취를 해결하지는 +않습니다. server token과 backup을 함께 얻으면 복구·복호화할 수 있으므로 recovery bundle +전체를 Secret으로 취급합니다. + +2026-08-09 live 실행에서 수동 LUKS header 복구 proof, recovery close·잔류 없음 검사, +closed validator, Secret encryption `Enabled/reencrypt_finished`, hash·integrity·API·node +검사와 post bundle 기록을 모두 통과했고 최신 bundle marker도 검증했습니다. 다만 일반 +lifecycle 자동화, 격리 restore drill, off-host 복제와 암호화 escrow 검증은 아직 +완료되지 않았으므로 관측성 Phase 4 gate는 열리지 않았습니다. + +Phase 4 Secret 생성 전에는 live `--expect-reencrypted`와 restore evidence `--check`를 매번 +독립 호출합니다. backend별 수동 복구, 일회용 격리 환경, 세 mode 결과 생성·등록·검사와 +파기 기준은 [k3s Secret 복구 drill](bootstrap/manual/k3s-secret-encryption-restore-drill.md), +중앙 보호 경계는 +[보안과 Secret 관리](/home/donghyeon/workspace/docs/platform/architecture/security-and-secrets.md)와 +[백업과 복구 설계](/home/donghyeon/workspace/docs/platform/architecture/backup-and-recovery.md)를 +따릅니다. 완료된 live hardening의 비민감 정정은 +[2026-08-08 복구 명령 원장](/home/donghyeon/workspace/docs/platform/runbooks/2026-08-08-k3s-local-recovery-command-log.md)의 +2026-08-09 addendum과 +[2026-08-09 로컬 정책 기록](/home/donghyeon/workspace/docs/platform/runbooks/2026-08-09-k3s-secret-encryption-local-policy.md)에 +남긴다. 별도의 Task 6 실행 runbook은 만들지 않았으며, 이를 암시하지 않는다. + +## 검증과 수동 적용 + +아무 리소스도 적용하지 않는 렌더 검증 명령은 다음과 같습니다. + +```sh +cd /home/donghyeon/workspace/platform +bash scripts/validate/render-phase1.sh +bash scripts/validate/render-phase2.sh +``` + +Phase 1 검증은 baseline과 OIDC profile을 포함한 manifest 여섯 개를 생성합니다. +최초 구축 스크립트는 그중 `gitea.yaml` baseline만, OIDC 수명주기 스크립트는 +`gitea-oidc.yaml` 하나만 각각 적용합니다. + +Helm이 시스템 `PATH`에 없다면 검증된 `3.19.4` 실행 파일의 절대 경로를 +`PLATFORM_HELM_BIN`으로 지정합니다. Phase 2 검증은 Phase 1을 먼저 검증한 뒤 +공식 AIStor Chart 패키지 다이제스트, Keycloak 단일 인스턴스·HTTP hostname/Ingress, +AIStor 1x1·900Gi·PVC 보호·ClusterIP, RootOnly 유지와 Secret 소스·렌더링 부재를 +확인합니다. 실제 클러스터의 ObjectStore CRD는 로컬 렌더에 필요하지 않습니다. + +Phase 1 상세 절차는 +[중앙 Phase 1 런북](/home/donghyeon/workspace/docs/platform/runbooks/2026-07-22-phase1-gitea-bootstrap.md)과 +[짧은 실행 진입점](bootstrap/manual/phase1-gitea.md)을 따릅니다. + +Keycloak에서 Gitea OIDC까지의 실제 실행 순서와 현재 완료·대기 경계는 +[중앙 Keycloak-Gitea OIDC 전환 런북](/home/donghyeon/workspace/docs/platform/runbooks/2026-07-23-keycloak-gitea-oidc-cutover.md)에 +기록합니다. 실행 진입점은 다음 네 개입니다. + +```sh +cd /home/donghyeon/workspace/platform + +# 완료: Keycloak DB·Operator·인스턴스 +bash scripts/bootstrap/apply-keycloak.sh --execute + +# 완료: realm·confidential Gitea client·OIDC Secret +bash scripts/bootstrap/configure-keycloak-gitea-oidc.sh --execute + +# 완료: id.learn.hyeonworks.com Host Nginx 전환 +sudo bash scripts/bootstrap/apply-host-nginx-keycloak.sh --execute + +# 완료: 고정 SHA 렌더 결과의 gitea-oidc.yaml만 적용하고 OIDC·브랜딩 검증 +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-gitea-oidc.sh --execute +``` + +Traefik 신뢰 경계의 적용 값과 첫 실패·복구 내용은 +[중앙 Traefik trust 런북](/home/donghyeon/workspace/docs/platform/runbooks/2026-07-23-traefik-forwarded-header-trust-boundary.md)에 +기록합니다. AIStor의 실제 배포 명령·결과·실패와 복구 과정은 +[중앙 AIStor 배포 기록](/home/donghyeon/workspace/docs/platform/runbooks/2026-07-23-aistor-deployment.md)과 +[Phase 2 수동 진입점](bootstrap/manual/phase2-keycloak-aistor.md)에 기록합니다. + +AIStor 재검증 진입점은 다음과 같습니다. + +```sh +cd /home/donghyeon/workspace/platform + +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-aistor.sh \ + --license-file /home/donghyeon/.secrets/aistor/minio.license \ + --root-config-file /home/donghyeon/.secrets/aistor/root.env \ + --execute + +bash scripts/validate/aistor-s3-smoke.sh --execute +``` + +첫 명령은 기존 Secret과 Retain PV를 회전·삭제하지 않고 계약과 readiness를 +재검증한다. 두 번째 명령은 임시 버킷과 객체를 만들었다가 삭제하며, 자격 증명 +payload는 출력하지 않는다. + +## 호스트 Nginx + +[Gitea Nginx 설정](infrastructure/networking/host-nginx/learn-services.conf)을 +기반으로 한 Gitea proxy는 현재 Host Nginx에 적용되어 있고 로컬·public HTTPS +health를 통과했습니다. `id.learn.hyeonworks.com`도 +[Keycloak 포함 설정](infrastructure/networking/host-nginx/learn-services-keycloak.conf)으로 +전환되어 공개 discovery issuer 검사를 통과했습니다. + +ID 전환은 `scripts/bootstrap/apply-host-nginx-keycloak.sh --execute`가 공개 +discovery 사전 검사, 기존 설정 백업, `nginx -t`, 다시 불러오기와 HTTPS 사후 +검사를 수행하도록 합니다. 실패 시 후보를 방치하지 않고 백업 설정을 복원합니다. +실제 전환 성공 뒤 `scripts/bootstrap/apply-gitea-oidc.sh --execute`로 Gitea +OIDC source와 브랜딩도 live Deployment에 적용했습니다. 세부 경계는 +[Nginx 수동 전환 절차](infrastructure/networking/host-nginx/README.md)를 따릅니다. + +## 저장소 규칙 + +- Kubernetes 리소스는 Kustomize `base`와 환경별 `overlays`로 구분합니다. +- 공식 Helm 차트 원본은 수정하지 않고 고정 버전, SHA-256, 저장소 관리 + values·patch로 설정합니다. +- 클러스터 진입점과 향후 GitOps 애플리케이션 선언은 `clusters/home`에서 + 관리합니다. +- 공통 컨트롤러와 기반 리소스는 `infrastructure`, 플랫폼 서비스는 + `services`에서 관리합니다. +- 반복 정책과 향후 서비스 메시 확장 지점은 `components`에서 관리합니다. +- 자동화는 `scripts`, 사람이 검토하는 최초 적용 절차는 `bootstrap/manual`, + 상세 운영 기록은 중앙 문서에 둡니다. +- 생성된 Helm 캐시와 렌더 산출물 대신 재현 가능한 선언만 버전 관리합니다. diff --git a/bootstrap/argocd/README.md b/bootstrap/argocd/README.md new file mode 100644 index 0000000..bbda140 --- /dev/null +++ b/bootstrap/argocd/README.md @@ -0,0 +1,25 @@ +# Argo CD 초기 구축 경계 + +Argo CD는 1차 Gitea 수동 구축과 플랫폼 저장소 푸시가 끝난 뒤 설치한다. 이 +디렉터리는 그 인계 지점을 보존하며, 현재 단계에는 설치 매니페스트가 없다. + +후속 단계의 기준은 다음과 같다. + +- Argo CD `v3.4.2`를 검토·고정한다. +- 해당 버전에 포함된 Kustomize `v5.8.1`, Helm `v3.19.4`와 로컬 검증 버전을 + 일치시킨다. +- `argocd-cm`의 `kustomize.buildOptions`에는 `--enable-helm`만 설정한다. +- `--load-restrictor LoadRestrictionsNone`는 사용하지 않는다. +- 공식 Chart 원본을 수정하거나 저장소에 복제하지 않는다. +- Application의 소스 경로는 각 실제 빌드 루트를 가리킨다. + - `infrastructure/controllers/cloudnative-pg` + - `services/platform-postgres` + - `services/gitea` +- Namespace, CRD/Operator, PostgreSQL, Gitea의 의존 순서를 별도 Application과 + 동기화 순서(sync wave)로 표현한다. +- 수동 생성한 Secret은 Secret 관리 ADR이 확정되기 전까지 Argo CD 관리 대상으로 + 가져오지 않는다. + +기존 수동 리소스를 인계하기 전에는 적용할 차이가 삭제나 재생성을 유발하지 않는지 확인하고, +특히 `Retain` Local PV/PVC와 PostgreSQL Cluster를 먼저 보호한다. + diff --git a/bootstrap/manual/k3s-secret-encryption-restore-drill.md b/bootstrap/manual/k3s-secret-encryption-restore-drill.md new file mode 100644 index 0000000..8a28c73 --- /dev/null +++ b/bootstrap/manual/k3s-secret-encryption-restore-drill.md @@ -0,0 +1,127 @@ +# k3s Secret 암호화 복구 drill + +이 문서는 승인된 k3s Secret hardening의 post recovery bundle을 실제 복구할 수 있는지 +검증하고, 파기 확인 뒤 운영 host에 비민감 evidence를 등록하는 수동 진입점이다. 운영 +server에서 restore를 실행하는 runbook이 아니며 서비스, datastore, backup 또는 Secret을 +자동으로 변경하지 않는다. + +## 승인과 책임 경계 + +- 운영 장애 복구와 drill은 서로 다른 변경 승인으로 다룬다. +- SQLite와 embedded etcd restore 명령은 이 저장소의 자동 실행 스크립트로 제공하지 않는다. +- drill은 운영 host와 network, datastore, hostname이 격리된 일회용 VM/host에서 post bundle + 복사본으로만 한다. +- recovery bundle에는 복구·복호화에 필요한 자료가 함께 있으므로 bundle 전체를 Secret으로 + 취급한다. 경로, token/config 내용, payload, hash와 escrow 위치를 terminal 결과나 원장에 + 기록하지 않는다. +- result만 반출하고 일회용 환경과 bundle 복사본을 파기한 뒤 운영 host에서 evidence를 + 등록한다. + +## backend별 수동 복구 기준 + +SQLite 장애 복구 순서는 k3s 중지, 현재 DB의 timestamp quarantine 이동, 같은 server +token·config와 검증된 backup DB 복원, k3s 시작이다. 현재 DB를 즉시 삭제하거나 backup으로 +덮어쓰지 않는다. 모든 단계는 별도 승인된 장애 runbook에서 사람이 대상과 rollback 지점을 +확인한다. + +embedded etcd는 현재 K3s 공식 +[`--cluster-reset-restore-path` snapshot 복구 절차](https://docs.k3s.io/cli/etcd-snapshot#restoring-snapshots)를 +사용한다. 단일 server와 다중 server 절차, token/config 일치 조건, reset 후 정상 시작 조건을 +실행 시점의 공식 문서와 다시 대조한다. 보조 tutorial을 복구 권위로 사용하지 않는다. + +## 일회용 환경의 격리 조건 + +복구 전에 다음 조건을 모두 만족시킨다. + +- 운영 server와 다른 hostname, network와 datastore를 사용한다. +- hypervisor private switch 또는 network namespace에 default route와 upstream DNS가 없다. +- 원본 운영 API/datastore로 향하는 route가 없다. +- host firewall egress가 default-deny다. +- LAN, Internet, Slack, AIStor와 운영 API 연결 시험이 모두 실패한다. +- 복구된 workload/controller가 격리망 밖으로 통신할 수 없고 외부 host port를 열 수 없다. +- 운영과 같은 정확한 k3s version, server token과 config를 post bundle 복사본에서 사용한다. + +복구 뒤에는 다음을 확인한다. + +- k3s API ready +- node Ready +- encryption status가 `Enabled/reencrypt_finished` +- server hash와 local integrity가 모두 일치 +- `kubectl get secrets --all-namespaces -o json` 조회가 오류 없이 완료 +- bundle metadata의 복구 전 전체 Secret object count와 복구 후 count가 일치 +- 위 격리 연결 시험을 다시 실행해 모두 실패 + +격리 조건과 live 검사가 모두 끝나기 전에는 result의 `isolation=pass`를 만들지 않는다. + +## 세 mode 사용 + +모든 mode는 먼저 일반 사용자의 현재 kube context를 확정한다. 전체 script를 `sudo`로 +실행하지 않는다. root 권한은 운영 host의 권위 파일 stat/read/install에만 좁게 사용되므로 +필요하면 같은 terminal에서 사전에 sudo credential을 갱신한다. + +격리 복구 host에서 post metadata와 결과 출력 위치를 직접 지정한다. 출력은 기존 파일을 +덮어쓰지 않는다. + +```sh +bash scripts/validate/k3s-secret-encryption-restore-evidence.sh \ + --emit-result --bundle-metadata BUNDLE_METADATA_FILE --output RESULT_FILE +``` + +script는 live API ready, node Ready, `Enabled/reencrypt_finished`, server hash, local integrity, +version/backend와 Secret object count를 직접 검사한다. 수동 격리 시험을 마친 현재 context를 +추가 확인한 뒤에만 result를 만든다. result에는 credential, 경로, 민감 filename이나 hash가 +없고 `destroyed`도 없다. + +결과 파일을 운영 host로 반입한 뒤 일회용 환경과 bundle 복사본을 먼저 파기한다. 그 다음 +운영 host에서 등록한다. + +```sh +bash scripts/validate/k3s-secret-encryption-restore-evidence.sh \ + --record --bundle-metadata BUNDLE_METADATA_FILE --result-file RESULT_FILE +``` + +`--record`는 post phase, metadata/result의 bundle-id·version·backend, 24시간 age와 운영 +host 권위 post metadata의 일곱 field 전체를 확인한다. evidence 대상이 이미 있으면 중단한다. +prompt에는 정확히 `DESTROYED default`를 입력한다. 파기하지 않았거나 확인할 수 없으면 +등록하지 않는다. + +Phase 4의 읽기 전용 gate는 다음과 같다. + +```sh +bash scripts/validate/k3s-secret-encryption-restore-evidence.sh --check +``` + +`--check`는 exact evidence field, 현재 권위 bundle-id/version/backend, live +`reencrypt_finished`와 local integrity, 30일 age, `destroyed=confirmed`를 모두 요구한다. +2026-08-09 live 전환은 `Enabled/reencrypt_finished`, hash·integrity·API·node 검사, +post bundle 기록과 최신 marker 검증까지 통과했다. 그러나 격리 restore·파기 evidence가 +아직 없고 lifecycle 자동화, off-host 복제와 암호화 escrow 검증도 미완료다. 따라서 실제 +운영 `--check`는 아직 성공으로 기록하지 않으며, 이 gate들이 완료되기 전에는 관측성 +Phase 4를 열지 않는다. + +## 입력 파일 보안 + +외부 metadata/result는 현재 사용자 소유 regular non-symlink, mode `0600`, non-empty여야 +한다. parser는 exact allowlist `KEY=VALUE` 한 줄만 받아들이며 duplicate/unknown/empty key, +control character, malformed line, trailing data, `$(`와 backtick을 거부한다. 파일을 shell로 +source하거나 `eval`하지 않는다. 입력 실패는 어떤 privileged install도 수행하기 전에 +종료한다. + +result와 evidence는 같은 directory의 mode `0600` temporary file을 완성하고 exact field를 +다시 검사한 뒤 기존 target을 덮어쓰지 않는 atomic install로 만든다. + +## 실행 원장 + +완료된 live hardening의 비민감 정정은 +[2026-08-08 복구 명령 원장](/home/donghyeon/workspace/docs/platform/runbooks/2026-08-08-k3s-local-recovery-command-log.md)의 +2026-08-09 addendum과 +[2026-08-09 로컬 정책 기록](/home/donghyeon/workspace/docs/platform/runbooks/2026-08-09-k3s-secret-encryption-local-policy.md)에 +있다. 아래 표는 아직 수행하지 않은 격리 restore drill의 실행 원장 형식이며, 완료된 +live hardening에 별도 Task 6 실행 runbook은 만들지 않았고, 이를 암시하지 않는다. + +| 시각 | 사전 상태 | backend | backup 검증 | 명령 | 종료 코드 | 후속 상태 | +|---|---|---|---|---|---:|---| + +backup 위치, token/hash payload, 민감 파일 hash, encryption config 내용과 escrow 위치는 +`검증 완료` 또는 `미완료`로만 쓴다. bundle-id, restore drill 시각과 pass/fail은 비밀값이 +아니므로 기록할 수 있다. diff --git a/bootstrap/manual/k3s-secret-encryption.md b/bootstrap/manual/k3s-secret-encryption.md new file mode 100644 index 0000000..416001c --- /dev/null +++ b/bootstrap/manual/k3s-secret-encryption.md @@ -0,0 +1,118 @@ +# k3s Secret 암호화 수동 운영 절차 + +> **현재 경계(2026-08-09):** bootstrap의 +> `--recovery-policy local-separate-disk-luks`, 물리 디스크 lineage validator, +> `LOCAL_RISK_ACCEPTED`, pre/post 동적 용량 gate는 구현·fixture 검증을 마쳤다. +> 수동 LUKS header 복구 proof, recovery close·잔류 없음 검사와 closed validator를 +> 통과한 뒤 live 전환도 완료했다. 현재 Secret encryption은 +> `Enabled/reencrypt_finished`이며 hash·integrity·API·node 검사와 post bundle 기록, +> 최신 bundle marker 검증이 모두 통과했다. lifecycle 자동화, 격리 restore drill, +> off-host 복제와 암호화 escrow 검증은 별도 미완료 gate이므로 관측성 Phase 4를 열지 않는다. + +이 문서는 단일 k3s server에서 Kubernetes Secret 저장 암호화를 fail-stop 방식으로 +전환하는 수동 절차다. 아래 상태 순서의 **live 전 역사적 기준선**은 +`v1.36.2+k3s1`, 단일 Ready 노드 `donghyeon-system-product-name`, SQLite, Secret +encryption Disabled, API `readyz` pass였다. 현재 live 상태는 위 banner의 +`Enabled/reencrypt_finished`다. 모든 단계는 운영자가 명시적으로 실행하고 확인한다. +자동화는 상태가 불명확하거나 검증이 실패하면 다음 단계로 진행하지 않는다. + +## 상태 순서 + +다음 순서와 각 화살표 사이의 검증을 바꾸지 않는다. + +```text +disabled_no_config + -> k3s secrets-encrypt enable + -> drop-in install + restart + -> transition_start + hashes_match + -> k3s secrets-encrypt rotate-keys + -> bounded wait for reencrypt_finished + -> final restart + -> Enabled/reencrypt_finished + server_hashes_match + local_integrity_match +``` + +각 상태 전환 전후에는 Task 2 validator를 해당 기대 상태로 실행해 version, 단일 +Ready server, datastore, API `readyz`, status와 hash/integrity를 확인한다. validator +출력이나 운영 로그에 server token, 비밀번호, encryption config 본문을 기록하지 +않는다. + +## 사전 판정과 소유 위치 + +이미 `enabled_stable`로 분류되면 Task 1 effective-source resolver를 사용하여 +`ExecStart`, `Environment`/`EnvironmentFile`, systemd drop-in, default 또는 +alternate config 및 그 config drop-in을 순서대로 판정한다. 여기서 +`secrets-encryption`과 provider의 **유효 소유 위치만** 확인한다. + +계획한 `40-secrets-encryption.yaml`이 아닌 기존 위치가 owner이면 그 위치를 그대로 +보존하고 config rewrite를 하지 않는다. 서로 충돌하는 두 owner, provider 판정 불가, +또는 기존 provider가 `aescbc`가 아닌 경우에는 자동 변경하지 않는다. 이 경우에는 +별도 ADR을 먼저 승인해야 한다. + +`enabled_stable`의 stage가 `reencrypt_finished`이면 rotation을 건너뛴다. stage가 +`start`이면 Phase 4 전에 `--rotate-existing`을 사용한 명시 승인 재암호화만 수행한다. + +## Bootstrap 진입 명령 + +인자 없는 명령은 상태만 읽고 변경하지 않는다. + +```sh +cd /home/donghyeon/workspace/platform +bash scripts/bootstrap/apply-k3s-secret-encryption.sh +``` + +실제 전환 명령은 recovery volume이 열린 상태에서 backup root를 직접 지정하고 로컬 +정책을 명시한다. 다음 명령은 maintenance 승인 전에는 실행하지 않는다. + +```sh +cd /home/donghyeon/workspace/platform +bash scripts/bootstrap/apply-k3s-secret-encryption.sh \ + --execute \ + --backup-root /srv/recovery/k3s \ + --recovery-policy local-separate-disk-luks +``` + +실행 확인 순서는 `APPLY ` → 자동 root·lineage·용량 검증 → +`RECOVERY ` → `ENCRYPTED ` → +`LOCAL_RISK_ACCEPTED `다. 어느 검사나 확인이 실패해도 다음 mutation으로 +진행하지 않는다. 비밀번호나 복구 키를 이 명령의 인자·환경변수로 전달하지 않는다. + +## 실행 단계 + +1. `disabled_no_config`을 validator로 확인한 뒤에만 `k3s secrets-encrypt enable`을 + 실행한다. +2. repository의 host artifact를 root 소유, mode `0644`로 설치한다. + + ```sh + install -o root -g root -m 0644 \ + infrastructure/security/k3s/40-secrets-encryption.yaml \ + /etc/rancher/k3s/config.yaml.d/40-secrets-encryption.yaml + ``` + +3. k3s를 재시작하고, `transition_start + hashes_match`가 validator로 확인될 때까지 + 중단한다. +4. 확인 뒤에만 `k3s secrets-encrypt rotate-keys`를 한 번 실행한다. +5. 제한된 시간 동안 `reencrypt_finished`를 기다린다. 시간 초과, API 실패, hash + mismatch 또는 local integrity mismatch이면 중단하고 조사한다. +6. 완료 상태를 확인한 후 final restart를 하고, + `Enabled/reencrypt_finished + server_hashes_match + local_integrity_match`를 + 다시 확인한다. + +> **금지 및 중단 조건** +> +> - `transition_start` 확인 전에는 `rotate-keys`를 실행하지 않는다. +> - 중간 stage에서는 다른 rotation 명령을 실행하지 않는다. +> - drop-in을 자동 삭제하지 않는다. +> - live datastore 자동 restore 금지: 복구 판단과 수행은 별도 승인 절차다. +> - server token 또는 encryption key material을 명령 인자, 로그, ticket, Git에 +> 남기지 않는다. + +post bundle의 격리 복구, 결과 반출·파기와 evidence 등록은 +[k3s Secret 암호화 복구 drill](k3s-secret-encryption-restore-drill.md)을 따른다. + +## 재시도와 복구 경계 + +상태가 `disabled_no_config`, `transition_start`, `enabled_stable` 중 하나로 명확히 +판정되지 않으면 재시도나 설정 변경을 하지 않는다. 특히 hash mismatch, provider +불명확, owner 충돌, API `readyz` 실패는 자동 보정 대상이 아니다. 라이브 datastore를 +되돌리거나 Secret을 변경하는 동작도 이 절차의 권한 밖이며, 별도 ADR과 명시 승인을 +필요로 한다. diff --git a/bootstrap/manual/phase1-gitea.md b/bootstrap/manual/phase1-gitea.md new file mode 100644 index 0000000..838e204 --- /dev/null +++ b/bootstrap/manual/phase1-gitea.md @@ -0,0 +1,50 @@ +# Phase 1 Gitea 수동 부트스트랩 진입점 + +상세 절차와 검증·롤백 기준의 단일 원본은 +[중앙 Phase 1 Gitea bootstrap runbook](/home/donghyeon/workspace/docs/platform/runbooks/2026-07-22-phase1-gitea-bootstrap.md)입니다. +이 파일은 저장소에서 실행 명령을 찾기 위한 짧은 진입점만 제공합니다. + +2026-07-23 기준 Phase 1 클러스터 적용과 내부 health, Gitea restricted +securityContext 검증을 완료했습니다. 이 진입점은 **신규 클러스터의 Gitea +baseline 최초 구축 전용**입니다. baseline에는 Keycloak OIDC, `id` host alias, +Keycloak 전용 egress 및 선언형 브랜딩을 포함하지 않습니다. + +기존 `Deployment/gitea` 또는 `Secret/gitea-keycloak-oidc`가 있는 클러스터에는 이 +스크립트를 재실행하지 않습니다. 전용 안전장치도 둘 중 하나를 발견하면 적용을 +거부합니다. Keycloak OIDC와 브랜딩을 포함한 현재 목표 상태는 public discovery +전환을 마친 뒤 `scripts/bootstrap/apply-gitea-oidc.sh --execute`로만 적용하며, +2026-07-23 실제 클러스터에는 이 전용 경로로 적용을 완료했습니다. 현재 +OIDC-enabled 클러스터에서 baseline 스크립트를 복구 수단으로 재실행하지 않습니다. + +## 실행 순서 + +저장소 루트 `/home/donghyeon/workspace/platform`에서 중앙 runbook을 확인한 +뒤 다음 순서를 지킵니다. + +1. 전체 렌더·정적 검증 + + ```sh + bash scripts/validate/render-phase1.sh + ``` + +2. 신규 클러스터 Phase 1 baseline 최초 적용 + + ```sh + bash scripts/bootstrap/apply-phase1-gitea.sh --execute + ``` + +`apply-phase1-gitea.sh`는 안전장치로 검증을 다시 실행합니다. 검증기가 Chart +SHA-256 확인을 마친 여섯 manifest를 제한된 `/tmp` handoff 경로로 받아 +`gitea.yaml` baseline만 적용하며, 함께 검증한 OIDC 목표 상태 +`gitea-oidc.yaml`은 적용하지 않습니다. 검증 뒤 Chart를 다시 내려받거나 +Kustomize를 다시 렌더링하지 않으며 취소·실패·신호 종료 때 임시 파일을 +정리합니다. + +내부 5단계에서는 `create-phase1-secrets.sh --execute`를 호출합니다. helper는 세 +Secret이 모두 없을 때에만 최초 생성하고, 모두 있으면 값이 같은 완전한 계약인지 +검증한 뒤 그대로 재사용합니다. 일부만 존재하면 중단합니다. 자격 증명 회전은 +별도 runbook이 마련될 때까지 지원하지 않으므로 이 helper를 갱신 용도로 단독 +실행하지 않습니다. + +OIDC 적용·자격 증명 회전·클러스터 검증·Nginx 전환·롤백 방법은 이 파일에 +복제하지 않고 중앙 runbook을 따릅니다. diff --git a/bootstrap/manual/phase2-keycloak-aistor.md b/bootstrap/manual/phase2-keycloak-aistor.md new file mode 100644 index 0000000..6b23352 --- /dev/null +++ b/bootstrap/manual/phase2-keycloak-aistor.md @@ -0,0 +1,139 @@ +# Phase 2 Keycloak·MinIO AIStor 수동 배포 진입점 + +상세 설계는 중앙 문서 `/home/donghyeon/workspace/docs/platform`에서 관리한다. +이 파일은 선언의 의존 순서와 검증 진입점만 기록한다. + +## 상태 + +2026-07-23 기준 Keycloak-only 경로는 DB Secret 두 사본, DatabaseRole·Database, +additive NetworkPolicy, Operator·Server `26.7.0`, Ingress, `hyeonworks` realm, +confidential `gitea` client와 OIDC Secret까지 실제 적용했다. Traefik 최소 trust도 +완료했다. Host Nginx의 `id.learn.hyeonworks.com` proxy와 Gitea OIDC·브랜딩 live +적용도 완료했고 OAuth source·정책·redirect·브랜딩 자동 검증을 통과했다. 실제 +realm 사용자의 브라우저 login/callback/logout, 비상 관리자 로그인과 재시작 +지속성 검증은 남아 있다. MinIO AIStor는 NetworkPolicy, namespace, AIStor 전용 +두 Secret, 900Gi Retain Local PV, 공식 Operator와 단일 ObjectStore까지 실제 +적용했다. `Initialized/green`, PVC Bound, ClusterIP-only와 인증된 S3 +put/get/delete checksum 검증을 통과했다. + +## 적용 전 렌더 검증 + +Helm `3.19.4`가 `PATH`에 없다면 검증된 실행 파일의 절대 경로를 지정한다. + +```sh +cd /home/donghyeon/workspace/platform +PLATFORM_HELM_BIN=/absolute/path/to/helm \ + bash scripts/validate/render-phase2.sh +``` + +이 명령은 Phase 1 검증을 먼저 수행한 뒤 Phase 2 전체를 로컬 렌더한다. 실제 +클러스터의 ObjectStore CRD 설치 여부와 무관하며 어떤 리소스도 적용하지 않는다. + +## Keycloak-only 적용 진입점 + +AIStor license와 ObjectStore 준비를 기다리지 않고 Keycloak만 먼저 배포할 때는 +다음 전용 스크립트를 사용한다. Helm과 AIStor Secret은 요구하지 않는다. + +```sh +cd /home/donghyeon/workspace/platform +bash scripts/bootstrap/apply-keycloak.sh --execute +``` + +스크립트는 Keycloak namespace, Keycloak DB Secret 두 개, 공용 PostgreSQL의 +`DatabaseRole`·`Database`·`NetworkPolicy`, Keycloak Operator, Keycloak CR과 +Ingress를 순서대로 렌더·검증·적용하고 각 readiness를 기다린다. 먼저 현재 +Kubernetes context와 API server를 출력한 뒤 `APPLY`를 정확히 입력해야 한다. +두 DB Secret이 모두 없으면 비밀번호를 화면에 표시하지 않고 두 번 입력받은 뒤 +`APPLY KEYCLOAK SECRETS`를 한 번 더 확인한다. 두 Secret이 이미 있으면 값을 +바꾸지 않고 계약만 검증해 재사용한다. + +자동 적용 전에 두 Secret을 안전하게 미리 생성하려면 다음 명시적 비대화형 모드를 +사용할 수 있다. + +```sh +kubectl apply --filename=infrastructure/namespaces/phase2/keycloak.yaml +bash scripts/bootstrap/create-keycloak-secrets.sh --generate --execute +bash scripts/bootstrap/apply-keycloak.sh --execute +``` + +`--generate --execute`는 OpenSSL CSPRNG로 32바이트 난수를 생성해 mode `0600` +임시 파일에만 기록한다. 생성값은 argv나 화면에 출력하지 않으며 두 플래그 자체를 +자동 생성 승인으로 취급한다. 두 Secret이 이미 있으면 새 값을 만들거나 회전하지 +않고 기존 계약을 검증해 그대로 재사용한다. +독립 실행하는 Secret 생성기는 `platform-data`와 `keycloak` namespace가 먼저 +존재해야 하므로 위 첫 명령을 생략하지 않는다. + +위 Keycloak-only 명령은 실제 완료됐으며 재실행 시 기존 Secret을 회전하지 않고 +계약과 readiness를 다시 검증한다. 실제 결과는 중앙 +`runbooks/2026-07-23-keycloak-gitea-oidc-cutover.md`에 기록한다. + + +중간 단계가 실패해도 namespace, Secret, DB 리소스, Operator 또는 Keycloak +리소스를 자동 삭제하지 않는다. DB reclaim policy는 `Retain`으로 유지하며 원인을 +해결한 뒤 같은 명령을 다시 실행하는 것이 복구 경로다. Host Nginx의 Keycloak +정적 hold 제거와 Gitea OIDC 인증 소스 등록은 이 스크립트 범위가 아니다. + +## AIStor 실제 적용 및 재검증 + +실제 적용에는 다음 진입점을 사용했다. + +```sh +cd /home/donghyeon/workspace/platform +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-aistor.sh \ + --license-file /home/donghyeon/.secrets/aistor/minio.license \ + --root-config-file /home/donghyeon/.secrets/aistor/root.env \ + --execute + +bash scripts/validate/aistor-s3-smoke.sh --execute +``` + +첫 스크립트가 수행하고 검증한 순서는 다음과 같다. + +1. Phase 1과 Keycloak-only 리소스가 정상이며 Host Nginx·Gitea OIDC 작업과 AIStor + 작업이 서로 독립임을 확인한다. +2. AIStor Operator·ObjectStore용 저장소 소유 NetworkPolicy를 먼저 렌더하고 + k3s CNI의 승인 흐름과 차단 흐름을 검증한다. webhook, + Kubernetes API, license 확인, DNS, ObjectStore 내부 통신과 승인된 S3 소비자를 + 빠뜨리지 않는다. + + - `infrastructure/controllers/aistor-operator` + - `services/minio-aistor` + +3. 정확히 `aistor`, `object-storage` namespace만 검토·적용한다. +4. `/srv/k3s/aistor`의 XFS mount·권한과 예상 밖 StorageClass 소비자가 없음을 + 확인한 뒤 `infrastructure/storage/aistor-local-pv`를 적용한다. +5. AIStor 두 Secret만 two-or-none으로 관리하는 전용 helper를 실행한다. + 기존 four-or-none `create-phase2-secrets.sh`는 현재 Keycloak-only 상태에서 + 실행하지 않는다. +6. NetworkPolicy gate 통과 후 `infrastructure/controllers/aistor-operator`와 해당 + 정책을 적용하고 CRD, 두 Operator, admission webhook이 준비될 때까지 기다린다. +7. `services/minio-aistor`와 해당 정책을 적용하고 하나의 서버·하나의 900Gi 볼륨, + `ClusterIP` S3/Console, 예상 PVC·PV binding을 확인한다. 실제 workload에서도 + 승인·차단 흐름을 다시 시험한다. +8. S3 put/get/delete와 checksum, 관리 포트 비노출을 확인한다. + +1~8의 자동 검증은 완료했다. ObjectStore Pod 재생성 뒤 기존 영구 객체가 유지되는지 +보는 별도 지속성 시험은 운영 데이터와 구분된 시험 버킷·승인 창을 정한 뒤 수행한다. + +Argo CD 인계와 Host Nginx 변경은 이 Phase 2 수동 적용 범위에 포함하지 않는다. +Istio와 sidecar injection도 적용하지 않는다. + +## Secret 안전 조건 + +- Keycloak-only 경로의 두 DB Secret 중 일부만 존재하면 + `create-keycloak-secrets.sh`가 생성·회전을 거부한다. 두 Secret은 + `platform-data`와 `keycloak` namespace에 동일한 `keycloak` 사용자와 동일한 + 비밀번호로 존재해야 한다. +- Keycloak-only 스크립트는 AIStor license와 ObjectStore root Secret을 읽거나 + 요구하거나 수정하지 않는다. +- Keycloak DB Secret과 AIStor 두 Secret은 모두 존재하지만 서로 독립 계약이다. +- 기존 four-or-none helper는 이 상태에서 중단하는 것이 정상이며 우회하거나 + Keycloak Secret을 삭제하지 않는다. +- AIStor 전용 helper는 AIStor 두 Secret이 모두 없거나 모두 존재할 때만 + 진행하고 one-of-two 상태에서는 자동 삭제·덮어쓰기 없이 중단한다. +- 두 AIStor Secret이 모두 존재하면 타입·정확한 키, license file 동일성, root + configuration 형식만 검증하고 값을 바꾸지 않는다. +- license payload, root 사용자·비밀번호, DB 비밀번호는 Git, argv, 표준 출력에 + 기록하지 않는다. +- bootstrap helper는 credential 또는 license 회전을 수행하지 않는다. diff --git a/bootstrap/manual/phase3-admin-surfaces.md b/bootstrap/manual/phase3-admin-surfaces.md new file mode 100644 index 0000000..4b4b623 --- /dev/null +++ b/bootstrap/manual/phase3-admin-surfaces.md @@ -0,0 +1,189 @@ +# Phase 3 비공개 관리 UI 적용 절차 + +상세 설계와 실행 원장은 /home/donghyeon/workspace/docs/platform에서 관리합니다. +이 문서는 실제 적용 순서와 입력 문자열만 요약합니다. + +## 현재 완료 경계 + +2026-07-24 기준 선언형 YAML, 고정 차트 검증, dry-run 스크립트는 구현·검증했습니다. +pgAdmin bootstrap password는 다음 경로에 `0600`으로 생성했습니다. + +- /home/donghyeon/.secrets/pgadmin/bootstrap-password + +private DNS, Gitea CoreDNS 전환, 공유기 DNS와 Tailscale split DNS 등록, +Cloudflare DNS-01 관리 인증서, Keycloak 관리 OIDC, AIStor Console OIDC +profile, pgAdmin, Host Nginx cutover와 자동 수용 시험을 실제 적용하고 +검증했습니다. + +현재 AIStor는 `green`, pgAdmin Deployment는 `1/1`, PVC는 2Gi SSD Local PV에 +`Bound`입니다. 인증서 갱신 dry-run과 S3 bucket/object CRUD도 통과했습니다. +남은 단계는 브라우저 OIDC 수동 시험뿐입니다. + +- /home/donghyeon/.secrets/certbot/cloudflare.ini + +공유기 DHCP DNS와 Tailscale 관리 화면도 서버에서 자동 변경하지 않습니다. + +## 1. 정적 렌더 + + cd /home/donghyeon/workspace/platform + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/validate/render-admin-services.sh + +pgAdmin Chart archive와 OCI digest, AIStor ObjectStore Chart archive를 검증한 뒤 +Secret 리소스가 없는 manifest만 렌더합니다. + +## 2. Keycloak 외부망 시간 초과 대조 + +실패 단말에서 시각을 기록하고 다음을 실행합니다. + + date --iso-8601=seconds + dig @1.1.1.1 A id.learn.hyeonworks.com + dig @1.1.1.1 AAAA id.learn.hyeonworks.com + curl -4 -vkI --connect-timeout 10 https://id.learn.hyeonworks.com/ + curl -4 -vk --connect-timeout 10 \ + https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration + +같은 시간대 Host Nginx access log에 요청이 없으면 서버 설정이 아니라 실패 +단말의 VPN·보안 필터·통신사 경로 문제로 판정합니다. + +## 3. private DNS + +상태: 2026-07-24 실제 적용 완료. Gitea live `hostAliases` 제거도 완료. + +먼저 dry-run을 실행한 뒤 실제 적용합니다. + + bash scripts/bootstrap/apply-private-dns.sh + bash scripts/bootstrap/apply-private-dns.sh --execute + +확인 문자열은 APPLY default입니다. 이후 공유기 DHCP DNS를 192.168.0.107, +Tailscale의 learn.hyeonworks.com 제한 nameserver를 100.92.240.34로 등록합니다. +공개 DNS에는 두 admin 도메인을 만들지 않습니다. + +서버의 공유기는 TP-Link 계열로 식별됐습니다. 정확한 TP-Link·Tailscale 메뉴와 +검증 방법은 infrastructure/networking/private-dns/host/README.md를 따릅니다. + +CoreDNS 전환 뒤 Gitea OIDC profile을 다시 적용하면 기존 Pod의 임시 +hostAliases도 제거됩니다. + + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-gitea-oidc.sh --execute + +## 4. 운영자 Secret 파일 + +Cloudflare Dashboard의 `My Profile > API Tokens > Create Token`에서 +`Edit Zone DNS` 템플릿을 선택합니다. 권한은 `Zone:DNS:Edit`, Zone resource는 +`Include > Specific zone > hyeonworks.com`으로 제한합니다. + +공식 절차: +https://developers.cloudflare.com/fundamentals/api/get-started/create-token/ + +파일 내용은 `dns_cloudflare_api_token = ...` 한 줄이며 저장소에 만들지 않습니다. + + sudo install -d -o root -g root -m 0700 /home/donghyeon/.secrets/certbot + sudoedit /home/donghyeon/.secrets/certbot/cloudflare.ini + sudo chown root:root /home/donghyeon/.secrets/certbot/cloudflare.ini + sudo chmod 0600 /home/donghyeon/.secrets/certbot/cloudflare.ini + +pgAdmin 내부 비상 관리자 비밀번호는 이미 준비했습니다. 재생성이 필요한 경우에만 +다음을 실행합니다. 실행하면 기존 비밀번호 파일이 교체됩니다. + + install -d -m 0700 /home/donghyeon/.secrets/pgadmin + umask 077 + openssl rand -base64 32 | tr -d '\n' \ + > /home/donghyeon/.secrets/pgadmin/bootstrap-password + chmod 0600 /home/donghyeon/.secrets/pgadmin/bootstrap-password + +파일 값은 출력하거나 runbook에 복사하지 않습니다. + +## 5. admin 인증서 + +상태: 2026-07-24 실제 발급 완료. 만료일 2026-10-21, 자동 갱신 등록. + +서비스 적용 전에 exact SAN 두 개만 발급합니다. + + bash scripts/bootstrap/apply-host-nginx-admin.sh \ + --execute \ + --certificate-only \ + --certbot-email you@example.com + +확인 문자열은 APPLY입니다. 최초 실행은 공식 Cloudflare Certbot snap plugin을 +설치하고 deploy hook을 등록합니다. wildcard 인증서는 만들지 않습니다. + +## 6. Keycloak client와 그룹 + +상태: 2026-07-24 실제 적용 완료. `donghyeon.kang` 두 관리자 그룹 배정. + +먼저 dry-run으로 대상을 확인합니다. donghyeon.kang은 실제 realm 사용자명으로 +바꿀 수 있습니다. + + bash scripts/bootstrap/configure-keycloak-admin-oidc.sh + bash scripts/bootstrap/configure-keycloak-admin-oidc.sh \ + --execute \ + --object-admin donghyeon.kang \ + --db-admin donghyeon.kang + +확인 문자열은 APPLY default입니다. 두 OIDC Secret의 값은 출력하지 않습니다. + +## 7. 관리 서비스 + +상태: 2026-07-24 실제 적용 완료. + +최종 검증 결과: + + object-storage/minio-aistor: green, drivesOnline=1 + platform-admin/pgadmin: 1/1 + platform-admin/pgadmin PVC: Bound, pgadmin-data-local-pv + storage-admin Traefik HTTP: 200 + db-admin Traefik HTTP: 302 + + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-admin-services.sh \ + --execute \ + --pgadmin-password-file \ + /home/donghyeon/.secrets/pgadmin/bootstrap-password + +확인 문자열은 APPLY default입니다. 실패 시 AIStor spec은 복원하고 pgAdmin은 +0 replica로 내리지만 pgAdmin PVC/PV와 AIStor 데이터는 삭제하지 않습니다. + +적용 중 pgAdmin rollout 뒤 출력이 잠시 없으면 AIStor health를 확인하는 +구간입니다. 현재 스크립트는 15초마다 상태와 경과 시간을 출력합니다. 300초 뒤 +실패하면 자동 rollback하므로 강제로 다시 적용하기 전에 중앙 runbook의 +`pgAdmin 완료 뒤 AIStor health 대기` 항목을 확인합니다. + +## 8. Nginx 최종 전환 + +상태: 2026-07-24 실제 적용 완료. + +활성 설정 SHA-256: + + f07d558d7ebbe09bdd61f3422fe8dcdaaaa6d1160d0872c06538d2bee193908f + + bash scripts/bootstrap/apply-host-nginx-admin.sh --execute + +확인 문자열은 APPLY입니다. 설정을 timestamp backup한 뒤 LAN 요청의 정상 응답, +허용 목록 밖 loopback 요청의 403, unknown SNI 거부와 인증서 갱신 dry-run을 +검사합니다. + +## 9. 수용 시험 + +상태: 2026-07-24 자동 시험 완료. + + READ-ONLY CHECK PASS + AISTOR_S3_SMOKE_PASS + ADMIN UI SMOKE PASS + + bash scripts/validate/admin-ui-smoke.sh --execute --run-s3 + +확인 문자열은 APPLY이며 S3 하위 검사에서도 APPLY를 한 번 더 입력합니다. +재검증이 필요할 때만 다시 실행합니다. + +마지막으로 LAN 또는 Tailscale 연결 상태에서 브라우저로 다음을 시험합니다. + +- https://storage-admin.learn.hyeonworks.com + - `/platform-object-admins` 사용자의 Keycloak OIDC 로그인 성공 + - 해당 group이 없는 사용자의 관리 기능 접근 거부 +- https://db-admin.learn.hyeonworks.com + - `/platform-db-admins` 사용자의 Keycloak OIDC 로그인 성공 + - 해당 group이 없는 사용자의 로그인 거부 + +pgAdmin 내부 비상 관리자 로그인은 OIDC 장애 대응용으로만 유지합니다. diff --git a/bootstrap/manual/phase4-observability-access.md b/bootstrap/manual/phase4-observability-access.md new file mode 100644 index 0000000..75595df --- /dev/null +++ b/bootstrap/manual/phase4-observability-access.md @@ -0,0 +1,701 @@ +# Phase 4 관측성 접근·알림 전환 절차 + +상세 설계와 실행 원장은 `/home/donghyeon/workspace/docs/platform`에서 관리합니다. +이 문서는 현재 live substrate를 보존하면서 rules·alerts와 Host Nginx를 마지막에 +전환하는 권위 실행 순서를 요약합니다. Secret 값, token, Cookie, 사용자 ID와 +webhook 원문은 출력하거나 문서에 복사하지 않습니다. + +## 현재 완료 경계 + +2026-08-15 기준 Blackbox substrate, metric target과 rules-alerts는 live입니다. + +- Prometheus active target `30`, healthy `30`, unhealthy `0` +- Grafana와 Blackbox Exporter Deployment 각각 `1/1` Ready +- Host Nginx는 Grafana deny-only guard 상태 +- Grafana OIDC workload와 `grafana-keycloak-oidc` Secret 참조는 live +- Slack Secret과 risk deployment evidence는 live이며 off-host Slack DR은 deferred +- platform dashboard ConfigMap `5`, platform PrometheusRule `4`, + `AlertmanagerConfig/platform-alertmanager` `1`이 rollback ID `20260814T145009Z`에서 accepted +- generated receiver는 + `observability/platform-alertmanager/platform-slack` exact singleton +- full Nginx cutover와 browser OIDC·Slack firing/resolved·external-client acceptance는 미실행 + +rules-alerts 실행 때 다음 두 선행 조건은 모두 충족됐습니다. 이후 재실행이나 복구에서도 +같은 gate를 생략하지 않습니다. + +1. 아래 runbook URL이 HTTPS `200`으로 도달하고 모든 alert의 `runbook_url`과 일치한다. +2. 운영자가 만든 Slack webhook 입력 파일과 recovery evidence가 준비된다. + +```text +https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md +``` + +## 1. 공통 rollback transaction + +하나의 shell에서 같은 rollback ID를 끝까지 유지합니다. + +```bash +cd /home/donghyeon/workspace/platform + +OBS_ROLLBACK_ID="$(date -u +%Y%m%dT%H%M%SZ)" +[[ "$OBS_ROLLBACK_ID" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] +[[ "$OBS_ROLLBACK_ID" != 20260814T080303Z ]] +OBS_ROLLBACK_ROOT="/var/lib/hyeonworks/platform-rollbacks/observability-$OBS_ROLLBACK_ID" +sudo -n /usr/bin/test ! -e "$OBS_ROLLBACK_ROOT" +sudo -n /usr/bin/test ! -L "$OBS_ROLLBACK_ROOT" +sudo -n /usr/bin/mkdir --mode=0700 -- "$OBS_ROLLBACK_ROOT" +[[ "$(sudo -n /usr/bin/stat -c '%F|%u:%g|%a' -- "$OBS_ROLLBACK_ROOT")" == \ + 'directory|0:0|700' ]] +export PLATFORM_OBSERVABILITY_ROLLBACK_ID="$OBS_ROLLBACK_ID" +[[ "$PLATFORM_OBSERVABILITY_ROLLBACK_ID" == "$OBS_ROLLBACK_ID" ]] +``` + +pre-absence, 단 한 번의 `mkdir`, metadata 또는 active-ID equality가 실패하면 STOP하고 그 +collision/error ID를 재사용하지 않습니다. 자동 rollback 뒤에도 생성된 root를 삭제하지 않습니다. + +## 2. 기존 substrate와 deny guard 확인 + +```bash +bash scripts/bootstrap/apply-private-dns.sh +bash scripts/bootstrap/apply-host-nginx-observability.sh +bash scripts/validate/validate-blackbox-edge-source.sh +``` + +private DNS와 metrics/deny guard 자체를 다시 적용해야 할 때만 다음 mutation을 +순서대로 실행합니다. 각 명령의 화면 지시와 정확히 일치하는 확인 문자열만 입력합니다. + +```bash +: "${CERTBOT_EMAIL:?set the operator-managed Certbot contact email}" + +bash scripts/bootstrap/apply-private-dns.sh --execute +bash scripts/bootstrap/apply-host-nginx-observability.sh \ + --execute --metrics-guard-only + +metrics_status="$(curl --disable --silent --show-error --output /dev/null \ + --write-out '%{http_code}' \ + --resolve git.learn.hyeonworks.com:443:127.0.0.1 \ + https://git.learn.hyeonworks.com/metrics)" +[[ "$metrics_status" == 404 ]] + +bash scripts/bootstrap/apply-host-nginx-observability.sh \ + --execute --certificate-only --certbot-email "$CERTBOT_EMAIL" +bash scripts/bootstrap/apply-host-nginx-observability.sh \ + --execute --grafana-deny-guard-only +``` + +DNS/guard를 다시 적용했는지와 무관하게, **새 rollback ID마다** Blackbox source +proof를 반드시 새로 만듭니다. Task 6 retry에서는 failed ID `20260814T080303Z`를 거부하고 +controller가 발급한 fresh ID와 현재 active environment가 exact equality인지 먼저 확인합니다. + +```bash +FRESH_TASK6_ROLLBACK_ID="$OBS_ROLLBACK_ID" +[[ "$FRESH_TASK6_ROLLBACK_ID" != 20260814T080303Z ]] +[[ "${PLATFORM_OBSERVABILITY_ROLLBACK_ID:?active rollback ID is required}" == \ + "$FRESH_TASK6_ROLLBACK_ID" ]] +set +e +bash scripts/validate/validate-blackbox-edge-source.sh \ + --execute --context default +BLACKBOX_RC=$? +set -e +printf 'BLACKBOX_RC=%d\n' "$BLACKBOX_RC" +[[ "$BLACKBOX_RC" -eq 0 ]] +[[ "$(sudo -n /usr/bin/stat -c '%F|%u:%g|%a|%h' -- \ + "/var/lib/hyeonworks/platform-rollbacks/observability-${FRESH_TASK6_ROLLBACK_ID}/blackbox-source-proof.env")" == \ + 'regular file|0:0|600|1' ]] +``` + +이 proof는 같은 rollback ID, active deny hash, 24시간 이내 시각과 세 private +hostname의 exact `403`에 결속되어야 합니다. validator는 한 번만 호출하고 operator가 exact +`PROVE BLACKBOX PRIVATE EDGE default`를 입력한 뒤 `BLACKBOX PRIVATE EDGE SOURCE PASS`, immediate +RC `0`, normalized proof metadata `regular|0:0|600|1`를 모두 확인합니다. 하나라도 실패하거나 +불명확하면 STOP하고 new ID를 보존하며 같은 ID로 validator나 Task 6를 재시도하지 않습니다. +proof content는 읽지 않습니다. + +preflight/live residue는 absolute zero가 아니라 다음 attested preexisting name-only baseline의 +unchanged 계약입니다. + +```text +/tmp/platform-k3s-encryption.Mskzy3 +/tmp/platform-observability-access-apply.oeNcfI +/tmp/platform-observability-access-apply.Im02dz +/tmp/platform-observability-slack-gate.LYhYbv +``` + +`Mskzy3`는 8/1 empty evidence, `oeNcfI`/`Im02dz`는 8/12 recorded evidence, +`LYhYbv`는 private filename 두 개만 attested된 failed-live evidence입니다. 네 root는 content를 +읽거나 삭제하지 않습니다. baseline name set unchanged, matching executable process `0`, current +preflight/live newly-created matching-root delta `0`을 요구합니다. unknown/new root는 broad +delete하지 않고 STOP/identity review합니다. + +## 3. Secret과 Grafana OIDC recovery evidence + +먼저 K3s encryption과 restore evidence를 각각 새 process에서 검사합니다. + +```bash +bash scripts/validate/k3s-secret-encryption.sh --expect-reencrypted +bash scripts/validate/k3s-secret-encryption-restore-evidence.sh --check +``` + +입력 파일은 현재 사용자 소유 `0600`, non-symlink, link count 1이어야 합니다. +값을 shell 변수, argv, stdout 또는 runbook에 넣지 않습니다. + +```bash +bash scripts/bootstrap/create-observability-secrets.sh \ + --execute --grafana-admin \ + --grafana-admin-user-file /home/donghyeon/.secrets/grafana/admin-user \ + --grafana-admin-password-file /home/donghyeon/.secrets/grafana/admin-password +``` + +Slack Secret bootstrap 직전에 별도 내장 Windows SSD의 기존 KDBX에 same-host encrypted +disaster-recovery copy를 준비합니다. 이것은 일반 K3s restart나 host reboot용 사본이 아니라 +datastore·Secret·bootstrap state 손실 때를 위한 것입니다. 기본 no-argument 실행은 고정 +contract만 출력하며 SSD, KDBX, webhook, sudo에 접근하지 않습니다. 지원되는 interface는 +다음 두 개뿐입니다. + +```bash +bash scripts/bootstrap/backup-slack-webhook-recovery.sh +bash scripts/bootstrap/backup-slack-webhook-recovery.sh \ + --execute \ + --slack-webhook-file /home/donghyeon/.secrets/alertmanager/slack-webhook +``` + +execute에서 `SLACK_KEEPASS_RECOVERY=NOOP`는 exact entry의 verified read-only no-op이고, +`SLACK_KEEPASS_RECOVERY=COMMITTED`는 durable pre-change backup을 만든 verified commit입니다. +둘 다 source-based unmount proof와 private work/socket/helper cleanup 뒤에만 성공하며 +`WINDOWS_SSD_UNMOUNTED=PASS`, `OFF_HOST_RECOVERY_SATISFIED=NO`를 출력합니다. lost response, +post-commit verification failure 또는 cleanup/unmount ambiguity는 자동 재시도하지 않고 +`SLACK_KEEPASS_RECOVERY=MANUAL_RECOVERY_REQUIRED`로 중단하며 main과 backup을 보존합니다. +webhook payload, KeePassXC master password, hash·encoding·size·URL component 또는 protected +KDBX output을 terminal, argv, environment, log, runbook이나 plaintext 파일에 남기지 않습니다. + +This local encrypted copy does not authorize RECOVERY SLACK default when the +approved gate requires off-host escrow. Do not continue the Secret bootstrap +until that independent prerequisite is literally true. + +Slack에는 서로 다른 두 경로가 있습니다. off-host disaster recovery를 완료로 판정할 때만 +strict recovery evidence를 검사합니다. + +```bash +bash scripts/bootstrap/create-observability-secrets.sh \ + --check-slack-recovery-evidence +``` + +현재 사용자가 승인한 operational risk path는 off-host Slack DR이 아직 deferred인 사실을 +기록하고 deployment evidence를 만듭니다. 이 경로는 DR-complete을 주장하지 않습니다. + +```bash +bash scripts/bootstrap/create-observability-secrets.sh \ + --execute --slack-webhook \ + --slack-webhook-file /home/donghyeon/.secrets/alertmanager/slack-webhook \ + --accept-no-off-host-slack-recovery +``` + +도구가 요구하는 정확한 확인은 `ACCEPT NO OFF-HOST SLACK RECOVERY default`입니다. +이 risk path 밖에서 kubectl로 Secret을 수동 생성하지 않습니다. off-host 복구 증거가 +없을 때 거짓 `RECOVERY SLACK default` 확인을 입력하지 않습니다. + +위 risk path 또는 실제 off-host recovery evidence가 준비된 경우 Slack Secret bootstrap은 +이미 완료된 상태이므로, deployment gate와 다음 checker만 실행합니다. + +```bash +bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh --execute +bash scripts/bootstrap/create-observability-secrets.sh \ + --check-grafana-recovery-evidence +bash scripts/bootstrap/create-observability-secrets.sh \ + --check-slack-deployment-evidence +bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh \ + --check-recovery-evidence +``` + +기존 Secret의 payload가 다르면 자동 rotation하지 않고 중단합니다. UID drift나 API +결과 불명도 자동 삭제로 처리하지 않습니다. + +Grafana admin/OIDC object가 이미 exact live state이면 도구는 credential을 회전하거나 +workload를 다시 쓰지 않고 기존 payload를 재사용하며 recovery evidence만 검증·갱신합니다. +exact state가 아닌데 권위 prior와 ownership을 증명할 수 없으면 자동 수렴시키지 않습니다. + +## 4. 권위 inventory로 rules-alerts handoff 생성 + +현재 Blackbox substrate가 이미 live이므로 `target-initial`을 지금 다시 캡처하지 +않습니다. 실행 당시 보존한 두 phase만 새 `0700` output root에 복제하고 renderer가 +schema, phase, mode, link count와 hash를 다시 검증하게 합니다. + +먼저 공개 runbook이 실제로 게시되었는지 확인합니다. `200`이 아니면 renderer와 +rules-alerts apply를 실행하지 않습니다. + +```bash +RUNBOOK_URL='https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' +runbook_status="$(curl --disable --silent --show-error --location --output /dev/null \ + --write-out '%{http_code}' --connect-timeout 3 --max-time 10 \ + "$RUNBOOK_URL")" +[[ "$runbook_status" == 200 ]] +``` + +```bash +SOURCE_METRIC_ROOT=/tmp/platform-observability-metrics.VUpsZn +METRIC_ROOT="$(mktemp -d /tmp/platform-observability-metrics.XXXXXX)" +chmod 0700 "$METRIC_ROOT" + +[[ "$(realpath --canonicalize-existing -- "$SOURCE_METRIC_ROOT")" == "$SOURCE_METRIC_ROOT" ]] +[[ "$(stat -c '%F|%u:%g|%a|%h' -- "$SOURCE_METRIC_ROOT")" == \ + 'directory|1000:1000|700|4' ]] +declare -A SOURCE_METRIC_IDENTITY=() +for phase in target-initial post-substrate; do + [[ -d "$SOURCE_METRIC_ROOT/$phase" && ! -L "$SOURCE_METRIC_ROOT/$phase" ]] + [[ "$(realpath --canonicalize-existing -- "$SOURCE_METRIC_ROOT/$phase")" == \ + "$SOURCE_METRIC_ROOT/$phase" ]] + [[ "$(stat -c '%F|%u:%g|%a|%h' -- "$SOURCE_METRIC_ROOT/$phase")" == \ + 'directory|1000:1000|700|2' ]] + install -d -m 0700 -- "$METRIC_ROOT/$phase" + for file in inventory.json inventory.sha256; do + source_file="$SOURCE_METRIC_ROOT/$phase/$file" + destination_file="$METRIC_ROOT/$phase/$file" + [[ -f "$source_file" && ! -L "$source_file" ]] + [[ "$(realpath --canonicalize-existing -- "$source_file")" == "$source_file" ]] + [[ "$(stat -c '%F|%u:%g|%a|%h' -- "$source_file")" == \ + 'regular file|1000:1000|600|1' ]] + SOURCE_METRIC_IDENTITY["$phase/$file"]="$(stat -c '%d:%i|%F|%u:%g|%a|%h|%s|%Y|%Z' -- \ + "$source_file")|$(sha256sum -- "$source_file" | awk '{print $1}')" + cp --no-dereference --reflink=never -- "$source_file" "$destination_file" + chmod 0600 "$destination_file" + [[ "$(stat -c '%F|%u:%g|%a|%h' -- "$destination_file")" == \ + 'regular file|1000:1000|600|1' ]] + cmp -s -- "$source_file" "$destination_file" + done +done + +[[ "$(find "$METRIC_ROOT" -mindepth 1 -maxdepth 2 -printf '%P\n' | LC_ALL=C sort)" == \ + $'post-substrate\npost-substrate/inventory.json\npost-substrate/inventory.sha256\ntarget-initial\ntarget-initial/inventory.json\ntarget-initial/inventory.sha256' ]] +for phase in target-initial post-substrate; do + for file in inventory.json inventory.sha256; do + source_file="$SOURCE_METRIC_ROOT/$phase/$file" + [[ "$(stat -c '%d:%i|%F|%u:%g|%a|%h|%s|%Y|%Z' -- "$source_file")|$(sha256sum -- \ + "$source_file" | awk '{print $1}')" == "${SOURCE_METRIC_IDENTITY["$phase/$file"]}" ]] + done +done + +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/validate/render-observability-access.sh \ + --component rules-alerts --verified-output-dir "$METRIC_ROOT" +``` + +failed transaction handoff `/tmp/platform-observability-metrics.LNzksC`는 read-only immutable +evidence로 보존하고 source, destination 또는 live apply input으로 재사용하지 않습니다. +기존 attested path/fingerprint identity만 보존·비교하며 inventory body나 private file content를 +다시 읽지 않습니다. fresh destination은 renderer 전 exact two phase directories만 가집니다. +renderer와 live apply는 이 절에서 byte-preserving copy와 metadata/hash/count revalidation을 +마친 fresh `$METRIC_ROOT`만 사용합니다. + +위 exact six-entry gate는 renderer 전 destination이 두 phase directory와 네 file만 갖는지 +확인합니다. 각 source/destination pair는 byte-equal이며 copy 뒤 source inode/metadata/size/hash가 +copy 전 fingerprint와 같아야 합니다. 이어지는 renderer는 current production pins와 exact target +counts `21/30`, checksum/schema/semantic contract, 그리고 reviewed rendered manifest set을 다시 +검증합니다. source/destination metadata, fingerprint, byte equality, pins, counts 또는 entry set +중 하나라도 다르면 fresh root를 apply input으로 사용하지 않고 STOP합니다. + +권위 inventory hash는 다음과 같습니다. + +```text +target-initial: 79688d017d38eec9a6f100f8d0f784a5474e79802046ef1c2c11b30d170b0b0c +post-substrate: b1c3049206a1a88165ee672ae9aceac7945673a3bb9c3cf3670b7f0d56c3f291 +``` + +이 두 SHA는 현재 cluster freshness artifact가 아니라 변경할 수 없는 historical +metric/label provenance pair입니다. `target-initial`과 `post-substrate`는 각각 exact +target count `21`과 `30`을 initial gate와 confirmation 뒤 first mutation 직전 +last gate 모두에서 다시 검증합니다. `captured_at_utc`는 exact UTC-second +형식과 유효한 UTC calendar로 parse되어야 하며, 현재보다 300초를 초과해 +미래인 시각만 거부합니다. 이 exact pair에는 과거 방향 24시간 상한을 +적용하지 않으며, timestamp나 checksum을 현재 시각에 맞게 다시 쓰거나 +inventory를 재수집해서는 안 됩니다. 이 예외는 2절 Blackbox source proof의 +기존 24시간 freshness 계약에는 적용되지 않습니다. + +## 5. rules-alerts 적용 — 2026-08-15 terminal PASS + +먼저 no-argument dry-run과 focused test를 실행합니다. 둘 중 하나라도 끝나지 않거나 +실패하면 mutation을 실행하지 않습니다. + +```bash +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-observability-access.sh +bash scripts/validate/test-apply-observability-access.sh +``` + +성공한 뒤에만 다음을 실행합니다. + +```bash +case $- in *e*) TASK6_APPLY_ERREXIT_WAS_SET=1 ;; *) TASK6_APPLY_ERREXIT_WAS_SET=0 ;; esac +set +e +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm bash scripts/bootstrap/apply-observability-access.sh --execute --rules-alerts --verified-output-dir "$METRIC_ROOT" +TASK6_APPLY_RC=$? +printf 'TASK6_APPLY_RC=%d\n' "$TASK6_APPLY_RC" +(( TASK6_APPLY_ERREXIT_WAS_SET == 0 )) || set -e +[[ "$TASK6_APPLY_RC" -eq 0 ]] +``` + +operator만 exact `APPLY` confirmation을 입력합니다. apply는 위 exact one-line command로 한 번만 +실행하고, 바로 다음 statement가 다른 command 없이 `TASK6_APPLY_RC=$?`를 capture합니다. +immediate printed RC `0`과 exact `OBSERVABILITY_ACCESS_RULES_ALERTS=PASS`를 모두 확인한 경우에만 +후속 acceptance를 진행합니다. nonzero, missing/ambiguous RC 또는 PASS, response loss, rollback +ambiguity는 STOP하고 rollback ID와 evidence를 보존하며 같은 ID로 재시도하지 않습니다. + +성공 조건은 다음 전부입니다. + +- Prometheus와 Alertmanager owner/controller Ready +- Prometheus API의 desired alert·record exact set과 evaluation health 정상 +- Alertmanager generated config의 `observability/platform-alertmanager/platform-slack` receiver exact singleton +- Grafana sidecar의 exact dashboard 5개와 source content hash 일치 +- 기존 target·Grafana·Blackbox·Probe·Ingress·PVC·Secret 보존 +- acceptance marker는 모든 증거 뒤에만 root-only로 기록 + +apply 도구는 deployment checker를 confirmation 전과 Slack Secret-consuming mutation 직전에 +두 번 호출한다. rules-alerts acceptance ledger schema는 +`platform-observability-rules-alerts-v2`이며 Slack deployment gate 값은 bare `RECOVERY` 또는 +`RISK_ACCEPTED`만 기록한다. 이는 operational acceptance이며 off-host Slack DR-complete을 +의미하지 않는다. + +실제 terminal transaction은 fresh rollback ID `20260814T145009Z`와 fresh handoff +`/tmp/platform-observability-metrics.dw5gLZ`를 사용했다. argv는 exact six-element array로 +attest됐고, operator가 exact `APPLY`를 입력한 단 한 번의 실행에서 다음 safe marker를 확인했다. + +```text +target-initial SHA-256 = 79688d017d38eec9a6f100f8d0f784a5474e79802046ef1c2c11b30d170b0b0c +post-substrate SHA-256 = b1c3049206a1a88165ee672ae9aceac7945673a3bb9c3cf3670b7f0d56c3f291 +OBSERVABILITY_ACCESS_RULES_ALERTS=PASS +TASK6_APPLY_RC=0 +``` + +payload-free terminal audit는 dashboard `5`, platform PrometheusRule `4`, 전체 desired rule +`23`(`22` alerts + `1` recording) healthy, runbook URL `22/22`, AlertmanagerConfig `1`, exact +NetworkPolicy, target `30/30`, Grafana·Blackbox·Prometheus·Alertmanager Ready와 qualified receiver +exact singleton을 확인했다. acceptance schema와 `RISK_ACCEPTED` gate, root-only ledger의 +object/mutation line `13/13` 및 metadata contract도 통과했다. 성공 transaction에는 rollback이 +호출되지 않았고 rollback root와 handoff는 Task 7 종료까지 보존한다. + +failed rollback ID `20260814T080303Z`와 argument paste가 파싱 전에 중단된 +`20260814T140953Z`는 immutable evidence로 보존하고 재사용하지 않는다. 후자는 +`--verified-output-dir` token이 줄바꿈으로 분리돼 usage RC `2`, shell-level RC `127`로 끝났으며 +ledger·acceptance·Kubernetes mutation은 생성되지 않았다. + +## 6. Task 7 operator boundary와 Host Nginx first cutover + +Task 7은 성공 rollback ID `20260814T145009Z`와 original handoff +`/tmp/platform-observability-metrics.dw5gLZ`를 그대로 보존합니다. active state는 exact deny-only +SHA-256 `dbef6d443bcba58b26a5351ea76f6d09f6da8c2ef07a806e22745cf26c88f518`, +desired full은 `7d2de2a92c3597a0859775da1d2ccbf5a3d72c0b2af5cac2439c82311361f801`여야 +합니다. full이 이미 active이거나 third state이면 STOP합니다. + +어떤 external preparation command보다 먼저 fixed PATH를 export하고 command cache를 비운 뒤 +reviewed command/launcher inventory를 byte-equal로 재검증합니다. ambient PATH command로 prep를 +시작하지 않습니다. exact canonical `/usr/bin/sudo`는 `regular|0:0|4755|1`을 요구하는 +유일한 owner-setuid 예외입니다. setgid·group/world write는 금지되고 다른 allowlisted +executable은 setuid/setgid를 모두 금지합니다. + +```bash +TASK7_OPERATOR_PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +PATH=$TASK7_OPERATOR_PATH +export PATH +hash -r + +cd /home/donghyeon/workspace/platform +TASK7_ID=20260814T145009Z +TASK7_METRIC_ROOT=/tmp/platform-observability-metrics.dw5gLZ +METRIC_ROOT=$TASK7_METRIC_ROOT +export PLATFORM_OBSERVABILITY_ROLLBACK_ID="$TASK7_ID" +[[ "$PLATFORM_OBSERVABILITY_ROLLBACK_ID" == "$TASK7_ID" ]] +[[ "$METRIC_ROOT" == "$TASK7_METRIC_ROOT" ]] +[[ "$(/usr/bin/readlink -f -- "$TASK7_METRIC_ROOT")" == "$TASK7_METRIC_ROOT" ]] + +TASK7_HOST_DRY=(/usr/bin/bash) +TASK7_HOST_DRY+=(/home/donghyeon/workspace/platform/scripts/bootstrap/apply-host-nginx-observability.sh) +[[ "${#TASK7_HOST_DRY[@]}" -eq 2 ]] +case $- in *e*) TASK7_HOST_DRY_ERREXIT=1 ;; *) TASK7_HOST_DRY_ERREXIT=0 ;; esac +set +e +"${TASK7_HOST_DRY[@]}" +TASK7_HOST_DRY_RC=$? +printf 'TASK7_HOST_DRY_RC=%d\n' "$TASK7_HOST_DRY_RC" +(( TASK7_HOST_DRY_ERREXIT == 0 )) || set -e +``` + +no-arg는 source/hash/no-contact만 검증합니다. certificate/SAN, DNS, Kubernetes, proof +ID/age, NodePort와 network boundary는 execute 내부의 `APPLY` prompt 전 live gate입니다. +dry-run은 Host mutation `0`과 다음 exact output을 요구합니다. + +```text +HOST_NGINX_ACTIVE_SHA256=dbef6d443bcba58b26a5351ea76f6d09f6da8c2ef07a806e22745cf26c88f518 +HOST_NGINX_GRAFANA_DENY_GUARD_SHA256=dbef6d443bcba58b26a5351ea76f6d09f6da8c2ef07a806e22745cf26c88f518 +HOST_NGINX_FULL_SHA256=7d2de2a92c3597a0859775da1d2ccbf5a3d72c0b2af5cac2439c82311361f801 +HOST_NGINX_CERTIFICATE_EXPECTED_SAN=grafana.learn.hyeonworks.com +HOST_NGINX_CERTIFICATE_SAN=NOT_CHECKED_DRY_RUN +HOST_NGINX_GRAFANA_PUBLIC_DNS=NOT_CHECKED_DRY_RUN +HOST_NGINX_OBSERVABILITY_DRY_RUN=PASS +TASK7_HOST_DRY_RC=0 +``` + +`HOST_NGINX_CERTIFICATE_EXACT_SAN=`나 `HOST_NGINX_GRAFANA_PUBLIC_DNS=ABSENT`를 dry-run +결과로 받으면 STOP합니다. source proof는 exact ID/deny hash/status/IP/time에 bind되며 +과거 24시간, 미래 300초 경계를 벗어나면 fresh rollback ID, fresh proof, complete Task 6를 +다시 수행합니다. 같은 ID에서 proof만 바꾸거나 timestamp를 다시 쓰지 않습니다. + +execute가 `APPLY`를 표시하기 전에 다음 네 path가 각각 `test -e`/`test -L` 모두에서 +absent임을 no-follow, name-only 순서로 입증해야 합니다. + +```text +/var/lib/hyeonworks/platform-rollbacks/observability-20260814T145009Z/host-nginx +/var/lib/hyeonworks/platform-rollbacks/observability-20260814T145009Z/host-nginx/stages.tsv +/var/lib/hyeonworks/platform-rollbacks/observability-20260814T145009Z/host-nginx/payloads +/var/lib/hyeonworks/platform-rollbacks/observability-20260814T145009Z/host-nginx/payloads/full-prior-0001.conf +``` + +일반 directory/file로 남은 ledger/payload도 reusable recovery state가 아니라 STOP residue입니다. +reviewed sudo identity를 다시 확인한 뒤에만 operator가 credential를 refresh합니다. + +```bash +/usr/bin/sudo -v +/usr/bin/sudo -n /usr/bin/true +printf 'SUDO_READY\n' + +TASK7_VOD=--verified +TASK7_VOD+=-output-dir +TASK7_HOST=(/usr/bin/bash) +TASK7_HOST+=(/home/donghyeon/workspace/platform/scripts/bootstrap/apply-host-nginx-observability.sh) +TASK7_HOST+=(--execute) +TASK7_HOST+=("$TASK7_VOD") +TASK7_HOST+=("$TASK7_METRIC_ROOT") +[[ "${#TASK7_HOST[@]}" -eq 5 ]] +printf 'TASK7_HOST_ARGC=%d\n' "${#TASK7_HOST[@]}" +case $- in *e*) TASK7_HOST_ERREXIT=1 ;; *) TASK7_HOST_ERREXIT=0 ;; esac +set +e +"${TASK7_HOST[@]}" +TASK7_HOST_RC=$? +printf 'TASK7_HOST_RC=%d\n' "$TASK7_HOST_RC" +(( TASK7_HOST_ERREXIT == 0 )) || set -e +``` + +operator만 exact `APPLY`를 입력합니다. 성공은 exact `HOST_NGINX_FULL_STAGE=PASS`와 +`TASK7_HOST_RC=0`이 모두 있을 때뿐입니다. `ALREADY_ACTIVE`, missing/ambiguous marker, +nonzero RC, response loss는 모두 실패이며 같은 ID로 재실행하지 않습니다. + +1. prompt 전 실패: active deny unchanged, Host ledger/mutation `0`, rollback N/A. +2. prompt 뒤 `rollback_armed=true` 전 실패: active config/reload mutation `0`, root-owned + ledger/payload 또는 timestamp backup staging은 남을 수 있으며 rollback N/A. 전체 evidence를 + 보존합니다. +3. active install 뒤 실패: exact `HOST_NGINX_OBSERVABILITY_ROLLBACK=PASS`와 deny hash + 복원을 요구합니다. +4. `ROLLBACK=FAIL`, `MANUAL_RECOVERY_REQUIRED=YES`, unknown stage/hash: 모든 후속 gate를 + STOP합니다. + +`stages.tsv`는 prior-payload recovery ledger이지 success marker가 아닙니다. ledger/payload/timestamp +staging 생성·검증·설치 중 하나라도 실패하면 현 ID/root를 보존하고 fresh ID, +source proof, complete Task 6를 다시 수행합니다. staged evidence를 repair/reuse하거나 +failed fresh ID를 재사용하지 않습니다. + +## 7. OIDC membership과 사람/external readiness + +Host PASS 뒤 mutation 전에 서로 다른 admin, viewer, no-group, membership-removal test +identity, local break-glass 접근, Slack firing/resolved view, LAN/Tailscale 밖 proxy-disabled +external client를 모두 준비합니다. 하나라도 없으면 membership와 smoke를 시작하지 +않습니다. username은 stdin으로만 받고 기록하지 않습니다. + +```bash +read -r -p 'Grafana organization admin realm username: ' OBS_ADMIN_USER +read -r -p 'Grafana viewer realm username: ' OBS_VIEWER_USER +TASK7_OIDC=(/usr/bin/bash /home/donghyeon/workspace/platform/scripts/bootstrap/configure-keycloak-grafana-oidc.sh) +TASK7_OIDC+=(--execute) +TASK7_OIDC+=(--admin "$OBS_ADMIN_USER") +TASK7_OIDC+=(--viewer "$OBS_VIEWER_USER") +case $- in *e*) TASK7_OIDC_ERREXIT=1 ;; *) TASK7_OIDC_ERREXIT=0 ;; esac +set +e +"${TASK7_OIDC[@]}" +TASK7_OIDC_RC=$? +unset OBS_ADMIN_USER OBS_VIEWER_USER TASK7_OIDC +printf 'TASK7_OIDC_RC=%d\n' "$TASK7_OIDC_RC" +(( TASK7_OIDC_ERREXIT == 0 )) || set -e +``` + +operator는 exact `APPLY default`와 `RECOVERY KEYCLOAK default`를 입력합니다. +`GRAFANA_OIDC_TRANSACTION=PASS`와 RC `0`을 모두 요구합니다. +`transaction_active=true` 전 실패는 managed Keycloak/OIDC Secret/membership mutation `0`, +rollback N/A입니다. active failure는 exact `GRAFANA_OIDC_ROLLBACK=PASS`를 요구합니다. +rollback FAIL, manual recovery 또는 unknown stage는 STOP입니다. OIDC rollback은 실행 중 private +snapshot을 사용하는 in-process rollback입니다. 성공 뒤 복원용 persistent Task 6 Keycloak +reversal ledger가 있다고 주장하거나 탐색하지 않습니다. + +## 8. monolithic observability smoke exactly once + +no-arg는 acceptance가 아닌 계획 확인으로 한 번만 실행합니다. + +```bash +TASK7_SMOKE_DRY=(/usr/bin/bash) +TASK7_SMOKE_DRY+=(/home/donghyeon/workspace/platform/scripts/validate/observability-smoke.sh) +[[ "${#TASK7_SMOKE_DRY[@]}" -eq 2 ]] +case $- in *e*) TASK7_SMOKE_DRY_ERREXIT=1 ;; *) TASK7_SMOKE_DRY_ERREXIT=0 ;; esac +set +e +"${TASK7_SMOKE_DRY[@]}" +TASK7_SMOKE_DRY_RC=$? +printf 'TASK7_SMOKE_DRY_RC=%d\n' "$TASK7_SMOKE_DRY_RC" +(( TASK7_SMOKE_DRY_ERREXIT == 0 )) || set -e +``` + +RC `0`, `OBSERVABILITY_SMOKE_DRY_RUN=PASS`, `HUMAN_EXTERNAL_CLIENT=required`, +`MUTATION=NOT_REQUESTED`를 요구합니다. 그 뒤 machine, OIDC human/session, Slack +firing/resolved, true external-client attestation을 하나의 execute에서만 수행합니다. + +```bash +TASK7_SMOKE=(/usr/bin/bash) +TASK7_SMOKE+=(/home/donghyeon/workspace/platform/scripts/validate/observability-smoke.sh) +TASK7_SMOKE+=(--execute) +case $- in *e*) TASK7_SMOKE_ERREXIT=1 ;; *) TASK7_SMOKE_ERREXIT=0 ;; esac +set +e +"${TASK7_SMOKE[@]}" +TASK7_SMOKE_RC=$? +printf 'TASK7_SMOKE_RC=%d\n' "$TASK7_SMOKE_RC" +(( TASK7_SMOKE_ERREXIT == 0 )) || set -e +``` + +operator만 requested identity와 exact dynamic confirmation을 입력합니다. 성공은 다음 전체 +marker와 immediate RC를 요구합니다. + +```text +OBSERVABILITY_MACHINE_ACCEPTANCE=PASS +OBSERVABILITY_OIDC_ACCEPTANCE=PASS +OBSERVABILITY_SLACK_ACCEPTANCE=PASS +OBSERVABILITY_EXTERNAL_BOUNDARY=PASS +OBSERVABILITY_SMOKE=PASS +TASK7_SMOKE_RC=0 +``` + +RC `2` 또는 `OBSERVABILITY_EXTERNAL_BOUNDARY=PARTIAL`은 Task 7을 `부분 구현`으로 남깁니다. +external result를 server-side, LAN 또는 Tailscale probe로 대체하지 않습니다. cleanup ambiguity는 +owned-object review 전 automatic rerun을 금지합니다. + +## 9. fresh inventory-only renderer root와 단일 회귀 pass + +fixed PATH를 다시 설치하고 `hash -r`, command inventory byte equality를 external prep 전에 +확인합니다. original `dw5gLZ`의 canonical path, owner/mode/nlink, exact entry set, two +inventory hash와 three Task 6 YAML fingerprint를 보존합니다. complete publication에 original +root를 사용하지 않습니다. + +```bash +TASK7_RENDER_ROOT="$(/usr/bin/mktemp -d /tmp/platform-observability-metrics.XXXXXX)" +/usr/bin/chmod 0700 "$TASK7_RENDER_ROOT" +for phase in target-initial post-substrate; do + /usr/bin/install -d -m 0700 -- "$TASK7_RENDER_ROOT/$phase" + for file in inventory.json inventory.sha256; do + source_file="$TASK7_METRIC_ROOT/$phase/$file" + destination_file="$TASK7_RENDER_ROOT/$phase/$file" + [[ -f "$source_file" && ! -L "$source_file" ]] + /usr/bin/cp --no-dereference --reflink=never -- "$source_file" "$destination_file" + /usr/bin/chmod 0600 "$destination_file" + /usr/bin/cmp -s -- "$source_file" "$destination_file" + done +done +unset source_file destination_file +``` + +destination은 exact six-entry topology, current owner, root/phase `0700`, file `0600`, nlink `1`, +byte equality와 known inventory hash를 요구합니다. copy 뒤 original fingerprint가 unchanged여야 +합니다. 실패한 fresh root는 evidence로 보존하고 repair/reuse하지 않습니다. + +core와 complete renderer, admin UI, AIStor S3, phase1, phase2, admin renderer를 다음 exact +array/envelope로 각각 한 번만 실행합니다. + +```bash +TASK7_CORE=(/usr/bin/bash) +TASK7_CORE+=(/home/donghyeon/workspace/platform/scripts/validate/render-observability-core.sh) +[[ "${#TASK7_CORE[@]}" -eq 2 ]] +case $- in *e*) TASK7_CORE_ERREXIT=1 ;; *) TASK7_CORE_ERREXIT=0 ;; esac +set +e +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm "${TASK7_CORE[@]}" +TASK7_CORE_RC=$? +(( TASK7_CORE_ERREXIT == 0 )) || set -e +printf 'TASK7_CORE_RC=%d\n' "$TASK7_CORE_RC" + +TASK7_VOD=--verified +TASK7_VOD+=-output-dir +TASK7_COMPLETE=(/usr/bin/bash) +TASK7_COMPLETE+=(/home/donghyeon/workspace/platform/scripts/validate/render-observability-access.sh) +TASK7_COMPLETE+=(--component complete) +TASK7_COMPLETE+=("$TASK7_VOD" "$TASK7_RENDER_ROOT") +[[ "${#TASK7_COMPLETE[@]}" -eq 6 ]] +case $- in *e*) TASK7_COMPLETE_ERREXIT=1 ;; *) TASK7_COMPLETE_ERREXIT=0 ;; esac +set +e +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm "${TASK7_COMPLETE[@]}" +TASK7_COMPLETE_RC=$? +(( TASK7_COMPLETE_ERREXIT == 0 )) || set -e +printf 'TASK7_COMPLETE_RC=%d\n' "$TASK7_COMPLETE_RC" + +TASK7_ADMIN_UI=(/usr/bin/bash) +TASK7_ADMIN_UI+=(/home/donghyeon/workspace/platform/scripts/validate/admin-ui-smoke.sh) +[[ "${#TASK7_ADMIN_UI[@]}" -eq 2 ]] +case $- in *e*) TASK7_ADMIN_UI_ERREXIT=1 ;; *) TASK7_ADMIN_UI_ERREXIT=0 ;; esac +set +e +"${TASK7_ADMIN_UI[@]}" +TASK7_ADMIN_UI_RC=$? +(( TASK7_ADMIN_UI_ERREXIT == 0 )) || set -e +printf 'TASK7_ADMIN_UI_RC=%d\n' "$TASK7_ADMIN_UI_RC" + +TASK7_AISTOR_S3=(/usr/bin/bash) +TASK7_AISTOR_S3+=(/home/donghyeon/workspace/platform/scripts/validate/aistor-s3-smoke.sh) +TASK7_AISTOR_S3+=(--execute) +[[ "${#TASK7_AISTOR_S3[@]}" -eq 3 ]] +case $- in *e*) TASK7_AISTOR_S3_ERREXIT=1 ;; *) TASK7_AISTOR_S3_ERREXIT=0 ;; esac +set +e +"${TASK7_AISTOR_S3[@]}" +TASK7_AISTOR_S3_RC=$? +(( TASK7_AISTOR_S3_ERREXIT == 0 )) || set -e +printf 'TASK7_AISTOR_S3_RC=%d\n' "$TASK7_AISTOR_S3_RC" + +TASK7_PHASE1=(/usr/bin/bash) +TASK7_PHASE1+=(/home/donghyeon/workspace/platform/scripts/validate/render-phase1.sh) +[[ "${#TASK7_PHASE1[@]}" -eq 2 ]] +case $- in *e*) TASK7_PHASE1_ERREXIT=1 ;; *) TASK7_PHASE1_ERREXIT=0 ;; esac +set +e +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm "${TASK7_PHASE1[@]}" +TASK7_PHASE1_RC=$? +(( TASK7_PHASE1_ERREXIT == 0 )) || set -e +printf 'TASK7_PHASE1_RC=%d\n' "$TASK7_PHASE1_RC" + +TASK7_PHASE2=(/usr/bin/bash) +TASK7_PHASE2+=(/home/donghyeon/workspace/platform/scripts/validate/render-phase2.sh) +[[ "${#TASK7_PHASE2[@]}" -eq 2 ]] +case $- in *e*) TASK7_PHASE2_ERREXIT=1 ;; *) TASK7_PHASE2_ERREXIT=0 ;; esac +set +e +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm "${TASK7_PHASE2[@]}" +TASK7_PHASE2_RC=$? +(( TASK7_PHASE2_ERREXIT == 0 )) || set -e +printf 'TASK7_PHASE2_RC=%d\n' "$TASK7_PHASE2_RC" + +TASK7_ADMIN_RENDER=(/usr/bin/bash) +TASK7_ADMIN_RENDER+=(/home/donghyeon/workspace/platform/scripts/validate/render-admin-services.sh) +[[ "${#TASK7_ADMIN_RENDER[@]}" -eq 2 ]] +case $- in *e*) TASK7_ADMIN_RENDER_ERREXIT=1 ;; *) TASK7_ADMIN_RENDER_ERREXIT=0 ;; esac +set +e +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm "${TASK7_ADMIN_RENDER[@]}" +TASK7_ADMIN_RENDER_RC=$? +(( TASK7_ADMIN_RENDER_ERREXIT == 0 )) || set -e +printf 'TASK7_ADMIN_RENDER_RC=%d\n' "$TASK7_ADMIN_RENDER_RC" +``` + +다섯 renderer의 exact Helm assignment을 생략하지 않습니다. 특히 phase1/phase2는 fixed +PATH에 Helm이 없으므로 `command -v helm` fallback을 허용하지 않습니다. 전체 RC `0`, +expected terminal PASS, complete seven-artifact publication, original fingerprint unchanged와 new +residue `0`을 요구합니다. 존재하지 않는 core smoke를 호출하지 않고 monolithic smoke +execute를 다시 실행하지 않습니다. + +## 10. rollback·STOP 경계 + +Host/rules recovery evidence는 성공 Task 6 root에서 각 transaction 소유 범위만 사용합니다. +OIDC는 persistent Task 6 reversal ledger가 아니라 in-process private snapshot으로만 rollback합니다. +PVC, Secret, CRD, PV, Loki/Tempo object·bucket은 자동 삭제하지 않습니다. API timeout, +response loss, UID drift, third-state, controller 비수렴, ledger mismatch, rollback ambiguity는 +`MANUAL_RECOVERY_REQUIRED=YES`로 STOP하고 evidence/root를 보존합니다. + +## 11. 전체 완료 판정 + +Task 7은 Host/OIDC/smoke/renderer의 실제 RC·marker·cleanup과 independent review가 모두 있을 +때만 완료로 표시합니다. admin, viewer, no-group, membership-removal, break-glass, Slack +firing/resolved, true external client 중 하나라도 미실행/실패면 `부분 구현`을 유지합니다. +Slack off-host DR은 Task 7 PASS와 무관하게 `deferred / not complete`이며 active exception을 +유지합니다. 실제 terminal evidence의 independent review 전에는 중앙 Task 7 Step 1–6 +checkbox를 체크하지 않고 Task 8을 시작하지 않습니다. diff --git a/clusters/home/applications/.gitkeep b/clusters/home/applications/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/clusters/home/projects/.gitkeep b/clusters/home/projects/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/components/mesh-ready/README.md b/components/mesh-ready/README.md new file mode 100644 index 0000000..06d9f4a --- /dev/null +++ b/components/mesh-ready/README.md @@ -0,0 +1,17 @@ +# 서비스 메시 확장 지점 + +Istio나 다른 서비스 메시는 현재 설치하지 않는다. 이 디렉터리는 후속 검토에서 +공통 Kustomize Component를 둘 위치만 예약한다. + +현재 리소스에는 다음 설정을 넣지 않는다. + +- 사이드카 주입용 namespace label +- `sidecar.istio.io/inject` annotation +- Istio `Gateway`, `VirtualService`, `DestinationRule` +- 애플리케이션 간 mTLS 강제 정책 + +향후 서비스 메시를 검토할 때에는 단일 노드의 CPU·메모리 비용, Host Nginx → Traefik +경로와의 역할 중복, PostgreSQL·AIStor 트래픽 영향, 장애 시 우회 경로를 먼저 +검증한다. 채택하더라도 서비스 base를 직접 수정하지 않고 이 위치의 선택적 +Component를 환경 Overlay에서 참조한다. + diff --git a/components/policies/.gitkeep b/components/policies/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/platform/plans/2026-08-14-observability-alertmanager-receiver-postcheck-design.md b/docs/platform/plans/2026-08-14-observability-alertmanager-receiver-postcheck-design.md new file mode 100644 index 0000000..3d8b5ea --- /dev/null +++ b/docs/platform/plans/2026-08-14-observability-alertmanager-receiver-postcheck-design.md @@ -0,0 +1,69 @@ +# Observability Alertmanager Receiver Postcheck Correction + +## Status + +Approved approach: require the exact Prometheus Operator-generated receiver identity and reject every other receiver topology. + +## Problem + +Task 6 applied the rules/alerts resources but its reconcile postcheck rejected the live Alertmanager receiver and rolled the transaction back. The rendered `AlertmanagerConfig` correctly uses the local receiver name `platform-slack`. Prometheus Operator v0.93.0 transforms that local name for the generated Alertmanager configuration using: + +```text +// +``` + +The runtime receiver returned by Alertmanager is therefore exactly: + +```text +observability/platform-alertmanager/platform-slack +``` + +The existing postcheck and its fake API response incorrectly expect the unqualified local name `platform-slack`. + +## Scope + +Change only the Task 6 runtime receiver oracle and its focused fake/test matrix. Do not change the `AlertmanagerConfig`, route, Slack Secret selector, webhook data, rendered resources, inventory files, or Kubernetes substrate. + +The failed rollback ID `20260814T080303Z`, its ledger, and the preserved handoff remain immutable evidence and must not be reused. + +## Runtime contract + +The Alertmanager `/api/v2/receivers` response must be a JSON array containing exactly one receiver object, and that object's name must be exactly: + +```text +observability/platform-alertmanager/platform-slack +``` + +Fail closed when the response is malformed or when the receiver set is empty, contains the raw local name, contains the qualified name more than once, contains a differently qualified name, or contains any additional receiver. + +This check remains part of the existing combined reconcile gate. Failure writes no acceptance marker and invokes the existing UID-bound rollback path. + +## Test-first implementation + +First change the fake receiver API to model the pinned Operator's qualified runtime name. On unchanged production, the existing success transaction must fail at reconcile; this is the required RED. + +Then add explicit behavioral cases: + +- exact qualified singleton: pass; +- raw local name only: fail and rollback; +- empty response or `null` only: fail and rollback; +- qualified duplicate: fail and rollback; +- qualified receiver plus any extra receiver: fail and rollback; +- wrong namespace, AlertmanagerConfig name, or local receiver segment: fail and rollback; +- malformed JSON/topology: fail and rollback. + +The minimal production change replaces the raw-name count predicate with exact singleton equality. No new production test seam is permitted. + +## Verification and live recovery + +Run Bash syntax checks, the complete focused apply suite, the focused sensitive-source scanner, and the bounded full source scan required by the source-freeze workflow. Perform an independent read-only review of the exact frozen hashes and confirm no Secret payload, webhook value, or rendered credential was read. + +After review, create a fresh source manifest and a new root-owned rollback ID. Revalidate Slack deployment evidence, encryption, restore evidence, inventory identity, handoff identity, API readiness, and rollback-root metadata before exactly one new Task 6 execution. Success requires the reconcile checks, acceptance marker, and payload-free live resource counts to pass before Task 7 begins. + +## Non-goals + +- accepting both raw and qualified names; +- dynamically discovering arbitrary receiver identities; +- changing Operator or Alertmanager versions; +- changing the Slack routing resource; +- retrying the failed rollback ID. diff --git a/docs/superpowers/plans/2026-08-14-observability-alertmanager-receiver-postcheck.md b/docs/superpowers/plans/2026-08-14-observability-alertmanager-receiver-postcheck.md new file mode 100644 index 0000000..2c05de1 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-observability-alertmanager-receiver-postcheck.md @@ -0,0 +1,380 @@ +# Observability Alertmanager Receiver Postcheck Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Correct Task 6's Alertmanager runtime receiver oracle, prove the correction against the pinned Operator behavior, and authorize one fresh rules/alerts transaction without weakening any Secret, rollback, inventory, or reconciliation boundary. + +**Architecture:** Keep the rendered `AlertmanagerConfig` unchanged and correct only the runtime `/api/v2/receivers` acceptance predicate. The postcheck accepts an exact singleton receiver named `observability/platform-alertmanager/platform-slack`; focused fakes model that downstream Operator transformation. A fresh source freeze and rollback ID are required before another live execution. + +**Tech Stack:** Bash 5.2, Python 3 JSON validation embedded in Bash, Kubernetes/kubectl, Prometheus Operator v0.93.0, Alertmanager v0.33.1, private file-backed source manifests. + +## Global Constraints + +- Never read, print, hash, copy, or disclose Secret data, the Slack webhook value, or a rendered credential. +- Do not change `services/observability/alerting/platform-alertmanager.yaml`, its raw route/receiver name, or its Secret selector. +- Runtime receiver identity is exactly `observability/platform-alertmanager/platform-slack`. +- `/api/v2/receivers` must contain exactly one receiver object; raw, absent, duplicate, differently qualified, extra, or malformed topologies fail closed. +- Failed rollback ID `20260814T080303Z`, its root ledger, and `/tmp/platform-observability-metrics.LNzksC` remain preserved and are never reused for mutation. +- The missing `kubectl.kubernetes.io/last-applied-configuration` warnings on the two pre-existing resources are not the root cause and require no source change. +- No live retry occurs before RED, GREEN, bounded scans, exact source freeze, and independent review all pass. +- This workspace is non-Git; do not commit, create a branch, or claim a commit. Freeze exact hashes/modes in reports instead. + +--- + +### Task 1: Production-faithful receiver regression + +**Files:** +- Modify: `scripts/validate/test-apply-observability-access.sh:230-240` +- Report: `.superpowers/sdd/2026-08-14-observability-alertmanager-receiver-postcheck/task-1-report.md` + +**Interfaces:** +- Consumes: fake kubectl `/api/v2/receivers` response and `run_apply` environment forwarding. +- Produces: a focused fake whose default response is the exact Operator-generated singleton and whose closed modes drive negative reconciliation cases. + +- [ ] **Step 1: Freeze the unchanged production/test identities** + + Record SHA-256, mode, UID:GID, nlink, type, and size for: + + ```text + scripts/bootstrap/apply-observability-access.sh + scripts/validate/test-apply-observability-access.sh + ``` + + Expected starting SHA-256 values: + + ```text + apply = 1e1c44e349e46229dfe79d461e711940e1f2b07ea8c0b91e907eeb7dc8740a11 + test = 6f8ec85010b6c0ae1fea0579a343c28a03d3278bb58b99513f58bef4be3f5644 + ``` + +- [ ] **Step 2: Change only the fake default and observe RED** + + Replace the fake's receiver response with the production-faithful literal: + + ```python + if raw_path.endswith("/proxy/api/v2/receivers"): + print(json.dumps([ + {"name": "observability/platform-alertmanager/platform-slack"}, + ])) + raise SystemExit(0) + ``` + + Do not edit production. Run: + + ```bash + bash -n scripts/bootstrap/apply-observability-access.sh + bash -n scripts/validate/test-apply-observability-access.sh + timeout --signal=TERM --kill-after=5s 240s \ + bash scripts/validate/test-apply-observability-access.sh + ``` + + Expected: syntax RC `0`; suite RC nonzero because a success transaction reaches reconcile, the unchanged raw-name predicate rejects the qualified singleton, and no acceptance marker is created. + +- [ ] **Step 3: Add closed fake receiver modes** + + Use only a test-file fake mode, defaulting to `qualified`: + + ```python + mode = os.environ.get("PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE", "qualified") + qualified = "observability/platform-alertmanager/platform-slack" + receiver_cases = { + "qualified": [{"name": qualified}], + "raw": [{"name": "platform-slack"}], + "empty": [], + "null": [{"name": "null"}], + "duplicate": [{"name": qualified}, {"name": qualified}], + "extra": [{"name": qualified}, {"name": "null"}], + "wrong-namespace": [{"name": "other/platform-alertmanager/platform-slack"}], + "wrong-config": [{"name": "observability/other/platform-slack"}], + "wrong-local": [{"name": "observability/platform-alertmanager/other"}], + } + if mode == "malformed": + print('{"name":') + elif mode in receiver_cases: + print(json.dumps(receiver_cases[mode])) + else: + raise SystemExit(76) + raise SystemExit(0) + ``` + + Forward `PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE` through `run_apply`. Retain the existing receiver-drop case by mapping it to `empty` or replacing that test with the explicit matrix; do not leave two contradictory controls. + +- [ ] **Step 4: Add exact behavioral assertions** + + Add one positive exact-singleton transaction and table-driven negative transactions for: + + ```text + raw empty null duplicate extra wrong-namespace wrong-config wrong-local malformed + ``` + + Each negative must assert transaction RC nonzero, `OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS`, absent `acceptance.env`, and no leftover newly-created fake state. The positive must still be RED until production changes. + +- [ ] **Step 5: Record Task 1 RED evidence** + + Write the exact baseline hashes, command, RC, expected failing leaf, acceptance absence, fake-state cleanup, and process/temp residue counts to the Task 1 report. Do not include API bodies beyond the safe receiver-name literals listed in this plan. + +--- + +### Task 2: Minimal exact-singleton production correction + +**Files:** +- Modify: `scripts/bootstrap/apply-observability-access.sh:1209-1215` +- Test: `scripts/validate/test-apply-observability-access.sh` +- Append: `.superpowers/sdd/2026-08-14-observability-alertmanager-receiver-postcheck/task-1-report.md` + +**Interfaces:** +- Consumes: JSON bytes returned from Alertmanager `/api/v2/receivers`. +- Produces: reconcile success only for the exact qualified singleton; all other topologies return failure to the existing rollback coordinator. + +- [ ] **Step 1: Implement the minimal predicate** + + Replace the raw-name counter with: + + ```python + receiver_payload = json.loads(receivers_api.read_text(encoding="utf-8")) + expected_receiver = "observability/platform-alertmanager/platform-slack" + if ( + not isinstance(receiver_payload, list) + or len(receiver_payload) != 1 + or not isinstance(receiver_payload[0], dict) + or receiver_payload[0].get("name") != expected_receiver + ): + raise SystemExit(1) + ``` + + Do not add a production environment seam, fallback raw name, prefix match, wildcard, or dynamic discovery. + +- [ ] **Step 2: Run GREEN syntax and full focused suite** + + Run exactly: + + ```bash + bash -n scripts/bootstrap/apply-observability-access.sh + bash -n scripts/validate/test-apply-observability-access.sh + timeout --signal=TERM --kill-after=5s 330s \ + bash scripts/validate/test-apply-observability-access.sh + ``` + + The receiver matrix adds nine complete rollback transactions to the prior 176-second suite. Set the test-only internal `SUITE_WALL_BOUND_SECONDS` to `300`; the measured unchanged-bound RED is `222 > 220`. Expected: both syntax RC `0`; focused suite RC `0` within the new internal 300-second bound and outer 330-second supervisor; exact terminal PASS; every receiver negative rolls back; no suite-owned orphan process or fixture residue. + +- [ ] **Step 3: Mutation-check the tests** + + In a private temporary copy only, substitute each of the following and prove at least one focused assertion fails for each mutation: + + ```text + expected_receiver = "platform-slack" + len(receiver_payload) >= 1 + receiver_payload[0].get("name", "").endswith("/platform-slack") + ``` + + Delete only the private temporary copy afterward. Do not edit production for this check. + +- [ ] **Step 4: Append GREEN evidence** + + Append final source/test hashes and modes, the full assertion count, terminal PASS, wall time, orphan count, and residue audit to the Task 1 report. + +--- + +### Task 3: Documentation, scans, review, and retry freeze + +**Files:** +- Modify: `bootstrap/manual/phase4-observability-access.md:242-247` +- Modify: `/home/donghyeon/workspace/docs/platform/plans/2026-08-14-observability-authoritative-metric-inventory-implementation.md` +- Regenerate: `.superpowers/sdd/2026-08-14-observability-authoritative-metric-inventory-implementation/task-3-brief.md` +- Append: `.superpowers/sdd/2026-08-14-observability-authoritative-metric-inventory-implementation/task-2-report.md` +- Append: `/home/donghyeon/workspace/docs/platform/runbooks/2026-08-13-observability-phase4-resume-worklog.md` +- Preserve old authority under: `.superpowers/sdd/2026-08-14-observability-alertmanager-receiver-postcheck/baseline/failed-task6-20260814T080303Z/` +- Regenerate: `.superpowers/sdd/2026-08-14-observability-slack-risk-acceptance-implementation/baseline/task-6-prelive-source-manifest.txt` +- Regenerate: `.superpowers/sdd/2026-08-14-observability-slack-risk-acceptance-implementation/baseline/task-6-prelive-source-manifest.sha256` + +**Interfaces:** +- Consumes: Task 2 exact hashes and passing evidence. +- Produces: reviewed execution authority for one fresh Task 6 transaction. + +- [ ] **Step 1: Correct human and agent acceptance wording** + + Replace only runtime acceptance wording from raw `platform-slack` to: + + ```text + generated receiver observability/platform-alertmanager/platform-slack = exact singleton + ``` + + Keep the `AlertmanagerConfig` source receiver documented as raw `platform-slack`. Record the failed live RC `1`, stage `reconcile`, rollback `PASS`, no acceptance claim, exact root cause, and preservation of rollback ID `20260814T080303Z`. + +- [ ] **Step 2: Regenerate the derived Task 3 brief deterministically** + + Run the canonical extractor once to the derived brief and once to a private comparison file: + + ```bash + EXTRACTOR=/home/donghyeon/.codex/plugins/cache/openai-curated-remote/superpowers/6.2.0/skills/subagent-driven-development/scripts/task-brief + CENTRAL_PLAN=/home/donghyeon/workspace/docs/platform/plans/2026-08-14-observability-authoritative-metric-inventory-implementation.md + DERIVED_BRIEF=/home/donghyeon/workspace/platform/.superpowers/sdd/2026-08-14-observability-authoritative-metric-inventory-implementation/task-3-brief.md + PRIVATE_BRIEF="$(mktemp /tmp/platform-task6-derived-brief.XXXXXX)" + "$EXTRACTOR" "$CENTRAL_PLAN" 3 "$DERIVED_BRIEF" + "$EXTRACTOR" "$CENTRAL_PLAN" 3 "$PRIVATE_BRIEF" + cmp -s -- "$DERIVED_BRIEF" "$PRIVATE_BRIEF" + ``` + + Require `cmp` RC `0`, record the derived SHA-256, then remove only `PRIVATE_BRIEF`. Do not manually edit the derived brief. + +- [ ] **Step 3: Run focused scanner gates** + + Run: + + ```bash + bash -n scripts/validate/scan-platform-sensitive-source.sh + timeout --signal=TERM --kill-after=5s 120s \ + bash scripts/validate/test-scan-platform-sensitive-source.sh + ``` + + Expected: RC `0` and `PLATFORM SENSITIVE SOURCE ASSERTION TEST PASS` with no credential output. + +- [ ] **Step 4: Run one new authoritative full scan** + + A source change authorizes exactly one new full scan. With tracing disabled, run: + + ```bash + TASK6_FIX_SCAN_ROOT="$(mktemp -d /tmp/platform-task6-receiver-fix-scan.XXXXXX)" + chmod 0700 "$TASK6_FIX_SCAN_ROOT" + : >"$TASK6_FIX_SCAN_ROOT/scan.log" + chmod 0600 "$TASK6_FIX_SCAN_ROOT/scan.log" + start_ms="$(date +%s%3N)" + set +e + timeout --signal=TERM --kill-after=5s 240s \ + bash scripts/validate/scan-platform-sensitive-source.sh \ + >"$TASK6_FIX_SCAN_ROOT/scan.log" 2>&1 + scan_rc=$? + set -e + end_ms="$(date +%s%3N)" + printf 'RC=%d\nWALL_MS=%d\n' "$scan_rc" "$((end_ms - start_ms))" \ + >"$TASK6_FIX_SCAN_ROOT/result" + chmod 0600 "$TASK6_FIX_SCAN_ROOT/result" + ``` + + Require RC `0`, exactly one `PLATFORM RENDERED SECRET SCAN PASS`, exactly one `PLATFORM SENSITIVE SOURCE SCAN PASS`, and zero new scanner/renderer processes or temporary artifacts. + +- [ ] **Step 5: Preserve the failed execution authority before regeneration** + + Attest the current prelive manifest/sidecar as regular, non-symlink, mode `0664`, UID:GID `1000:1000`, nlink `1`; copy them byte-for-byte into the failed-transaction baseline directory with mode `0600`; verify `cmp -s` and record both old hashes. Amend the central plan to state that this preserved copy is the historical authority for rollback ID `20260814T080303Z` and the canonical prelive path is superseded only for the fresh retry. + +- [ ] **Step 6: Regenerate and strictly validate the canonical prelive manifest** + + Rebuild the existing exact 15-row manifest grammar: + + ```text + sha256|mode4|uid:gid|nlink|regular file|size|/canonical/absolute/path + ``` + + Keep exactly the verifier's `EXPECTED_ROWS`, each once and in order. Regenerate the canonical sidecar, then invoke the unchanged verifier with literal absolute paths and externally reviewed lowercase SHA values. Require 15 rows, 15 canonical unique paths, no row mismatch, and sidecar binding PASS. + +- [x] **Step 7: Independent read-only review** + + Review exact source/test/docs/manifest hashes against this plan and the approved design. Required verdicts: spec compliance, Critical/Important/Minor counts, TDD RED provenance, exact-singleton contract, rollback preservation, scan evidence, and live retry Ready YES/NO. Any Critical or Important finding returns to the same implementer for a bounded fix/re-review loop. + +--- + +### Task 4: One fresh live Task 6 transaction + +**Files:** +- Preserve: `/var/lib/hyeonworks/platform-rollbacks/observability-20260814T080303Z` +- Preserve read-only: `/tmp/platform-observability-metrics.LNzksC` +- Create: one fresh `/var/lib/hyeonworks/platform-rollbacks/observability-` root +- Create and revalidate: one fresh `/tmp/platform-observability-metrics.XXXXXX` root +- Append after terminal result: Task 2 report, central worklog, central implementation plan +- Create after terminal success: `task-6-final-source-manifest.txt` and `.sha256` + +**Interfaces:** +- Consumes: reviewed Task 3 freeze and a fresh byte-preserved/revalidated private handoff. +- Produces: Task 6 acceptance schema `platform-observability-rules-alerts-v2` or a preserved fail-closed transaction with no retry. + +- [x] **Step 1: Create, attest, and bind a fresh rollback root** + + In the existing attached tmux operator pane, refresh sudo and generate one strict UTC ID. Reject + `20260814T080303Z`; require its exact rollback path pre-ABSENT and non-symlink; invoke + `sudo -n /usr/bin/mkdir --mode=0700 -- "$TASK6_ROLLBACK_ROOT"` exactly once; then require + `directory|0:0|700` and export it as `PLATFORM_OBSERVABILITY_ROLLBACK_ID`. Require exact equality + between the fresh ID and active exported ID. Any collision, create error, metadata drift, or + equality failure stops and forbids retry with that ID. + +- [x] **Step 2: Re-establish per-ID and global gates** + + For the exact active new ID, run the Blackbox private-edge proof exactly once. Only the operator + enters exact confirmation `PROVE BLACKBOX PRIVATE EDGE default`. Require exact + `BLACKBOX PRIVATE EDGE SOURCE PASS`, immediate RC `0`, and normalized proof metadata + `regular|0:0|600|1` (`regular file|0:0|600|1` from the exact `stat` fields). Any missing, + ambiguous, or nonzero result stops the transaction, preserves the new ID, and forbids validator + or Task 6 retry under that ID. Then require encryption RC `0`, restore RC `0`, Slack deployment + gate `RISK_ACCEPTED`, Secret name-only presence, API `readyz`, and rollback-root metadata. + + Residue is a delta contract. Preserve without reading/deleting the exact preexisting roots + `/tmp/platform-k3s-encryption.Mskzy3`, + `/tmp/platform-observability-access-apply.oeNcfI`, + `/tmp/platform-observability-access-apply.Im02dz`, and + `/tmp/platform-observability-slack-gate.LYhYbv`. The attested classifications are respectively + 8/1 empty evidence, two 8/12 recorded evidence roots, and today's failed-live evidence containing + only two private filenames. Require the name-only baseline unchanged, matching executable + processes `0`, and current preflight/live newly-created matching-root delta `0`. Unknown/new + residue stops for identity review and is never broadly deleted. Do not read Secret data, proof + contents, or these evidence-root contents. + +- [x] **Step 3: Create a fresh handoff and revalidate source authority** + + Run the strict absolute three-argument source-manifest verifier with the newly reviewed verifier + and manifest hashes. Preserve `/tmp/platform-observability-metrics.LNzksC` read-only and never use + it as source, destination, renderer input, or apply input. From canonical preserved source + `/tmp/platform-observability-metrics.VUpsZn`, create a new mode `0700` + `/tmp/platform-observability-metrics.XXXXXX` root, copy the two phase inventory pairs with + `cp --no-dereference --reflink=never`, and set files mode `0600`. Before rendering, require the + new destination to contain exactly the two phase directories and no other entry; the renderer + then adds only the reviewed manifest set. Recheck source/destination + canonical path, owner/mode/nlink, exact two inventory pins/counts `21/30`, exact entry set and + rendered file set, and source identity unchanged. Bind the resulting exact path to `$METRIC_ROOT`. + For `LNzksC`, preserve and compare only the already-attested path/fingerprint identity; do not + reopen inventory bodies or private file content. Any copy or identity gate failure stops the + transaction without using either handoff for apply. + +- [x] **Step 4: Execute exactly once** + + Send this as one unsplit line to the verified idle tmux pane: + + ```bash + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm bash scripts/bootstrap/apply-observability-access.sh --execute --rules-alerts --verified-output-dir "$METRIC_ROOT" + ``` + + The operator types exactly `APPLY` at the prompt. Immediately after return, run `echo "TASK6_APPLY_RC=$?"`. Any nonzero RC, missing PASS, response loss, or rollback ambiguity stops and preserves the new ID without retry. + +- [x] **Step 5: Validate terminal acceptance payload-free** + + Require: + + ```text + AlertmanagerConfig platform-alertmanager = 1 + platform PrometheusRule exact set = 4 + dashboard ConfigMap exact set = 5 + receiver list = [observability/platform-alertmanager/platform-slack] + desired rules evaluation health = ok + Grafana / Blackbox / target readiness unchanged + acceptance schema = platform-observability-rules-alerts-v2 + acceptance slack_deployment_gate = RISK_ACCEPTED + ``` + + Verify acceptance and ledger metadata only; never read the Slack URL or Secret data. + + Terminal evidence: rollback ID `20260814T145009Z`, fresh handoff + `/tmp/platform-observability-metrics.dw5gLZ`, exact six-element argv attestation, inventory + pins/counts `21/30`, apply RC `0`, exact terminal PASS, dashboard `5`, platform rules `4`, desired + rule health `23/23`, runbook URL `22/22`, AlertmanagerConfig `1`, receiver exact qualified + singleton, target `30/30`, Ready workloads, v2 acceptance with `RISK_ACCEPTED`, ledger + object/mutation lines `13/13`, and residue delta `0`. Success did not invoke rollback. + +- [x] **Step 6: Terminal documentation and final manifest** + + Append the new rollback ID, RC, exact resource counts, inventory pins/counts, qualified receiver identity, acceptance schema, and residue result. Check only genuinely completed Task 6 steps. Regenerate the supported final manifest/sidecar from terminal bytes and run the strict verifier. Task 7 may begin only after final independent review returns Ready YES. + + Post-review closeout: final manifest SHA-256 + `52c2230f23d0cd7733c2ae685737e7c93d837182b01a4c25ff755e780df308f3`, sidecar file SHA-256 + `655d5b1efc2a6ce04715f11fdc646db392ef3951b24b5fe5f2b21c9703c15ee3`, exact rows/unique paths + `15/15`, strict literal absolute three-argument verifier RC `0` twice. Independent terminal review + returned Critical/Important/Minor `0/0/1`, Spec `PASS`, Quality `Approved`, Task 6 complete `YES`, + and Task 7 start `YES`. The historical-numbering Minor is deferred to Task 8. Task 7 has not been + executed by this closeout. diff --git a/docs/superpowers/plans/2026-08-15-observability-structured-slack-notifications.md b/docs/superpowers/plans/2026-08-15-observability-structured-slack-notifications.md new file mode 100644 index 0000000..1c175bf --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-observability-structured-slack-notifications.md @@ -0,0 +1,373 @@ +# Structured Slack Observability Notifications Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace Alertmanager's terse default Slack attachment with a structured, severity-aware message and route `InfoInhibitor` to a null receiver while preserving the existing webhook, transactional apply, and rollback boundaries. + +**Architecture:** Keep formatting inside `AlertmanagerConfig/platform-alertmanager`; do not add a formatter service. Strengthen the existing source, core-render, access-render, and apply assertions so the same exact configuration is enforced at every boundary. Runtime acceptance changes from one qualified receiver to the exact qualified pair for Slack and null. + +**Tech Stack:** Kubernetes `AlertmanagerConfig` v1alpha1, Prometheus Alertmanager 0.33.1 Go templates, Bash, embedded Python/PyYAML, existing renderer and rules-alerts transaction. + +## Global Constraints + +- Approved design: `/home/donghyeon/workspace/docs/platform/plans/2026-08-15-observability-structured-slack-notifications-design.md`. +- The Slack Secret selector remains exactly `alertmanager-slack-webhook` / `url`; never read, print, copy, decode, or persist its payload. +- Do not set a Slack `channel`, automatic mention, webhook literal, or new credential source. +- Route grouping is the exact set `cluster`, `namespace`, `alertname`, `severity`; timing remains `30s`, `5m`, `4h`. +- Source receivers are exactly `platform-null` and `platform-slack`; `platform-null` has no integrations. +- Runtime receivers are the order-independent exact set `observability/platform-alertmanager/platform-null` and `observability/platform-alertmanager/platform-slack`. +- `InfoInhibitor` routes to `platform-null`; its inhibition rule targets `severity=info` with equality on `namespace`. Preserve the upstream conditional semantics rather than claiming all info alerts are always dropped. +- Colors are resolved=`good`, emergency/critical=`danger`, warning=`warning`, and info/unknown=`#439FE0`. +- Target text uses all labels after removing group-label names, sorted by label name; missing values use explicit fallbacks. +- No alert expression, threshold, duration, NetworkPolicy, dashboard JSON, retention setting, or Slack destination changes. +- No new formatter workload or generic automation script. +- This is a non-Git workspace. Record starting/final SHA-256 and metadata in task reports instead of creating commits or worktrees. +- Production changes require a real failing test first. Run each expensive focused suite at most once for RED and once for final GREEN unless a genuine failure requires a fix. +- Do not execute live apply until source review is clean. Direct `kubectl apply` is forbidden; use the existing rules-alerts transaction exactly once with a fresh rollback ID and fresh verified handoff. + +## Exact AlertmanagerConfig Contract + +The canonical source and every exact oracle use this `spec` value. YAML block scalars preserve the template text exactly. + +```yaml +spec: + route: + receiver: platform-slack + groupBy: + - cluster + - namespace + - alertname + - severity + groupWait: 30s + groupInterval: 5m + repeatInterval: 4h + routes: + - receiver: platform-null + matchers: + - name: alertname + matchType: "=" + value: InfoInhibitor + inhibitRules: + - sourceMatch: + - name: alertname + matchType: "=" + value: InfoInhibitor + targetMatch: + - name: severity + matchType: "=" + value: info + equal: + - namespace + receivers: + - name: platform-null + - name: platform-slack + slackConfigs: + - apiURL: + name: alertmanager-slack-webhook + key: url + sendResolved: true + linkNames: false + mrkdwnIn: + - text + - fields + fallback: >- + {{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }} + title: >- + {{ if eq .Status "firing" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }} + titleLink: https://grafana.learn.hyeonworks.com/ + color: >- + {{ if eq .Status "resolved" }}good{{ else if or (eq .CommonLabels.severity "emergency") (eq .CommonLabels.severity "critical") }}danger{{ else if eq .CommonLabels.severity "warning" }}warning{{ else }}#439FE0{{ end }} + fields: + - title: Status + value: '{{ .Status | toUpper }}' + short: true + - title: Severity + value: '{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}' + short: true + - title: Location + value: '{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}' + short: true + - title: Alert count + value: '{{ len .Alerts }}' + short: true + text: |- + {{ range .Alerts }} + *Alert status:* {{ .Status | toUpper }} + *Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }} + *Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }} + *Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }} + *Started:* {{ .StartsAt.Format "2006-01-02T15:04:05Z07:00" }} + {{ if eq .Status "resolved" }}*Ended:* {{ .EndsAt.Format "2006-01-02T15:04:05Z07:00" }}{{ end }} + {{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }} + + {{ end }} + · + footer: hyeonworks observability · Alertmanager +``` + +--- + +### Task 1: Source and Core Renderer Exact Contract + +**Files:** +- Modify: `scripts/validate/test-observability-alerting.sh` +- Modify: `scripts/validate/test-render-observability-core.sh` +- Modify: `services/observability/alerting/platform-alertmanager.yaml` +- Modify: `scripts/validate/render-observability-core.sh` +- Create: `.superpowers/sdd/2026-08-15-observability-structured-slack-notifications/task-1-report.md` + +**Interfaces:** +- Consumes: the exact AlertmanagerConfig contract above. +- Produces: source YAML and core renderer assertion that accept only that contract; later tasks rely on `assert_alertmanager_routing_contract` as the canonical oracle. + +- [ ] **Step 1: Record the starting identities** + +Record SHA-256, mode, owner, nlink, and size for the four modified files in the task report. Starting SHA-256 values must be: + +```text +platform-alertmanager.yaml 76db5e6dd3653bc8afb9ecffe3472fc57eb07c5a2b33d218f3f4a41fb202caa2 +test-observability-alerting.sh c5c578435cfeaac0d4221dd335797e09aaa010a9f63c397d21936a252e461db2 +render-observability-core.sh 14460b985489ca59ba2377a61300a187d84fe795c8307fe2ba32bee9674d4bb2 +test-render-observability-core.sh 9dd935e403968bf063ec0ddd44cd32ba853fe9f121a2764fbb01206d3c561d35 +``` + +- [ ] **Step 2: Write the failing source and core tests** + +In `test-observability-alerting.sh`, replace the old exact `alert` oracle with the exact contract above, retaining exact metadata and API identity. In `test-render-observability-core.sh`, update the valid Alertmanager fixture to the exact contract and add independently hand-authored negative fixtures for: + +```text +missing severity from groupBy +missing title +color that maps critical to warning +InfoInhibitor child route pointing to platform-slack +missing inhibit rule +platform-null containing a Slack integration +channel override +text containing @channel +extra receiver +``` + +Each negative must exercise `assert_alertmanager_routing_contract` and expect rejection. + +- [ ] **Step 3: Run RED against unchanged production** + +Run: + +```bash +bash -n scripts/validate/test-observability-alerting.sh +bash -n scripts/validate/test-render-observability-core.sh +bash scripts/validate/test-observability-alerting.sh +bash scripts/validate/test-render-observability-core.sh +``` + +Expected: both syntax checks return 0; at least one behavioral command returns nonzero because production still has the old three-key group and terse singleton receiver. Capture the exact first failing leaf. A syntax or fixture-construction error is not an acceptable RED. + +- [ ] **Step 4: Implement the minimal source and canonical oracle** + +Use `apply_patch`. Replace only the `spec` in `platform-alertmanager.yaml` with the exact contract above. Replace `render-observability-core.sh` mode `alertmanager_routing`'s `expected_config` with the exact full object, including receiver order, every template string, child route, and inhibit rule. Do not change the NetworkPolicy oracle or Alertmanager global-reference checks. + +- [ ] **Step 5: Run GREEN and mutation checks** + +Run both commands from Step 3 again. Expected exact terminal markers: + +```text +OBSERVABILITY ALERTING STATIC CONTRACT PASS +OBSERVABILITY ALERTMANAGER GLOBAL RENDER CONTRACT PASS +OBSERVABILITY CORE STATIC RENDER PASS +``` + +Confirm every negative fixture returns nonzero while the canonical fixture returns zero. Record assertion counts and final identities in `task-1-report.md`. + +### Task 2: Access Renderer and Transaction Runtime Contract + +**Files:** +- Modify: `scripts/validate/test-render-observability-access.sh` +- Modify: `scripts/validate/render-observability-access.sh` +- Modify: `scripts/validate/test-apply-observability-access.sh` +- Modify: `scripts/bootstrap/apply-observability-access.sh` +- Create: `.superpowers/sdd/2026-08-15-observability-structured-slack-notifications/task-2-report.md` + +**Interfaces:** +- Consumes: Task 1's `assert_alertmanager_routing_contract` and exact source configuration. +- Produces: an access handoff that rejects altered templates/routes and a transaction postcheck that accepts only the qualified receiver pair. + +- [ ] **Step 1: Record starting identities** + +Starting SHA-256 values must be: + +```text +render-observability-access.sh ae5373b1b20be42e573d58e7384dcc654b1d3cac75a6c269cf684f4a14f3ca1a +test-render-observability-access.sh 5dfdc52b9f72b6d2e754893e799f5baf27c2b87e701958452dc170627ff21435 +apply-observability-access.sh f797c3c678e45adb08c407f9a55b6da10fa49d93038cd0b301f589555b11263c +test-apply-observability-access.sh dd78ef4601c89652b9270a7fcb257232a9f54431f65b96669062cc0efa920e4f +``` + +- [ ] **Step 2: Write access RED cases** + +Update the valid fixtures to the exact Task 1 contract. In `test-render-observability-access.sh`, add mutations that remove `title`, change `groupBy`, route `InfoInhibitor` to Slack, add `channel`, and add an extra receiver. Each must fail at the published handoff assertion with no partial output mutation. + +Run: + +```bash +bash -n scripts/validate/test-render-observability-access.sh +bash scripts/validate/test-render-observability-access.sh +``` + +Expected: syntax 0 and behavioral nonzero against the current loose `rules-alerts` access assertion; capture the leaf showing an invalid structured contract was accepted. + +- [ ] **Step 3: Write apply RED cases** + +Update the fake `/api/v2/receivers` default to the qualified pair. Define literal cases: + +```python +qualified_pair = [ + {"name": "observability/platform-alertmanager/platform-null"}, + {"name": "observability/platform-alertmanager/platform-slack"}, +] +``` + +Add one accepted reversed-order case and rejected cases for old Slack singleton, null singleton, empty list, raw pair, duplicate member, extra receiver, malformed JSON, and wrong namespace/config/local name for either member. Add a structured-handoff mutation that must fail before prompt, ledger creation, dry-run, or apply. + +Run the smallest named positive case or the bounded focused suite until the new qualified pair reaches the old singleton predicate. Expected RED is nonzero at the receiver postcheck, not a fixture or syntax error. + +- [ ] **Step 4: Strengthen production access and apply checks** + +In `render-observability-access.sh`, replace the one-line shell wrapper with: + +```bash +assert_access_rules_alerts_contract() { + (( $# == 3 )) || return 1 + _access_assert rules-alerts "$@" || return 1 + assert_alertmanager_routing_contract "$3" +} +``` + +Keep the existing dashboard, rule, runbook, scope, and NetworkPolicy checks. + +In the apply postcheck Python, replace singleton logic with: + +```python +expected_receivers = { + "observability/platform-alertmanager/platform-null", + "observability/platform-alertmanager/platform-slack", +} +if ( + not isinstance(receiver_payload, list) + or len(receiver_payload) != 2 + or any(not isinstance(item, dict) or set(item) != {"name"} for item in receiver_payload) + or {item["name"] for item in receiver_payload} != expected_receivers +): + raise SystemExit(1) +``` + +Do not change rollback, ledger, resource rows, Slack gate, Secret name/key validation, or acceptance schema. + +- [ ] **Step 5: Run final focused GREEN once** + +Run syntax for all four files, then: + +```bash +bash scripts/validate/test-render-observability-access.sh +timeout 330s bash scripts/validate/test-apply-observability-access.sh +``` + +Expected: access terminal PASS; apply terminal `APPLY OBSERVABILITY ACCESS TEST PASS`, the documented assertion count, suite wall at most 300 seconds, outer RC0, and relevant orphan count 0. Verify every receiver mutation fails at its intended leaf. Record final identities and evidence in `task-2-report.md`. + +### Task 3: Integrated Schema, Template, and Independent Source Review + +**Files:** +- Modify only if the RED demands it: `scripts/validate/test-observability-alerting.sh` +- Create: `.superpowers/sdd/2026-08-15-observability-structured-slack-notifications/task-3-report.md` + +**Interfaces:** +- Consumes: Tasks 1-2 frozen candidate bytes. +- Produces: pre-live Ready/Not Ready verdict and exact candidate identities. + +- [ ] **Step 1: Run complete syntax and source-focused matrix** + +Run Bash syntax on all eight changed scripts. Run `test-observability-alerting.sh` without live mode once, then the already-green core/access/apply commands only if their final Task reports do not contain unambiguous RC0 evidence for the same hashes. Do not rerun an expensive suite merely to duplicate evidence. + +- [ ] **Step 2: Validate CRD admission without mutation** + +Run: + +```bash +PLATFORM_KUBECTL_BIN=/usr/local/bin/kubectl \ + bash scripts/validate/test-observability-alerting.sh --server-dry-run +``` + +Expected: both server-side dry runs and all static markers pass. This is schema evidence only. + +- [ ] **Step 3: Render real templates with pinned Alertmanager 0.33.1** + +Extract `fallback`, `title`, `color`, all field values, and `text` from the candidate YAML without exposing any Secret value. Use `/bin/amtool` in the existing Alertmanager 0.33.1 container with `amtool template render`; pass a hand-authored JSON fixture through `/dev/stdin`, not a persistent pod file. The fixture must cover: + +```text +warning firing with cluster and namespace +resolved alert with EndsAt +missing cluster, namespace, summary, description, and runbook +two alerts with different non-group labels +``` + +Assert literal results include the correct severity color, `unknown-cluster`, `cluster-scoped`, sorted target labels, both fallbacks, FIRING count, RESOLVED title, runbook when present, Grafana links, and footer. No Secret or webhook URL enters the fixture or output. + +- [ ] **Step 4: Perform independent read-only review** + +The reviewer reads the design, plan, task reports, final changed files, and current CRD. Required verdict fields are Spec PASS/FAIL, Critical/Important/Minor counts, and Ready for one transactional live apply YES/NO. Any Critical or Important finding must be fixed with a new failing test and re-reviewed before Task 4. + +- [ ] **Step 5: Freeze pre-live identities** + +Record SHA-256 and metadata for all eight source/test files plus the design, plan, and Task reports. Confirm no webhook literal, decoded Secret, `channel`, `@channel`, or `@here` occurs in governed source. Record process/temp residue and mark Task 3 Ready only when all checks pass. + +### Task 4: Transactional Rollout, Synthetic Delivery, and Documentation + +**Files:** +- Modify after observed results: `/home/donghyeon/workspace/docs/platform/observability/03-alerting-and-slack.md` +- Modify after observed results: `/home/donghyeon/workspace/docs/platform/observability/07-build-history-and-decisions.md` +- Create: `.superpowers/sdd/2026-08-15-observability-structured-slack-notifications/task-4-report.md` + +**Interfaces:** +- Consumes: Task 3 Ready verdict and frozen candidate identities. +- Produces: live structured Slack configuration, observed FIRING/RESOLVED acceptance, cleanup evidence, and final operator documentation. + +- [ ] **Step 1: Prepare one fresh transaction** + +Use a fresh UTC rollback ID unequal to every previous ID and create its root once as root-owned mode 0700. Preserve all prior rollback roots. Generate one fresh private handoff from the authoritative immutable inventory pair: + +```text +target-initial 79688d017d38eec9a6f100f8d0f784a5474e79802046ef1c2c11b30d170b0b0c 21 +post-substrate b1c3049206a1a88165ee672ae9aceac7945673a3bb9c3cf3670b7f0d56c3f291 30 +``` + +Render rules-alerts into the fresh root and verify exact source hashes, metadata, topology, and new structured AlertmanagerConfig. Run the no-argument dry mode once. Do not reuse a prior handoff or rollback ID. + +- [ ] **Step 2: Execute the existing transaction once** + +Build a six-token Bash array to avoid paste splitting: + +```bash +TASK4_CMD=(bash) +TASK4_CMD+=(scripts/bootstrap/apply-observability-access.sh) +TASK4_CMD+=(--execute) +TASK4_CMD+=(--rules-alerts) +TASK4_CMD+=(--verified-output-dir) +TASK4_CMD+=("$METRIC_ROOT") +``` + +Export the fresh `PLATFORM_OBSERVABILITY_ROLLBACK_ID` and exact `PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm`. Invoke `"${TASK4_CMD[@]}"` once, enter exact `APPLY`, and make the immediate next statement `TASK4_APPLY_RC=$?`. Require RC0, exact transaction PASS, both inventory SHA markers, no rollback ambiguity, and accepted v2 ledger. A nonzero or ambiguous result means preserve evidence and stop; never retry the same ID. + +- [ ] **Step 3: Verify runtime reconciliation** + +Require Alertmanager CR Available/Reconciled, StatefulSet ready, `alertmanager_config_last_reload_successful == 1`, no relevant operator/config-reloader error, production rules healthy, and the exact qualified receiver pair. AlertmanagerConfig itself has no usable status condition and must not be reported as Available/Reconciled. + +- [ ] **Step 4: Verify one warning FIRING and RESOLVED message** + +Create one uniquely named, temporary warning `PrometheusRule` with the required `observability.hyeonworks.com/instance=home` selector label, literal summary/description/runbook, and a unique target label. Wait until Prometheus and Alertmanager each show exactly one matching firing alert. The user verifies that Slack contains the agreed title, severity, fields, target labels, summary, details, start time, runbook, Grafana links, and footer. + +Delete the temporary rule once. Wait for the matching RESOLVED message and have the user verify its resolved title, green color, end time, and same context. The synthetic rule name and labels must never collide with production rules. + +- [ ] **Step 5: Clean up and document only observed facts** + +Require synthetic resource absent, matching pending/firing/active alerts zero, production rules healthy, receiver pair exact, and no mutator process or task-created temp residue. Confirm `InfoInhibitor` is not delivered to Slack. Document the conditional upstream info-inhibition semantics; do not claim every info alert is always suppressed. + +Update chapter 03 with the exact source contract and observed FIRING/RESOLVED results. Update chapter 07 with the original terse-message problem, the decision to keep formatting in Alertmanager, and the final transaction result. Record final file identities and any limitation, including long grouped Slack text, in `task-4-report.md`. diff --git a/docs/superpowers/plans/2026-08-15-observability-task7-cutover-correction.md b/docs/superpowers/plans/2026-08-15-observability-task7-cutover-correction.md new file mode 100644 index 0000000..b30165e --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-observability-task7-cutover-correction.md @@ -0,0 +1,1290 @@ +# Observability Task 7 Cutover Correction Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Correct the fail-closed Host Nginx/Task 6 evidence interface, preserve the successful Task 6 authority, and complete Task 7 through one Host cutover, one OIDC membership transaction, one monolithic smoke, and one bounded regression pass. + +**Architecture:** Change only the Host Nginx production script and its focused test for the v2 evidence, first-cutover, command-boundary, and no-contact defects. Keep OIDC, smoke, and renderer production behavior unchanged; correct their orchestration in the central/manual documents, use a fresh inventory-only renderer root, then freeze exact non-Git identities before live execution. + +**Tech Stack:** Bash 5.2, Python 3.12, Nginx/systemd, Kubernetes/k3s/kubectl, Keycloak Admin API, Prometheus/Alertmanager/Grafana, hardened private `/tmp` handoffs, SHA-256 non-Git source manifests. + +## Global Constraints + +- Do not read, print, hash, copy, or disclose Kubernetes Secret values, Slack webhook data, OAuth codes/state, passwords, cookies, bearer tokens, or private recovery payloads. +- Preserve rollback ID `20260814T145009Z` and `/tmp/platform-observability-metrics.dw5gLZ` through Task 7; never use that handoff as complete-render publication output. +- Preserve failed IDs `20260814T080303Z`, `20260814T140953Z`, `/tmp/platform-observability-metrics.LNzksC`, and all historical evidence without mutation or deletion. +- The accepted inventory pins remain `79688d017d38eec9a6f100f8d0f784a5474e79802046ef1c2c11b30d170b0b0c` and `b1c3049206a1a88165ee672ae9aceac7945673a3bb9c3cf3670b7f0d56c3f291`. +- The exact Nginx deny/full pins remain `dbef6d443bcba58b26a5351ea76f6d09f6da8c2ef07a806e22745cf26c88f518` and `7d2de2a92c3597a0859775da1d2ccbf5a3d72c0b2af5cac2439c82311361f801`. +- Full cutover accepts only `platform-observability-rules-alerts-v2`; legacy v1 has no fallback. +- First full cutover requires exact deny-only active state. `ALREADY_ACTIVE` is a STOP condition, not success. +- Before the first full-cutover live preflight or APPLY prompt, the `host-nginx` subtree, ledger, payload + directory, and first payload path must each be absent under both existence and symlink tests. Normal + existing files/directories are residue and fail closed just like symlinks. +- A proof older than 24 hours requires a fresh rollback ID, fresh proof, and fresh Task 6 acceptance. Never recreate only the proof or retry Host under the frozen ID. +- Run `observability-smoke.sh --execute` exactly once, after membership reconciliation and external-client readiness. RC `2` remains partial implementation. +- OIDC rollback is in-process automatic rollback; do not claim a persistent Keycloak reversal ledger under the Task 6 root. +- Every live command gets immediate RC capture. Interpret rollback by stage: pre-transaction/pre-active-config failures have rollback N/A; active transaction failures require exact rollback PASS; ambiguity/manual recovery stops all later tasks. +- Any Host ledger/payload/timestamp staging failure consumes the rollback ID for Host purposes: preserve it, + create a fresh ID, and repeat source proof plus complete Task 6. Never repair staging or retry Host in place. +- Before any operator-shell preparation command, either install the exact production PATH, export it, run + `hash -r`, and verify the reviewed command inventory, or use literal absolute paths for every preparation + executable. Do not mix an ambient-PATH setup command into a fixed-boundary procedure. +- Do not mark existing Task 7 checkboxes until the corresponding live result is observed and independently reviewed. Task 8 remains unstarted. +- This workspace is non-Git. Do not commit, create a branch, or claim worktree cleanliness; record exact file metadata and SHA-256 instead. After freeze this implementation plan is immutable authority; track progress only in reports. + +--- + +### Task 1: Make the Host acceptance fixture production-faithful + +**Files:** +- Modify: `scripts/validate/test-apply-host-nginx-observability.sh:102-143` +- Modify: `scripts/bootstrap/apply-host-nginx-observability.sh:34,372-423,517-612` +- Create: `.superpowers/sdd/2026-08-15-observability-task7-cutover-correction/task-1-report.md` + +**Interfaces:** +- Consumes: Task 6 `access-rules-alerts/acceptance.env`, the two metric inventory hashes, and the same rollback ID. +- Produces: `validate_full_evidence()` acceptance only for an exact nine-field v2 record and one of two exact Slack gate tuples. + +- [ ] **Step 1: Freeze the starting identities** + + Record SHA-256, mode, UID:GID, nlink, type, and size for the two files. Required starting hashes are: + + ```text + apply = 518119a5e11fe8462c5d41921534351f78a2dd721c1bf69ac638e5bffe83536c + test = 1c429f7eb2cb73ec25dacd9fe4fdc5e063f62866c6e36732a3e9116fa68b2f53 + ``` + + If either hash differs, stop for source-drift review before editing. + +- [ ] **Step 2: Change the focused fixture to v2 and add a legacy-v1 rejection test** + + Replace the acceptance writer with an exact helper whose default is the successful Task 6 risk tuple: + + ```bash + write_acceptance_v2() { + local path=$1 initial_sha=$2 post_sha=$3 + local gate=${4:-RISK_ACCEPTED} ref uid + case "$gate" in + RECOVERY) + ref=strict-recovery-evidence-v1 + uid=not-applicable + ;; + RISK_ACCEPTED) + ref=2026-08-14-observability-slack-recovery-risk-acceptance-design + uid=1000 + ;; + *) return 2 ;; + esac + cat >"$path" <`/`` placeholders, smoke `argc=3`, and complete renderer `argc=6`. Require + root-owned, non-writable executable targets, safe PATH lineage, exact cardinality, no duplicate/missing row, + and no username/Secret value. Add a verifier launcher row for literal `/usr/bin/python3`, the absolute + verifier path and expected metadata, fixed digest sentinel ``, and the absolute + manifest/sidecar pair (`argc=5`, counting the interpreter as argv[0]). The sentinel must occur exactly once + in that row; the prelive/final manifest's own 64-character lowercase digest may not occupy that argv slot + or be copied into another manifest-row file. Other independently required file/executable SHA metadata is + unaffected. Generate a private comparison independently, require `cmp -s`, then remove only that + comparison. Repeat the comparison immediately before live Host execution. + + This split is mandatory because `task-7-command-inventory.txt` is manifest row 21. Only after all 21 rows + and the sidecar exist may independent review write the actual lowercase manifest digest to the non-governed + `task-5-report`/review handoff. The launcher reads no digest from the governed inventory; the controller + supplies the report's reviewed literal directly as verifier argv[2]. Add tests that replace the governed + verifier row's sentinel slot with a 64-character manifest digest and require rejection, require sentinel + exact-once plus exact argc/path/metadata shape, and refuse verifier invocation until the external report + pin exists and matches the reviewed value. + + The command inventory validator uses the same non-widening metadata policy as Task 2: exact canonical + `/usr/bin/sudo` is the only row permitted and required to carry owner-setuid and must be exact + `regular|0:0|4755|1`; every other executable row forbids both setuid and setgid as well as group/world + write. A private comparison with any sudo or non-sudo bit/path/owner/link drift must fail. + + Use these paths, once each and in this order: + + ```text + /home/donghyeon/workspace/platform/scripts/bootstrap/apply-host-nginx-observability.sh + /home/donghyeon/workspace/platform/scripts/validate/test-apply-host-nginx-observability.sh + /home/donghyeon/workspace/platform/scripts/validate/test-host-nginx-observability-source.sh + /home/donghyeon/workspace/platform/infrastructure/networking/host-nginx/learn-services-admin.conf + /home/donghyeon/workspace/platform/infrastructure/networking/host-nginx/learn-services-metrics-guard.conf + /home/donghyeon/workspace/platform/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf + /home/donghyeon/workspace/platform/infrastructure/networking/host-nginx/learn-services-observability.conf + /home/donghyeon/workspace/platform/scripts/bootstrap/apply-observability-access.sh + /home/donghyeon/workspace/platform/scripts/validate/test-apply-observability-access.sh + /home/donghyeon/workspace/platform/scripts/validate/render-observability-access.sh + /home/donghyeon/workspace/platform/scripts/validate/test-render-observability-access.sh + /home/donghyeon/workspace/platform/scripts/bootstrap/configure-keycloak-grafana-oidc.sh + /home/donghyeon/workspace/platform/scripts/validate/test-configure-keycloak-grafana-oidc.sh + /home/donghyeon/workspace/platform/scripts/validate/observability-smoke.sh + /home/donghyeon/workspace/platform/scripts/validate/test-observability-smoke.sh + /home/donghyeon/workspace/platform/bootstrap/manual/phase4-observability-access.md + /home/donghyeon/workspace/docs/platform/plans/2026-08-14-observability-slack-risk-acceptance-implementation.md + /home/donghyeon/workspace/docs/platform/plans/2026-08-15-observability-task7-cutover-correction-design.md + /home/donghyeon/workspace/platform/docs/superpowers/plans/2026-08-15-observability-task7-cutover-correction.md + /home/donghyeon/workspace/docs/platform/runbooks/2026-08-13-observability-phase4-resume-worklog.md + /home/donghyeon/workspace/platform/.superpowers/sdd/2026-08-15-observability-task7-cutover-correction/baseline/task-7-command-inventory.txt + ``` + + The verifier, manifest, and sidecar themselves are not rows. Create the sidecar with the literal canonical + manifest path, pin the verifier separately as specified in Step 3, and run the descriptor-retaining strict + launcher twice with the externally reviewed lowercase manifest SHA and literal absolute paths. Require RC + `0` both times and unchanged verifier/manifest/sidecar descriptor/path identities after each invocation. + +- [ ] **Step 5: Independent pre-live review** + + Review spec coverage, C/I/M, v2 tuple matrix, first-cutover behavior, fixed PATH and command/launcher + residue rejection, exact 21-row inventory, self-reference-free sentinel row plus external digest pin, + separately pinned verifier identity and rebind tests, both + correction authority documents, RED provenance, mutation evidence, original handoff + preservation, corrected OIDC/smoke sequencing, scanner result, and manifest/sidecar identity. Required + verdict: Critical `0`, Important `0`, Spec YES, Quality Approved, Ready for Host Task 7 YES. Any other + verdict returns to the responsible source task and requires a new scan/freeze after correction. + +--- + +### Task 6: Execute the single Host Nginx full cutover + +**Files:** +- Preserve: `/var/lib/hyeonworks/platform-rollbacks/observability-20260814T145009Z` +- Preserve: `/tmp/platform-observability-metrics.dw5gLZ` +- Append after outcome: `.superpowers/sdd/2026-08-15-observability-task7-cutover-correction/task-6-report.md` + +**Interfaces:** +- Consumes: Task 5 exact reviewed freeze, same Task 6 ID/root/handoff, current source proof, active deny-only config. +- Produces: full Host Nginx PASS or a preserved fail-closed transaction that is never retried under this ID. + +- [ ] **Step 1: Bind the successful Task 6 identity in the existing operator shell** + + Before any external preparation command, install the reviewed operator boundary and invalidate Bash's + command cache: + + ```bash + TASK7_OPERATOR_PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=$TASK7_OPERATOR_PATH + export PATH + hash -r + ``` + + Using the exact Task 5 generation procedure and only commands resolved under that boundary, regenerate the + command/launcher inventory into a fresh private comparison and require byte equality with the reviewed + inventory. Require every lexical/canonical command row to match before continuing. If this comparison + cannot be completed first, stop; do not run an ambient-PATH prep command. Then run short assignments only: + + ```bash + cd /home/donghyeon/workspace/platform + TASK7_ID=20260814T145009Z + TASK7_METRIC_ROOT=/tmp/platform-observability-metrics.dw5gLZ + export PLATFORM_OBSERVABILITY_ROLLBACK_ID="$TASK7_ID" + [[ "$PLATFORM_OBSERVABILITY_ROLLBACK_ID" == "$TASK7_ID" ]] + [[ "$(/usr/bin/readlink -f -- "$TASK7_METRIC_ROOT")" == "$TASK7_METRIC_ROOT" ]] + [[ "$(/usr/bin/stat -c '%F|%u:%g|%a|%h' -- "$TASK7_METRIC_ROOT")" == 'directory|1000:1000|700|4' ]] + ``` + + Use literal `/usr/bin/find`, `/usr/bin/sort`, `/usr/bin/stat`, and `/usr/bin/sha256sum` for the remaining + preparation checks. Require the root name-only topology to be exactly two phase directories plus + `dashboards.yaml`, `rules.yaml`, and `alertmanager.yaml`. Verify both known inventory hashes and metadata + without printing inventory bodies. + +- [ ] **Step 2: Revalidate the frozen source and no-contact dry-run** + + Run the Task 5 descriptor-retaining strict launcher twice, exactly as independently reviewed: literal + `/usr/bin/python3`, literal absolute verifier path, literal reviewed 64-character manifest SHA, and literal + absolute prelive manifest/sidecar paths. The SHA must come from the non-governed task-5 review handoff, + never from the command-inventory sentinel or another manifest-row file. Require RC `0` and unchanged separately pinned verifier + SHA/type/owner/mode/nlink/size/canonical-path identity before and after both invocations. + + Then build `TASK7_HOST_DRY=(/usr/bin/bash + /home/donghyeon/workspace/platform/scripts/bootstrap/apply-host-nginx-observability.sh)` and invoke it once + under the same errexit save/`set +e`/immediate RC/restore envelope used by execute. Require: + + ```text + HOST_NGINX_ACTIVE_SHA256=dbef6d443bcba58b26a5351ea76f6d09f6da8c2ef07a806e22745cf26c88f518 + HOST_NGINX_GRAFANA_DENY_GUARD_SHA256=dbef6d443bcba58b26a5351ea76f6d09f6da8c2ef07a806e22745cf26c88f518 + HOST_NGINX_FULL_SHA256=7d2de2a92c3597a0859775da1d2ccbf5a3d72c0b2af5cac2439c82311361f801 + HOST_NGINX_CERTIFICATE_EXPECTED_SAN=grafana.learn.hyeonworks.com + HOST_NGINX_CERTIFICATE_SAN=NOT_CHECKED_DRY_RUN + HOST_NGINX_GRAFANA_PUBLIC_DNS=NOT_CHECKED_DRY_RUN + HOST_NGINX_OBSERVABILITY_DRY_RUN=PASS + TASK7_HOST_DRY_RC=0 + ``` + + Also require legacy dry-run claims `HOST_NGINX_CERTIFICATE_EXACT_SAN=` and + `HOST_NGINX_GRAFANA_PUBLIC_DNS=ABSENT` to be absent. Any different/unreadable/already-full active hash stops + before sudo or execute. + +- [ ] **Step 3: Refresh sudo and build a paste-safe argv array** + + Run `/usr/bin/sudo -v`, then require `/usr/bin/sudo -n /usr/bin/true` and print only `SUDO_READY`. + + Build the command without placing the long option in pasted text: + + ```bash + TASK7_VOD=--verified + TASK7_VOD+=-output-dir + TASK7_HOST=(/usr/bin/bash) + TASK7_HOST+=(/home/donghyeon/workspace/platform/scripts/bootstrap/apply-host-nginx-observability.sh) + TASK7_HOST+=(--execute) + TASK7_HOST+=("$TASK7_VOD") + TASK7_HOST+=("$TASK7_METRIC_ROOT") + [[ "${#TASK7_HOST[@]}" -eq 5 ]] + printf 'TASK7_HOST_ARGC=%d\n' "${#TASK7_HOST[@]}" + ``` + + Do not print `%q` for usernames or sensitive data; this Host argv contains only safe paths/options. + +- [ ] **Step 4: Execute exactly once and capture immediate RC** + + ```bash + case $- in *e*) TASK7_HOST_ERREXIT=1 ;; *) TASK7_HOST_ERREXIT=0 ;; esac + set +e + "${TASK7_HOST[@]}" + TASK7_HOST_RC=$? + printf 'TASK7_HOST_RC=%d\n' "$TASK7_HOST_RC" + (( TASK7_HOST_ERREXIT == 0 )) || set -e + ``` + + Before entering anything, require exact pre-prompt stage and previous/candidate hashes. Only the operator enters `APPLY`. + + The prompt itself is admissible only after no-follow absence checks have proved all four paths absent under + both `test -e` and `test -L`: + + ```text + /var/lib/hyeonworks/platform-rollbacks/observability-20260814T145009Z/host-nginx + /var/lib/hyeonworks/platform-rollbacks/observability-20260814T145009Z/host-nginx/stages.tsv + /var/lib/hyeonworks/platform-rollbacks/observability-20260814T145009Z/host-nginx/payloads + /var/lib/hyeonworks/platform-rollbacks/observability-20260814T145009Z/host-nginx/payloads/full-prior-0001.conf + ``` + + An existing normal path or symlink is a pre-prompt STOP, not resumable ledger state. + + Success requires exact `HOST_NGINX_FULL_STAGE=PASS` and RC `0`. `ALREADY_ACTIVE`, missing PASS, nonzero or ambiguous RC is failure. + +- [ ] **Step 5: Classify failure without retry** + + - Pre-prompt failure: active deny unchanged, Host ledger absent, rollback N/A. + - Post-prompt/pre-arm failure: active deny and reload count unchanged; root-owned ledger/payload or timestamp + backup may remain and must be preserved; rollback N/A. + - Armed failure: require `HOST_NGINX_OBSERVABILITY_ROLLBACK=PASS` and active deny hash restored. + - `ROLLBACK=FAIL`, `MANUAL_RECOVERY_REQUIRED=YES`, unknown active hash, response loss, or missing evidence: STOP for manual recovery. + + If any residue is present before prompt, or any ledger directory/file, payload, or timestamp-backup staging + step fails, preserve the entire root and start a new Task 6 design/execution with a fresh ID. The same route + applies when source proof is stale/future. Do not alter this root or proof, recreate only one object, or + invoke the Host command again under `20260814T145009Z`. + +- [ ] **Step 6: Record the Host terminal result** + + Record safe argv cardinality, dry/execute RC, exact stage/hashes, whether APPLY was entered, Nginx active/test/reload outcome, three stable LAN/local postchecks, rollback invocation/outcome, original Task 6 evidence metadata unchanged, and process/temp residue delta. Do not mark later Task 7 steps yet. + +--- + +### Task 7: Reconcile memberships and run the monolithic acceptance once + +**Files:** +- Append: `.superpowers/sdd/2026-08-15-observability-task7-cutover-correction/task-7-report.md` +- Preserve unchanged: `scripts/bootstrap/configure-keycloak-grafana-oidc.sh` +- Preserve unchanged: `scripts/validate/observability-smoke.sh` + +**Interfaces:** +- Consumes: Host full PASS, prepared human identities, Slack view, and a real external client. +- Produces: OIDC transaction PASS plus one complete machine/OIDC/Slack/external smoke PASS. + +- [ ] **Step 1: Prove human/external readiness before mutation** + + Prepare four distinct realm usernames for admin, viewer, no-group, and membership-removal testing; local break-glass access; a Slack channel view; and an outside-LAN/Tailscale client with proxy disabled. If any is unavailable, stop without running membership or smoke. + +- [ ] **Step 2: Build and execute the OIDC membership argv once** + + ```bash + read -r -p 'Grafana organization admin realm username: ' OBS_ADMIN_USER + read -r -p 'Grafana viewer realm username: ' OBS_VIEWER_USER + TASK7_OIDC=(/usr/bin/bash /home/donghyeon/workspace/platform/scripts/bootstrap/configure-keycloak-grafana-oidc.sh) + TASK7_OIDC+=(--execute) + TASK7_OIDC+=(--admin "$OBS_ADMIN_USER") + TASK7_OIDC+=(--viewer "$OBS_VIEWER_USER") + case $- in *e*) TASK7_OIDC_ERREXIT=1 ;; *) TASK7_OIDC_ERREXIT=0 ;; esac + set +e + "${TASK7_OIDC[@]}" + TASK7_OIDC_RC=$? + unset OBS_ADMIN_USER OBS_VIEWER_USER TASK7_OIDC + printf 'TASK7_OIDC_RC=%d\n' "$TASK7_OIDC_RC" + (( TASK7_OIDC_ERREXIT == 0 )) || set -e + ``` + + Enter exact `APPLY default` and `RECOVERY KEYCLOAK default`. Require `GRAFANA_OIDC_TRANSACTION=PASS` and + RC `0`. A failure before `transaction_active=true` has no managed Keycloak/OIDC Secret/membership mutation + and rollback marker N/A. An active transaction failure requires `GRAFANA_OIDC_ROLLBACK=PASS`; rollback + FAIL/manual recovery or unknown stage stops. Do not claim or search for a persistent Task 6 Keycloak ledger. + +- [ ] **Step 3: Run the smoke no-arg plan once before execute** + + Build an absolute two-element argv array and use the errexit save/`set +e`/immediate RC/restore envelope: + + ```bash + TASK7_SMOKE_DRY=(/usr/bin/bash) + TASK7_SMOKE_DRY+=(/home/donghyeon/workspace/platform/scripts/validate/observability-smoke.sh) + ``` + + Invoke it exactly once. Require no-arg RC `0`, exact `OBSERVABILITY_SMOKE_DRY_RUN=PASS`, + `HUMAN_EXTERNAL_CLIENT=required`, and `MUTATION=NOT_REQUESTED`. This is not acceptance and does not replace + execute. + +- [ ] **Step 4: Execute the smoke exactly once with immediate RC** + + ```bash + TASK7_SMOKE=(/usr/bin/bash) + TASK7_SMOKE+=(/home/donghyeon/workspace/platform/scripts/validate/observability-smoke.sh) + TASK7_SMOKE+=(--execute) + case $- in *e*) TASK7_SMOKE_ERREXIT=1 ;; *) TASK7_SMOKE_ERREXIT=0 ;; esac + set +e + "${TASK7_SMOKE[@]}" + TASK7_SMOKE_RC=$? + printf 'TASK7_SMOKE_RC=%d\n' "$TASK7_SMOKE_RC" + (( TASK7_SMOKE_ERREXIT == 0 )) || set -e + ``` + + Only the operator supplies the requested identities and exact confirmations. Confirm actual admin/viewer/denied/break-glass/session-revoke/relogin behavior, Slack firing then resolved message, and run the script-rendered `--resolve` command from the true external client before entering its exact dynamic token. + +- [ ] **Step 5: Require the full acceptance marker set** + + Success requires all: + + ```text + OBSERVABILITY_MACHINE_ACCEPTANCE=PASS + OBSERVABILITY_OIDC_ACCEPTANCE=PASS + OBSERVABILITY_SLACK_ACCEPTANCE=PASS + OBSERVABILITY_EXTERNAL_BOUNDARY=PASS + OBSERVABILITY_SMOKE=PASS + TASK7_SMOKE_RC=0 + ``` + + Machine evidence must include LAN and Tailscale Grafana success, denied local/public boundary, external `403`, public metrics `404`, unknown SNI rejection, LAN NodePorts refused, loopback NodePort exact Traefik identity, exact `30/30` targets, datasource/trace link, persistence, network-policy denial, and Certbot/Nginx health. + + RC `2` or `OBSERVABILITY_EXTERNAL_BOUNDARY=PARTIAL` leaves Task 7 incomplete. Any cleanup ambiguity/manual recovery stops and forbids automatic rerun; review exact owned-object cleanup first. + +- [ ] **Step 6: Record payload-free OIDC/smoke evidence** + + Record roles and PASS/FAIL only, never identity values or browser/session data. Record temporary object kinds and cleanup disposition, not payloads. Verify Task 6 root/handoff metadata unchanged and no smoke-owned process/root remains. + +--- + +### Task 8: Run complete rendering from a fresh clone and existing regressions once + +**Files:** +- Create: one fresh `/tmp/platform-observability-metrics.XXXXXX` renderer root +- Preserve: `/tmp/platform-observability-metrics.dw5gLZ` +- Append: `.superpowers/sdd/2026-08-15-observability-task7-cutover-correction/task-8-report.md` + +**Interfaces:** +- Consumes: Task 7 final acceptance and byte-preserved inventories from the Task 6 handoff. +- Produces: complete renderer and existing platform regression RCs without a second smoke. + +- [ ] **Step 1: Fingerprint the original handoff without changing it** + + Reinstall `TASK7_OPERATOR_PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`, export + `PATH`, run `hash -r`, and require the independently regenerated command/launcher inventory to remain + byte-equal before any external preparation command. From this point, the procedure below nevertheless uses + literal absolute paths for every preparation executable. + + Require exact canonical root, UID:GID `1000:1000`, root mode `0700`, phase modes `0700`, file modes `0600`, nlink `1`, exact root/phase entry sets, and the two known inventory hashes. Record device/inode/type/owner/mode/nlink/size/mtime/ctime/SHA for both inventories and checksums plus the three Task 6 YAML files. Use only literal `/usr/bin/readlink`, `/usr/bin/stat`, `/usr/bin/find`, `/usr/bin/sort`, and `/usr/bin/sha256sum` for these checks. + +- [ ] **Step 2: Create the fresh inventory-only root** + + ```bash + TASK7_RENDER_ROOT="$(/usr/bin/mktemp -d /tmp/platform-observability-metrics.XXXXXX)" + /usr/bin/chmod 0700 "$TASK7_RENDER_ROOT" + for phase in target-initial post-substrate; do + /usr/bin/install -d -m 0700 -- "$TASK7_RENDER_ROOT/$phase" + for file in inventory.json inventory.sha256; do + source_file="$TASK7_METRIC_ROOT/$phase/$file" + destination_file="$TASK7_RENDER_ROOT/$phase/$file" + [[ -f "$source_file" && ! -L "$source_file" ]] + /usr/bin/cp --no-dereference --reflink=never -- "$source_file" "$destination_file" + /usr/bin/chmod 0600 "$destination_file" + /usr/bin/cmp -s -- "$source_file" "$destination_file" + done + done + unset source_file destination_file + ``` + + Require the exact six-entry topology, current-user ownership, modes, nlinks, byte equality, known hashes, and unchanged original fingerprints. Any failure preserves the new root and stops; do not repair or reuse it. + +- [ ] **Step 3: Run core and complete access renderers once** + + Build and invoke the core renderer exactly once with an absolute two-element argv and exact Helm + environment: + + ```bash + TASK7_CORE=(/usr/bin/bash) + TASK7_CORE+=(/home/donghyeon/workspace/platform/scripts/validate/render-observability-core.sh) + [[ "${#TASK7_CORE[@]}" -eq 2 ]] + case $- in *e*) TASK7_CORE_ERREXIT=1 ;; *) TASK7_CORE_ERREXIT=0 ;; esac + set +e + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm "${TASK7_CORE[@]}" + TASK7_CORE_RC=$? + (( TASK7_CORE_ERREXIT == 0 )) || set -e + printf 'TASK7_CORE_RC=%d\n' "$TASK7_CORE_RC" + ``` + + Require `OBSERVABILITY CORE STATIC RENDER PASS` and `TASK7_CORE_RC=0`. Do not invoke the core renderer in + Step 4 or through another script. + + Then build and invoke the complete access renderer exactly once: + + ```bash + TASK7_VOD=--verified + TASK7_VOD+=-output-dir + TASK7_COMPLETE=(/usr/bin/bash) + TASK7_COMPLETE+=(/home/donghyeon/workspace/platform/scripts/validate/render-observability-access.sh) + TASK7_COMPLETE+=(--component complete) + TASK7_COMPLETE+=("$TASK7_VOD" "$TASK7_RENDER_ROOT") + [[ "${#TASK7_COMPLETE[@]}" -eq 6 ]] + case $- in *e*) TASK7_COMPLETE_ERREXIT=1 ;; *) TASK7_COMPLETE_ERREXIT=0 ;; esac + set +e + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm "${TASK7_COMPLETE[@]}" + TASK7_COMPLETE_RC=$? + (( TASK7_COMPLETE_ERREXIT == 0 )) || set -e + printf 'TASK7_COMPLETE_RC=%d\n' "$TASK7_COMPLETE_RC" + ``` + + Require `OBSERVABILITY ACCESS STATIC RENDER PASS` and `TASK7_COMPLETE_RC=0`; verify seven YAML outputs and + original handoff fingerprints unchanged. Do not invoke the complete renderer again. + +- [ ] **Step 4: Run the remaining regressions once each** + + The remaining set has exactly five commands; neither renderer from Step 3 is a member. Build each absolute + argv and give every invocation its own errexit save/`set +e`/immediate RC/restore envelope: + + ```bash + TASK7_ADMIN_UI=(/usr/bin/bash) + TASK7_ADMIN_UI+=(/home/donghyeon/workspace/platform/scripts/validate/admin-ui-smoke.sh) + [[ "${#TASK7_ADMIN_UI[@]}" -eq 2 ]] + case $- in *e*) TASK7_ADMIN_UI_ERREXIT=1 ;; *) TASK7_ADMIN_UI_ERREXIT=0 ;; esac + set +e + "${TASK7_ADMIN_UI[@]}" + TASK7_ADMIN_UI_RC=$? + (( TASK7_ADMIN_UI_ERREXIT == 0 )) || set -e + printf 'TASK7_ADMIN_UI_RC=%d\n' "$TASK7_ADMIN_UI_RC" + + TASK7_AISTOR_S3=(/usr/bin/bash) + TASK7_AISTOR_S3+=(/home/donghyeon/workspace/platform/scripts/validate/aistor-s3-smoke.sh) + TASK7_AISTOR_S3+=(--execute) + [[ "${#TASK7_AISTOR_S3[@]}" -eq 3 ]] + case $- in *e*) TASK7_AISTOR_S3_ERREXIT=1 ;; *) TASK7_AISTOR_S3_ERREXIT=0 ;; esac + set +e + "${TASK7_AISTOR_S3[@]}" + TASK7_AISTOR_S3_RC=$? + (( TASK7_AISTOR_S3_ERREXIT == 0 )) || set -e + printf 'TASK7_AISTOR_S3_RC=%d\n' "$TASK7_AISTOR_S3_RC" + + TASK7_PHASE1=(/usr/bin/bash) + TASK7_PHASE1+=(/home/donghyeon/workspace/platform/scripts/validate/render-phase1.sh) + [[ "${#TASK7_PHASE1[@]}" -eq 2 ]] + case $- in *e*) TASK7_PHASE1_ERREXIT=1 ;; *) TASK7_PHASE1_ERREXIT=0 ;; esac + set +e + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm "${TASK7_PHASE1[@]}" + TASK7_PHASE1_RC=$? + (( TASK7_PHASE1_ERREXIT == 0 )) || set -e + printf 'TASK7_PHASE1_RC=%d\n' "$TASK7_PHASE1_RC" + + TASK7_PHASE2=(/usr/bin/bash) + TASK7_PHASE2+=(/home/donghyeon/workspace/platform/scripts/validate/render-phase2.sh) + [[ "${#TASK7_PHASE2[@]}" -eq 2 ]] + case $- in *e*) TASK7_PHASE2_ERREXIT=1 ;; *) TASK7_PHASE2_ERREXIT=0 ;; esac + set +e + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm "${TASK7_PHASE2[@]}" + TASK7_PHASE2_RC=$? + (( TASK7_PHASE2_ERREXIT == 0 )) || set -e + printf 'TASK7_PHASE2_RC=%d\n' "$TASK7_PHASE2_RC" + + TASK7_ADMIN_RENDER=(/usr/bin/bash) + TASK7_ADMIN_RENDER+=(/home/donghyeon/workspace/platform/scripts/validate/render-admin-services.sh) + [[ "${#TASK7_ADMIN_RENDER[@]}" -eq 2 ]] + case $- in *e*) TASK7_ADMIN_RENDER_ERREXIT=1 ;; *) TASK7_ADMIN_RENDER_ERREXIT=0 ;; esac + set +e + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm "${TASK7_ADMIN_RENDER[@]}" + TASK7_ADMIN_RENDER_RC=$? + (( TASK7_ADMIN_RENDER_ERREXIT == 0 )) || set -e + printf 'TASK7_ADMIN_RENDER_RC=%d\n' "$TASK7_ADMIN_RENDER_RC" + ``` + + The fixed operator `PATH` excludes `/home/donghyeon/.local/bin`, and both phase renderers fall back to + `command -v helm` when `PLATFORM_HELM_BIN` is unset. Therefore their exact inline assignments above are + mandatory and PATH fallback is forbidden. Before execution, run the Task 4 fenced-Bash structural checker + against both authority documents. Require its current-document RC `0`, then require nonzero RC for four + identity-checked private mutations that respectively remove the phase1 assignment, remove the phase2 + assignment, replace the phase1 value, or replace the phase2 value. Exercise each mutation in both document + positions with the other document unchanged; require all eight final gates nonzero. Remove only those + private copies. + + Do not run nonexistent `observability-core-smoke.sh`. Do not run `observability-smoke.sh --execute` again. Require every RC `0`, expected terminal PASS, and no new residue. + +- [ ] **Step 5: Preserve regression evidence through closeout** + + Preserve the original Task 6 handoff, fresh successful Task 7 renderer root, scanner root, rollback root, terminal status/logs, and name-only residue baseline through Task 9 review. Cleanup belongs to the later Task 8 final-closeout plan, not this task. + +--- + +### Task 9: Freeze terminal results, update status truthfully, and review Task 7 completion + +**Files:** +- Modify after observed results: `bootstrap/manual/phase4-observability-access.md` +- Modify after observed results: `/home/donghyeon/workspace/docs/platform/plans/2026-08-14-observability-slack-risk-acceptance-implementation.md` +- Append after observed results: `/home/donghyeon/workspace/docs/platform/runbooks/2026-08-13-observability-phase4-resume-worklog.md` +- Create: `.superpowers/sdd/2026-08-15-observability-task7-cutover-correction/baseline/task-7-final-source-manifest.txt` +- Create: `.superpowers/sdd/2026-08-15-observability-task7-cutover-correction/baseline/task-7-final-source-manifest.sha256` +- Create: `.superpowers/sdd/2026-08-15-observability-task7-cutover-correction/task-9-report.md` + +**Interfaces:** +- Consumes: exact terminal RCs/markers and preserved pre-live authority. +- Produces: evidence-backed Task 7 completion or an explicit partial/failed state; Task 8 remains separate. + +- [ ] **Step 1: Write only observed terminal facts** + + Record Host dry/execute markers and RC, active full hash, rollback invocation state, OIDC transaction/rollback RC, one smoke marker set and RC, external-client result, renderer/platform regression RCs, original handoff preservation, fresh renderer root, temporary-object cleanup, and residue delta. Do not record usernames, external client address beyond the safe public edge already printed by smoke, Secret-derived data, or response bodies. + +- [ ] **Step 2: Update Task 7 checkboxes conditionally** + + Mark a Step 1–6 checkbox only when its exact evidence exists. Mark all Task 7 steps complete only if Tasks 6–8 of this plan are RC `0` with every required marker and no unresolved cleanup/recovery. Otherwise leave failed/unrun steps unchecked and state `부분 구현` with the exact blocker. Do not mark any Task 8 checkbox. + +- [ ] **Step 3: State the remaining Slack DR exception** + + Even after Task 7 PASS, retain: + + ```text + Phase 4 operational acceptance: complete with active Slack DR exception + Slack off-host disaster recovery: deferred / not complete + ``` + + Never convert `RISK_ACCEPTED` into recovery-complete language. + +- [ ] **Step 4: Regenerate and strictly verify the final source manifest** + + Regenerate the same exact 21 paths in the same order after terminal documentation changes, bind the final + sidecar, and independently review the new literal final manifest SHA. Regenerate and compare the command + inventory first with the same exact-once `` sentinel; reject any actual manifest + digest in its verifier row. Only after the final manifest is complete may the non-governed task-9 + report/review handoff pin its actual digest. Re-pin the still-excluded verifier's canonical absolute path, SHA, type, owner, mode, + nlink, and size. Run the descriptor-retaining launcher twice using literal `/usr/bin/python3`, the literal + absolute verifier, the reviewed literal SHA, and the literal absolute final manifest/sidecar pair; require + RC `0` plus unchanged descriptor/path identities before and after each invocation. Record prelive/final + manifest and sidecar file hashes separately; do not overwrite or delete the prelive pair. + +- [ ] **Step 5: Final independent Task 7 review** + + Verify design/spec coverage, source quality, RED/GREEN/mutation provenance, fixed command boundary, v2 exact tuple, proof age/ID, prompt-before empty Host residue, Host state/rollback classification, OIDC truth, exactly one smoke per token/argv structural document assertion and exactly one live smoke, public/Tailscale/external evidence, `30/30` targets, NodePort boundary, exactly one core plus one complete renderer, exact phase1/phase2 Helm assignments with structural mutation rejection, clone/original preservation, every regression RC, self-reference-free 21-row manifest/sidecar binding and external digest pin, separate verifier pin/rebind defense, mutator/process/temp residue, and accurate docs/check boxes. + + Required terminal verdicts: + + ```text + Critical=0 + Important=0 + Spec=YES + Quality=APPROVED + Task7=COMPLETE + ReadyToStartTask8=YES + ``` + + Any Critical/Important finding or missing evidence returns to the responsible task; do not claim completion because the day ended or because only server-side checks passed. + +--- + +## Execution Handoff + +Plan implementation must use either `superpowers:subagent-driven-development` with a fresh implementer and spec/quality review per task, or `superpowers:executing-plans` with explicit checkpoints. The recommended route is subagent-driven development because Tasks 1–5 are independently reviewable before any live mutation, while Tasks 6–9 require controller/operator coordination. diff --git a/infrastructure/controllers/aistor-operator/README.md b/infrastructure/controllers/aistor-operator/README.md new file mode 100644 index 0000000..37c4f93 --- /dev/null +++ b/infrastructure/controllers/aistor-operator/README.md @@ -0,0 +1,56 @@ +# AIStor Operator + +상태(2026-07-23): 실제 클러스터 적용 완료. ObjectStore Operator, AdminJob +Operator와 admission webhook이 모두 Ready `1/1`이며 단일 ObjectStore를 +`Initialized/green`으로 관리한다. + +이 디렉터리는 AIStor Operator의 유일한 Kustomize 빌드 루트다. 업스트림 템플릿을 +복제하거나 수정하지 않고 공식 Chart를 렌더링한다. 홈 클러스터용 재정의 값은 +`values/home.yaml`에 둔다. + +## 고정된 원본 + +| 항목 | 값 | +| --- | --- | +| Chart 저장소 | `https://helm.min.io/` | +| Chart | `aistor-operator` | +| Chart 버전 | `5.10.0` | +| 애플리케이션 버전 | `v20260721224148.0.0` | +| Chart 패키지 SHA-256 | `e5534f5ae4f6f12a3528a8cda5954d73280cba1a8e979e1628fbf3aac76babd1` | + +Chart 캐시는 Git에서 제외된 `.helm/charts/` 아래에 작성된다. 모든 입력 파일이 이 +빌드 루트 아래에 있으므로 Kustomize의 기본 `RootOnly` 로드 제한으로 충분하다. + +## 라이선스 계약 + +Chart 값 `license`는 빈 문자열로 유지해야 한다. 이 빌드 루트를 적용하기 전에 +`aistor` namespace에 다음 키를 가진 `Secret/minio-license`를 생성한다. + +| Secret | 유형 | 필수 키 | +| --- | --- | --- | +| `aistor/minio-license` | `Opaque` | `minio.license` | + +라이선스 본문을 Git, values 파일, Shell 기록 또는 렌더링 산출물에 절대 남기지 +않는다. 이 값을 비워 두면 Chart는 미리 생성한 Secret을 참조하지만 라이선스 +Secret 자체를 렌더링하지 않는다. + +Operator, admin-job Operator와 admission webhook은 각각 +`donghyeon-system-product-name`에서 1 replica로 시작한다. 더 이상 권장되지 않는 +AIHub, Prompt 및 Warp는 비활성화한다. TokenReview는 Chart의 표준 Kubernetes 인증 +방식이므로 활성 상태를 유지한다. + +Helm으로 직접 설치하지 않고 Kustomize 렌더와 보호 스크립트로만 적용한다. + +```sh +kubectl kustomize --enable-helm infrastructure/controllers/aistor-operator +``` + +공식 Chart가 webhook `caBundle`을 렌더하지만 실행 중인 Operator가 이 값을 +주입·회전한다. 재적용 때 server-side apply 소유권 충돌이 발생하지 않도록 +`kustomization.yaml`에서 초기 빈 `caBundle` 필드를 제거한다. 실제 적용과 +readiness 재검증은 `scripts/bootstrap/apply-aistor.sh --execute`가 수행한다. + +## 공식 참고 문서 + +- +- diff --git a/infrastructure/controllers/aistor-operator/base/README.md b/infrastructure/controllers/aistor-operator/base/README.md new file mode 100644 index 0000000..62be41f --- /dev/null +++ b/infrastructure/controllers/aistor-operator/base/README.md @@ -0,0 +1,5 @@ +# 기본 확장 지점 + +업스트림 리소스는 컨트롤러 디렉터리 루트에 선언된 Helm Chart에서 가져온다. +Chart와 무관하고 환경에 종속되지 않는 보조 리소스가 필요할 때만 이곳에 추가하며, +Chart 템플릿을 저장소에 복제하지 않는다. diff --git a/infrastructure/controllers/aistor-operator/kustomization.yaml b/infrastructure/controllers/aistor-operator/kustomization.yaml new file mode 100644 index 0000000..11e91e1 --- /dev/null +++ b/infrastructure/controllers/aistor-operator/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: aistor + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: aistor-operator + repo: https://helm.min.io/ + version: 5.10.0 + releaseName: aistor + namespace: aistor + valuesFile: values/home.yaml + includeCRDs: true + skipTests: true + +# The Operator injects and rotates this runtime CA. Do not let the Helm render +# claim field ownership with an empty value, which would break idempotent SSA. +patches: + - target: + group: admissionregistration.k8s.io + version: v1 + kind: MutatingWebhookConfiguration + name: object-store-operator-webhook + patch: |- + - op: remove + path: /webhooks/0/clientConfig/caBundle diff --git a/infrastructure/controllers/aistor-operator/overlays/home/README.md b/infrastructure/controllers/aistor-operator/overlays/home/README.md new file mode 100644 index 0000000..15921b0 --- /dev/null +++ b/infrastructure/controllers/aistor-operator/overlays/home/README.md @@ -0,0 +1,4 @@ +# 홈 환경 Overlay 확장 지점 + +홈 클러스터 Chart 설정은 `../../values/home.yaml`에 둔다. 업스트림 Chart 값으로 +표현할 수 없는 Kustomize Patch만 이 디렉터리에 둔다. diff --git a/infrastructure/controllers/aistor-operator/values/home.yaml b/infrastructure/controllers/aistor-operator/values/home.yaml new file mode 100644 index 0000000..8dbaaeb --- /dev/null +++ b/infrastructure/controllers/aistor-operator/values/home.yaml @@ -0,0 +1,37 @@ +# The official chart always looks for Secret/aistor/minio-license and mounts the +# minio.license key. Keep this empty so Helm never renders the license into Git. +license: "" + +global: + fipsMode: false + clusterDomain: cluster.local + operator: + replicas: 1 + nodeSelector: + kubernetes.io/hostname: donghyeon-system-product-name + resources: + requests: + cpu: 100m + memory: 128Mi + ephemeral-storage: 100Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 1Gi + +operators: + adminjob: + disabled: false + aihub: + disabled: true + object-store: + disabled: false + replicas: 1 + tokenValidation: TokenReview + webhook: + enabled: true + replicas: 1 + prompt: + disabled: true + warp: + disabled: true diff --git a/infrastructure/controllers/cloudnative-pg/README.md b/infrastructure/controllers/cloudnative-pg/README.md new file mode 100644 index 0000000..b19b655 --- /dev/null +++ b/infrastructure/controllers/cloudnative-pg/README.md @@ -0,0 +1,32 @@ +# CloudNativePG Operator + +이 디렉터리는 Operator의 유일한 Kustomize 빌드 루트다. 공식 +`cloudnative-pg` Helm Chart를 복제하거나 수정하지 않고 렌더링한다. 홈 클러스터 +변경 사항은 `values/home.yaml`에 둔다. + +## 고정된 원본 + +- Chart 저장소: +- Chart 버전: `0.29.0` +- Operator 애플리케이션 버전: `1.30.0` +- 공식 저장소 인덱스에 있는 Chart 패키지 SHA-256: + `668e065ff53508d58238788fd35b355a925060843629a951df0e6a9362e6d32f` + +Operator는 전체 클러스터를 감시하지만, 자체 namespace 리소스는 `cnpg-system`에서 +실행된다. CRD와 클러스터 RBAC은 Chart가 설치한다. 모니터링 스택과 관련 CRD가 +준비될 때까지 PodMonitor와 Grafana Dashboard 리소스는 비활성화한다. + +저장소 공통 규칙에 따라 다음과 같이 렌더링한다. + +```bash +kubectl kustomize --enable-helm infrastructure/controllers/cloudnative-pg +``` + +Kustomize는 Git에서 제외된 이 디렉터리의 `.helm/charts/` 아래에 Chart를 내려받는다. +values 파일도 빌드 루트 아래에 있으므로 기본 `RootOnly` 로드 제한으로 충분하다. +`--load-restrictor=LoadRestrictionsNone`는 사용하지 않는다. + +## 공식 참고 문서 + +- +- diff --git a/infrastructure/controllers/cloudnative-pg/base/README.md b/infrastructure/controllers/cloudnative-pg/base/README.md new file mode 100644 index 0000000..02806b4 --- /dev/null +++ b/infrastructure/controllers/cloudnative-pg/base/README.md @@ -0,0 +1,5 @@ +# 기본 확장 지점 + +업스트림 Operator는 컨트롤러 디렉터리 루트에 선언된 Helm Chart로 공급한다. +Chart와 무관하고 환경에 종속되지 않는 보조 리소스가 실제로 필요할 때만 이곳에 +추가하며, Chart 템플릿을 저장소에 복제하지 않는다. diff --git a/infrastructure/controllers/cloudnative-pg/kustomization.yaml b/infrastructure/controllers/cloudnative-pg/kustomization.yaml new file mode 100644 index 0000000..ef1b350 --- /dev/null +++ b/infrastructure/controllers/cloudnative-pg/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: cnpg-system + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: cloudnative-pg + repo: https://cloudnative-pg.github.io/charts + version: 0.29.0 + releaseName: cloudnative-pg + namespace: cnpg-system + valuesFile: values/home.yaml + includeCRDs: true + skipTests: true diff --git a/infrastructure/controllers/cloudnative-pg/overlays/home/README.md b/infrastructure/controllers/cloudnative-pg/overlays/home/README.md new file mode 100644 index 0000000..ba1de2f --- /dev/null +++ b/infrastructure/controllers/cloudnative-pg/overlays/home/README.md @@ -0,0 +1,5 @@ +# 홈 환경 Overlay + +현재 홈 클러스터용 Operator Overlay는 `../../values/home.yaml`의 Chart values +파일로 표현한다. Helm values와 내려받은 Chart가 Kustomize의 기본 로드 제한 범위 +안에 있도록 컨트롤러 디렉터리를 빌드 루트로 유지한다. diff --git a/infrastructure/controllers/cloudnative-pg/values/home.yaml b/infrastructure/controllers/cloudnative-pg/values/home.yaml new file mode 100644 index 0000000..ad94344 --- /dev/null +++ b/infrastructure/controllers/cloudnative-pg/values/home.yaml @@ -0,0 +1,31 @@ +# Home-cluster overrides for the official CloudNativePG operator chart. +# Chart: cloudnative-pg 0.29.0 / appVersion 1.30.0 +replicaCount: 1 + +crds: + create: true + +config: + create: true + secret: false + clusterWide: true + +rbac: + create: true + aggregateClusterRoles: false + +serviceAccount: + create: true + +monitoring: + podMonitorEnabled: false + grafanaDashboard: + create: false + +resources: + requests: + cpu: 50m + memory: 100Mi + limits: + cpu: 500m + memory: 256Mi diff --git a/infrastructure/controllers/keycloak-operator/README.md b/infrastructure/controllers/keycloak-operator/README.md new file mode 100644 index 0000000..1ef2c2f --- /dev/null +++ b/infrastructure/controllers/keycloak-operator/README.md @@ -0,0 +1,35 @@ +# Keycloak Operator + +상태: `26.7.0` 실제 적용, Operator와 Keycloak Server 각각 `1/1` Ready +namespace는 `baseline` enforce·`restricted` warn이다. 공식 Operator 적용 시 +restricted 필드 경고가 있었지만 baseline 정책에서 rollout은 성공했으므로 이를 +restricted 준수 완료로 표현하지 않는다. + +이 디렉터리는 Keycloak Operator의 유일한 Kustomize 빌드 루트다. 공식 Keycloak +Kubernetes 리소스를 직접 참조하며 업스트림 매니페스트를 복제하거나 수정하지 +않는다. + +## 고정된 원본 + +- 저장소: +- 리소스 경로: `kubernetes` +- Keycloak 및 Operator 버전: `26.7.0` +- 설치 모드: `keycloak` namespace 범위 + +업스트림 namespace 범위 Deployment는 각 컨트롤러의 감시 namespace를 +`JOSDK_WATCH_CURRENT`로 설정한다. 이 단일 인스턴스 홈 클러스터에서는 프리뷰 +상태인 `cluster-wide` Overlay로 전환하지 않는다. + +이 디렉터리에서 다음과 같이 렌더링한다. + +```bash +kubectl kustomize infrastructure/controllers/keycloak-operator +``` + +remote 참조가 Kustomize 캐시에 없으면 네트워크 접근이 필요하다. Argo CD가 고정된 +GitHub 원본을 가져올 수 있어야 한다. + +## 공식 참고 문서 + +- +- diff --git a/infrastructure/controllers/keycloak-operator/base/README.md b/infrastructure/controllers/keycloak-operator/base/README.md new file mode 100644 index 0000000..5cccf33 --- /dev/null +++ b/infrastructure/controllers/keycloak-operator/base/README.md @@ -0,0 +1,5 @@ +# 기본 확장 지점 + +공식 namespace 범위 Operator는 컨트롤러 디렉터리 루트의 Kustomization에서 +참조한다. 환경에 종속되지 않는 보조 리소스만 이곳에 추가하며, 생성된 CRD, +RBAC 또는 Operator Deployment를 저장소에 복제하지 않는다. diff --git a/infrastructure/controllers/keycloak-operator/kustomization.yaml b/infrastructure/controllers/keycloak-operator/kustomization.yaml new file mode 100644 index 0000000..d8c89fc --- /dev/null +++ b/infrastructure/controllers/keycloak-operator/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: keycloak + +resources: + - github.com/keycloak/keycloak-k8s-resources/kubernetes?ref=26.7.0 + +labels: + - pairs: + app.kubernetes.io/part-of: platform + includeSelectors: false diff --git a/infrastructure/controllers/keycloak-operator/overlays/home/README.md b/infrastructure/controllers/keycloak-operator/overlays/home/README.md new file mode 100644 index 0000000..971c11d --- /dev/null +++ b/infrastructure/controllers/keycloak-operator/overlays/home/README.md @@ -0,0 +1,5 @@ +# 홈 환경 Overlay 확장 지점 + +Keycloak Operator 26.7.0에는 홈 환경 전용 Patch가 필요하지 않다. 고정한 업스트림 +리소스를 로컬 Kustomize와 향후 Argo CD에서 일관되게 렌더링하도록 컨트롤러 +디렉터리를 유일한 빌드 루트로 유지한다. diff --git a/infrastructure/controllers/prometheus-operator-crds/README.md b/infrastructure/controllers/prometheus-operator-crds/README.md new file mode 100644 index 0000000..3cd4f5c --- /dev/null +++ b/infrastructure/controllers/prometheus-operator-crds/README.md @@ -0,0 +1,5 @@ +# Prometheus Operator CRDs + +`prometheus-operator-crds` chart `31.0.0`만 독립적으로 렌더한다. CRD는 +cluster-scoped API와 저장 객체의 스키마 경계이므로 workload 실패 시에도 자동 삭제하지 +않는다. `kube-prometheus-stack` 쪽에서는 CRD 생성을 별도로 비활성화한다. diff --git a/infrastructure/controllers/prometheus-operator-crds/kustomization.yaml b/infrastructure/controllers/prometheus-operator-crds/kustomization.yaml new file mode 100644 index 0000000..fde5b14 --- /dev/null +++ b/infrastructure/controllers/prometheus-operator-crds/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: prometheus-operator-crds + repo: https://prometheus-community.github.io/helm-charts + version: 31.0.0 + releaseName: prometheus-operator-crds + namespace: observability + valuesFile: values/home.yaml + includeCRDs: true + skipTests: true diff --git a/infrastructure/controllers/prometheus-operator-crds/values/home.yaml b/infrastructure/controllers/prometheus-operator-crds/values/home.yaml new file mode 100644 index 0000000..1d79f92 --- /dev/null +++ b/infrastructure/controllers/prometheus-operator-crds/values/home.yaml @@ -0,0 +1,22 @@ +crds: + annotations: {} + alertmanagerconfigs: + enabled: true + alertmanagers: + enabled: true + podmonitors: + enabled: true + probes: + enabled: true + prometheusagents: + enabled: true + prometheuses: + enabled: true + prometheusrules: + enabled: true + scrapeconfigs: + enabled: true + servicemonitors: + enabled: true + thanosrulers: + enabled: true diff --git a/infrastructure/namespaces/admin-tools/README.md b/infrastructure/namespaces/admin-tools/README.md new file mode 100644 index 0000000..bd4b079 --- /dev/null +++ b/infrastructure/namespaces/admin-tools/README.md @@ -0,0 +1,7 @@ +# 관리 도구 네임스페이스 + +`platform-admin`은 pgAdmin처럼 운영자가 사용하는 비공개 관리 도구만 +수용합니다. 애플리케이션, 데이터베이스, 모니터링 워크로드와 분리하며 +Pod Security `restricted`를 강제합니다. + +이 디렉터리에는 Secret을 두지 않습니다. diff --git a/infrastructure/namespaces/admin-tools/kustomization.yaml b/infrastructure/namespaces/admin-tools/kustomization.yaml new file mode 100644 index 0000000..dca4a51 --- /dev/null +++ b/infrastructure/namespaces/admin-tools/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml diff --git a/infrastructure/namespaces/admin-tools/namespace.yaml b/infrastructure/namespaces/admin-tools/namespace.yaml new file mode 100644 index 0000000..7c0e874 --- /dev/null +++ b/infrastructure/namespaces/admin-tools/namespace.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: platform-admin + labels: + app.kubernetes.io/part-of: platform + platform.hyeonworks.com/purpose: private-admin-tools + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest diff --git a/infrastructure/namespaces/base/README.md b/infrastructure/namespaces/base/README.md new file mode 100644 index 0000000..337f079 --- /dev/null +++ b/infrastructure/namespaces/base/README.md @@ -0,0 +1,17 @@ +# 플랫폼 네임스페이스 + +이 base는 첫 Gitea 배포에 필요한 namespace만 선언한다. + +- `cnpg-system`: CloudNativePG Operator +- `platform-data`: 공통 플랫폼 PostgreSQL Cluster +- `gitea`: Gitea 애플리케이션 + +초기 구축 중 업스트림 Operator와 Helm Chart가 차단되지 않도록 Pod Security +Admission은 `baseline`을 강제한다. 더 엄격한 `restricted` 프로파일 위반은 감사 +이벤트와 kubectl 경고를 통해 계속 표시한다. namespace에서 `restricted` 강제로 +전환하기 전에는 렌더링된 매니페스트 검사와 런타임 Smoke Test를 먼저 통과해야 +한다. + +`latest` 정책 버전은 이 단일 노드 클러스터에 설치된 Kubernetes 버전을 따른다. +향후 클러스터 업그레이드와 정책 변경을 분리해서 진행해야 한다면 먼저 Kubernetes +minor 버전으로 고정한다. diff --git a/infrastructure/namespaces/base/cnpg-system.yaml b/infrastructure/namespaces/base/cnpg-system.yaml new file mode 100644 index 0000000..7b9e712 --- /dev/null +++ b/infrastructure/namespaces/base/cnpg-system.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cnpg-system + labels: + app.kubernetes.io/part-of: platform + platform.hyeonworks.com/purpose: database-controller + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest diff --git a/infrastructure/namespaces/base/gitea.yaml b/infrastructure/namespaces/base/gitea.yaml new file mode 100644 index 0000000..a5de492 --- /dev/null +++ b/infrastructure/namespaces/base/gitea.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gitea + labels: + app.kubernetes.io/part-of: platform + platform.hyeonworks.com/purpose: source-control + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest diff --git a/infrastructure/namespaces/base/kustomization.yaml b/infrastructure/namespaces/base/kustomization.yaml new file mode 100644 index 0000000..a8a2e5f --- /dev/null +++ b/infrastructure/namespaces/base/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cnpg-system.yaml + - platform-data.yaml + - gitea.yaml diff --git a/infrastructure/namespaces/base/platform-data.yaml b/infrastructure/namespaces/base/platform-data.yaml new file mode 100644 index 0000000..1b0e1a7 --- /dev/null +++ b/infrastructure/namespaces/base/platform-data.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: platform-data + labels: + app.kubernetes.io/part-of: platform + platform.hyeonworks.com/purpose: platform-databases + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest diff --git a/infrastructure/namespaces/observability/README.md b/infrastructure/namespaces/observability/README.md new file mode 100644 index 0000000..22733e9 --- /dev/null +++ b/infrastructure/namespaces/observability/README.md @@ -0,0 +1,17 @@ +# Observability namespaces + +This kustomization owns exactly the `observability` and `observability-agent` +Namespace objects. Both opt into observability discovery through +`observability.hyeonworks.com/enabled: "true"`. + +`observability` enforces, audits, and warns at `restricted:latest`. +`observability-agent` enforces `privileged:latest` so the narrowly scoped agent +workloads can use their approved read-only host mounts, while audit and warn +remain `restricted:latest`. + +`discovery-namespaces.txt` is an inventory for the later apply workflow. The +five listed namespaces already exist and are deliberately not rendered here. +That workflow may own only the +`observability.hyeonworks.com/enabled` discovery label on those namespaces. It +must preserve their existing Pod Security and purpose labels, and rollback must +restore or remove only that exact discovery-label key. diff --git a/infrastructure/namespaces/observability/discovery-namespaces.txt b/infrastructure/namespaces/observability/discovery-namespaces.txt new file mode 100644 index 0000000..61f2125 --- /dev/null +++ b/infrastructure/namespaces/observability/discovery-namespaces.txt @@ -0,0 +1,5 @@ +kube-system +gitea +keycloak +platform-data +object-storage diff --git a/infrastructure/namespaces/observability/kustomization.yaml b/infrastructure/namespaces/observability/kustomization.yaml new file mode 100644 index 0000000..7e24c32 --- /dev/null +++ b/infrastructure/namespaces/observability/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - observability.yaml + - observability-agent.yaml diff --git a/infrastructure/namespaces/observability/observability-agent.yaml b/infrastructure/namespaces/observability/observability-agent.yaml new file mode 100644 index 0000000..ea33b0d --- /dev/null +++ b/infrastructure/namespaces/observability/observability-agent.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: observability-agent + labels: + observability.hyeonworks.com/enabled: "true" + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest diff --git a/infrastructure/namespaces/observability/observability.yaml b/infrastructure/namespaces/observability/observability.yaml new file mode 100644 index 0000000..f11f49f --- /dev/null +++ b/infrastructure/namespaces/observability/observability.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: observability + labels: + observability.hyeonworks.com/enabled: "true" + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest diff --git a/infrastructure/namespaces/overlays/home/kustomization.yaml b/infrastructure/namespaces/overlays/home/kustomization.yaml new file mode 100644 index 0000000..774a422 --- /dev/null +++ b/infrastructure/namespaces/overlays/home/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base diff --git a/infrastructure/namespaces/phase2/README.md b/infrastructure/namespaces/phase2/README.md new file mode 100644 index 0000000..6859568 --- /dev/null +++ b/infrastructure/namespaces/phase2/README.md @@ -0,0 +1,22 @@ +# Phase 2 네임스페이스 + +상태: `keycloak`만 실제 적용, `aistor`·`object-storage` 미적용 +현재 AIStor NetworkPolicy gate가 남아 있으므로 디렉터리 전체를 apply하지 않고 +Keycloak-only 스크립트가 `keycloak.yaml` 하나만 정확히 적용했다. + +이 빌드 루트는 Phase 1 namespace 빌드 루트를 변경하지 않고 인증과 오브젝트 +스토리지 단계에 추가되는 namespace를 선언한다. + +- `keycloak`: Keycloak Operator와 단일 Keycloak 인스턴스 +- `aistor`: AIStor Operator와 `minio-license` Secret 계약 +- `object-storage`: standalone AIStor ObjectStore와 해당 PVC + +Phase 2의 컨트롤러나 서비스를 적용하기 전에 이 디렉터리를 별도로 렌더링한다. + +```sh +kubectl kustomize infrastructure/namespaces/phase2 +``` + +이 namespace들은 Pod Security `baseline` 프로파일을 강제하고 `restricted` +위반을 감사하고 경고한다. `restricted` 강제로 전환하기 전에는 고정된 업스트림 +Operator의 렌더링과 런타임 시험을 먼저 통과해야 한다. diff --git a/infrastructure/namespaces/phase2/aistor.yaml b/infrastructure/namespaces/phase2/aistor.yaml new file mode 100644 index 0000000..3801bff --- /dev/null +++ b/infrastructure/namespaces/phase2/aistor.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: aistor + labels: + app.kubernetes.io/part-of: platform + platform.hyeonworks.com/purpose: object-storage-control-plane + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest diff --git a/infrastructure/namespaces/phase2/keycloak.yaml b/infrastructure/namespaces/phase2/keycloak.yaml new file mode 100644 index 0000000..01c1de7 --- /dev/null +++ b/infrastructure/namespaces/phase2/keycloak.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: keycloak + labels: + app.kubernetes.io/part-of: platform + platform.hyeonworks.com/purpose: identity + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest diff --git a/infrastructure/namespaces/phase2/kustomization.yaml b/infrastructure/namespaces/phase2/kustomization.yaml new file mode 100644 index 0000000..54b1d29 --- /dev/null +++ b/infrastructure/namespaces/phase2/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - keycloak.yaml + - aistor.yaml + - object-storage.yaml diff --git a/infrastructure/namespaces/phase2/object-storage.yaml b/infrastructure/namespaces/phase2/object-storage.yaml new file mode 100644 index 0000000..0aebec9 --- /dev/null +++ b/infrastructure/namespaces/phase2/object-storage.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: object-storage + labels: + app.kubernetes.io/part-of: platform + platform.hyeonworks.com/purpose: object-storage-data-plane + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest diff --git a/infrastructure/networking/aistor/README.md b/infrastructure/networking/aistor/README.md new file mode 100644 index 0000000..74ff3ea --- /dev/null +++ b/infrastructure/networking/aistor/README.md @@ -0,0 +1,20 @@ +# AIStor 네트워크 정책 + +이 디렉터리는 AIStor 제어 영역(`aistor`)과 데이터 영역(`object-storage`)의 +NetworkPolicy를 한 곳에서 관리한다. + +- 두 namespace의 모든 Pod는 기본적으로 ingress와 egress가 차단된다. +- DNS와 k3s Kubernetes API 접근만 명시적으로 허용한다. +- API server에서 admission webhook `8443/TCP`로 들어오는 트래픽을 허용한다. +- ObjectStore Pod와 Operator 사이에는 관리 포트만 허용한다. +- S3 `9000/TCP`와 HTTP Console `9090/TCP`는 `object-storage` namespace 내부 + 클라이언트만 사용할 수 있다. +- Operator와 sidecar 관리에는 `4221/TCP`, `4444/TCP`만 허용한다. +- 외부 인터넷, Traefik, NodePort, LoadBalancer, Host Nginx 경로는 열지 않는다. + +`learning-project`가 S3를 사용하게 될 때에는 이 파일을 넓히지 않고, 소비 +namespace와 ServiceAccount 또는 Pod label을 고정한 별도 정책을 추가한다. + +```bash +kubectl kustomize infrastructure/networking/aistor +``` diff --git a/infrastructure/networking/aistor/kustomization.yaml b/infrastructure/networking/aistor/kustomization.yaml new file mode 100644 index 0000000..00fff3b --- /dev/null +++ b/infrastructure/networking/aistor/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - network-policies.yaml diff --git a/infrastructure/networking/aistor/network-policies.yaml b/infrastructure/networking/aistor/network-policies.yaml new file mode 100644 index 0000000..74272b2 --- /dev/null +++ b/infrastructure/networking/aistor/network-policies.yaml @@ -0,0 +1,313 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: aistor-default-deny + namespace: aistor + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage-control-plane +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: aistor-allow-dns + namespace: aistor + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage-control-plane +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: aistor-allow-kubernetes-api + namespace: aistor + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage-control-plane +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - protocol: TCP + port: 443 + - to: + - ipBlock: + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 6443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: aistor-allow-api-server-admission-webhook + namespace: aistor + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage-control-plane +spec: + podSelector: + matchLabels: + aistor.min.io/name: object-store-webhook + policyTypes: + - Ingress + ingress: + - from: + - ipBlock: + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 8443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: aistor-allow-object-store-management + namespace: aistor + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage-control-plane +spec: + podSelector: + matchExpressions: + - key: aistor.min.io/name + operator: In + values: + - adminjob-operator + - object-store-operator + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: object-storage + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + ports: + - protocol: TCP + port: 4221 + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: object-storage + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + ports: + - protocol: TCP + port: 9000 + - protocol: TCP + port: 4444 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-allow-labeled-clients + namespace: object-storage + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage +spec: + podSelector: + matchLabels: + platform.hyeonworks.com/aistor-client: "true" + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: object-storage + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + ports: + - protocol: TCP + port: 80 + - protocol: TCP + port: 9000 + - protocol: TCP + port: 9090 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-default-deny + namespace: object-storage + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-allow-dns + namespace: object-storage + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-allow-kubernetes-api + namespace: object-storage + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - protocol: TCP + port: 443 + - to: + - ipBlock: + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 6443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-allow-internal-clients + namespace: object-storage + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage +spec: + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + - podSelector: + matchLabels: + platform.hyeonworks.com/aistor-client: "true" + ports: + - protocol: TCP + port: 9000 + - protocol: TCP + port: 9090 + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: object-storage + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + ports: + - protocol: TCP + port: 9000 + - protocol: TCP + port: 9090 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-allow-aistor-operator + namespace: object-storage + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage +spec: + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: aistor + podSelector: + matchExpressions: + - key: aistor.min.io/name + operator: In + values: + - adminjob-operator + - object-store-operator + ports: + - protocol: TCP + port: 9000 + - protocol: TCP + port: 4444 + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: aistor + podSelector: + matchLabels: + aistor.min.io/name: object-store-operator + ports: + - protocol: TCP + port: 4221 diff --git a/infrastructure/networking/host-nginx/README.md b/infrastructure/networking/host-nginx/README.md new file mode 100644 index 0000000..5e41b1e --- /dev/null +++ b/infrastructure/networking/host-nginx/README.md @@ -0,0 +1,309 @@ +# 호스트 Nginx 전환 + +이 디렉터리에는 호스트 수준 Nginx 신뢰 경계의 설정 원본이 있다. 이 파일은 +Kubernetes 리소스가 아니며 Argo CD에서 조정하지 않는다. +2026-07-23 현재 활성 설정은 Gitea와 Keycloak을 함께 proxy하는 +[learn-services-keycloak.conf](./learn-services-keycloak.conf)이며 SHA-256은 +`5c5cd74b4992f537fd27c50cf2209573a80a9904e0b19b58c3154717be6ff4a5`다. +전환 전 설정은 +`/etc/nginx/sites-available/learn-services.before-keycloak-20260723160519`에 +백업했다. Nginx proxy 상태는 두 번의 probe 뒤 안정화됐고 Gitea health, Keycloak +discovery issuer, HTTPS cookie·redirect와 미등록 hostname 거부 검사를 통과했다. + +후속 Gitea OIDC·브랜딩 rollout도 manifest SHA-256 `d25a757...a157`로 완료했으며 +OAuth source·정책·authorization-code redirect·브랜딩 자동 검증을 통과했다. 실제 +realm 사용자의 브라우저 login/callback/logout, 비상 관리자 실제 로그인과 Pod +재시작 뒤 설정 지속성은 별도 수용 시험으로 남아 있다. 인증이 필요한 Git +clone/push/reclone 시험도 아직 남아 있다. + +## 계약 + +- 공개 포트 `80`과 `443`은 Host Nginx에서 종료한다. +- TLS는 Host Nginx에서만 종료한다. +- `git.learn.hyeonworks.com`은 HTTP를 통해 Traefik의 loopback NodePort인 + `127.0.0.1:30080`으로 proxy한다. +- Host Nginx는 들어오는 `Host`, `X-Forwarded-*`, `X-Real-IP` 값을 교체한다. 특히 + 신뢰할 수 없는 클라이언트가 제공한 `X-Forwarded-For` chain을 이어 붙이지 않는다. +- `id.learn.hyeonworks.com`도 같은 loopback NodePort의 Keycloak Host route로 + proxy한다. +- Traefik HTTPS NodePort인 `30443`은 이 경로에서 사용하지 않는다. + +2026-07-23 현재 kube-proxy의 `nodePort-addresses=127.0.0.0/8` 설정을 적용했다. +서버의 loopback `127.0.0.1:30080`은 Traefik에 도달하지만, 서버와 같은 LAN의 +노트북에서 `192.168.0.107:30080` 및 `30443` 연결은 모두 거부되는 것을 확인했다. +따라서 Host Nginx를 우회하는 LAN NodePort 경로는 현재 닫혀 있다. + +현재 설정 원본은 +[learn-services-keycloak.conf](./learn-services-keycloak.conf)다. 전환 전 +[learn-services.conf](./learn-services.conf)는 Keycloak 정적 hold가 포함된 +rollback 기준으로 보존한다. 등록되지 않은 TLS hostname이 첫 번째 virtual host인 +Gitea로 흘러가지 않도록 별도의 `default_server`가 `ssl_reject_handshake on`으로 +handshake를 거부한다. + +활성 Keycloak 설정은 같은 Gitea proxy를 보존하면서 `id.learn.hyeonworks.com`만 +`http://127.0.0.1:30080`의 Traefik Host route로 바꾼다. 이는 Keycloak을 모든 +서비스 앞의 인증 middleware로 두는 구성이 아니다. Gitea가 Keycloak을 독립 OIDC +Provider로 사용하는 데 필요한 네트워크 reverse proxy다. + +2026-07-23 확인한 인증서는 CN이 `git.learn.hyeonworks.com`이고 SAN에 +`git.learn.hyeonworks.com`, `id.learn.hyeonworks.com`을 모두 포함한다. 발급자는 +Let's Encrypt YE2, 유효기간은 2026-07-18부터 2026-10-16까지다. Snap Certbot +5.7.0과 `snap.certbot.renew.timer` 활성 상태도 확인했다. 이 값은 점검 시점의 +스냅샷이므로 설정 또는 인증서가 변경될 때 다시 검사한다. + +## 사전 조건 + +다음 검사를 모두 통과하기 전에는 Nginx를 전환하지 않는다. + +1. Gitea 워크로드, Service 및 Ingress가 Ready 상태다. +2. Ingress는 `git.learn.hyeonworks.com`을 사용하며 Kubernetes TLS block이 없다. +3. 다음 Traefik 직접 probe가 Gitea health 응답을 반환한다. + + ```sh + curl --fail-with-body \ + --header 'Host: git.learn.hyeonworks.com' \ + http://127.0.0.1:30080/api/healthz + ``` + +4. 별도 LAN 호스트에서 `192.168.0.107:30080`과 `192.168.0.107:30443`의 TCP + 연결이 모두 거부되거나 timeout되는지 확인한다. + + ```sh + nc -vz -w 3 192.168.0.107 30080 + nc -vz -w 3 192.168.0.107 30443 + ``` + + TCP 연결이나 HTTP 응답이 하나라도 성공하면 이 차단 조건을 통과하지 못한 것이다. Traefik `404`는 + 포트가 차단된 것이 아니라 정상적으로 도달했다는 증거다. 포트 하나라도 + 도달할 수 있으면 전환을 중지하고 `../traefik/README.md`의 제한 지침을 따른다. + +5. 활성 인증서가 이 파일에 남아 있는 두 hostname을 모두 포함한다. + + ```sh + sudo certbot certificates + ``` + +6. `sudo nginx -t`와 Nginx 서비스 상태 검사를 통과한다. + +## 최초 Gitea 전환 절차(과거 기록) + +다음 절차는 Keycloak 전환 전 Gitea-only 설정을 처음 적용했을 때의 기록이다. +현재 활성 Keycloak 설정에 이 스크립트를 재실행하지 않는다. 당시에는 Argo CD가 +아니라 의도적인 호스트 작업으로 저장소 루트에서 다음 스크립트를 실행했다. + +```sh +cd /home/donghyeon/workspace/platform +sudo bash scripts/bootstrap/apply-host-nginx-gitea.sh --execute +``` + +스크립트가 출력한 활성/후보 SHA-256과 백업 경로를 확인한 뒤 prompt에 정확히 +`APPLY`를 입력한다. 스크립트는 다음 작업을 한 단위로 수행한다. + +- 기존 활성 파일을 timestamp가 붙은 root 소유 파일로 백업하고 해시를 검증한다. +- 후보를 root:root, mode 0644로 설치한 뒤 `nginx -t`, reload, active 상태를 검사한다. +- reload 직후 기존 placeholder fingerprint만 bounded retry하고, 정상 Gitea health + JSON을 두 번 연속 확인해야 다음 검사로 진행한다. 다른 `200` 비JSON 응답은 + 오라우팅으로 즉시 실패한다. +- loopback TLS 경로의 Gitea health, HTTP→HTTPS 301, 로그인 쿠키의 `Secure`, + Keycloak hold 응답, 미등록 TLS hostname 거부를 검사한다. +- 활성 파일 변경 뒤 포착 가능한 오류가 발생하거나 INT/TERM signal로 중단되면 + 정확한 백업을 자동 복원하고 `nginx -t`와 reload를 다시 수행한다. +- SIGKILL 또는 전원 장애처럼 trap이 실행될 수 없는 중단은 자동 복구 대상이 아니며, + 재접속 후 출력된 백업 경로로 수동 복구한다. + +서버에서는 NAT hairpin이 지원되지 않아 공인 주소를 향한 요청이 timeout될 수 있다. +따라서 서버 로컬 검증은 스크립트처럼 `--resolve ...:127.0.0.1`을 사용한다. 실제 +공개 경로는 외부망(예: 모바일 핫스팟)에 연결된 별도 클라이언트 또는 외부 probe에서 +`--resolve` 없이 검증한다. 아래 명령과 정적 hold 기대값은 최초 Gitea-only 전환 +당시의 검사이며 현재 Keycloak 공개 경로의 수용 기준이 아니다. + +```sh +curl --fail-with-body https://git.learn.hyeonworks.com/api/healthz +curl --fail-with-body https://id.learn.hyeonworks.com/ +``` + +두 번째 응답은 계속 `Keycloak domain reached Nginx successfully`여야 한다. 실패 시 +스크립트가 출력한 정확한 백업 경로를 사용해 다음 순서로 수동 복구한다. + +```sh +sudo install -o root -g root -m 0644 BACKUP_PATH /etc/nginx/sites-available/learn-services +sudo nginx -t +sudo systemctl reload nginx +``` + +전환 기록에는 활성/후보/백업 SHA-256, 실제 백업 경로, `nginx -t`, reload, 로컬 +health와 redirect, 값은 숨긴 Cookie 속성, 외부 health, Git clone/push/reclone 결과를 +남긴다. + +2026-07-23 1차 전환의 reload readiness race와 자동 롤백, 2차 전환 성공, 공개 +경로 검증 결과는 +[중앙 실행 기록](../../../../docs/platform/runbooks/2026-07-23-host-nginx-gitea-cutover.md)에 +보존한다. 사용자 인증이 필요한 Git clone/push/reclone은 아직 남아 있으므로 그 +결과도 같은 문서에 추가한다. + +## Keycloak proxy 전환 + +### 완료 상태와 선행 조건 + +Keycloak 후보는 2026-07-23 활성화했다. 다음 항목은 전환 전에 모두 통과한 +선행 조건이다. + +1. `keycloak` namespace의 공식 Keycloak Operator와 Server `26.7.0`이 Ready다. +2. `hyeonworks` realm의 내부 discovery가 JSON으로 응답하고 issuer가 정확히 + `https://id.learn.hyeonworks.com/realms/hyeonworks`다. + + ```sh + curl --disable --noproxy '*' \ + --fail-with-body --silent --show-error \ + --header 'Host: id.learn.hyeonworks.com' \ + --header 'X-Forwarded-Host: id.learn.hyeonworks.com' \ + --header 'X-Forwarded-Proto: https' \ + --header 'X-Forwarded-Port: 443' \ + http://127.0.0.1:30080/realms/hyeonworks/.well-known/openid-configuration \ + | jq --exit-status \ + '.issuer == "https://id.learn.hyeonworks.com/realms/hyeonworks"' + ``` + +3. Traefik `web` entrypoint가 Host Nginx 경로에서 실제로 관측한 한 주소 + `10.42.0.1/32`만 신뢰한다. `forwardedHeaders.insecure`와 `websecure` trust는 + 없어야 한다. + + ```sh + kubectl -n kube-system get helmchartconfig traefik \ + -o jsonpath='{.spec.valuesContent}' + + kubectl -n kube-system get deployment traefik -o json \ + | jq --raw-output \ + '.spec.template.spec.containers[] + | select(.name == "traefik") + | .args[]' \ + | rg 'forwardedHeaders|accesslog' + ``` + + 기대하는 trust runtime 인자는 다음 한 줄이다. + + ```text + --entryPoints.web.forwardedHeaders.trustedIPs=10.42.0.1/32 + ``` + +4. Keycloak Ingress backend는 `keycloak-service:8080`뿐이며 관리 포트 `9000`은 + Ingress, NodePort, LoadBalancer와 Host Nginx 후보에 연결되지 않는다. + + ```sh + kubectl -n keycloak get ingress keycloak-http -o wide + kubectl -n keycloak get service keycloak-service -o wide + rg -n '9000|keycloak-service|proxy_pass' \ + infrastructure/networking/host-nginx/learn-services-keycloak.conf + ``` + + `keycloak-service` 자체가 내부 `ClusterIP`에서 `9000`을 제공하는 것은 + Operator의 관리 interface 계약이다. 실패 조건은 이 포트를 외부 경로에 연결한 + Ingress, NodePort, LoadBalancer 또는 Nginx `proxy_pass`가 존재하는 경우다. + +5. Gitea health, NodePort loopback 경계, 두 hostname을 포함하는 인증서, + Nginx active 상태와 `nginx -t`가 계속 통과한다. + +2026-07-23 내부 `hyeonworks` discovery의 issuer·endpoint, Traefik의 +`10.42.0.1/32` 최소 trust와 관리 포트 `9000` 미노출을 확인한 뒤 public +cutover를 완료했다. 활성 Nginx SHA-256은 +`5c5cd74b4992f537fd27c50cf2209573a80a9904e0b19b58c3154717be6ff4a5`다. + +### 실제 실행 명령과 결과 + +저장소 루트에서 다음 root 작업을 실행했다. + +```sh +cd /home/donghyeon/workspace/platform +sudo bash scripts/bootstrap/apply-host-nginx-keycloak.sh --execute +``` + +스크립트가 보여 준 후보·활성 파일과 SHA-256, backup 경로를 확인한 뒤 prompt에 +정확히 다음을 입력했다. + +```text +APPLY +``` + +스크립트는 후보 SHA-256 +`5c5cd74b4992f537fd27c50cf2209573a80a9904e0b19b58c3154717be6ff4a5`와 +전환 전 활성 SHA-256 +`de7ebd4f69cd7d2204ee633e074bf6a4370a6f5e3f3fac9067b099d5d75269b5`를 +고정 gate로 확인한다. 예상하지 않은 활성 설정이면 덮어쓰지 않고 중단한다. + +실제 backup은 +`/etc/nginx/sites-available/learn-services.before-keycloak-20260723160519`이며, +두 번의 probe 뒤 proxy 상태가 안정화됐다. Gitea health, Keycloak discovery +issuer, HTTPS cookie·redirect와 미등록 hostname 거부 자동 검사를 모두 통과했다. + +### 자동 backup, 검증과 rollback + +전환 스크립트는 다음을 한 단위로 수행한다. + +- 변경 전에 직접 Traefik 경로의 Gitea health와 `hyeonworks` discovery issuer를 + 검증한다. +- 활성 `/etc/nginx/sites-available/learn-services`를 + `learn-services.before-keycloak-`로 백업하고 digest를 보존한다. +- 후보를 `root:root 0644`로 설치하고 `nginx -t`가 성공한 경우에만 reload한다. +- reload 뒤 Gitea health와 Keycloak discovery가 함께 수렴할 때까지 bounded + retry한다. +- Git과 ID hostname의 HTTP→HTTPS `301`, Gitea 로그인 Cookie의 `Secure`, + Keycloak discovery issuer·endpoint, 미등록 TLS hostname 거부와 node IP의 + `30080/30443` 차단을 검사한다. +- 활성 파일을 바꾼 뒤 오류 또는 INT/TERM이 발생하면 정확한 backup을 복원하고 + `nginx -t`, reload, Gitea health와 기존 Keycloak static hold가 돌아왔는지 + 다시 확인한다. + +자동 rollback이 성공하면 전환 전 Gitea proxy와 Keycloak static hold 상태로 +복귀한다. SIGKILL이나 전원 장애처럼 trap이 실행되지 않은 경우에는 스크립트가 +출력한 정확한 backup 경로를 사용한다. + +```sh +sudo install -o root -g root -m 0644 \ + BACKUP_PATH \ + /etc/nginx/sites-available/learn-services +sudo nginx -t +sudo systemctl reload nginx +``` + +전환 성공 후 Gitea OIDC source와 브랜딩을 포함한 manifest도 적용했고 자동 +수용 검사를 통과했다. 실제 realm 사용자의 브라우저 login/callback/logout, +비상 관리자 실제 로그인과 Pod 재시작 뒤 설정 지속성은 별도 단계다. 완료 상태와 +후속 수용 기준은 +[중앙 Keycloak·Gitea OIDC 실행 기록](../../../../docs/platform/runbooks/2026-07-23-keycloak-gitea-oidc-cutover.md)에 +보존한다. + +## Observability 단계 전환 + +Observability 전환은 기존 site 전체를 임의로 재생성하지 않고 다음 네 mode만 사용한다. +인자 없는 호출은 공개 Grafana DNS 부재와 source/active SHA-256만 읽는 dry-run이다. + +```sh +cd /home/donghyeon/workspace/platform +bash scripts/bootstrap/apply-host-nginx-observability.sh +bash scripts/bootstrap/apply-host-nginx-observability.sh --execute --metrics-guard-only +bash scripts/bootstrap/apply-host-nginx-observability.sh \ + --execute --certificate-only --certbot-email you@example.com +bash scripts/bootstrap/apply-host-nginx-observability.sh --execute --grafana-deny-guard-only +bash scripts/bootstrap/apply-host-nginx-observability.sh \ + --execute --verified-output-dir "$METRIC_ROOT" +``` + +모든 config mutation은 `PLATFORM_OBSERVABILITY_ROLLBACK_ID`가 가리키는 root-only +rollback root 아래 `host-nginx/stages.tsv`와 `host-nginx/payloads/`에 직전 active +bytes와 양쪽 SHA-256을 먼저 기록한다. 실패 시 timestamp backup이 아니라 이 payload를 +hash 검증해 복원한다. full mode는 active deny-guard SHA 외에도 같은 rollback ID의 +`blackbox-source-proof.env`, `access-rules-alerts/acceptance.env`와 +`$METRIC_ROOT/{target-initial,post-substrate}/inventory.sha256` 결합이 정확해야만 prompt를 +표시한다. 인증서 private key와 Cloudflare token은 ledger에 기록하지 않는다. + +## 범위 경계 + +Debian/Ubuntu의 `sites-enabled/*` 파일은 Nginx의 `http` context에서 include되므로 +`map` directive를 사용할 수 있다. 이 include 구조가 변경되면 설치 전에 다시 +검증한다. 두 hostname 중 어느 쪽에도 활성 server block을 하나 더 만들지 말고 기존 +`learn-services` 파일을 하나의 단위로 교체한다. + +NodePort 주소 제한은 k3s의 +`/etc/rancher/k3s/config.yaml.d/30-nodeport-loopback.yaml`에서 관리한다. 변경 +후에는 Host Nginx의 `127.0.0.1:30080` 접근과 LAN의 node-IP 접근 거부를 항상 함께 +재검증한다. Router firewall/NAT는 이 저장소가 자동으로 변경하지 않는다. diff --git a/infrastructure/networking/host-nginx/learn-services-admin.conf b/infrastructure/networking/host-nginx/learn-services-admin.conf new file mode 100644 index 0000000..f2daf15 --- /dev/null +++ b/infrastructure/networking/host-nginx/learn-services-admin.conf @@ -0,0 +1,165 @@ +# Phase 3 candidate for /etc/nginx/sites-available/learn-services. +# Host Nginx terminates TLS. Traefik receives HTTP only on loopback NodePort. + +map $http_upgrade $hyeonworks_connection_upgrade { + default upgrade; + '' close; +} + +server { + listen 443 ssl default_server; + listen [::]:443 ssl default_server ipv6only=on; + ssl_reject_handshake on; +} + +server { + server_name git.learn.hyeonworks.com; + + location / { + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $hyeonworks_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_redirect off; + client_max_body_size 512m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name id.learn.hyeonworks.com; + + location / { + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host id.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host id.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + proxy_redirect off; + client_max_body_size 10m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name storage-admin.learn.hyeonworks.com; + + location / { + allow 192.168.0.0/24; + allow 100.64.0.0/10; + deny all; + + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host storage-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host storage-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $hyeonworks_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_redirect off; + client_max_body_size 5g; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name db-admin.learn.hyeonworks.com; + + location / { + allow 192.168.0.0/24; + allow 100.64.0.0/10; + deny all; + + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host db-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host db-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + proxy_redirect off; + client_max_body_size 100m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + listen 80; + listen [::]:80; + server_name git.learn.hyeonworks.com; + return 301 https://git.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name id.learn.hyeonworks.com; + return 301 https://id.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name storage-admin.learn.hyeonworks.com; + return 301 https://storage-admin.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name db-admin.learn.hyeonworks.com; + return 301 https://db-admin.learn.hyeonworks.com$request_uri; +} diff --git a/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf b/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf new file mode 100644 index 0000000..1d1e36b --- /dev/null +++ b/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf @@ -0,0 +1,214 @@ +# Phase 3 candidate for /etc/nginx/sites-available/learn-services. +# Host Nginx terminates TLS. Traefik receives HTTP only on loopback NodePort. + +map $http_upgrade $hyeonworks_connection_upgrade { + default upgrade; + '' close; +} + +server { + listen 443 ssl default_server; + listen [::]:443 ssl default_server ipv6only=on; + ssl_reject_handshake on; +} + +server { + server_name git.learn.hyeonworks.com; + + location = /metrics { + return 404; + } + + location / { + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $hyeonworks_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_redirect off; + client_max_body_size 512m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name id.learn.hyeonworks.com; + + location / { + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host id.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host id.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + proxy_redirect off; + client_max_body_size 10m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name storage-admin.learn.hyeonworks.com; + + location / { + allow 192.168.0.0/24; + allow 100.64.0.0/10; + deny all; + + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host storage-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host storage-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $hyeonworks_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_redirect off; + client_max_body_size 5g; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name db-admin.learn.hyeonworks.com; + + location / { + allow 192.168.0.0/24; + allow 100.64.0.0/10; + deny all; + + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host db-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host db-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + proxy_redirect off; + client_max_body_size 100m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + listen 80; + listen [::]:80; + server_name git.learn.hyeonworks.com; + return 301 https://git.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name id.learn.hyeonworks.com; + return 301 https://id.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name storage-admin.learn.hyeonworks.com; + return 301 https://storage-admin.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name db-admin.learn.hyeonworks.com; + return 301 https://db-admin.learn.hyeonworks.com$request_uri; +} + +server { + server_name grafana.learn.hyeonworks.com; + + location = /metrics { + return 404; + } + + location / { + allow 192.168.0.0/24; + allow 100.64.0.0/10; + deny all; + + proxy_pass http://127.0.0.1:1; + proxy_http_version 1.1; + proxy_set_header Host grafana.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host grafana.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $hyeonworks_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + proxy_redirect off; + client_max_body_size 10m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/grafana.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/grafana.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + listen 80; + listen [::]:80; + server_name grafana.learn.hyeonworks.com; + return 301 https://grafana.learn.hyeonworks.com$request_uri; +} diff --git a/infrastructure/networking/host-nginx/learn-services-keycloak.conf b/infrastructure/networking/host-nginx/learn-services-keycloak.conf new file mode 100644 index 0000000..49f00fe --- /dev/null +++ b/infrastructure/networking/host-nginx/learn-services-keycloak.conf @@ -0,0 +1,104 @@ +# Phase 2 candidate for /etc/nginx/sites-available/learn-services. +# +# Host Nginx is the only public TLS endpoint. Both virtual hosts forward to the +# loopback-only Traefik HTTP NodePort; Keycloak is an OIDC provider, not an +# authentication proxy in front of Gitea. + +map $http_upgrade $gitea_connection_upgrade { + default upgrade; + '' close; +} + +server { + listen 443 ssl default_server; + listen [::]:443 ssl default_server ipv6only=on; + ssl_reject_handshake on; +} + +server { + server_name git.learn.hyeonworks.com; + + location / { + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + + proxy_set_header Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $gitea_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_redirect off; + + client_max_body_size 512m; + } + + listen 443 ssl; # managed by Certbot + listen [::]:443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot +} + +server { + server_name id.learn.hyeonworks.com; + + location / { + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + + proxy_set_header Host id.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host id.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + proxy_redirect off; + + client_max_body_size 10m; + } + + listen 443 ssl; # managed by Certbot + listen [::]:443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot +} + +server { + if ($host = git.learn.hyeonworks.com) { + return 301 https://$host$request_uri; + } # managed by Certbot + + listen 80; + listen [::]:80; + + server_name git.learn.hyeonworks.com; + return 404; # managed by Certbot +} + +server { + if ($host = id.learn.hyeonworks.com) { + return 301 https://$host$request_uri; + } # managed by Certbot + + listen 80; + listen [::]:80; + + server_name id.learn.hyeonworks.com; + return 404; # managed by Certbot +} diff --git a/infrastructure/networking/host-nginx/learn-services-metrics-guard.conf b/infrastructure/networking/host-nginx/learn-services-metrics-guard.conf new file mode 100644 index 0000000..e139f38 --- /dev/null +++ b/infrastructure/networking/host-nginx/learn-services-metrics-guard.conf @@ -0,0 +1,169 @@ +# Phase 3 candidate for /etc/nginx/sites-available/learn-services. +# Host Nginx terminates TLS. Traefik receives HTTP only on loopback NodePort. + +map $http_upgrade $hyeonworks_connection_upgrade { + default upgrade; + '' close; +} + +server { + listen 443 ssl default_server; + listen [::]:443 ssl default_server ipv6only=on; + ssl_reject_handshake on; +} + +server { + server_name git.learn.hyeonworks.com; + + location = /metrics { + return 404; + } + + location / { + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $hyeonworks_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_redirect off; + client_max_body_size 512m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name id.learn.hyeonworks.com; + + location / { + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host id.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host id.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + proxy_redirect off; + client_max_body_size 10m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name storage-admin.learn.hyeonworks.com; + + location / { + allow 192.168.0.0/24; + allow 100.64.0.0/10; + deny all; + + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host storage-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host storage-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $hyeonworks_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_redirect off; + client_max_body_size 5g; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name db-admin.learn.hyeonworks.com; + + location / { + allow 192.168.0.0/24; + allow 100.64.0.0/10; + deny all; + + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host db-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host db-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + proxy_redirect off; + client_max_body_size 100m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + listen 80; + listen [::]:80; + server_name git.learn.hyeonworks.com; + return 301 https://git.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name id.learn.hyeonworks.com; + return 301 https://id.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name storage-admin.learn.hyeonworks.com; + return 301 https://storage-admin.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name db-admin.learn.hyeonworks.com; + return 301 https://db-admin.learn.hyeonworks.com$request_uri; +} diff --git a/infrastructure/networking/host-nginx/learn-services-observability.conf b/infrastructure/networking/host-nginx/learn-services-observability.conf new file mode 100644 index 0000000..3f2553c --- /dev/null +++ b/infrastructure/networking/host-nginx/learn-services-observability.conf @@ -0,0 +1,214 @@ +# Phase 3 candidate for /etc/nginx/sites-available/learn-services. +# Host Nginx terminates TLS. Traefik receives HTTP only on loopback NodePort. + +map $http_upgrade $hyeonworks_connection_upgrade { + default upgrade; + '' close; +} + +server { + listen 443 ssl default_server; + listen [::]:443 ssl default_server ipv6only=on; + ssl_reject_handshake on; +} + +server { + server_name git.learn.hyeonworks.com; + + location = /metrics { + return 404; + } + + location / { + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $hyeonworks_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_redirect off; + client_max_body_size 512m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name id.learn.hyeonworks.com; + + location / { + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host id.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host id.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + proxy_redirect off; + client_max_body_size 10m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name storage-admin.learn.hyeonworks.com; + + location / { + allow 192.168.0.0/24; + allow 100.64.0.0/10; + deny all; + + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host storage-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host storage-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $hyeonworks_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_redirect off; + client_max_body_size 5g; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + server_name db-admin.learn.hyeonworks.com; + + location / { + allow 192.168.0.0/24; + allow 100.64.0.0/10; + deny all; + + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host db-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host db-admin.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + proxy_redirect off; + client_max_body_size 100m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/storage-admin.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + listen 80; + listen [::]:80; + server_name git.learn.hyeonworks.com; + return 301 https://git.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name id.learn.hyeonworks.com; + return 301 https://id.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name storage-admin.learn.hyeonworks.com; + return 301 https://storage-admin.learn.hyeonworks.com$request_uri; +} + +server { + listen 80; + listen [::]:80; + server_name db-admin.learn.hyeonworks.com; + return 301 https://db-admin.learn.hyeonworks.com$request_uri; +} + +server { + server_name grafana.learn.hyeonworks.com; + + location = /metrics { + return 404; + } + + location / { + allow 192.168.0.0/24; + allow 100.64.0.0/10; + deny all; + + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + proxy_set_header Host grafana.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host grafana.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $hyeonworks_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + proxy_redirect off; + client_max_body_size 10m; + } + + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/letsencrypt/live/grafana.learn.hyeonworks.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/grafana.learn.hyeonworks.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; +} + +server { + listen 80; + listen [::]:80; + server_name grafana.learn.hyeonworks.com; + return 301 https://grafana.learn.hyeonworks.com$request_uri; +} diff --git a/infrastructure/networking/host-nginx/learn-services.conf b/infrastructure/networking/host-nginx/learn-services.conf new file mode 100644 index 0000000..3d796dd --- /dev/null +++ b/infrastructure/networking/host-nginx/learn-services.conf @@ -0,0 +1,94 @@ +# Candidate configuration for /etc/nginx/sites-available/learn-services. +# +# This file is source-controlled configuration, not the active host configuration. +# Review the accompanying README before installing it. + +map $http_upgrade $gitea_connection_upgrade { + default upgrade; + '' close; +} + +# Do not let an unknown TLS hostname fall through to the first named virtual +# host. Nginx can reject the handshake without presenting either certificate. +server { + listen 443 ssl default_server; + listen [::]:443 ssl default_server ipv6only=on; + ssl_reject_handshake on; +} + +server { + server_name git.learn.hyeonworks.com; + + # TLS terminates here. The hop to the k3s Traefik web NodePort is HTTP. + location / { + proxy_pass http://127.0.0.1:30080; + proxy_http_version 1.1; + + # Replace all externally supplied forwarding headers at the trust boundary. + proxy_set_header Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Host git.learn.hyeonworks.com; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + + # Keep Git operations and WebSocket-capable endpoints streaming. + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $gitea_connection_upgrade; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_redirect off; + + client_max_body_size 512m; + } + + listen 443 ssl; # managed by Certbot + listen [::]:443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot +} + +# Keycloak is intentionally left on its current static response until its phase. +server { + server_name id.learn.hyeonworks.com; + + location / { + default_type text/plain; + return 200 "Keycloak domain reached Nginx successfully\n"; + } + + listen 443 ssl; # managed by Certbot + listen [::]:443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/git.learn.hyeonworks.com/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/git.learn.hyeonworks.com/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot +} + +server { + if ($host = git.learn.hyeonworks.com) { + return 301 https://$host$request_uri; + } # managed by Certbot + + listen 80; + listen [::]:80; + + server_name git.learn.hyeonworks.com; + return 404; # managed by Certbot +} + +server { + if ($host = id.learn.hyeonworks.com) { + return 301 https://$host$request_uri; + } # managed by Certbot + + listen 80; + listen [::]:80; + + server_name id.learn.hyeonworks.com; + return 404; # managed by Certbot +} diff --git a/infrastructure/networking/host-nginx/reload-nginx.sh b/infrastructure/networking/host-nginx/reload-nginx.sh new file mode 100755 index 0000000..915dceb --- /dev/null +++ b/infrastructure/networking/host-nginx/reload-nginx.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +/usr/sbin/nginx -t +/usr/bin/systemctl reload nginx diff --git a/infrastructure/networking/private-dns/host/README.md b/infrastructure/networking/private-dns/host/README.md new file mode 100644 index 0000000..6a80bc2 --- /dev/null +++ b/infrastructure/networking/private-dns/host/README.md @@ -0,0 +1,53 @@ +# 호스트 Split DNS + +두 dnsmasq 프로세스가 서로 다른 주소에만 바인딩됩니다. + +- LAN: 192.168.0.107:53 +- Tailscale: 100.92.240.34:53 + +공유기 DHCP DNS에는 192.168.0.107을 지정하고, Tailscale 관리 화면에는 +learn.hyeonworks.com 제한 nameserver로 100.92.240.34를 등록합니다. +이 두 관리 화면 변경은 자동화하지 않습니다. + +현재 gateway `192.168.0.1`의 MAC OUI와 관리 페이지는 TP-Link 계열이다. +dnsmasq 실제 적용이 성공한 뒤 다음 순서로 설정한다. + +## TP-Link LAN + +1. LAN 단말에서 http://192.168.0.1/ 에 로그인한다. +2. `Advanced > Network > DHCP Server`에 DNS 필드가 있으면 Primary DNS를 + `192.168.0.107`로 지정하고 Secondary DNS는 비워 둔다. +3. 해당 메뉴에 DNS 필드가 없으면 `Advanced > Network > Internet`에서 + `Use the following DNS addresses`를 선택하고 Primary DNS를 + `192.168.0.107`로 지정한다. +4. 저장한 뒤 공유기를 재부팅하고 단말의 Wi-Fi를 다시 연결해 DHCP lease를 + 갱신한다. + +공개 DNS를 Secondary에 넣으면 단말이 private resolver를 우회할 수 있으므로 +사용하지 않는다. 정확한 메뉴는 TP-Link firmware에 따라 둘 중 하나다. + +## Tailscale + +1. https://login.tailscale.com/admin/dns 에 로그인한다. +2. `Nameservers > Add nameserver > Custom`을 선택한다. +3. nameserver에 `100.92.240.34`를 입력한다. +4. `Restrict to search domain`을 켜고 `learn.hyeonworks.com`을 입력해 저장한다. +5. 단말에서 Tailscale DNS 수락을 켜고 다시 연결한다. + +검증: + + nslookup git.learn.hyeonworks.com + nslookup id.learn.hyeonworks.com + nslookup storage-admin.learn.hyeonworks.com + nslookup db-admin.learn.hyeonworks.com + nslookup grafana.learn.hyeonworks.com + +LAN에서는 다섯 이름이 `192.168.0.107`, Tailscale 경로에서는 +`100.92.240.34`로 응답해야 한다. + +공식 참고: + +- TP-Link DNS 설정: https://www.tp-link.com/ca/support/faq/1712/ +- Tailscale split DNS: https://tailscale.com/learn/why-split-dns + +공개 DNS에는 storage-admin, db-admin, grafana A/AAAA 레코드를 만들지 않습니다. diff --git a/infrastructure/networking/private-dns/host/dnsmasq-lan.conf b/infrastructure/networking/private-dns/host/dnsmasq-lan.conf new file mode 100644 index 0000000..2ae042d --- /dev/null +++ b/infrastructure/networking/private-dns/host/dnsmasq-lan.conf @@ -0,0 +1,16 @@ +# Hyeonworks LAN 전용 dnsmasq 인스턴스 +port=53 +listen-address=192.168.0.107 +bind-interfaces +no-hosts +no-resolv +server=1.1.1.1 +server=1.0.0.1 +cache-size=1000 +domain-needed +bogus-priv +address=/git.learn.hyeonworks.com/192.168.0.107 +address=/id.learn.hyeonworks.com/192.168.0.107 +address=/storage-admin.learn.hyeonworks.com/192.168.0.107 +address=/db-admin.learn.hyeonworks.com/192.168.0.107 +address=/grafana.learn.hyeonworks.com/192.168.0.107 diff --git a/infrastructure/networking/private-dns/host/dnsmasq-tailscale.conf b/infrastructure/networking/private-dns/host/dnsmasq-tailscale.conf new file mode 100644 index 0000000..3b52d17 --- /dev/null +++ b/infrastructure/networking/private-dns/host/dnsmasq-tailscale.conf @@ -0,0 +1,16 @@ +# Hyeonworks Tailscale 전용 dnsmasq 인스턴스 +port=53 +listen-address=100.92.240.34 +bind-interfaces +no-hosts +no-resolv +server=1.1.1.1 +server=1.0.0.1 +cache-size=1000 +domain-needed +bogus-priv +address=/git.learn.hyeonworks.com/100.92.240.34 +address=/id.learn.hyeonworks.com/100.92.240.34 +address=/storage-admin.learn.hyeonworks.com/100.92.240.34 +address=/db-admin.learn.hyeonworks.com/100.92.240.34 +address=/grafana.learn.hyeonworks.com/100.92.240.34 diff --git a/infrastructure/networking/private-dns/host/hyeonworks-dnsmasq-lan.service b/infrastructure/networking/private-dns/host/hyeonworks-dnsmasq-lan.service new file mode 100644 index 0000000..4537a3e --- /dev/null +++ b/infrastructure/networking/private-dns/host/hyeonworks-dnsmasq-lan.service @@ -0,0 +1,23 @@ +[Unit] +Description=Hyeonworks LAN private DNS +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/sbin/dnsmasq --keep-in-foreground --user=dnsmasq --group=nogroup --conf-file=/etc/dnsmasq-hyeonworks/lan.conf --pid-file=/run/hyeonworks-dnsmasq-lan.pid +Restart=on-failure +RestartSec=2s +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_SETGID CAP_SETUID + +[Install] +WantedBy=multi-user.target diff --git a/infrastructure/networking/private-dns/host/hyeonworks-dnsmasq-tailscale.service b/infrastructure/networking/private-dns/host/hyeonworks-dnsmasq-tailscale.service new file mode 100644 index 0000000..8f14bf5 --- /dev/null +++ b/infrastructure/networking/private-dns/host/hyeonworks-dnsmasq-tailscale.service @@ -0,0 +1,23 @@ +[Unit] +Description=Hyeonworks Tailscale private DNS +After=network-online.target tailscaled.service +Wants=network-online.target tailscaled.service + +[Service] +Type=simple +ExecStart=/usr/sbin/dnsmasq --keep-in-foreground --user=dnsmasq --group=nogroup --conf-file=/etc/dnsmasq-hyeonworks/tailscale.conf --pid-file=/run/hyeonworks-dnsmasq-tailscale.pid +Restart=on-failure +RestartSec=2s +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_SETGID CAP_SETUID + +[Install] +WantedBy=multi-user.target diff --git a/infrastructure/networking/private-dns/kubernetes/README.md b/infrastructure/networking/private-dns/kubernetes/README.md new file mode 100644 index 0000000..7cd370a --- /dev/null +++ b/infrastructure/networking/private-dns/kubernetes/README.md @@ -0,0 +1,8 @@ +# k3s 내부 Split DNS + +k3s CoreDNS의 기존 /etc/coredns/custom/*.server 확장점을 이용해 +learn.hyeonworks.com의 다섯 서비스 이름을 호스트 Nginx가 있는 +192.168.0.107로 해석합니다. + +이 구성이 적용되고 Pod 검증이 끝나면 Gitea의 임시 hostAliases는 +필요하지 않습니다. 다른 DNS 영역은 기존 CoreDNS 전달 경로를 유지합니다. diff --git a/infrastructure/networking/private-dns/kubernetes/coredns-custom.yaml b/infrastructure/networking/private-dns/kubernetes/coredns-custom.yaml new file mode 100644 index 0000000..1c040a5 --- /dev/null +++ b/infrastructure/networking/private-dns/kubernetes/coredns-custom.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-custom + namespace: kube-system + labels: + app.kubernetes.io/name: coredns + app.kubernetes.io/component: private-dns + app.kubernetes.io/part-of: platform +data: + learn-hyeonworks.server: | + learn.hyeonworks.com:53 { + errors + cache 30 + hosts { + 192.168.0.107 git.learn.hyeonworks.com + 192.168.0.107 id.learn.hyeonworks.com + 192.168.0.107 storage-admin.learn.hyeonworks.com + 192.168.0.107 db-admin.learn.hyeonworks.com + 192.168.0.107 grafana.learn.hyeonworks.com + fallthrough + } + forward . /etc/resolv.conf + } diff --git a/infrastructure/networking/private-dns/kubernetes/kustomization.yaml b/infrastructure/networking/private-dns/kubernetes/kustomization.yaml new file mode 100644 index 0000000..fa4194c --- /dev/null +++ b/infrastructure/networking/private-dns/kubernetes/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - coredns-custom.yaml diff --git a/infrastructure/networking/traefik/README.md b/infrastructure/networking/traefik/README.md new file mode 100644 index 0000000..d9606d4 --- /dev/null +++ b/infrastructure/networking/traefik/README.md @@ -0,0 +1,287 @@ +# Traefik 경계 계약과 적용 기록 + +Traefik은 k3s가 관리하는 클러스터 내부 Ingress Controller다. 공개 요청은 반드시 +Host Nginx에서 TLS를 종료한 뒤 loopback NodePort를 통해 Traefik의 `web` +entrypoint로 들어온다. 이 저장소는 두 번째 Ingress Controller를 설치하지 않으며, +애플리케이션 Ingress에 클러스터 내부 TLS를 중복 구성하지 않는다. + +2026-07-23 현재 `kube-system/traefik` `HelmChartConfig`에는 trust overlay가 실제로 +적용돼 있다. Host Nginx 경유 관측에서 확인한 `ClientHost` `10.42.0.1` 한 주소만 +`10.42.0.1/32`로 신뢰하며, Gitea의 site manifest도 외부 HTTPS URL을 생성한다. + +## 현재 live 상태 + +| 항목 | 확인된 값 | +|---|---| +| k3s Traefik Chart | `40.1.3+up40.1.0` | +| Traefik 이미지 | `v3.7.4` | +| `HelmChartConfig` | trust overlay와 일치, live 적용됨 | +| Service 유형 | `NodePort`, `externalTrafficPolicy: Cluster` | +| `web` | Service `80`, NodePort `30080` | +| `websecure` | Service `443`, NodePort `30443` | +| NodePort bind 범위 | `127.0.0.0/8` | +| JSON access log | 활성화, request header 기록 제외 | +| 관측 `ClientHost` | `10.42.0.1` | +| `web` trusted CIDR | `10.42.0.1/32` | +| `websecure` forwarded-header trust | 없음 | +| `forwardedHeaders.insecure` | 없음 | +| Gitea site manifest | `start_url`과 icon URL 모두 `https://git.learn.hyeonworks.com/` 기준 | +| 실제 ingress 경로 | `Host Nginx :443 -> 127.0.0.1:30080 -> Traefik web` | + +다음 명령으로 변할 수 있는 live 상태를 다시 확인한다. + +```sh +kubectl -n kube-system get helmchartconfig.helm.cattle.io/traefik + +kubectl -n kube-system get service/traefik \ + -o custom-columns='NAME:.metadata.name,TYPE:.spec.type,PORTS:.spec.ports[*].port,NODEPORTS:.spec.ports[*].nodePort' + +kubectl -n kube-system get deployment/traefik -o json | + jq -r '.spec.template.spec.containers[] | select(.name == "traefik") | .args[]' +``` + +`web=30080`, `websecure=30443`, Service `NodePort` 중 하나라도 다르면 Host Nginx를 +새 포트로 임의 변경하지 말고 중지한다. 선언과 live 상태가 왜 달라졌는지 먼저 +확인한다. + +## 트래픽과 노출 경계 + +- 애플리케이션 Ingress가 hostname에서 Service로 이어지는 routing을 소유한다. +- 모든 Ingress는 `spec.ingressClassName: traefik`과 `web` entrypoint를 명시한다. +- 공개 TLS는 Host Nginx가 종료하므로 애플리케이션 Ingress에 `spec.tls`를 넣지 않는다. +- `websecure` NodePort `30443`은 Service 계약상 고정하지만 현재 Host Nginx upstream은 + 사용하지 않는다. 이 entrypoint에는 forwarded-header trust도 설정하지 않는다. +- Traefik Dashboard와 관리 endpoint는 공개하지 않는다. +- k3s drop-in의 `nodeport-addresses=127.0.0.0/8`이 LAN에서 NodePort에 직접 + 접근하는 우회 경로를 차단한다. + +서버 node IP와 별도 LAN 클라이언트에서는 다음 연결이 거부되거나 timeout이어야 +한다. Traefik `404`도 TCP 연결에 성공했다는 뜻이므로 실패다. + +```sh +nc -vz -w 3 192.168.0.107 30080 +nc -vz -w 3 192.168.0.107 30443 +``` + +반대로 서버 loopback에서는 두 포트가 listening 상태여야 하며 Host 기반 Gitea +health가 통과해야 한다. + +```sh +nc -vz -w 3 127.0.0.1 30080 +nc -vz -w 3 127.0.0.1 30443 + +curl --fail-with-body \ + --header 'Host: git.learn.hyeonworks.com' \ + http://127.0.0.1:30080/api/healthz +``` + +UFW는 현재 inactive다. 인터넷 측 고포트 차단 여부는 LAN 결과에서 추론하지 않고 +router 규칙 또는 별도 외부망 검사로 확인한다. + +## 선언 구조와 각 overlay의 역할 + +```text +infrastructure/networking/traefik/ +├── base/ +│ └── helm-chart-config.yaml +├── overlays/ +│ ├── baseline/ +│ │ └── service-boundary-only-patch.yaml +│ ├── observe/ +│ └── trust/ +│ └── trusted-proxy-cidr-patch.yaml +└── scripts/ + ├── apply-observe.sh + ├── observe-client-host.sh + ├── apply-trust.sh + ├── rollback-to-observe.sh + └── validate.sh +``` + +세 overlay는 모두 Service `NodePort`, `externalTrafficPolicy: Cluster`와 +`30080/30443`을 명시적으로 소유한다. + +- `baseline`: Service 경계만 남긴다. access log와 forwarded-header trust는 없다. +- `observe`: Service 경계와 header를 버리는 JSON access log를 적용한다. trust는 없다. +- `trust`: observe 설정에 `web.forwardedHeaders.trustedIPs=10.42.0.1/32`만 추가한다. + +루트 `kustomization.yaml`은 의도적으로 안전한 `observe` overlay를 가리킨다. 현재 +live 상태는 `trust`이므로 루트에 단순히 `kubectl apply -k`를 실행하면 trust 제거를 +요청하게 된다. 상태 전환은 아래 guarded script와 정확한 overlay를 사용한다. +Chart 원본이나 k3s가 소유한 `HelmChart`는 직접 수정하지 않는다. + +## 첫 observe 적용 실패와 복구 + +첫 observe 적용 때 `HelmChartConfig`에는 access log만 있고 Traefik Service values가 +없었다. k3s Helm Controller가 전체 Chart를 기본값으로 다시 조정하면서 다음 drift가 +발생했다. + +```text +기존: NodePort web=30080, websecure=30443 +변경: LoadBalancer web=31251, websecure=30997 +``` + +이어진 loopback listener 검사가 실패했다. 당시 실패 처리도 새 +`HelmChartConfig`를 삭제했을 뿐, desired state에 없던 수동 Service spec은 복원하지 +못했다. Gitea·PostgreSQL·PV/PVC는 건드리지 않고 Traefik Service만 다음 명령으로 +즉시 원래 경계에 복구했다. + +```sh +kubectl -n kube-system patch service traefik \ + --type=merge \ + --patch '{"spec":{"type":"NodePort","externalTrafficPolicy":"Cluster","ports":[{"name":"web","port":80,"protocol":"TCP","targetPort":"web","nodePort":30080},{"name":"websecure","port":443,"protocol":"TCP","targetPort":"websecure","nodePort":30443}]}}' +``` + +그 뒤 다음을 영구 보완했다. + +- `base`, `baseline`, `observe`, `trust`가 Service type과 정확한 NodePort를 선언한다. +- `baseline` overlay를 추가해 access log나 trust 없이도 NodePort desired state를 + 유지한다. +- observe 실패 시 `HelmChartConfig`를 삭제하지 않고 baseline을 적용한다. +- trust 실패 또는 표준 trust 롤백 시 observe를 적용한다. +- rollout 뒤 NodePort listener와 Gitea health가 수렴할 때까지 bounded wait를 한다. +- 검증기는 세 overlay에서 LoadBalancer 부재와 `30080/30443`을 강제한다. + +따라서 `HelmChartConfig` 삭제는 더 이상 롤백 방법이 아니다. 삭제하면 Chart 기본값이 +다시 Service를 소유해 같은 drift를 재발시킬 수 있다. + +## 전달 헤더 최소 신뢰 적용 결과 + +Host Nginx는 외부 요청의 기존 forwarded chain을 이어 붙이지 않고 신뢰 경계에서 +다음 값을 새로 만든다. + +- `Host`는 선택한 공개 hostname으로 고정한다. +- `X-Real-IP`와 `X-Forwarded-For`는 Nginx가 실제로 본 client address로 교체한다. +- `X-Forwarded-Proto`는 `https`, `X-Forwarded-Port`는 `443`으로 고정한다. + +observe 단계에서 다음 probe가 Host Nginx를 반드시 통과하는 고유 요청을 만들고 +Traefik JSON access log의 한 router 기록만 읽었다. request header와 자격 증명은 +로그에 남기지 않았다. + +```sh +bash infrastructure/networking/traefik/scripts/observe-client-host.sh +``` + +확인 결과는 다음과 같다. + +```text +ClientHost: 10.42.0.1 +Minimum trusted CIDR: 10.42.0.1/32 +``` + +Pod CIDR 전체, loopback 전체 또는 LAN CIDR을 추정해 넓히지 않고 이 한 주소만 trust +overlay에 기록했다. 적용 명령과 승인 문자열은 다음과 같았다. + +```sh +bash infrastructure/networking/traefik/scripts/apply-trust.sh \ + --observed-client-host '10.42.0.1' \ + --execute +``` + +```text +APPLY default TRUST 10.42.0.1/32 +``` + +현재 runtime에는 다음 trust 인자 하나만 존재한다. + +```text +--entryPoints.web.forwardedHeaders.trustedIPs=10.42.0.1/32 +``` + +`entryPoints.websecure.forwardedHeaders.*`와 `forwardedHeaders.insecure` 인자는 없다. +적용 후 `/assets/site-manifest.json`의 `start_url`과 두 icon URL이 모두 HTTPS로 +확인됐고 Gitea health의 status·database·cache 검사도 통과했다. + +## 검증과 상태 전환 + +소스와 세 overlay의 정적 계약은 다음 명령으로 검증한다. + +```sh +cd /home/donghyeon/workspace/platform +bash infrastructure/networking/traefik/scripts/validate.sh +``` + +검증기는 다음 조건을 강제한다. + +- 세 overlay의 Service가 `NodePort`, `externalTrafficPolicy: Cluster`, + `30080/30443`을 유지한다. +- observe와 trust access log는 JSON이고 request header를 기록하지 않는다. +- trust CIDR은 관측한 단일 host `/32` 또는 `/128` 형식이다. +- `forwardedHeaders.insecure`, `websecure` trust, `LoadBalancer`가 없다. + +새 설치처럼 `HelmChartConfig`가 없거나 이미 observe 상태인 경우에는 다음 guarded +script로 observe 구성을 확인하거나 적용한다. + +```sh +bash infrastructure/networking/traefik/scripts/apply-observe.sh --execute +# 승인: APPLY <현재-context> OBSERVE +``` + +현재 live trust에서 다시 관측하려면 먼저 아래 표준 롤백으로 observe를 적용한 뒤 +probe를 실행한다. trust 상태에서 `apply-observe.sh`를 바로 실행하지 않는다. + +```sh +bash infrastructure/networking/traefik/scripts/rollback-to-observe.sh --execute +bash infrastructure/networking/traefik/scripts/observe-client-host.sh +``` + +관측값이 달라지면 기존 CIDR을 넓히지 말고 trust patch를 exact host CIDR로 갱신한 뒤 +`apply-trust.sh`를 실행한다. Chart, 이미지, context, API server, NodePort 경계 또는 +재관측 값이 기대와 다르면 스크립트가 적용을 중단한다. + +### 롤백 + +trust만 제거하고 JSON access log를 남기는 표준 롤백은 observe overlay를 적용한다. + +```sh +bash infrastructure/networking/traefik/scripts/rollback-to-observe.sh --execute +# 승인: ROLLBACK <현재-context> OBSERVE +``` + +observe 적용 자체가 실패하면 `apply-observe.sh`가 NodePort-only baseline overlay를 +적용한다. access log와 trust를 모두 제거해야 하는 명시적 유지보수에서는 live +context와 대상 overlay를 재확인한 뒤 baseline을 적용한다. + +```sh +kubectl apply --kustomize \ + infrastructure/networking/traefik/overlays/baseline +``` + +어느 경우에도 `HelmChartConfig`를 삭제해 롤백하지 않는다. baseline 또는 observe를 +적용해 Service `30080/30443`을 계속 desired state로 남긴다. + +k3s·kube-proxy·CNI·Service traffic policy나 Host Nginx 경로를 바꾸면 +`ClientHost`가 달라질 수 있다. 이때는 observe로 돌아가 다시 관측하고 정확한 한 +주소만 trust한다. + +## 종단 간 인수 조건 + +다음 로컬 검사는 Host Nginx와 Traefik을 함께 통과해야 한다. + +```sh +curl --fail-with-body \ + --resolve git.learn.hyeonworks.com:443:127.0.0.1 \ + https://git.learn.hyeonworks.com/api/healthz + +curl --fail-with-body \ + --resolve git.learn.hyeonworks.com:443:127.0.0.1 \ + https://git.learn.hyeonworks.com/assets/site-manifest.json | + jq -e ' + .start_url == "https://git.learn.hyeonworks.com/" and + ([.icons[].src | startswith("https://git.learn.hyeonworks.com/")] | all) + ' +``` + +별도 LAN 클라이언트에서 `192.168.0.107:30080/30443`이 거부되는지 다시 확인하고, +독립 외부망에서는 공개 HTTP→HTTPS redirect와 두 서비스의 HTTPS 응답을 검사한다. +서버에서 공인 FQDN으로 향하는 NAT hairpin timeout만으로 공개 실패를 판정하지 않는다. + +구현 근거는 [k3s HelmChartConfig](https://docs.k3s.io/helm), +[k3s 내장 Traefik](https://docs.k3s.io/networking/networking-services), +[Traefik forwarded headers](https://doc.traefik.io/traefik/reference/install-configuration/entrypoints/), +[Traefik access log](https://doc.traefik.io/traefik/observe/logs-and-access-logs/), +[Traefik Chart 40.1.0 values](https://github.com/traefik/traefik-helm-chart/blob/v40.1.0/traefik/values.yaml)다. + +첫 실패, 수동 복구, 영구 보완, 관측값과 trust 적용의 전체 명령·출력은 +[중앙 실행 기록](../../../../docs/platform/runbooks/2026-07-23-traefik-forwarded-header-trust-boundary.md)에 +보존한다. diff --git a/infrastructure/networking/traefik/base/helm-chart-config.yaml b/infrastructure/networking/traefik/base/helm-chart-config.yaml new file mode 100644 index 0000000..7f778c6 --- /dev/null +++ b/infrastructure/networking/traefik/base/helm-chart-config.yaml @@ -0,0 +1,45 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: traefik + namespace: kube-system + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/managed-by: kustomize +spec: + failurePolicy: abort + valuesContent: |- + deployment: + podAnnotations: + prometheus.io/path: /metrics + prometheus.io/port: "9100" + prometheus.io/scrape: "true" + service: + spec: + type: NodePort + externalTrafficPolicy: Cluster + ports: + web: + nodePort: 30080 + websecure: + nodePort: 30443 + metrics: + prometheus: + service: + enabled: true + serviceMonitor: + enabled: true + additionalLabels: + observability.hyeonworks.com/instance: home + jobLabel: app.kubernetes.io/name + interval: 30s + scrapeTimeout: 10s + logs: + access: + enabled: true + format: json + fields: + general: + defaultmode: keep + headers: + defaultmode: drop diff --git a/infrastructure/networking/traefik/base/kustomization.yaml b/infrastructure/networking/traefik/base/kustomization.yaml new file mode 100644 index 0000000..3246cb5 --- /dev/null +++ b/infrastructure/networking/traefik/base/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - helm-chart-config.yaml diff --git a/infrastructure/networking/traefik/kustomization.yaml b/infrastructure/networking/traefik/kustomization.yaml new file mode 100644 index 0000000..aae6a80 --- /dev/null +++ b/infrastructure/networking/traefik/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - overlays/observe diff --git a/infrastructure/networking/traefik/overlays/baseline/kustomization.yaml b/infrastructure/networking/traefik/overlays/baseline/kustomization.yaml new file mode 100644 index 0000000..4bc6720 --- /dev/null +++ b/infrastructure/networking/traefik/overlays/baseline/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +patches: + - path: service-boundary-only-patch.yaml + target: + group: helm.cattle.io + version: v1 + kind: HelmChartConfig + name: traefik + namespace: kube-system diff --git a/infrastructure/networking/traefik/overlays/baseline/service-boundary-only-patch.yaml b/infrastructure/networking/traefik/overlays/baseline/service-boundary-only-patch.yaml new file mode 100644 index 0000000..1f5d5de --- /dev/null +++ b/infrastructure/networking/traefik/overlays/baseline/service-boundary-only-patch.yaml @@ -0,0 +1,28 @@ +- op: replace + path: /spec/valuesContent + value: |- + deployment: + podAnnotations: + prometheus.io/path: /metrics + prometheus.io/port: "9100" + prometheus.io/scrape: "true" + service: + spec: + type: NodePort + externalTrafficPolicy: Cluster + ports: + web: + nodePort: 30080 + websecure: + nodePort: 30443 + metrics: + prometheus: + service: + enabled: true + serviceMonitor: + enabled: true + additionalLabels: + observability.hyeonworks.com/instance: home + jobLabel: app.kubernetes.io/name + interval: 30s + scrapeTimeout: 10s diff --git a/infrastructure/networking/traefik/overlays/observe/kustomization.yaml b/infrastructure/networking/traefik/overlays/observe/kustomization.yaml new file mode 100644 index 0000000..774a422 --- /dev/null +++ b/infrastructure/networking/traefik/overlays/observe/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base diff --git a/infrastructure/networking/traefik/overlays/trust/kustomization.yaml b/infrastructure/networking/traefik/overlays/trust/kustomization.yaml new file mode 100644 index 0000000..c7a7a04 --- /dev/null +++ b/infrastructure/networking/traefik/overlays/trust/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +patches: + - path: trusted-proxy-cidr-patch.yaml + target: + group: helm.cattle.io + version: v1 + kind: HelmChartConfig + name: traefik + namespace: kube-system diff --git a/infrastructure/networking/traefik/overlays/trust/trusted-proxy-cidr-patch.yaml b/infrastructure/networking/traefik/overlays/trust/trusted-proxy-cidr-patch.yaml new file mode 100644 index 0000000..df422a3 --- /dev/null +++ b/infrastructure/networking/traefik/overlays/trust/trusted-proxy-cidr-patch.yaml @@ -0,0 +1,41 @@ +- op: replace + path: /spec/valuesContent + value: |- + deployment: + podAnnotations: + prometheus.io/path: /metrics + prometheus.io/port: "9100" + prometheus.io/scrape: "true" + service: + spec: + type: NodePort + externalTrafficPolicy: Cluster + ports: + web: + nodePort: 30080 + forwardedHeaders: + trustedIPs: + # 2026-07-23 Host Nginx 경유 probe에서 관측한 Traefik ClientHost이다. + - "10.42.0.1/32" + websecure: + nodePort: 30443 + metrics: + prometheus: + service: + enabled: true + serviceMonitor: + enabled: true + additionalLabels: + observability.hyeonworks.com/instance: home + jobLabel: app.kubernetes.io/name + interval: 30s + scrapeTimeout: 10s + logs: + access: + enabled: true + format: json + fields: + general: + defaultmode: keep + headers: + defaultmode: drop diff --git a/infrastructure/networking/traefik/scripts/apply-observe.sh b/infrastructure/networking/traefik/scripts/apply-observe.sh new file mode 100755 index 0000000..beeb623 --- /dev/null +++ b/infrastructure/networking/traefik/scripts/apply-observe.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +set +x +umask 077 + +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +usage() { + cat <<'USAGE' +Usage: bash infrastructure/networking/traefik/scripts/apply-observe.sh --execute + +Applies JSON access logging to the k3s-managed Traefik HelmChartConfig. +It does not trust any forwarded header. The existing loopback-only +NodePort 30080/30443 boundary is declared explicitly so Helm reconciliation +cannot replace it with chart defaults. +USAGE +} + +[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || { + usage + exit 2 +} + +require_commands kubectl jq rg curl nc python3 awk cmp find bash + +bash "${SCRIPT_DIR}/validate.sh" + +work_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-traefik-observe.XXXXXX")" +rollback_required=false +selected_context="" + +cleanup() { + local exit_code=$? + + trap - EXIT + if [[ "$exit_code" -ne 0 && "$rollback_required" == "true" ]]; then + printf '\nROLLBACK: restoring the durable NodePort-only baseline.\n' >&2 + set +e + kubectl --context "$selected_context" apply --kustomize "$BASELINE_OVERLAY" + wait_for_runtime baseline + kubectl --context "$selected_context" --namespace kube-system \ + rollout status deployment/traefik --timeout=5m + wait_for_nodeport_boundary_and_health + printf 'ROLLBACK complete. NodePort 30080/30443 remains pinned.\n' >&2 + set -e + fi + + case "$work_dir" in + /tmp/platform-traefik-observe.*|"${TMPDIR:-/tmp}"/platform-traefik-observe.*) + rm -rf -- "$work_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected directory: %s\n' \ + "$work_dir" >&2 + ;; + esac + exit "$exit_code" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +observe_render="${work_dir}/observe.yaml" +render_overlay "$OBSERVE_OVERLAY" "$observe_render" + +assert_live_baseline +assert_nodeport_boundary_and_health + +selected_context="$(current_context)" +selected_api_server="$(current_api_server)" +printf '\nKubernetes context: %s\nAPI server: %s\nTarget node: %s\n' \ + "$selected_context" "$selected_api_server" "$TARGET_NODE" + +already_applied=false +if kubectl --namespace kube-system \ + get helmchartconfig.helm.cattle.io traefik >/dev/null 2>&1; then + assert_live_hcc_matches_overlay "$OBSERVE_OVERLAY" \ + "${work_dir}/live-observe-compare.yaml" + already_applied=true + printf 'The live HelmChartConfig already matches the observation overlay.\n' +fi + +printf 'Type APPLY %s OBSERVE to enable JSON access logs: ' "$selected_context" +read -r confirmation +[[ "$confirmation" == "APPLY ${selected_context} OBSERVE" ]] || \ + fail "cancelled" + +[[ "$(current_context)" == "$selected_context" ]] || \ + fail "kubectl context changed after confirmation" +[[ "$(current_api_server)" == "$selected_api_server" ]] || \ + fail "Kubernetes API server changed after confirmation" +assert_live_baseline +assert_nodeport_boundary_and_health + +if [[ "$already_applied" == "false" ]]; then + rollback_required=true + kubectl apply --filename "$observe_render" +fi + +wait_for_runtime observe +kubectl --namespace kube-system rollout status deployment/traefik --timeout=5m +assert_runtime observe +assert_live_hcc_matches_overlay "$OBSERVE_OVERLAY" \ + "${work_dir}/post-apply-observe.yaml" +assert_live_baseline +wait_for_nodeport_boundary_and_health + +manifest_start_url="$( + curl --fail-with-body --silent --show-error \ + --resolve "${GITEA_HOST}:443:127.0.0.1" \ + "$GITEA_MANIFEST_URL" | + jq --raw-output '.start_url' +)" + +rollback_required=false +printf '\nOBSERVATION PHASE READY\n' +printf 'Traefik JSON access logging: enabled\n' +printf 'forwardedHeaders trust: absent\n' +printf 'Current Gitea manifest start_url: %s\n' "$manifest_start_url" +printf 'Next: bash %s/observe-client-host.sh\n' "$SCRIPT_DIR" diff --git a/infrastructure/networking/traefik/scripts/apply-trust.sh b/infrastructure/networking/traefik/scripts/apply-trust.sh new file mode 100755 index 0000000..08c6ed7 --- /dev/null +++ b/infrastructure/networking/traefik/scripts/apply-trust.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +set +x +umask 077 + +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +usage() { + cat <<'USAGE' +Usage: + bash infrastructure/networking/traefik/scripts/apply-trust.sh \ + --observed-client-host --execute + +The IP must be the ClientHost printed by observe-client-host.sh. The trust +overlay must already contain that exact IP as /32 (IPv4) or /128 (IPv6). +USAGE +} + +[[ "$#" -eq 3 && "$1" == "--observed-client-host" && "$3" == "--execute" ]] || { + usage + exit 2 +} + +readonly REVIEWED_CLIENT_HOST="$2" + +require_commands kubectl jq rg curl nc python3 awk sort date cmp find bash + +reviewed_cidr="$(host_to_exact_cidr "$REVIEWED_CLIENT_HOST")" +declared_cidr="$(source_trusted_proxy_cidr)" +declared_cidr="$(normalize_exact_host_cidr "$declared_cidr")" + +[[ "$declared_cidr" != "$SENTINEL_TRUSTED_PROXY_CIDR" ]] || \ + fail "trust overlay still contains the non-routable sentinel CIDR" +[[ "$declared_cidr" == "$reviewed_cidr" ]] || \ + fail "declared CIDR ${declared_cidr} does not match ClientHost ${REVIEWED_CLIENT_HOST}" + +bash "${SCRIPT_DIR}/validate.sh" + +work_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-traefik-trust.XXXXXX")" +rollback_required=false +selected_context="" + +cleanup() { + local exit_code=$? + + trap - EXIT + if [[ "$exit_code" -ne 0 && "$rollback_required" == "true" ]]; then + printf '\nROLLBACK: restoring the access-log-only observation overlay.\n' >&2 + set +e + kubectl --context "$selected_context" apply --kustomize "$OBSERVE_OVERLAY" + wait_for_runtime observe + kubectl --context "$selected_context" --namespace kube-system \ + rollout status deployment/traefik --timeout=5m + wait_for_nodeport_boundary_and_health + printf 'ROLLBACK complete. Forwarded-header trust removal was requested.\n' >&2 + set -e + fi + + case "$work_dir" in + /tmp/platform-traefik-trust.*|"${TMPDIR:-/tmp}"/platform-traefik-trust.*) + rm -rf -- "$work_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected directory: %s\n' \ + "$work_dir" >&2 + ;; + esac + exit "$exit_code" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +trust_render="${work_dir}/trust.yaml" +render_overlay "$TRUST_OVERLAY" "$trust_render" + +assert_live_baseline +assert_runtime observe +assert_live_hcc_matches_overlay "$OBSERVE_OVERLAY" \ + "${work_dir}/live-observe.yaml" +assert_nodeport_boundary_and_health + +fresh_client_host="$(observe_host_nginx_client_host)" +fresh_cidr="$(host_to_exact_cidr "$fresh_client_host")" +[[ "$fresh_cidr" == "$reviewed_cidr" ]] || \ + fail "fresh ClientHost ${fresh_client_host} differs from reviewed ${REVIEWED_CLIENT_HOST}" + +selected_context="$(current_context)" +selected_api_server="$(current_api_server)" +printf '\nKubernetes context: %s\nAPI server: %s\nTarget node: %s\n' \ + "$selected_context" "$selected_api_server" "$TARGET_NODE" +printf 'Fresh ClientHost: %s\nExact trusted CIDR: %s\n' \ + "$fresh_client_host" "$declared_cidr" +printf 'Type APPLY %s TRUST %s to continue: ' \ + "$selected_context" "$declared_cidr" +read -r confirmation +[[ "$confirmation" == "APPLY ${selected_context} TRUST ${declared_cidr}" ]] || \ + fail "cancelled" + +[[ "$(current_context)" == "$selected_context" ]] || \ + fail "kubectl context changed after confirmation" +[[ "$(current_api_server)" == "$selected_api_server" ]] || \ + fail "Kubernetes API server changed after confirmation" +assert_live_baseline +assert_runtime observe +assert_nodeport_boundary_and_health +assert_live_hcc_matches_overlay "$OBSERVE_OVERLAY" \ + "${work_dir}/pre-apply-observe.yaml" + +rollback_required=true +kubectl apply --filename "$trust_render" + +wait_for_runtime trust "$declared_cidr" +kubectl --namespace kube-system rollout status deployment/traefik --timeout=5m +assert_runtime trust "$declared_cidr" +assert_live_hcc_matches_overlay "$TRUST_OVERLAY" \ + "${work_dir}/post-apply-trust.yaml" +assert_live_baseline +wait_for_nodeport_boundary_and_health +assert_manifest_https + +rollback_required=false +printf '\nTRUST PHASE READY\n' +printf 'Traefik web trusted CIDR: %s\n' "$declared_cidr" +printf 'Traefik websecure trusted CIDR: absent\n' +printf 'forwardedHeaders.insecure: absent\n' +printf 'Gitea health and HTTPS site-manifest checks: PASS\n' +printf 'Repeat the 30080/30443 refusal check from a separate LAN client.\n' diff --git a/infrastructure/networking/traefik/scripts/common.sh b/infrastructure/networking/traefik/scripts/common.sh new file mode 100755 index 0000000..ab08969 --- /dev/null +++ b/infrastructure/networking/traefik/scripts/common.sh @@ -0,0 +1,464 @@ +#!/usr/bin/env bash + +# 이 파일은 같은 디렉터리의 실행 스크립트에서만 source한다. + +readonly TRAEFIK_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)" +readonly BASELINE_OVERLAY="${TRAEFIK_ROOT}/overlays/baseline" +readonly OBSERVE_OVERLAY="${TRAEFIK_ROOT}/overlays/observe" +readonly TRUST_OVERLAY="${TRAEFIK_ROOT}/overlays/trust" +readonly TRUST_PATCH="${TRUST_OVERLAY}/trusted-proxy-cidr-patch.yaml" +readonly SENTINEL_TRUSTED_PROXY_CIDR="192.0.2.1/32" + +readonly TARGET_NODE="donghyeon-system-product-name" +readonly K3S_NODEPORT_CONFIG="/etc/rancher/k3s/config.yaml.d/30-nodeport-loopback.yaml" +readonly EXPECTED_K3S_CHART="https://%{KUBERNETES_API}%/static/charts/traefik-40.1.3+up40.1.0.tgz" +readonly EXPECTED_CHART_LABEL="traefik-40.1.3_up40.1.0" +readonly EXPECTED_TRAEFIK_IMAGE="rancher/mirrored-library-traefik:3.7.4" +readonly GITEA_HOST="git.learn.hyeonworks.com" +readonly GITEA_HEALTH_URL="http://127.0.0.1:30080/api/healthz" +readonly GITEA_HTTPS_HEALTH_URL="https://${GITEA_HOST}/api/healthz" +readonly GITEA_MANIFEST_URL="https://${GITEA_HOST}/assets/site-manifest.json" + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +require_commands() { + local command_name + + for command_name in "$@"; do + command -v "$command_name" >/dev/null 2>&1 || \ + fail "${command_name} is required" + done +} + +render_overlay() { + local overlay="$1" + local output="$2" + + kubectl kustomize "$overlay" >"$output" + [[ -s "$output" ]] || fail "rendered manifest is empty: ${overlay}" +} + +manifest_values_content() { + local manifest="$1" + + awk ' + /^ valuesContent: \|-$/ { + found = 1 + next + } + found { + sub(/^ /, "") + print + } + ' "$manifest" +} + +source_trusted_proxy_cidr() { + local -a values=() + + mapfile -t values < <( + awk -F'"' '/^[[:space:]]*-[[:space:]]*"/ { print $2 }' "$TRUST_PATCH" + ) + [[ "${#values[@]}" -eq 1 ]] || \ + fail "trust patch must contain exactly one quoted trusted CIDR" + printf '%s\n' "${values[0]}" +} + +host_to_exact_cidr() { + local host="$1" + + python3 - "$host" <<'PY' +import ipaddress +import sys + +value = sys.argv[1] +if "/" in value: + raise SystemExit("ClientHost must be one IP address, not a CIDR") + +address = ipaddress.ip_address(value) +if address.is_unspecified or address.is_multicast: + raise SystemExit("ClientHost cannot be unspecified or multicast") + +prefix = 32 if address.version == 4 else 128 +print(f"{address.compressed}/{prefix}") +PY +} + +normalize_exact_host_cidr() { + local cidr="$1" + + python3 - "$cidr" <<'PY' +import ipaddress +import sys + +network = ipaddress.ip_network(sys.argv[1], strict=True) +required_prefix = 32 if network.version == 4 else 128 +if network.prefixlen != required_prefix: + raise SystemExit( + f"trusted proxy range must be one exact host /{required_prefix}, " + f"not {network.with_prefixlen}" + ) +print(network.with_prefixlen) +PY +} + +current_context() { + kubectl config current-context +} + +current_api_server() { + kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}' +} + +deployment_args() { + kubectl --namespace kube-system get deployment traefik --output=json | + jq --raw-output ' + .spec.template.spec.containers[] + | select(.name == "traefik") + | .args[] + ' +} + +runtime_matches() { + local mode="$1" + local trusted_cidr="${2:-}" + local args + + args="$(deployment_args 2>/dev/null)" || return 1 + if rg --quiet --ignore-case -- 'forwardedheaders\.insecure' <<<"$args"; then + return 1 + fi + + if [[ "$mode" != "baseline" ]]; then + rg --quiet --fixed-strings --line-regexp -- '--accesslog=true' <<<"$args" || \ + return 1 + rg --quiet --fixed-strings --line-regexp -- '--accesslog.format=json' <<<"$args" || \ + return 1 + fi + + case "$mode" in + observe) + ! rg --quiet --ignore-case -- 'forwardedheaders\.trustedips' <<<"$args" + ;; + trust) + rg --quiet --fixed-strings --line-regexp -- \ + "--entryPoints.web.forwardedHeaders.trustedIPs=${trusted_cidr}" <<<"$args" || \ + return 1 + ! rg --quiet --ignore-case -- \ + 'entrypoints\.websecure\.forwardedheaders\.(trustedips|insecure)' <<<"$args" + ;; + baseline) + ! rg --quiet --ignore-case -- \ + 'accesslog|forwardedheaders\.(trustedips|insecure)' <<<"$args" + ;; + *) + return 1 + ;; + esac +} + +assert_runtime() { + local mode="$1" + local trusted_cidr="${2:-}" + local args + local trusted_count + + args="$(deployment_args)" + rg --quiet --fixed-strings --line-regexp -- '--accesslog=true' <<<"$args" || \ + fail "Traefik runtime is missing --accesslog=true" + rg --quiet --fixed-strings --line-regexp -- '--accesslog.format=json' <<<"$args" || \ + fail "Traefik runtime is missing JSON access-log format" + if rg --quiet --ignore-case -- 'forwardedheaders\.insecure' <<<"$args"; then + fail "forwardedHeaders.insecure must never be present" + fi + + trusted_count="$( + rg --count --ignore-case -- 'forwardedheaders\.trustedips' <<<"$args" || true + )" + trusted_count="${trusted_count:-0}" + + case "$mode" in + observe) + [[ "$trusted_count" == "0" ]] || \ + fail "observation phase must not trust forwarded headers" + ;; + trust) + [[ "$trusted_count" == "1" ]] || \ + fail "trust phase must render exactly one trustedIPs argument" + rg --quiet --fixed-strings --line-regexp -- \ + "--entryPoints.web.forwardedHeaders.trustedIPs=${trusted_cidr}" <<<"$args" || \ + fail "web entrypoint does not contain the reviewed exact-host CIDR" + if rg --quiet --ignore-case -- \ + 'entrypoints\.websecure\.forwardedheaders\.(trustedips|insecure)' <<<"$args"; then + fail "websecure must not receive forwarded-header trust" + fi + ;; + *) + fail "unsupported runtime assertion mode: ${mode}" + ;; + esac +} + +wait_for_runtime() { + local mode="$1" + local trusted_cidr="${2:-}" + local attempt + + for ((attempt = 1; attempt <= 120; attempt++)); do + if runtime_matches "$mode" "$trusted_cidr"; then + return 0 + fi + sleep 2 + done + fail "Traefik runtime did not reach ${mode} state within 240 seconds" +} + +assert_live_baseline() { + local chart + local chart_label + local image + local nodeport_matches + local service_json + + kubectl get node "$TARGET_NODE" >/dev/null + + chart="$( + kubectl --namespace kube-system get helmchart.helm.cattle.io traefik \ + --output=jsonpath='{.spec.chart}' + )" + [[ "$chart" == "$EXPECTED_K3S_CHART" ]] || \ + fail "unexpected packaged Traefik chart: ${chart}" + + chart_label="$( + kubectl --namespace kube-system get deployment traefik \ + --output=jsonpath='{.metadata.labels.helm\.sh/chart}' + )" + [[ "$chart_label" == "$EXPECTED_CHART_LABEL" ]] || \ + fail "unexpected live Traefik chart label: ${chart_label}" + + image="$( + kubectl --namespace kube-system get deployment traefik --output=json | + jq --raw-output ' + .spec.template.spec.containers[] + | select(.name == "traefik") + | .image + ' + )" + [[ "$image" == "$EXPECTED_TRAEFIK_IMAGE" ]] || \ + fail "unexpected live Traefik image: ${image}" + + [[ -r "$K3S_NODEPORT_CONFIG" ]] || \ + fail "cannot read the k3s nodeport-addresses drop-in: ${K3S_NODEPORT_CONFIG}" + rg --quiet --fixed-strings --line-regexp -- \ + ' - "nodeport-addresses=127.0.0.0/8"' "$K3S_NODEPORT_CONFIG" || \ + fail "k3s nodeport-addresses is not pinned to 127.0.0.0/8" + + nodeport_matches="$( + rg --no-heading --line-number -- 'nodeport-addresses[=:]' \ + /etc/rancher/k3s/config.yaml \ + /etc/rancher/k3s/config.yaml.d 2>/dev/null || true + )" + [[ "$(wc -l <<<"$nodeport_matches" | tr -d '[:space:]')" == "1" ]] || \ + fail "nodeport-addresses must have exactly one k3s configuration owner" + rg --quiet --fixed-strings -- "$K3S_NODEPORT_CONFIG" <<<"$nodeport_matches" || \ + fail "nodeport-addresses is owned by an unexpected k3s configuration file" + + service_json="$( + kubectl --namespace kube-system get service traefik --output=json + )" + jq --exit-status ' + .spec.type == "NodePort" + and .spec.externalTrafficPolicy == "Cluster" + and (.spec.ports | length) == 2 + and any(.spec.ports[]; + .name == "web" + and .port == 80 + and .nodePort == 30080 + and .protocol == "TCP") + and any(.spec.ports[]; + .name == "websecure" + and .port == 443 + and .nodePort == 30443 + and .protocol == "TCP") + ' >/dev/null <<<"$service_json" || \ + fail "Traefik Service no longer matches the 80/30080 and 443/30443 boundary" +} + +assert_health_body() { + local body="$1" + + jq --exit-status ' + .status == "pass" + and (.checks["database:ping"] | length) > 0 + and all(.checks["database:ping"][]; .status == "pass") + and (.checks["cache:ping"] | length) > 0 + and all(.checks["cache:ping"][]; .status == "pass") + ' >/dev/null <<<"$body" || fail "Gitea database/cache health is not pass" +} + +assert_nodeport_boundary_and_health() { + local body + local node_ip + local port + + node_ip="$( + kubectl get node "$TARGET_NODE" --output=json | + jq --raw-output ' + [.status.addresses[] | select(.type == "InternalIP") | .address] + | if length == 1 then .[0] else empty end + ' + )" + [[ -n "$node_ip" && "$node_ip" != "127.0.0.1" ]] || \ + fail "could not resolve exactly one non-loopback node InternalIP" + + for port in 30080 30443; do + nc -z -w 3 127.0.0.1 "$port" >/dev/null 2>&1 || \ + fail "loopback NodePort is not listening: 127.0.0.1:${port}" + if nc -z -w 3 "$node_ip" "$port" >/dev/null 2>&1; then + fail "NodePort escaped the loopback boundary: ${node_ip}:${port}" + fi + done + + body="$( + curl --fail-with-body --silent --show-error \ + --header "Host: ${GITEA_HOST}" \ + "$GITEA_HEALTH_URL" + )" + assert_health_body "$body" + + body="$( + curl --fail-with-body --silent --show-error \ + --resolve "${GITEA_HOST}:443:127.0.0.1" \ + "$GITEA_HTTPS_HEALTH_URL" + )" + assert_health_body "$body" +} + +wait_for_nodeport_boundary_and_health() { + local attempt + + for ((attempt = 1; attempt <= 60; attempt++)); do + if assert_nodeport_boundary_and_health >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + + # 마지막 검사는 오류 원인을 숨기지 않고 그대로 출력한다. + assert_nodeport_boundary_and_health + fail "Traefik NodePort boundary and Gitea health did not recover within 120 seconds" +} + +observe_host_nginx_client_host() { + local http_code + local logs + local matched + local nonce + local observed + local probe_path + local router_count + local since + local -a client_hosts=() + local attempt + + since="$(date --utc '+%Y-%m-%dT%H:%M:%SZ')" + nonce="$(date --utc '+%Y%m%dT%H%M%S')-${BASHPID}" + probe_path="/api/healthz/traefik-source-${nonce}" + + http_code="$( + curl --silent --show-error \ + --resolve "${GITEA_HOST}:443:127.0.0.1" \ + --output /dev/null \ + --write-out '%{http_code}' \ + "https://${GITEA_HOST}${probe_path}" + )" + [[ "$http_code" == "404" ]] || \ + fail "unique Host Nginx observation request returned HTTP ${http_code}, expected 404" + + matched="" + for ((attempt = 1; attempt <= 20; attempt++)); do + logs="$( + kubectl --namespace kube-system logs deployment/traefik \ + --since-time "$since" + )" + matched="$( + jq --raw-input --compact-output --arg path "$probe_path" ' + fromjson? + | select(.RequestPath == $path) + ' <<<"$logs" + )" + [[ -n "$matched" ]] && break + sleep 1 + done + [[ -n "$matched" ]] || \ + fail "the unique request was not found in Traefik JSON access logs" + + router_count="$( + jq --slurp ' + [ + .[] + | select( + ((.RouterName // "") | ascii_downcase | contains("gitea")) + ) + ] + | length + ' <<<"$matched" + )" + [[ "$router_count" -ge 1 ]] || \ + fail "the observation log did not traverse a Gitea router" + + mapfile -t client_hosts < <( + jq --raw-output ' + select((.RouterName // "") | ascii_downcase | contains("gitea")) + | .ClientHost // empty + ' <<<"$matched" | + sort --unique + ) + [[ "${#client_hosts[@]}" -eq 1 && -n "${client_hosts[0]}" ]] || \ + fail "expected one distinct Traefik ClientHost for the unique request" + + observed="${client_hosts[0]}" + host_to_exact_cidr "$observed" >/dev/null + printf '%s\n' "$observed" +} + +assert_manifest_https() { + local body + local attempt + + for ((attempt = 1; attempt <= 30; attempt++)); do + body="$( + curl --fail-with-body --silent --show-error \ + --resolve "${GITEA_HOST}:443:127.0.0.1" \ + "$GITEA_MANIFEST_URL" + )" + if jq --exit-status --arg expected "https://${GITEA_HOST}/" ' + .start_url == $expected + and all(.icons[]; (.src | startswith("https://"))) + ' >/dev/null <<<"$body"; then + return 0 + fi + sleep 2 + done + fail "Gitea site manifest did not stabilize on HTTPS URLs" +} + +assert_live_hcc_matches_overlay() { + local overlay="$1" + local render_file="$2" + local actual + local expected + + render_overlay "$overlay" "$render_file" + expected="$(manifest_values_content "$render_file")" + actual="$( + kubectl --namespace kube-system \ + get helmchartconfig.helm.cattle.io traefik --output=json | + jq --raw-output '.spec.valuesContent' + )" + [[ "$actual" == "$expected" ]] || \ + fail "live Traefik HelmChartConfig does not match the expected overlay" +} diff --git a/infrastructure/networking/traefik/scripts/observe-client-host.sh b/infrastructure/networking/traefik/scripts/observe-client-host.sh new file mode 100755 index 0000000..9bed7b6 --- /dev/null +++ b/infrastructure/networking/traefik/scripts/observe-client-host.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +set +x +umask 077 + +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +usage() { + cat <<'USAGE' +Usage: bash infrastructure/networking/traefik/scripts/observe-client-host.sh + +Sends one unique HTTPS request through Host Nginx and extracts the corresponding +ClientHost from Traefik JSON access logs. It performs no cluster mutation. +USAGE +} + +[[ "$#" -eq 0 ]] || { + usage + exit 2 +} + +require_commands kubectl jq rg curl nc python3 awk sort date + +work_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-traefik-clienthost.XXXXXX")" +cleanup() { + case "$work_dir" in + /tmp/platform-traefik-clienthost.*|"${TMPDIR:-/tmp}"/platform-traefik-clienthost.*) + rm -rf -- "$work_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected directory: %s\n' \ + "$work_dir" >&2 + ;; + esac +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +bash "${SCRIPT_DIR}/validate.sh" +assert_live_baseline +assert_runtime observe +assert_live_hcc_matches_overlay "$OBSERVE_OVERLAY" \ + "${work_dir}/live-observe.yaml" +assert_nodeport_boundary_and_health + +observed_client_host="$(observe_host_nginx_client_host)" +trusted_proxy_cidr="$(host_to_exact_cidr "$observed_client_host")" + +printf '\nTRAEFIK SOURCE OBSERVED\n' +printf 'ClientHost: %s\n' "$observed_client_host" +printf 'Minimum trusted CIDR: %s\n' "$trusted_proxy_cidr" +printf 'Record that CIDR in:\n%s\n' "$TRUST_PATCH" +printf 'Replace only the sentinel %s, then run apply-trust.sh with this ClientHost.\n' \ + "$SENTINEL_TRUSTED_PROXY_CIDR" diff --git a/infrastructure/networking/traefik/scripts/rollback-to-observe.sh b/infrastructure/networking/traefik/scripts/rollback-to-observe.sh new file mode 100755 index 0000000..962b899 --- /dev/null +++ b/infrastructure/networking/traefik/scripts/rollback-to-observe.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +set +x +umask 077 + +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +usage() { + cat <<'USAGE' +Usage: + bash infrastructure/networking/traefik/scripts/rollback-to-observe.sh --execute + +Removes forwarded-header trust while retaining JSON access logging. +USAGE +} + +[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || { + usage + exit 2 +} + +require_commands kubectl jq rg curl nc python3 awk cmp find bash + +bash "${SCRIPT_DIR}/validate.sh" +assert_live_baseline +kubectl --namespace kube-system \ + get helmchartconfig.helm.cattle.io traefik >/dev/null +assert_nodeport_boundary_and_health + +selected_context="$(current_context)" +selected_api_server="$(current_api_server)" +printf '\nKubernetes context: %s\nAPI server: %s\nTarget node: %s\n' \ + "$selected_context" "$selected_api_server" "$TARGET_NODE" +printf 'Type ROLLBACK %s OBSERVE to remove forwarded-header trust: ' \ + "$selected_context" +read -r confirmation +[[ "$confirmation" == "ROLLBACK ${selected_context} OBSERVE" ]] || \ + fail "cancelled" + +[[ "$(current_context)" == "$selected_context" ]] || \ + fail "kubectl context changed after confirmation" +[[ "$(current_api_server)" == "$selected_api_server" ]] || \ + fail "Kubernetes API server changed after confirmation" +assert_live_baseline + +kubectl apply --kustomize "$OBSERVE_OVERLAY" +wait_for_runtime observe +kubectl --namespace kube-system rollout status deployment/traefik --timeout=5m +assert_runtime observe + +work_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-traefik-rollback.XXXXXX")" +cleanup() { + case "$work_dir" in + /tmp/platform-traefik-rollback.*|"${TMPDIR:-/tmp}"/platform-traefik-rollback.*) + rm -rf -- "$work_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected directory: %s\n' \ + "$work_dir" >&2 + ;; + esac +} +trap cleanup EXIT + +assert_live_hcc_matches_overlay "$OBSERVE_OVERLAY" \ + "${work_dir}/post-rollback-observe.yaml" +assert_live_baseline +wait_for_nodeport_boundary_and_health + +printf '\nROLLBACK COMPLETE\n' +printf 'JSON access logging remains enabled.\n' +printf 'forwardedHeaders trust is absent from both entrypoints.\n' diff --git a/infrastructure/networking/traefik/scripts/validate.sh b/infrastructure/networking/traefik/scripts/validate.sh new file mode 100755 index 0000000..f8cba55 --- /dev/null +++ b/infrastructure/networking/traefik/scripts/validate.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +set +x +umask 077 + +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +assert_count() { + local file="$1" + local pattern="$2" + local expected="$3" + local description="$4" + local actual + + actual="$(rg --count --no-filename -- "$pattern" "$file" || true)" + actual="${actual:-0}" + [[ "$actual" == "$expected" ]] || \ + fail "${description}: expected ${expected}, found ${actual}" +} + +require_commands kubectl rg awk cmp python3 find bash + +render_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-traefik-render.XXXXXX")" +cleanup() { + case "$render_dir" in + /tmp/platform-traefik-render.*|"${TMPDIR:-/tmp}"/platform-traefik-render.*) + rm -rf -- "$render_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected directory: %s\n' \ + "$render_dir" >&2 + ;; + esac +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +root_render="${render_dir}/root.yaml" +baseline_render="${render_dir}/baseline.yaml" +observe_render="${render_dir}/observe.yaml" +trust_render="${render_dir}/trust.yaml" + +render_overlay "$TRAEFIK_ROOT" "$root_render" +render_overlay "$BASELINE_OVERLAY" "$baseline_render" +render_overlay "$OBSERVE_OVERLAY" "$observe_render" +render_overlay "$TRUST_OVERLAY" "$trust_render" + +cmp --silent "$root_render" "$observe_render" || \ + fail "the Traefik root must render the observation phase" + +for manifest in "$baseline_render" "$observe_render" "$trust_render"; do + assert_count "$manifest" '^apiVersion: helm\.cattle\.io/v1$' 1 \ + "HelmChartConfig API version" + assert_count "$manifest" '^kind: HelmChartConfig$' 1 \ + "HelmChartConfig kind" + assert_count "$manifest" '^ name: traefik$' 1 \ + "HelmChartConfig name" + assert_count "$manifest" '^ namespace: kube-system$' 1 \ + "HelmChartConfig namespace" + assert_count "$manifest" '^ failurePolicy: abort$' 1 \ + "Helm failure policy must preserve the running release on upgrade failure" + assert_count "$manifest" '^ deployment:$' 1 \ + "Traefik Deployment values root" + assert_count "$manifest" '^ podAnnotations:$' 1 \ + "Traefik Pod annotations block" + assert_count "$manifest" '^ prometheus\.io/path: /metrics$' 1 \ + "Traefik legacy scrape path preservation" + assert_count "$manifest" '^ prometheus\.io/port: "9100"$' 1 \ + "Traefik legacy scrape port preservation" + assert_count "$manifest" '^ prometheus\.io/scrape: "true"$' 1 \ + "Traefik legacy scrape enablement preservation" + assert_count "$manifest" '[Ii]nsecure' 0 \ + "insecure forwarded-header mode" + assert_count "$manifest" '^ service:$' 1 \ + "Traefik Service values root" + assert_count "$manifest" '^ type: NodePort$' 1 \ + "loopback NodePort Service type preservation" + assert_count "$manifest" '^ externalTrafficPolicy: Cluster$' 1 \ + "Traefik externalTrafficPolicy preservation" + assert_count "$manifest" '^ nodePort: 30080$' 1 \ + "Traefik web NodePort preservation" + assert_count "$manifest" '^ nodePort: 30443$' 1 \ + "Traefik websecure NodePort preservation" + assert_count "$manifest" '^ metrics:$' 1 \ + "Traefik metrics values root" + assert_count "$manifest" '^ prometheus:$' 1 \ + "Traefik Prometheus metrics block" + assert_count "$manifest" '^ serviceMonitor:$' 1 \ + "Traefik ServiceMonitor block" + assert_count "$manifest" '^ jobLabel: app\.kubernetes\.io/name$' 1 \ + "Traefik ServiceMonitor job label" + assert_count "$manifest" '^ observability\.hyeonworks\.com/instance: home$' 1 \ + "Traefik ServiceMonitor selector label" + assert_count "$manifest" '^ interval: 30s$' 1 \ + "Traefik ServiceMonitor interval" + assert_count "$manifest" '^ scrapeTimeout: 10s$' 1 \ + "Traefik ServiceMonitor timeout" + assert_count "$manifest" '^ enabled: true$' 2 \ + "Traefik metrics Service and ServiceMonitor enablement" + assert_count "$manifest" '^[[:space:]]*(type|serviceType):[[:space:]]*LoadBalancer' 0 \ + "LoadBalancer exposure" +done + +assert_count "$baseline_render" '^[[:space:]]*logs:' 0 \ + "baseline access logs" +assert_count "$baseline_render" '^[[:space:]]*forwardedHeaders:' 0 \ + "baseline forwarded-header trust" + +for manifest in "$observe_render" "$trust_render"; do + assert_count "$manifest" '^ enabled: true$' 1 \ + "access log enablement" + assert_count "$manifest" '^ format: json$' 1 \ + "JSON access log format" + assert_count "$manifest" '^ defaultmode: keep$' 1 \ + "access-log general-field policy" + assert_count "$manifest" '^ defaultmode: drop$' 1 \ + "access-log header policy" +done + +assert_count "$observe_render" '^[[:space:]]*forwardedHeaders:' 0 \ + "observation-phase forwarded-header trust" +assert_count "$observe_render" '^[[:space:]]*trustedIPs:' 0 \ + "observation-phase trusted IP list" + +assert_count "$trust_render" '^ ports:$' 1 \ + "trust-phase ports values root" +assert_count "$trust_render" '^ web:$' 1 \ + "trust-phase web entrypoint" +assert_count "$trust_render" '^ forwardedHeaders:$' 1 \ + "trust-phase forwarded-header block" +assert_count "$trust_render" '^ trustedIPs:$' 1 \ + "trust-phase trusted IP list" + +trusted_cidr="$(source_trusted_proxy_cidr)" +normalized_cidr="$(normalize_exact_host_cidr "$trusted_cidr")" +[[ "$trusted_cidr" == "$normalized_cidr" ]] || \ + fail "trusted proxy CIDR must use canonical exact-host notation" +assert_count "$trust_render" \ + "^[[:space:]]*-[[:space:]]*\"${trusted_cidr//./\\.}\"[[:space:]]*$" 1 \ + "rendered exact-host trusted proxy CIDR" + +while IFS= read -r -d '' script_path; do + bash -n "$script_path" +done < <( + find "$SCRIPT_DIR" -maxdepth 1 -type f -name '*.sh' -print0 +) + +printf 'Traefik observation and trust overlays rendered successfully.\n' +printf 'Access logs are JSON and request headers are dropped.\n' +printf 'No insecure mode, websecure trust, or LoadBalancer exposure was found.\n' +printf 'The existing NodePort 30080/30443 boundary is declared in both phases.\n' +if [[ "$trusted_cidr" == "$SENTINEL_TRUSTED_PROXY_CIDR" ]]; then + printf 'Trust overlay remains intentionally blocked by sentinel CIDR %s.\n' \ + "$SENTINEL_TRUSTED_PROXY_CIDR" +else + printf 'Trust overlay contains reviewed exact-host CIDR %s.\n' "$trusted_cidr" +fi diff --git a/infrastructure/security/k3s/40-secrets-encryption.yaml b/infrastructure/security/k3s/40-secrets-encryption.yaml new file mode 100644 index 0000000..c94ae06 --- /dev/null +++ b/infrastructure/security/k3s/40-secrets-encryption.yaml @@ -0,0 +1,2 @@ +secrets-encryption: true +secrets-encryption-provider: aescbc diff --git a/infrastructure/security/k3s/README.md b/infrastructure/security/k3s/README.md new file mode 100644 index 0000000..d57fb9c --- /dev/null +++ b/infrastructure/security/k3s/README.md @@ -0,0 +1,34 @@ +# k3s Secret 암호화 host drop-in + +`40-secrets-encryption.yaml`은 Kubernetes API에 적용하는 Kustomize resource가 +아니라, k3s server host의 설정 조각이다. bootstrap 담당자는 이 파일을 다음처럼 +live drop-in 경로에 설치한다. + +```sh +install -o root -g root -m 0644 \ + infrastructure/security/k3s/40-secrets-encryption.yaml \ + /etc/rancher/k3s/config.yaml.d/40-secrets-encryption.yaml +``` + +따라서 이 디렉터리에는 `kustomization.yaml`을 만들지 않는다. 이 파일은 정확히 +`secrets-encryption`과 `secrets-encryption-provider` 두 key만 가지며, Kubernetes +manifest metadata나 다른 host 설정을 넣지 않는다. + +실제 적용 순서, validator 판정, 서버 token 취급, 재암호화 대기와 중단 조건은 +[`bootstrap/manual/k3s-secret-encryption.md`](../../../bootstrap/manual/k3s-secret-encryption.md)를 +따른다. 이 artifact 자체를 자동 삭제하거나 기존 운영 설정을 덮어쓰지 않는다. + +## 로컬 recovery 하드웨어 계약 + +`local-recovery.env`는 로컬 암호화 recovery 저장소에 사용할 디스크 identity, +mount 경로와 용량 기준만 담는 비밀 없는 계약 파일이다. 이 파일은 shell에서 +`source`하지 않는다. `scripts/lib/k3s-local-recovery.sh`의 parser가 허용된 key가 +정확히 한 번씩 있는지, 각 행의 `KEY=VALUE` 형식과 값 문법이 맞는지를 확인한 뒤 +데이터로만 읽는다. 알 수 없는 key, 중복·누락 key, 상대 장치 경로, 공백·인용부호·명령 +치환을 포함한 값은 모두 실패 처리한다. + +용량 helper는 Bash signed 64-bit 정수 범위를 넘기기 전에 실패한다. 내부 recovery +공간은 각 backup phase의 125%를 두 번과 10GiB 최소 여유로 계산한다. 외부 NTFS +공간 판정은 최종 container와 header-proof용 완전 할당 copy, 10GiB 최소 여유, +전체 용량의 20% 잔여 공간을 함께 예약한다. 이 단계는 순수 parser·계산만 제공하며 +mount, format, package 설치, `cryptsetup`, k3s 변경을 수행하지 않는다. diff --git a/infrastructure/security/secrets/README.md b/infrastructure/security/secrets/README.md new file mode 100644 index 0000000..35914a3 --- /dev/null +++ b/infrastructure/security/secrets/README.md @@ -0,0 +1,92 @@ +# 플랫폼 Secret 계약 + +상태: Phase 1·Keycloak DB·Gitea OIDC·AIStor Secret 적용 + +이 디렉터리에는 Secret 값이나 Secret 매니페스트를 저장하지 않는다. 수동 +초기 구축 스크립트가 터미널 또는 지정된 파일에서 값을 읽어 Kubernetes API로 +직접 전달한다. + +## Phase 1 Secret 계약 + +| Namespace | 이름 | 타입 | 키 | 소비자 | +| --- | --- | --- | --- | --- | +| `platform-data` | `gitea-db-credentials` | `kubernetes.io/basic-auth` | `username`, `password` | CloudNativePG `DatabaseRole` 및 최초 `initdb` | +| `gitea` | `gitea-db-credentials` | `kubernetes.io/basic-auth` | `username`, `password` | Gitea | +| `gitea` | `gitea-admin` | `kubernetes.io/basic-auth` | `username`, `password` | Gitea 초기 관리자 생성 | + +두 namespace의 `gitea-db-credentials`는 동일한 `username=gitea`와 동일한 +비밀번호를 가져야 한다. Kubernetes Secret은 namespace를 넘어서 참조할 수 +없기 때문에 두 객체가 필요하다. `scripts/bootstrap/create-phase1-secrets.sh`는 +세 Secret이 모두 없을 때만 최초 생성하고, 모두 있으면 계약을 검증해 재사용하며, +일부만 존재하면 중단한다. + +## Keycloak-only Secret 계약 + +| Namespace | 이름 | 타입 | 키 | 소비자 | +| --- | --- | --- | --- | --- | +| `platform-data` | `keycloak-db-credentials` | `kubernetes.io/basic-auth` | `username`, `password` | CloudNativePG `DatabaseRole` | +| `keycloak` | `keycloak-db-credentials` | `kubernetes.io/basic-auth` | `username`, `password` | Keycloak | +| `gitea` | `gitea-keycloak-oidc` | `Opaque` | `key`, `secret` | Gitea OAuth2 source | + +두 DB Secret은 `username=keycloak`과 동일한 비밀번호를 가져야 하며 +`platform-data` 객체에는 `cnpg.io/reload=true` 라벨이 필요하다. +`create-keycloak-secrets.sh`는 두 DB Secret만 two-or-none으로 생성·검증한다. +`configure-keycloak-gitea-oidc.sh`는 Keycloak의 기존 confidential client secret을 +새로 회전하지 않고 Gitea namespace의 `gitea-keycloak-oidc`에 전달한다. `key`는 +client ID `gitea`이며 `secret` payload는 출력·디코드·문서화하지 않는다. 세 객체는 +실제 적용됐고 타입과 정확한 key 이름만 검증했다. + +```sh +bash scripts/bootstrap/create-keycloak-secrets.sh --generate --execute +bash scripts/bootstrap/configure-keycloak-gitea-oidc.sh --execute +``` + +Keycloak Operator가 만든 `keycloak/keycloak-initial-admin`도 실제 존재한다. 이 +Secret은 저장소가 렌더하는 자격 증명 manifest가 아니다. named 관리자와 MFA를 +검증한 뒤 초기 관리자의 보존·비활성화 정책을 별도 운영 절차로 결정한다. + +## AIStor Secret 계약 + +| Namespace | 이름 | 타입 | 키 | 소비자 | +| --- | --- | --- | --- | --- | +| `aistor` | `minio-license` | `Opaque` | `minio.license` | AIStor Operator | +| `object-storage` | `aistor-root-configuration` | `Opaque` | `config.env` | AIStor ObjectStore | + +두 AIStor Secret은 실제 적용됐다. `minio.license`는 사용자가 지정한 읽기 가능한 +절대 regular file에서 직접 읽는다. `config.env`에는 강한 `MINIO_ROOT_USER`, +`MINIO_ROOT_PASSWORD` export를 각각 이중 따옴표 형식으로 저장한다. + +```sh +export MINIO_ROOT_USER="" +export MINIO_ROOT_PASSWORD="" +``` + +실제 값은 이 문서나 Git에 기록하지 않는다. AIStor 전용 +`create-aistor-secrets.sh`는 두 객체를 two-or-none으로 생성·검증하고, 현재 +클러스터에서는 기존 값을 회전하지 않고 재사용한다. + +기존 `create-phase2-secrets.sh`는 Keycloak DB 두 객체와 AIStor 두 객체를 +four-or-none으로 관리한다. 현재 Keycloak DB 두 객체만 의도적으로 존재하므로 이 +스크립트는 실행하지 않는다. Keycloak Secret을 삭제하거나 검사를 우회하지 않는다. +기존 four-or-none helper는 호환용으로만 남긴다. AIStor 작업에는 전용 helper만 +사용하며 one-of-two 상태에서는 자동 삭제·보정·덮어쓰기 없이 중단한다. + +## 공통 운영 규칙 + +- 평문, Base64 값, 실제 Secret YAML, 라이선스 본문, 개인 키를 Git에 커밋하지 + 않는다. +- 스크립트는 호출자가 `bash -x`를 사용해도 명령 추적을 즉시 비활성화한다. +- 최초 생성 경로의 임시 파일은 `mktemp` 전용 디렉터리에 `umask 077`로 만들고 + 정상 종료·오류·신호 종료 시 삭제한다. +- 비밀값 대신 권한이 제한된 임시 파일 경로만 `kubectl` 인자로 전달한다. 생성 + YAML은 두 `kubectl` 프로세스 사이의 파이프로만 전달하며 터미널에 출력하지 + 않는다. +- Secret 이름과 키만 선언형 리소스에서 참조한다. +- `gitea-admin`은 Keycloak OIDC 도입 후에도 비상용 로컬 관리자 계정으로 + 유지한다. `initialOnlyRequireReset` 모드이므로 Secret 값만 바꿔도 이미 생성된 + 관리자 계정 비밀번호는 바뀌지 않는다. +- Argo CD 전환 전 별도 ADR로 SOPS, Sealed Secrets 또는 외부 Secret 동기화 + 방식을 선택한다. +- 초기 구축 스크립트는 DB 비밀번호, 관리자 비밀번호, AIStor root 자격 증명, + 라이선스를 회전하지 않는다. 회전은 소비자 재시작과 실제 접속 검증을 포함한 + 별도 운영 절차로 수행한다. diff --git a/infrastructure/storage/aistor-local-pv/README.md b/infrastructure/storage/aistor-local-pv/README.md new file mode 100644 index 0000000..d143ab7 --- /dev/null +++ b/infrastructure/storage/aistor-local-pv/README.md @@ -0,0 +1,38 @@ +# AIStor 정적 Local PersistentVolume + +상태(2026-07-23): 실제 클러스터 적용 및 바인딩 완료 + +이 기반 구성은 `/srv/k3s/aistor`에 마운트된 전용 XFS 파일시스템을 하나의 정적 +Local PV에 연결한다. + +| 리소스 | 값 | +| --- | --- | +| PersistentVolume | `aistor-data-local-pv` | +| StorageClass | `aistor-local-xfs-retain` | +| Kubernetes에 표시되는 용량 | `900Gi` | +| 접근 모드 | `ReadWriteOnce` | +| Reclaim 정책 | `Retain` | +| 바인딩 모드 | `WaitForFirstConsumer` | +| 필수 노드 | `donghyeon-system-product-name` | + +ObjectStore pool은 이 StorageClass에 정확히 하나의 `900Gi` Claim을 요청한다. +`WaitForFirstConsumer`는 해당 Claim과 Local PV node affinity를 함께 반영한다. +StorageClass와 PV 모두 `Retain`을 사용하므로 Claim을 삭제해도 XFS 데이터는 +지워지지 않지만, 복구와 재바인딩은 수동 작업이다. + +적용하기 전에 호스트에서 다음 항목을 모두 확인한다. + +```sh +findmnt -no SOURCE,TARGET,FSTYPE,OPTIONS /srv/k3s/aistor +df -hT /srv/k3s/aistor +kubectl get node donghyeon-system-product-name +``` + +첫 번째 명령은 전용 XFS 파티션이 정확한 경로에 마운트됐다고 출력해야 한다. +PV의 `900Gi` 용량은 하드 파일시스템 quota가 아니라 Kubernetes 바인딩 +메타데이터다. 이 단일 노드 Local PV는 복제나 재해 복구를 제공하지 않는다. + +현재 `data0-minio-aistor-pool-0-0` PVC가 `aistor-data-local-pv`에 Bound 상태이며 +ObjectStore health는 `green`이다. `/srv/k3s/aistor`의 기존 XFS 메타데이터 사용량은 +AIStor 객체 데이터와 별개일 수 있으므로 `du` 합계와 `df` 사용량을 동일시하지 +않는다. diff --git a/infrastructure/storage/aistor-local-pv/aistor-data-pv.yaml b/infrastructure/storage/aistor-local-pv/aistor-data-pv.yaml new file mode 100644 index 0000000..e7b84bf --- /dev/null +++ b/infrastructure/storage/aistor-local-pv/aistor-data-pv.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: aistor-data-local-pv + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage + platform.hyeonworks.com/storage-tier: hdd + platform.hyeonworks.com/filesystem: xfs +spec: + capacity: + storage: 900Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: aistor-local-xfs-retain + local: + path: /srv/k3s/aistor + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - donghyeon-system-product-name diff --git a/infrastructure/storage/aistor-local-pv/kustomization.yaml b/infrastructure/storage/aistor-local-pv/kustomization.yaml new file mode 100644 index 0000000..466a22d --- /dev/null +++ b/infrastructure/storage/aistor-local-pv/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - storage-class.yaml + - aistor-data-pv.yaml diff --git a/infrastructure/storage/aistor-local-pv/storage-class.yaml b/infrastructure/storage/aistor-local-pv/storage-class.yaml new file mode 100644 index 0000000..031cd90 --- /dev/null +++ b/infrastructure/storage/aistor-local-pv/storage-class.yaml @@ -0,0 +1,12 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: aistor-local-xfs-retain + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage + platform.hyeonworks.com/storage-tier: hdd +provisioner: kubernetes.io/no-provisioner +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false diff --git a/infrastructure/storage/observability-local-pv/README.md b/infrastructure/storage/observability-local-pv/README.md new file mode 100644 index 0000000..c798709 --- /dev/null +++ b/infrastructure/storage/observability-local-pv/README.md @@ -0,0 +1,32 @@ +# Observability Local PersistentVolumes + +This kustomization reserves six workload-specific Local PV binding contracts on +the root SSD. Each owner has a dedicated `kubernetes.io/no-provisioner` +StorageClass with `WaitForFirstConsumer`, `Retain`, and volume expansion +disabled. Every PV is `ReadWriteOnce`, retains its data after claim deletion, +and is affined to `donghyeon-system-product-name`. + +| Owner | Host path | Capacity | StorageClass | Directory owner | +| --- | --- | ---: | --- | --- | +| Prometheus | `/srv/k3s/ssd/observability/prometheus` | `20Gi` | `ssd-local-observability-prometheus-retain` | `1000:2000` | +| Grafana | `/srv/k3s/ssd/observability/grafana` | `2Gi` | `ssd-local-observability-grafana-retain` | `472:472` | +| Alertmanager | `/srv/k3s/ssd/observability/alertmanager` | `1Gi` | `ssd-local-observability-alertmanager-retain` | `1000:2000` | +| Alloy | `/srv/k3s/ssd/observability/alloy` | `2Gi` | `ssd-local-observability-alloy-retain` | `473:473` | +| Loki | `/srv/k3s/ssd/observability/loki` | `5Gi` | `ssd-local-observability-loki-retain` | `10001:10001` | +| Tempo | `/srv/k3s/ssd/observability/tempo` | `5Gi` | `ssd-local-observability-tempo-retain` | `10001:10001` | + +The declared 35 GiB is a Kubernetes binding contract, not an ext4 directory +quota. Project quota is not active. All six directories share root-filesystem +free space, so the later operational protections remain mandatory: + +- Prometheus `retentionSize=16GB` is not a whole-filesystem cap and does not cap + WAL plus head data. +- Loki's 512 MB setting limits WAL replay RAM; it is not a disk quota. +- Loki `disk_full_threshold=0.90` uses `Statfs` on the WAL filesystem. +- Tempo 2.10.7 has no WAL byte limit or disk-full guard. +- Root-filesystem 75%, 85%, and 90% alerts, apply/smoke `du` and `df` caps, and + acceptance of the root-90%-full ingest-stop runbook are still required. + +Run `bash scripts/bootstrap/prepare-observability-local-paths.sh` to inspect the +read-only plan. The separately confirmed `--execute` mode prepares the exact +paths at mode `0770`; it does not apply Kubernetes resources. diff --git a/infrastructure/storage/observability-local-pv/kustomization.yaml b/infrastructure/storage/observability-local-pv/kustomization.yaml new file mode 100644 index 0000000..622b4a3 --- /dev/null +++ b/infrastructure/storage/observability-local-pv/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - storage-classes.yaml + - persistent-volumes.yaml diff --git a/infrastructure/storage/observability-local-pv/persistent-volumes.yaml b/infrastructure/storage/observability-local-pv/persistent-volumes.yaml new file mode 100644 index 0000000..1c66ed6 --- /dev/null +++ b/infrastructure/storage/observability-local-pv/persistent-volumes.yaml @@ -0,0 +1,155 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: observability-prometheus-local-pv + labels: + app.kubernetes.io/part-of: observability + app.kubernetes.io/component: prometheus +spec: + capacity: + storage: 20Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: ssd-local-observability-prometheus-retain + local: + path: /srv/k3s/ssd/observability/prometheus + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - donghyeon-system-product-name +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: observability-grafana-local-pv + labels: + app.kubernetes.io/part-of: observability + app.kubernetes.io/component: grafana +spec: + capacity: + storage: 2Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: ssd-local-observability-grafana-retain + local: + path: /srv/k3s/ssd/observability/grafana + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - donghyeon-system-product-name +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: observability-alertmanager-local-pv + labels: + app.kubernetes.io/part-of: observability + app.kubernetes.io/component: alertmanager +spec: + capacity: + storage: 1Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: ssd-local-observability-alertmanager-retain + local: + path: /srv/k3s/ssd/observability/alertmanager + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - donghyeon-system-product-name +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: observability-alloy-local-pv + labels: + app.kubernetes.io/part-of: observability + app.kubernetes.io/component: alloy +spec: + capacity: + storage: 2Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: ssd-local-observability-alloy-retain + local: + path: /srv/k3s/ssd/observability/alloy + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - donghyeon-system-product-name +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: observability-loki-local-pv + labels: + app.kubernetes.io/part-of: observability + app.kubernetes.io/component: loki +spec: + capacity: + storage: 5Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: ssd-local-observability-loki-retain + local: + path: /srv/k3s/ssd/observability/loki + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - donghyeon-system-product-name +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: observability-tempo-local-pv + labels: + app.kubernetes.io/part-of: observability + app.kubernetes.io/component: tempo +spec: + capacity: + storage: 5Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: ssd-local-observability-tempo-retain + local: + path: /srv/k3s/ssd/observability/tempo + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - donghyeon-system-product-name diff --git a/infrastructure/storage/observability-local-pv/storage-classes.yaml b/infrastructure/storage/observability-local-pv/storage-classes.yaml new file mode 100644 index 0000000..4121dcb --- /dev/null +++ b/infrastructure/storage/observability-local-pv/storage-classes.yaml @@ -0,0 +1,53 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ssd-local-observability-prometheus-retain +provisioner: kubernetes.io/no-provisioner +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ssd-local-observability-grafana-retain +provisioner: kubernetes.io/no-provisioner +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ssd-local-observability-alertmanager-retain +provisioner: kubernetes.io/no-provisioner +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ssd-local-observability-alloy-retain +provisioner: kubernetes.io/no-provisioner +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ssd-local-observability-loki-retain +provisioner: kubernetes.io/no-provisioner +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ssd-local-observability-tempo-retain +provisioner: kubernetes.io/no-provisioner +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false diff --git a/infrastructure/storage/pgadmin-local-pv/README.md b/infrastructure/storage/pgadmin-local-pv/README.md new file mode 100644 index 0000000..1a1150c --- /dev/null +++ b/infrastructure/storage/pgadmin-local-pv/README.md @@ -0,0 +1,12 @@ +# pgAdmin SSD Local PV + +pgAdmin 설정 데이터베이스와 세션 상태를 위한 2Gi Local PV입니다. + +- 호스트 경로: /srv/k3s/ssd/pgadmin +- 노드: donghyeon-system-product-name +- StorageClass: ssd-local-pgadmin-retain +- 회수 정책: Retain + +PVC나 배포를 제거해도 PV와 호스트 데이터는 자동 삭제하지 않습니다. +호스트 디렉터리는 적용 스크립트가 소유권과 파일시스템 경계를 검증한 뒤 +생성합니다. diff --git a/infrastructure/storage/pgadmin-local-pv/kustomization.yaml b/infrastructure/storage/pgadmin-local-pv/kustomization.yaml new file mode 100644 index 0000000..05533e3 --- /dev/null +++ b/infrastructure/storage/pgadmin-local-pv/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - storage-class.yaml + - pgadmin-pv.yaml diff --git a/infrastructure/storage/pgadmin-local-pv/pgadmin-pv.yaml b/infrastructure/storage/pgadmin-local-pv/pgadmin-pv.yaml new file mode 100644 index 0000000..a1384ff --- /dev/null +++ b/infrastructure/storage/pgadmin-local-pv/pgadmin-pv.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pgadmin-data-local-pv + labels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/part-of: platform + platform.hyeonworks.com/storage-tier: ssd +spec: + capacity: + storage: 2Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: ssd-local-pgadmin-retain + local: + path: /srv/k3s/ssd/pgadmin + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - donghyeon-system-product-name diff --git a/infrastructure/storage/pgadmin-local-pv/storage-class.yaml b/infrastructure/storage/pgadmin-local-pv/storage-class.yaml new file mode 100644 index 0000000..9a487e1 --- /dev/null +++ b/infrastructure/storage/pgadmin-local-pv/storage-class.yaml @@ -0,0 +1,11 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ssd-local-pgadmin-retain + labels: + app.kubernetes.io/part-of: platform + platform.hyeonworks.com/storage-tier: ssd +provisioner: kubernetes.io/no-provisioner +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false diff --git a/infrastructure/storage/ssd-local-pv/README.md b/infrastructure/storage/ssd-local-pv/README.md new file mode 100644 index 0000000..6eb8514 --- /dev/null +++ b/infrastructure/storage/ssd-local-pv/README.md @@ -0,0 +1,33 @@ +# SSD 정적 Local PersistentVolume + +첫 배포에서는 서버의 루트 SSD에 워크로드별 Local PV 두 개를 예약한다. + +| 워크로드 | StorageClass | 로컬 경로 | 선언 용량 | +| --- | --- | --- | ---: | +| 플랫폼 PostgreSQL | `ssd-local-postgres-retain` | `/srv/k3s/ssd/platform-postgres` | `20Gi` | +| Gitea | `ssd-local-gitea-retain` | `/srv/k3s/ssd/gitea` | `50Gi` | + +별도의 StorageClass를 사용하여 PostgreSQL의 20 GiB Claim이 Gitea의 50 GiB 볼륨에 +잘못 바인딩되는 것을 방지한다. `WaitForFirstConsumer`를 사용하면 Scheduler가 +Local PV의 node affinity를 반영할 수 있다. 두 PV 모두 `Retain`을 사용하므로 +Claim을 삭제해도 디스크의 데이터는 삭제되지 않는다. + +PVC는 의도적으로 이 인프라 디렉터리가 아니라 소유자와 함께 둔다. + +- CloudNativePG는 `Cluster`의 storage 명세에서 PostgreSQL 인스턴스 PVC를 생성한다. + 이 PVC는 `ssd-local-postgres-retain`과 `20Gi`를 요청해야 한다. +- Gitea 서비스 배포가 직접 또는 공식 Chart를 통해 자체 PVC를 소유한다. 이 PVC는 + `ssd-local-gitea-retain`과 `50Gi`를 요청해야 한다. + +PV를 생성하기 전에 `scripts/bootstrap/prepare-ssd-local-paths.sh`를 수동으로 +실행한다. 이 스크립트는 `/srv`가 루트 파일시스템에 있는지 확인한 후 정확히 두 +디렉터리만 준비한다. + +Local PV의 `capacity` 필드는 바인딩 메타데이터이며 파일시스템 quota가 아니다. +PostgreSQL과 Gitea는 루트 SSD의 여유 공간을 공유한다. 루트 파일시스템을 +모니터링하고, 하드 제한이 필요해지면 파일시스템 또는 블록 장치 quota를 별도로 +추가한다. + +Local PV는 이 워크로드들을 `donghyeon-system-product-name` 노드에 종속시킨다. +`Retain`은 자동 파일 삭제를 막지만 백업이나 고가용성을 제공하지 않는다. Released +상태의 PV를 재사용하는 작업은 명시적인 복구 절차로 수행해야 한다. diff --git a/infrastructure/storage/ssd-local-pv/gitea-pv.yaml b/infrastructure/storage/ssd-local-pv/gitea-pv.yaml new file mode 100644 index 0000000..8c68f51 --- /dev/null +++ b/infrastructure/storage/ssd-local-pv/gitea-pv.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: gitea-local-pv + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: source-control + platform.hyeonworks.com/storage-tier: ssd +spec: + capacity: + storage: 50Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: ssd-local-gitea-retain + local: + path: /srv/k3s/ssd/gitea + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - donghyeon-system-product-name diff --git a/infrastructure/storage/ssd-local-pv/kustomization.yaml b/infrastructure/storage/ssd-local-pv/kustomization.yaml new file mode 100644 index 0000000..de7ab15 --- /dev/null +++ b/infrastructure/storage/ssd-local-pv/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - storage-classes.yaml + - platform-postgres-pv.yaml + - gitea-pv.yaml diff --git a/infrastructure/storage/ssd-local-pv/platform-postgres-pv.yaml b/infrastructure/storage/ssd-local-pv/platform-postgres-pv.yaml new file mode 100644 index 0000000..b96c087 --- /dev/null +++ b/infrastructure/storage/ssd-local-pv/platform-postgres-pv.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: platform-postgres-local-pv + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: database + platform.hyeonworks.com/storage-tier: ssd +spec: + capacity: + storage: 20Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: ssd-local-postgres-retain + local: + path: /srv/k3s/ssd/platform-postgres + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - donghyeon-system-product-name diff --git a/infrastructure/storage/ssd-local-pv/storage-classes.yaml b/infrastructure/storage/ssd-local-pv/storage-classes.yaml new file mode 100644 index 0000000..2a7dc3d --- /dev/null +++ b/infrastructure/storage/ssd-local-pv/storage-classes.yaml @@ -0,0 +1,23 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ssd-local-postgres-retain + labels: + app.kubernetes.io/part-of: platform + platform.hyeonworks.com/storage-tier: ssd +provisioner: kubernetes.io/no-provisioner +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ssd-local-gitea-retain + labels: + app.kubernetes.io/part-of: platform + platform.hyeonworks.com/storage-tier: ssd +provisioner: kubernetes.io/no-provisioner +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false diff --git a/scripts/bootstrap/apply-admin-services.sh b/scripts/bootstrap/apply-admin-services.sh new file mode 100755 index 0000000..cd759e3 --- /dev/null +++ b/scripts/bootstrap/apply-admin-services.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +set +x +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)" +readonly TARGET_NODE="donghyeon-system-product-name" +readonly PG_PATH="/srv/k3s/ssd/pgadmin" +readonly PG_SECRET="pgadmin-bootstrap" +readonly PG_OIDC_SECRET="pgadmin-keycloak-oidc" +readonly AI_OIDC_SECRET="aistor-keycloak-oidc" +readonly NAMES="admin-namespace pgadmin-local-pv coredns-custom aistor-admin-oidc pgadmin" +execute=false +password_file="" +work="" +rollback_armed=false +bootstrap_existed=false + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'USAGE' +사용법: + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-admin-services.sh + + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-admin-services.sh \ + --execute \ + --pgadmin-password-file /home/donghyeon/.secrets/pgadmin/bootstrap-password + +인자 없이 실행하면 공식 차트 hash와 manifest만 검증합니다. +--execute는 Local PV, pgAdmin bootstrap Secret, pgAdmin, AIStor OIDC profile을 +적용합니다. 비밀번호와 OIDC Secret 값은 출력하지 않습니다. +USAGE +} + +while (( $# > 0 )); do + case "$1" in + --execute) + execute=true + shift + ;; + --pgadmin-password-file) + (( $# >= 2 )) || fail "--pgadmin-password-file 값이 필요합니다" + password_file="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage >&2 + fail "지원하지 않는 인자: $1" + ;; + esac +done + +[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요" +for cmd in awk curl find findmnt install jq kubectl mktemp od rg seq sha256sum sleep sort stat tail tr wc; do + command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다" +done + +work="$(mktemp -d /tmp/platform-admin-apply.XXXXXX)" + +rollback() { + set +e + printf '\nROLLBACK: 관리 UI 트래픽을 내리고 AIStor 이전 spec을 복원합니다.\n' >&2 + kubectl -n platform-admin scale deployment/pgadmin --replicas=0 >/dev/null 2>&1 || true + kubectl -n platform-admin delete ingress pgadmin --ignore-not-found >/dev/null 2>&1 + kubectl -n object-storage delete ingress minio-aistor-console --ignore-not-found >/dev/null 2>&1 + kubectl apply -f "$work/objectstore-before.json" >/dev/null 2>&1 || true + for statefulset_name in $( + kubectl -n object-storage get objectstore minio-aistor \ + -o jsonpath='{range .status.pools[*]}{.ssName}{"\n"}{end}' 2>/dev/null + ); do + kubectl -n object-storage rollout restart "statefulset/$statefulset_name" >/dev/null 2>&1 || true + kubectl -n object-storage rollout status "statefulset/$statefulset_name" \ + --timeout=300s >/dev/null 2>&1 || true + done + if [[ "$bootstrap_existed" == true ]]; then + kubectl apply -f "$work/pgadmin-bootstrap-before.yaml" >/dev/null 2>&1 || true + else + kubectl -n platform-admin delete secret "$PG_SECRET" --ignore-not-found >/dev/null 2>&1 + fi + rollback_armed=false + printf 'ROLLBACK complete. pgAdmin PVC/PV와 AIStor 데이터는 삭제하지 않았습니다.\n' >&2 +} + +cleanup() { + rc=$? + trap - EXIT INT TERM + if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then + rollback + fi + case "$work" in + /tmp/platform-admin-apply.*) rm -rf -- "$work" ;; + esac + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +unexpected_error() { + rc=$? + line="$1" + trap - ERR + printf 'ERROR: 예상하지 못한 명령 실패(line=%s, exit=%s)\n' "$line" "$rc" >&2 + exit "$rc" +} +trap 'unexpected_error "$LINENO"' ERR + +if [[ ! -v PLATFORM_HELM_BIN ]]; then + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm +fi +PLATFORM_HELM_BIN="$PLATFORM_HELM_BIN" \ + bash "$ROOT/scripts/validate/render-admin-services.sh" --verified-output-dir "$work" + +for name in $NAMES; do + file="$work/$name.yaml" + [[ -f "$file" && ! -L "$file" && -O "$file" && -s "$file" ]] || fail "안전하지 않은 handoff: $file" + [[ "$(stat -c '%a' "$file")" == 600 ]] || fail "handoff 권한이 0600이 아닙니다" + sha256sum "$file" | awk '{print $1}' >"$work/$name.sha256" +done + +context="$(kubectl config current-context)" +api="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" +ready="$(kubectl get node "$TARGET_NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')" +[[ "$ready" == True ]] || fail "대상 노드가 Ready가 아닙니다" +printf 'Current context: %s\n' "$context" +printf 'API server: %s\n' "$api" +for name in $NAMES; do + printf 'SHA-256 %-20s %s\n' "$name" "$(cat "$work/$name.sha256")" +done + +if [[ "$execute" == false ]]; then + printf 'DRY RUN PASS: --execute를 지정하지 않아 클러스터를 변경하지 않았습니다.\n' + exit 0 +fi + +[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다" +[[ "$password_file" == /* && -f "$password_file" && ! -L "$password_file" ]] || \ + fail "pgAdmin 비밀번호 파일은 일반 파일인 절대 경로여야 합니다" +[[ "$(stat -c '%a' "$password_file")" == 600 ]] || fail "pgAdmin 비밀번호 파일 권한은 0600이어야 합니다" +password_bytes="$(wc -c <"$password_file" | tr -d '[:space:]')" +(( password_bytes >= 16 && password_bytes <= 256 )) || fail "pgAdmin 비밀번호는 16~256 bytes여야 합니다" +last_byte="$(tail -c 1 "$password_file" | od -An -t x1 | tr -d '[:space:]')" +[[ "$last_byte" != 0a && "$last_byte" != 0d ]] || \ + fail "pgAdmin 비밀번호 파일 끝에 개행이 없어야 합니다" + +for contract in "object-storage $AI_OIDC_SECRET client-id client-secret" \ + "platform-admin $PG_OIDC_SECRET client-id client-secret"; do + set -- $contract + namespace="$1" + secret="$2" + key_a="$3" + key_b="$4" + kubectl -n "$namespace" get "secret/$secret" >/dev/null 2>&1 || \ + fail "$namespace/$secret Secret이 없습니다. Keycloak OIDC 구성을 먼저 실행하세요" + keys="$(kubectl -n "$namespace" get "secret/$secret" \ + -o go-template='{{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}' | LC_ALL=C sort)" + [[ "$keys" == "$key_a"$'\n'"$key_b" ]] || fail "$namespace/$secret key 계약이 다릅니다" +done + +core_data="$(kubectl -n kube-system get configmap coredns-custom \ + -o jsonpath='{.data.learn-hyeonworks\.server}' 2>/dev/null)" || \ + fail "coredns-custom이 없습니다. apply-private-dns.sh를 먼저 실행하세요" +for host in git.learn.hyeonworks.com id.learn.hyeonworks.com \ + storage-admin.learn.hyeonworks.com db-admin.learn.hyeonworks.com; do + printf '%s\n' "$core_data" | rg -q -F "$host" || fail "CoreDNS에 $host가 없습니다" +done + +printf 'Type APPLY %s to deploy private admin services: ' "$context" +read -r answer +[[ "$answer" == "APPLY $context" ]] || fail "취소했습니다" +[[ "$(kubectl config current-context)" == "$context" ]] || fail "context가 바뀌었습니다" +[[ "$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" == "$api" ]] || \ + fail "API server가 바뀌었습니다" +for name in $NAMES; do + current_sha="$(sha256sum "$work/$name.yaml" | awk '{print $1}')" + [[ "$current_sha" == "$(cat "$work/$name.sha256")" ]] || fail "$name handoff가 확인 뒤 바뀌었습니다" +done + +printf '[1/6] sudo 인증 확인\n' +sudo -v +printf '[2/6] pgAdmin SSD filesystem 경계와 Local PV 경로 확인\n' +root_source="$(sudo findmnt --kernel --first-only --noheadings --output SOURCE --target /)" +srv_source="$(sudo findmnt --kernel --first-only --noheadings --output SOURCE --target /srv)" +srv_target="$(sudo findmnt --kernel --first-only --noheadings --output TARGET --target /srv)" +[[ "$srv_source" == "$root_source" && "$srv_target" == / ]] || \ + fail "/srv가 root SSD 파일시스템 경계가 아닙니다" +if sudo test -L "$PG_PATH"; then + fail "$PG_PATH가 심볼릭 링크입니다" +fi +if sudo test -e "$PG_PATH"; then + sudo test -d "$PG_PATH" || fail "$PG_PATH가 디렉터리가 아닙니다" +else + sudo install -d -o root -g root -m 0750 "$PG_PATH" +fi +[[ "$(sudo findmnt --kernel --first-only --noheadings --output SOURCE --target "$PG_PATH")" == "$root_source" ]] || \ + fail "$PG_PATH가 root SSD에 있지 않습니다" +[[ "$(sudo findmnt --kernel --first-only --noheadings --output TARGET --target "$PG_PATH")" == / ]] || \ + fail "$PG_PATH 아래에 다른 mount가 있습니다" + +printf '[3/6] rollback용 AIStor spec과 기존 pgAdmin Secret 상태 저장\n' +kubectl -n object-storage get objectstore minio-aistor -o json | \ + jq '{apiVersion,kind,metadata:{name:.metadata.name,namespace:.metadata.namespace},spec}' \ + >"$work/objectstore-before.json" +if kubectl -n platform-admin get secret "$PG_SECRET" -o yaml >"$work/pgadmin-bootstrap-before.yaml" 2>/dev/null; then + bootstrap_existed=true +fi + +rollback_armed=true +printf '[4/6] namespace, Local PV, Secret, AIStor OIDC와 pgAdmin 적용\n' +kubectl apply -f "$work/admin-namespace.yaml" +kubectl apply -f "$work/pgadmin-local-pv.yaml" + +kubectl -n platform-admin create secret generic "$PG_SECRET" \ + --from-file="password=$password_file" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl -n platform-admin label secret "$PG_SECRET" \ + app.kubernetes.io/name=pgadmin4 \ + app.kubernetes.io/component=bootstrap-credential \ + app.kubernetes.io/part-of=platform \ + app.kubernetes.io/managed-by=bootstrap-script \ + --overwrite >/dev/null + +kubectl apply -f "$work/aistor-admin-oidc.yaml" +kubectl apply -f "$work/pgadmin.yaml" + +printf '[5/6] pgAdmin rollout, EndpointSlice와 AIStor health 대기\n' +kubectl -n platform-admin rollout status deployment/pgadmin --timeout=300s +kubectl -n platform-admin wait --for=jsonpath='{.endpoints[0].conditions.ready}'=true \ + endpointslice --selector=kubernetes.io/service-name=pgadmin --timeout=120s + +healthy=false +for attempt in $(seq 1 60); do + health="$(kubectl -n object-storage get objectstore minio-aistor -o jsonpath='{.status.healthStatus}' 2>/dev/null)" + if [[ "$health" == green ]]; then + healthy=true + break + fi + if (( attempt == 1 || attempt % 3 == 0 )); then + printf 'AIStor health 대기: status=%s elapsed=%ss/300s\n' \ + "${health:-unknown}" "$((attempt * 5))" + fi + sleep 5 +done +[[ "$healthy" == true ]] || fail "AIStor ObjectStore가 green으로 복귀하지 않았습니다" + +printf '[6/6] Traefik Host routing 확인\n' +pg_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --noproxy '*' -H 'Host: db-admin.learn.hyeonworks.com' http://127.0.0.1:30080/)" +ai_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --noproxy '*' -H 'Host: storage-admin.learn.hyeonworks.com' http://127.0.0.1:30080/)" +[[ "$pg_code" == 200 || "$pg_code" == 302 || "$pg_code" == 303 ]] || fail "pgAdmin Traefik 응답 실패: $pg_code" +[[ "$ai_code" == 200 || "$ai_code" == 302 || "$ai_code" == 303 ]] || fail "AIStor Console Traefik 응답 실패: $ai_code" + +rollback_armed=false +printf 'ADMIN SERVICES APPLY SUCCESS\n' +printf 'pgAdmin PVC/PV: Retain, AIStor S3 API: cluster-internal only\n' diff --git a/scripts/bootstrap/apply-aistor.sh b/scripts/bootstrap/apply-aistor.sh new file mode 100755 index 0000000..595c685 --- /dev/null +++ b/scripts/bootstrap/apply-aistor.sh @@ -0,0 +1,470 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +# 호출자가 bash -x로 실행해도 Secret 경로와 향후 입력이 추적되지 않도록 한다. +set +x + +readonly EXPECTED_HELM_VERSION="v3.19.4" +readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1" +readonly TARGET_NODE="donghyeon-system-product-name" +readonly EXPECTED_API_SERVICE_IP="10.43.0.1" +readonly EXPECTED_API_ENDPOINT_IP="192.168.0.107" +readonly EXPECTED_AISTOR_DEVICE="/dev/sdb3" +readonly EXPECTED_AISTOR_MOUNT="/srv/k3s/aistor" +readonly EXPECTED_STORAGE_CLASS="aistor-local-xfs-retain" +readonly EXPECTED_PV="aistor-data-local-pv" +readonly EXPECTED_OBJECTSTORE="minio-aistor" +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly -a VERIFIED_MANIFEST_NAMES=( + phase2-namespaces + aistor-local-pv + keycloak-operator + platform-postgres-keycloak + keycloak + aistor-operator + minio-aistor + aistor-network-policies +) +readonly -a AISTOR_CRDS=( + customresourcedefinition/adminjobs.aistor.min.io + customresourcedefinition/objectstores.aistor.min.io + customresourcedefinition/policybindings.sts.min.io +) + +license_file="" +root_config_file="" +generate_root_config=false +execute_requested=false +mutation_started=false +current_step="preflight" + +report_retained_state() { + if [[ "$mutation_started" == true ]]; then + printf '%s\n' \ + "SAFE STOP during ${current_step}." \ + 'No Namespace, Secret, PV, PVC, Operator, ObjectStore, or XFS data was deleted.' \ + 'The PV reclaim policy remains Retain and PVC protection remains enabled.' \ + 'Diagnose the failed wait or policy, then rerun this script.' >&2 + fi +} + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + report_retained_state + exit 1 +} + +on_error() { + local status="$1" + local line="$2" + + trap - ERR + set +e + printf 'ERROR: command failed at line %s (exit %s).\n' "$line" "$status" >&2 + report_retained_state + exit "$status" +} + +on_signal() { + local status="$1" + + trap - INT TERM + set +e + printf 'INTERRUPTED: stopping without deleting cluster or XFS state.\n' >&2 + report_retained_state + exit "$status" +} + +usage() { + cat <<'USAGE' +Usage: + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-aistor.sh \ + --license-file /home/donghyeon/.secrets/aistor/minio.license \ + --root-config-file /home/donghyeon/.secrets/aistor/root.env \ + --generate-root-config \ + --execute + +Renders, verifies, and applies the internal-only MinIO AIStor path: + aistor and object-storage namespaces + two out-of-Git Secret contracts + one 900Gi Retain Local PV on /srv/k3s/aistor + AIStor Operator 5.10.0 and CRDs + one-server, one-drive ObjectStore 1.0.16 + default-deny NetworkPolicies with only required internal paths + +It does not configure Host Nginx, Traefik Ingress, NodePort, LoadBalancer, +public DNS, credential rotation, or deletion. +USAGE +} + +while (( $# > 0 )); do + case "$1" in + --license-file) + (( $# >= 2 )) || fail "--license-file requires a path" + license_file="$2" + shift 2 + ;; + --root-config-file) + (( $# >= 2 )) || fail "--root-config-file requires a path" + root_config_file="$2" + shift 2 + ;; + --generate-root-config) + generate_root_config=true + shift + ;; + --execute) + execute_requested=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage >&2 + fail "unsupported argument: $1" + ;; + esac +done + +[[ "$execute_requested" == true ]] || { + usage >&2 + exit 2 +} +[[ "$license_file" == /* ]] || fail "--license-file must be an absolute path" +[[ "$root_config_file" == /* ]] || \ + fail "--root-config-file must be an absolute path" + +for command_name in cmp curl df find findmnt jq kubectl mktemp mountpoint \ + rg sed seq sha256sum sleep stat tail tr wc; do + command -v "$command_name" >/dev/null 2>&1 || \ + fail "${command_name} is required" +done + +if [[ -n "${PLATFORM_HELM_BIN:-}" ]]; then + [[ "$PLATFORM_HELM_BIN" == /* ]] || \ + fail "PLATFORM_HELM_BIN must be an absolute path" + [[ -f "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \ + fail "PLATFORM_HELM_BIN is not an executable file: ${PLATFORM_HELM_BIN}" + readonly HELM_BIN="$PLATFORM_HELM_BIN" +else + HELM_BIN="$(command -v helm 2>/dev/null)" || \ + fail "Helm ${EXPECTED_HELM_VERSION} is required" + readonly HELM_BIN +fi + +[[ "$("$HELM_BIN" version --template '{{.Version}}')" == "$EXPECTED_HELM_VERSION" ]] || \ + fail "Helm must be exactly ${EXPECTED_HELM_VERSION}" +kustomize_version="$( + kubectl version --client --output=yaml | + sed -n 's/^kustomizeVersion: //p' +)" +[[ "$kustomize_version" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \ + fail "expected Kustomize ${EXPECTED_KUSTOMIZE_VERSION}, found ${kustomize_version:-unknown}" + +umask 077 +render_temp_dir="$(mktemp -d /tmp/platform-phase2-apply.XXXXXX)" + +cleanup() { + case "$render_temp_dir" in + /tmp/platform-phase2-apply.*) + rm -rf -- "$render_temp_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected render directory: %s\n' \ + "$render_temp_dir" >&2 + ;; + esac +} +trap cleanup EXIT +trap 'on_error "$?" "$LINENO"' ERR +trap 'on_signal 130' INT +trap 'on_signal 143' TERM + +cd -- "$REPOSITORY_ROOT" +PLATFORM_HELM_BIN="$HELM_BIN" \ + bash scripts/validate/render-phase2.sh \ + --verified-output-dir "$render_temp_dir" + +declare -A verified_manifest_sha256=() +for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do + manifest_path="${render_temp_dir}/${manifest_name}.yaml" + [[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \ + fail "verified manifest is missing or unsafe: ${manifest_path}" + [[ "$(stat --format='%a' -- "$manifest_path")" == "600" ]] || \ + fail "verified manifest must have mode 0600: ${manifest_path}" + checksum_output="$(sha256sum -- "$manifest_path")" + verified_manifest_sha256["$manifest_name"]="${checksum_output%% *}" +done +verified_entry_count="$( + find "$render_temp_dir" -mindepth 1 -maxdepth 1 | + wc -l | tr -d '[:space:]' +)" +[[ "$verified_entry_count" == "${#VERIFIED_MANIFEST_NAMES[@]}" ]] || \ + fail "verified handoff must contain exactly eight manifest files" + +verify_manifest_unchanged() { + local manifest_name="$1" + local manifest_path="${render_temp_dir}/${manifest_name}.yaml" + local checksum_output + local actual_sha256 + + [[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \ + fail "verified manifest became missing or unsafe: ${manifest_path}" + checksum_output="$(sha256sum -- "$manifest_path")" + actual_sha256="${checksum_output%% *}" + [[ "$actual_sha256" == "${verified_manifest_sha256[$manifest_name]}" ]] || \ + fail "verified manifest changed before apply: ${manifest_name}.yaml" +} + +current_context="$(kubectl config current-context)" +api_server="$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" +node_ready="$( + kubectl get node "$TARGET_NODE" \ + --output='go-template={{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}true{{end}}{{end}}' +)" +[[ "$node_ready" == "true" ]] || fail "target node is not Ready: ${TARGET_NODE}" + +cluster_api_ip="$( + kubectl --namespace default get service kubernetes \ + --output=jsonpath='{.spec.clusterIP}' +)" +[[ "$cluster_api_ip" == "$EXPECTED_API_SERVICE_IP" ]] || \ + fail "Kubernetes API Service IP changed: expected ${EXPECTED_API_SERVICE_IP}, found ${cluster_api_ip}" +cluster_api_endpoint="$( + kubectl --namespace default get endpointslice \ + --selector=kubernetes.io/service-name=kubernetes \ + --output=jsonpath='{.items[0].endpoints[0].addresses[0]}' +)" +[[ "$cluster_api_endpoint" == "$EXPECTED_API_ENDPOINT_IP" ]] || \ + fail "Kubernetes API endpoint changed: expected ${EXPECTED_API_ENDPOINT_IP}, found ${cluster_api_endpoint}" + +mountpoint --quiet "$EXPECTED_AISTOR_MOUNT" || \ + fail "${EXPECTED_AISTOR_MOUNT} is not a mountpoint" +mount_source="$(findmnt --noheadings --output SOURCE --target "$EXPECTED_AISTOR_MOUNT" | tr -d '[:space:]')" +mount_fstype="$(findmnt --noheadings --output FSTYPE --target "$EXPECTED_AISTOR_MOUNT" | tr -d '[:space:]')" +mount_options="$(findmnt --noheadings --output OPTIONS --target "$EXPECTED_AISTOR_MOUNT")" +[[ "$mount_source" == "$EXPECTED_AISTOR_DEVICE" ]] || \ + fail "AIStor mount source changed: expected ${EXPECTED_AISTOR_DEVICE}, found ${mount_source}" +[[ "$mount_fstype" == "xfs" ]] || \ + fail "AIStor mount must be XFS, found ${mount_fstype}" +[[ ",${mount_options}," == *,rw,* ]] || fail "AIStor XFS mount is not writable" +[[ -d "$EXPECTED_AISTOR_MOUNT" && ! -L "$EXPECTED_AISTOR_MOUNT" ]] || \ + fail "AIStor mount path must be a non-symlink directory" + +available_bytes="$( + df --block-size=1 --output=avail "$EXPECTED_AISTOR_MOUNT" | + tail -n 1 | tr -d '[:space:]' +)" +minimum_bytes=$((900 * 1024 * 1024 * 1024)) +(( available_bytes >= minimum_bytes )) || \ + fail "AIStor XFS has less than 900Gi available" + +if ! kubectl get persistentvolume "$EXPECTED_PV" >/dev/null 2>&1; then + [[ -z "$(find "$EXPECTED_AISTOR_MOUNT" -mindepth 1 -maxdepth 1 -print -quit)" ]] || \ + fail "initial AIStor XFS root is not empty; refusing to bind an unknown data directory" +fi + +unexpected_consumers="$( + kubectl get persistentvolumeclaim --all-namespaces --output=json | + jq --arg storage_class "$EXPECTED_STORAGE_CLASS" \ + '[.items[] | select(.spec.storageClassName == $storage_class)] | length' +)" +if (( unexpected_consumers > 0 )); then + existing_expected_claim="$( + kubectl --namespace object-storage get persistentvolumeclaim \ + --ignore-not-found --output=json | + jq --arg storage_class "$EXPECTED_STORAGE_CLASS" \ + '[.items[] | select(.spec.storageClassName == $storage_class)] | length' + )" + [[ "$unexpected_consumers" == "1" && "$existing_expected_claim" == "1" ]] || \ + fail "the AIStor StorageClass has an unexpected PVC consumer" +fi + +if kubectl get customresourcedefinition objectstores.aistor.min.io >/dev/null 2>&1; then + unexpected_objectstores="$( + kubectl get objectstores.aistor.min.io --all-namespaces --output=json | + jq --arg name "$EXPECTED_OBJECTSTORE" \ + '[.items[] | select(.metadata.namespace != "object-storage" or .metadata.name != $name)] | length' + )" + [[ "$unexpected_objectstores" == "0" ]] || \ + fail "an unexpected AIStor ObjectStore already exists" +fi + +[[ -f "$license_file" && ! -L "$license_file" && -O "$license_file" && -s "$license_file" ]] || \ + fail "license file must be a non-empty, current-user-owned regular file" +[[ "$(stat --format='%a' -- "$license_file")" == "600" ]] || \ + fail "license file must have mode 0600" + +printf '\nKubernetes context: %s\nAPI server: %s\nTarget node: %s\n' \ + "$current_context" "$api_server" "$TARGET_NODE" +printf 'XFS: %s -> %s (%s, at least 900Gi available)\n' \ + "$mount_source" "$EXPECTED_AISTOR_MOUNT" "$mount_fstype" +printf '%s\n' \ + 'Scope: internal-only AIStor Operator, 900Gi Retain Local PV, one ObjectStore, and NetworkPolicies.' \ + 'Excluded: Host Nginx, Traefik, NodePort, LoadBalancer, public DNS, rotation, and deletion.' \ + 'Failure boundary: all applied state and XFS data are retained; rerunning is the recovery path.' +[[ -t 0 ]] || fail "an interactive terminal is required" +printf 'Type APPLY AISTOR %s to start the cluster mutation: ' "$current_context" +read -r confirmation +[[ "$confirmation" == "APPLY AISTOR ${current_context}" ]] || fail "cancelled" + +assert_cluster_identity() { + [[ "$(kubectl config current-context)" == "$current_context" ]] || \ + fail "kubectl context changed after confirmation" + [[ "$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" == "$api_server" ]] || \ + fail "Kubernetes API server changed after confirmation" + kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \ + fail "target node disappeared after confirmation: ${TARGET_NODE}" + mountpoint --quiet "$EXPECTED_AISTOR_MOUNT" || \ + fail "AIStor XFS mount disappeared after confirmation" +} + +assert_cluster_identity +for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do + verify_manifest_unchanged "$manifest_name" +done + +mutation_started=true + +current_step="[1/7] AIStor namespaces" +printf '\n%s\n' "$current_step" +assert_cluster_identity +verify_manifest_unchanged phase2-namespaces +kubectl apply --dry-run=server \ + --filename="${render_temp_dir}/phase2-namespaces.yaml" >/dev/null +kubectl apply --filename="${render_temp_dir}/phase2-namespaces.yaml" + +current_step="[2/7] AIStor Secret contracts" +printf '\n%s\n' "$current_step" +assert_cluster_identity +secret_args=( + --license-file "$license_file" + --root-config-file "$root_config_file" + --execute +) +if [[ "$generate_root_config" == true ]]; then + secret_args+=(--generate-root-config) +fi +bash scripts/bootstrap/create-aistor-secrets.sh "${secret_args[@]}" + +current_step="[3/7] 900Gi Retain Local PV" +printf '\n%s\n' "$current_step" +assert_cluster_identity +verify_manifest_unchanged aistor-local-pv +kubectl apply --dry-run=server \ + --filename="${render_temp_dir}/aistor-local-pv.yaml" >/dev/null +kubectl apply --filename="${render_temp_dir}/aistor-local-pv.yaml" + +current_step="[4/7] AIStor Operator and CRDs" +printf '\n%s\n' "$current_step" +assert_cluster_identity +verify_manifest_unchanged aistor-operator +kubectl apply --server-side \ + --filename="${render_temp_dir}/aistor-operator.yaml" +kubectl wait --for=condition=Established "${AISTOR_CRDS[@]}" --timeout=5m +kubectl --namespace aistor rollout status \ + deployment/adminjob-operator --timeout=10m +kubectl --namespace aistor rollout status \ + deployment/object-store-operator --timeout=10m +kubectl --namespace aistor rollout status \ + deployment/object-store-webhook --timeout=10m + +current_step="[5/7] One-node, one-drive AIStor ObjectStore" +printf '\n%s\n' "$current_step" +assert_cluster_identity +verify_manifest_unchanged minio-aistor +kubectl apply --server-side --dry-run=server \ + --filename="${render_temp_dir}/minio-aistor.yaml" >/dev/null +kubectl apply --server-side \ + --filename="${render_temp_dir}/minio-aistor.yaml" + +statefulset_name="" +for _ in $(seq 1 180); do + statefulset_names="$( + kubectl --namespace object-storage get statefulset \ + --selector="aistor.min.io/objectStore=${EXPECTED_OBJECTSTORE}" \ + --output=name + )" + statefulset_count="$(printf '%s\n' "$statefulset_names" | sed '/^$/d' | wc -l | tr -d '[:space:]')" + if [[ "$statefulset_count" == "1" ]]; then + statefulset_name="$statefulset_names" + break + fi + sleep 2 +done +[[ -n "$statefulset_name" ]] || \ + fail "the ObjectStore Operator did not create exactly one StatefulSet" +kubectl --namespace object-storage rollout status "$statefulset_name" --timeout=15m +kubectl --namespace object-storage wait \ + --for=condition=Ready pod \ + --selector="aistor.min.io/objectStore=${EXPECTED_OBJECTSTORE}" \ + --timeout=10m + +current_step="[6/7] AIStor default-deny NetworkPolicies" +printf '\n%s\n' "$current_step" +assert_cluster_identity +verify_manifest_unchanged aistor-network-policies +kubectl apply --dry-run=server \ + --filename="${render_temp_dir}/aistor-network-policies.yaml" >/dev/null +kubectl apply \ + --filename="${render_temp_dir}/aistor-network-policies.yaml" +kubectl --namespace object-storage wait \ + --for=condition=Ready pod \ + --selector="aistor.min.io/objectStore=${EXPECTED_OBJECTSTORE}" \ + --timeout=5m + +current_step="[7/7] Storage, service, and exposure acceptance" +printf '\n%s\n' "$current_step" +assert_cluster_identity +pvc_json="$( + kubectl --namespace object-storage get persistentvolumeclaim --output=json | + jq --arg storage_class "$EXPECTED_STORAGE_CLASS" \ + '{apiVersion, kind, items: [.items[] | select(.spec.storageClassName == $storage_class)]}' +)" +[[ "$(jq '.items | length' <<<"$pvc_json")" == "1" ]] || \ + fail "expected exactly one AIStor PVC" +[[ "$(jq -r '.items[0].status.phase' <<<"$pvc_json")" == "Bound" ]] || \ + fail "AIStor PVC is not Bound" +[[ "$(jq -r '.items[0].spec.volumeName' <<<"$pvc_json")" == "$EXPECTED_PV" ]] || \ + fail "AIStor PVC did not bind the expected Local PV" +[[ "$(jq -r '.items[0].spec.resources.requests.storage' <<<"$pvc_json")" == "900Gi" ]] || \ + fail "AIStor PVC request is not 900Gi" + +for service_name in minio minio-aistor-console minio-aistor-hl; do + service_type="$( + kubectl --namespace object-storage get service "$service_name" \ + --output=jsonpath='{.spec.type}' + )" + [[ "$service_type" == "ClusterIP" ]] || \ + fail "${service_name} must remain ClusterIP" +done +[[ "$( + kubectl --namespace object-storage get service minio \ + --output=jsonpath='{.spec.ports[0].port}:{.spec.ports[0].targetPort}' +)" == "80:9000" ]] || fail "S3 Service must map 80/TCP to 9000/TCP" +[[ "$( + kubectl --namespace object-storage get service minio-aistor-console \ + --output=jsonpath='{.spec.ports[0].port}:{.spec.ports[0].targetPort}' +)" == "9090:9090" ]] || fail "Console Service must map 9090/TCP to 9090/TCP" +[[ "$( + kubectl --namespace object-storage get service minio-aistor-hl \ + --output=jsonpath='{.spec.clusterIP}' +)" == "None" ]] || fail "AIStor headless Service must remain headless" +[[ -z "$( + kubectl --namespace object-storage get service --output=json | + jq -r '.items[].spec.ports[]? | select(.nodePort != null) | .nodePort' +)" ]] || fail "an AIStor service unexpectedly has a NodePort" +[[ -z "$(kubectl --namespace object-storage get ingress --output=name)" ]] || \ + fail "AIStor must not have an Ingress" + +printf '\nAISTOR APPLY SUCCESS\n' +printf 'ObjectStore: object-storage/%s\n' "$EXPECTED_OBJECTSTORE" +printf 'Storage: %s -> %s (900Gi PVC, Retain)\n' \ + "$EXPECTED_AISTOR_DEVICE" "$EXPECTED_AISTOR_MOUNT" +printf '%s\n' \ + 'Exposure: ClusterIP only; no Host Nginx, Traefik, NodePort, or public DNS.' \ + "Root credential file: ${root_config_file}" \ + 'Next: run the authenticated S3 write/read smoke test.' diff --git a/scripts/bootstrap/apply-gitea-oidc.sh b/scripts/bootstrap/apply-gitea-oidc.sh new file mode 100755 index 0000000..1629ec2 --- /dev/null +++ b/scripts/bootstrap/apply-gitea-oidc.sh @@ -0,0 +1,673 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +# Do not inherit xtrace: redirect headers contain an OIDC state value. +set +x +umask 077 + +readonly EXPECTED_HELM_VERSION="v3.19.4" +readonly TARGET_NODE="donghyeon-system-product-name" +readonly EXPECTED_NODE_INTERNAL_IP="192.168.0.107" +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly GITEA_NAMESPACE="gitea" +readonly GITEA_DEPLOYMENT="gitea" +readonly GITEA_SERVICE="gitea-http" +readonly GITEA_HOST="git.learn.hyeonworks.com" +readonly GITEA_ROOT_URL="https://${GITEA_HOST}" +readonly GITEA_HEALTH_URL="${GITEA_ROOT_URL}/api/healthz" +readonly GITEA_LOGIN_URL="${GITEA_ROOT_URL}/user/login" +readonly GITEA_SIGNUP_URL="${GITEA_ROOT_URL}/user/sign_up" +readonly GITEA_OIDC_START_URL="${GITEA_ROOT_URL}/user/oauth2/keycloak" +readonly GITEA_OIDC_CALLBACK_ENCODED="https%3A%2F%2Fgit.learn.hyeonworks.com%2Fuser%2Foauth2%2Fkeycloak%2Fcallback" +readonly KEYCLOAK_HOST="id.learn.hyeonworks.com" +readonly KEYCLOAK_ISSUER="https://${KEYCLOAK_HOST}/realms/hyeonworks" +readonly KEYCLOAK_DISCOVERY_URL="${KEYCLOAK_ISSUER}/.well-known/openid-configuration" +readonly KEYCLOAK_AUTH_ENDPOINT="${KEYCLOAK_ISSUER}/protocol/openid-connect/auth" +readonly OIDC_SECRET_NAME="gitea-keycloak-oidc" +readonly -a VERIFIED_MANIFEST_NAMES=( + namespaces + ssd-local-pv + cnpg-operator + platform-postgres + gitea + gitea-oidc +) + +render_temp_dir="" +runtime_temp_dir="" +gitea_manifest_sha256="not-rendered" +apply_started=0 + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'USAGE' +Usage: bash scripts/bootstrap/apply-gitea-oidc.sh --execute + +Keycloak 공개 discovery, Gitea OIDC Secret 계약, 현재 Gitea 상태를 먼저 +검사합니다. 고정된 Chart SHA를 검증하는 render-phase1.sh의 0600 handoff에서 +gitea-oidc.yaml 하나만 적용한 뒤 OIDC, 외부 인증 전용 가입 정책, 브랜딩을 확인합니다. + +실패해도 Kubernetes 리소스를 삭제하거나 이전 버전으로 롤백하지 않습니다. +Secret 값, 토큰, OIDC state가 포함된 전체 Location은 출력하지 않습니다. +USAGE +} + +cleanup() { + local cleanup_rc=$? + + trap - EXIT + set +e + if (( cleanup_rc != 0 )); then + if (( apply_started == 1 )); then + printf '\nRETAINED STATE: Gitea apply가 시작된 뒤 검증에 실패했습니다.\n' >&2 + printf '자동 삭제와 롤백은 수행하지 않았으며 현재 클러스터 상태를 보존했습니다.\n' >&2 + printf '적용 대상으로 고정했던 gitea-oidc.yaml SHA-256: %s\n' \ + "$gitea_manifest_sha256" >&2 + printf '확인: kubectl --namespace gitea get deployment,pod,service,endpointslice,ingress\n' >&2 + else + printf '\nNO MUTATION: 사전 검사 또는 렌더링 단계에서 중단되어 Gitea를 적용하지 않았습니다.\n' >&2 + fi + fi + + if [[ -n "$render_temp_dir" ]]; then + case "$render_temp_dir" in + /tmp/platform-phase1-apply.*) + rm -rf -- "$render_temp_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected render path: %s\n' \ + "$render_temp_dir" >&2 + ;; + esac + fi + if [[ -n "$runtime_temp_dir" ]]; then + case "$runtime_temp_dir" in + /tmp/gitea-oidc-apply.*) + rm -rf -- "$runtime_temp_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected runtime path: %s\n' \ + "$runtime_temp_dir" >&2 + ;; + esac + fi + + exit "$cleanup_rc" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || { + usage + exit 2 +} +[[ -t 0 ]] || fail "an interactive terminal is required" +[[ "$(pwd -P)" == "$REPOSITORY_ROOT" ]] || \ + fail "run from ${REPOSITORY_ROOT}" + +for command_name in \ + kubectl curl jq rg sha256sum stat find wc tr sort mktemp chmod mkdir rm awk; do + command -v "$command_name" >/dev/null 2>&1 || \ + fail "${command_name} is required" +done + +if [[ -n "${PLATFORM_HELM_BIN:-}" ]]; then + [[ "$PLATFORM_HELM_BIN" == /* ]] || \ + fail "PLATFORM_HELM_BIN must be an absolute path" + [[ -f "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \ + fail "PLATFORM_HELM_BIN is not executable: ${PLATFORM_HELM_BIN}" + readonly HELM_BIN="$PLATFORM_HELM_BIN" +else + HELM_BIN="$(command -v helm 2>/dev/null)" || \ + fail "Helm ${EXPECTED_HELM_VERSION} is required" + readonly HELM_BIN +fi +[[ "$("$HELM_BIN" version --template '{{.Version}}')" == "$EXPECTED_HELM_VERSION" ]] || \ + fail "Helm must be exactly ${EXPECTED_HELM_VERSION}" + +readonly CURRENT_CONTEXT="$(kubectl config current-context)" +readonly API_SERVER="$( + kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}' +)" +[[ -n "$CURRENT_CONTEXT" ]] || fail "kubectl current-context is empty" +[[ -n "$API_SERVER" ]] || fail "the selected Kubernetes API server is empty" + +https_get() { + local host="$1" + local url="$2" + local output_file="$3" + + curl \ + --disable \ + --silent \ + --show-error \ + --fail-with-body \ + --noproxy '*' \ + --resolve "${host}:443:127.0.0.1" \ + --connect-timeout 3 \ + --max-time 20 \ + --header 'Cache-Control: no-cache' \ + --output "$output_file" \ + "$url" +} + +check_target_node() { + local internal_ip + local ready_status + + kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \ + fail "target node is missing: ${TARGET_NODE}" + ready_status="$( + kubectl get node "$TARGET_NODE" \ + --output=jsonpath='{.status.conditions[?(@.type=="Ready")].status}' + )" + [[ "$ready_status" == "True" ]] || \ + fail "target node is not Ready: ${TARGET_NODE}" + internal_ip="$( + kubectl get node "$TARGET_NODE" \ + --output=jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' + )" + [[ "$internal_ip" == "$EXPECTED_NODE_INTERNAL_IP" ]] || \ + fail "target node InternalIP is ${internal_ip:-missing}, expected ${EXPECTED_NODE_INTERNAL_IP}" +} + +check_oidc_secret_contract() { + local secret_type + local secret_keys + + # These output expressions inspect only the Secret type and data key names. + # They never select, decode, compare, or print either data value. + secret_type="$( + kubectl --namespace "$GITEA_NAMESPACE" get secret "$OIDC_SECRET_NAME" \ + --output=jsonpath='{.type}' + )" + secret_keys="$( + kubectl --namespace "$GITEA_NAMESPACE" get secret "$OIDC_SECRET_NAME" \ + --output=go-template='{{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}' \ + | LC_ALL=C sort + )" + + [[ "$secret_type" == "Opaque" ]] || \ + fail "${GITEA_NAMESPACE}/${OIDC_SECRET_NAME} type must be Opaque" + [[ "$secret_keys" == $'key\nsecret' ]] || \ + fail "${GITEA_NAMESPACE}/${OIDC_SECRET_NAME} must contain exactly key and secret" +} + +check_public_discovery() { + local discovery_file="$1" + + https_get "$KEYCLOAK_HOST" "$KEYCLOAK_DISCOVERY_URL" "$discovery_file" + jq --exit-status \ + --arg issuer "$KEYCLOAK_ISSUER" \ + --arg authorization_endpoint "$KEYCLOAK_AUTH_ENDPOINT" \ + 'type == "object" and + .issuer == $issuer and + .authorization_endpoint == $authorization_endpoint and + (.token_endpoint | type == "string" and startswith($issuer + "/"))' \ + "$discovery_file" >/dev/null 2>&1 || \ + fail "local-SNI Keycloak discovery is not the expected JSON issuer" +} + +check_gitea_health() { + local health_file="$1" + + kubectl --namespace "$GITEA_NAMESPACE" wait \ + --for=condition=Available "deployment/${GITEA_DEPLOYMENT}" \ + --timeout=30s >/dev/null + kubectl --namespace "$GITEA_NAMESPACE" wait \ + --for=jsonpath='{.endpoints[0].conditions.ready}'=true \ + endpointslice \ + --selector="kubernetes.io/service-name=${GITEA_SERVICE}" \ + --timeout=30s >/dev/null + + https_get "$GITEA_HOST" "$GITEA_HEALTH_URL" "$health_file" + jq --exit-status '.status == "pass"' "$health_file" >/dev/null || \ + fail "Gitea public health response is not status=pass JSON" +} + +verify_rendered_manifest_unchanged() { + local manifest_path="${render_temp_dir}/gitea-oidc.yaml" + local checksum_output + local actual_sha256 + + [[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \ + fail "verified Gitea manifest is missing or unsafe" + [[ "$(stat --format='%a' -- "$manifest_path")" == "600" ]] || \ + fail "verified Gitea manifest must have mode 0600" + checksum_output="$(sha256sum -- "$manifest_path")" + actual_sha256="${checksum_output%% *}" + [[ "$actual_sha256" == "$gitea_manifest_sha256" ]] || \ + fail "verified gitea-oidc.yaml changed after confirmation" +} + +check_auth_source() { + local auth_list_file="$1" + local auth_error_file="$2" + local keycloak_count + local active_oauth2_count + + if ! kubectl --namespace "$GITEA_NAMESPACE" exec \ + "deployment/${GITEA_DEPLOYMENT}" \ + --container gitea \ + -- gitea admin auth list \ + --vertical-bars \ + --min-width 1 \ + --tab-width 1 \ + --padding 0 \ + --pad-char ' ' \ + >"$auth_list_file" 2>"$auth_error_file"; then + fail "gitea admin auth list failed; its output was retained only in the private temp directory" + fi + + read -r keycloak_count active_oauth2_count < <( + awk -F '|' ' + function trim(value) { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + return value + } + NF == 4 { + name = trim($2) + type = trim($3) + enabled = trim($4) + if (name == "keycloak") { + keycloak_count++ + if (type == "OAuth2" && enabled == "true") { + active_oauth2_count++ + } + } + } + END { + print keycloak_count + 0, active_oauth2_count + 0 + } + ' "$auth_list_file" + ) + + [[ "$keycloak_count" == "1" && "$active_oauth2_count" == "1" ]] || \ + fail "exactly one active OAuth2 auth source named keycloak was not found" +} + +check_app_ini_policy() { + local app_ini_error_file="$1" + + if ! kubectl --namespace "$GITEA_NAMESPACE" exec \ + "deployment/${GITEA_DEPLOYMENT}" \ + --container gitea \ + -- awk ' + function trim(value) { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + return value + } + /^[[:space:]]*\[/ { + section = $0 + gsub(/^[[:space:]]*\[|\][[:space:]]*$/, "", section) + section = tolower(section) + next + } + /^[[:space:]]*[#;]/ || /^[[:space:]]*$/ { + next + } + { + split($0, pair, "=") + key = toupper(trim(pair[1])) + value = $0 + sub(/^[^=]*=/, "", value) + value = trim(value) + if (section == "service" && key == "DISABLE_REGISTRATION" && tolower(value) == "false") disabled++ + if (section == "service" && key == "ALLOW_ONLY_EXTERNAL_REGISTRATION" && tolower(value) == "true") external_only++ + if (section == "service" && key == "SHOW_REGISTRATION_BUTTON" && tolower(value) == "false") button_hidden++ + if (section == "service" && key == "ENABLE_PASSWORD_SIGNIN_FORM" && tolower(value) == "true") password_signin++ + if (section == "oauth2_client" && key == "ENABLE_AUTO_REGISTRATION" && tolower(value) == "true") oidc_jit++ + if (section == "oauth2_client" && key == "USERNAME" && value == "preferred_username") username_claim++ + if (section == "oauth2_client" && key == "ACCOUNT_LINKING" && value == "login") account_linking++ + if (section == "oauth2_client" && key == "OPENID_CONNECT_SCOPES" && value == "profile email") oidc_scopes++ + } + END { + exit !(disabled == 1 && + external_only == 1 && + button_hidden == 1 && + password_signin == 1 && + oidc_jit == 1 && + username_claim == 1 && + account_linking == 1 && + oidc_scopes == 1) + } + ' /data/gitea/conf/app.ini >/dev/null 2>"$app_ini_error_file"; then + fail "live app.ini does not satisfy the external-registration-only OIDC policy" + fi +} + +check_login_html() { + local login_html_file="$1" + local signup_html_file="$2" + local signup_headers_file="$3" + local signup_status + local header_line + local field_name + local signup_location="" + local signup_location_count=0 + + https_get "$GITEA_HOST" "$GITEA_LOGIN_URL" "$login_html_file" + rg --quiet --fixed-strings 'href="/user/oauth2/keycloak"' "$login_html_file" || \ + fail "Gitea login HTML does not contain the Keycloak OIDC link" + rg --quiet --fixed-strings 'href="/assets/css/hyeonworks.css"' "$login_html_file" || \ + fail "Gitea login HTML does not contain the Hyeonworks stylesheet" + rg --quiet --fixed-strings 'name="theme-color" content="#0f172a"' "$login_html_file" || \ + fail "Gitea login HTML does not contain the Hyeonworks theme marker" + rg --quiet --fixed-strings 'hw-brand-link' "$login_html_file" || \ + fail "Gitea login HTML does not contain the Hyeonworks navigation marker" + if rg --quiet --fixed-strings 'href="/user/sign_up"' "$login_html_file"; then + fail "Gitea login HTML still exposes a local sign-up link" + fi + + if ! signup_status="$( + curl \ + --disable \ + --silent \ + --show-error \ + --noproxy '*' \ + --resolve "${GITEA_HOST}:443:127.0.0.1" \ + --connect-timeout 3 \ + --max-time 20 \ + --header 'Cache-Control: no-cache' \ + --output "$signup_html_file" \ + --dump-header "$signup_headers_file" \ + --write-out '%{http_code}' \ + "$GITEA_SIGNUP_URL" + )"; then + fail "Gitea sign-up endpoint transport check failed" + fi + + case "$signup_status" in + 200) + for field_name in user_name email password retype; do + if rg --quiet --fixed-strings "name=\"${field_name}\"" "$signup_html_file"; then + fail "Gitea sign-up HTML still exposes a local registration input" + fi + done + ;; + 404) + ;; + 301|302|303|307|308) + while IFS= read -r header_line; do + header_line="${header_line%$'\r'}" + case "$header_line" in + [Ll][Oo][Cc][Aa][Tt][Ii][Oo][Nn]:*) + signup_location="${header_line#*:}" + signup_location="${signup_location#"${signup_location%%[![:space:]]*}"}" + ((signup_location_count += 1)) + ;; + esac + done <"$signup_headers_file" + [[ "$signup_location_count" == "1" ]] || \ + fail "Gitea sign-up redirect must contain exactly one Location header" + case "$signup_location" in + /user/login|"${GITEA_ROOT_URL}/user/login") + ;; + *) + fail "Gitea sign-up redirect does not target the same-origin login page" + ;; + esac + ;; + *) + fail "Gitea sign-up endpoint returned an unexpected HTTP status: ${signup_status}" + ;; + esac +} + +check_oidc_redirect() { + local headers_file="$1" + local status + local header_line + local location="" + local location_count=0 + local query + local parameter + local parameter_name + local parameter_value + local -a query_parameters=() + local client_id_count=0 + local response_type_count=0 + local redirect_uri_count=0 + local state_count=0 + + status="$( + curl \ + --disable \ + --silent \ + --show-error \ + --noproxy '*' \ + --resolve "${GITEA_HOST}:443:127.0.0.1" \ + --connect-timeout 3 \ + --max-time 20 \ + --output /dev/null \ + --dump-header "$headers_file" \ + --write-out '%{http_code}' \ + "$GITEA_OIDC_START_URL" + )" + case "$status" in + 302|303|307) + ;; + *) + fail "Gitea OIDC start returned HTTP ${status}, expected 302, 303, or 307" + ;; + esac + + while IFS= read -r header_line; do + header_line="${header_line%$'\r'}" + case "$header_line" in + [Ll][Oo][Cc][Aa][Tt][Ii][Oo][Nn]:*) + location="${header_line#*:}" + location="${location#"${location%%[![:space:]]*}"}" + ((location_count += 1)) + ;; + esac + done <"$headers_file" + + [[ "$location_count" == "1" && -n "$location" ]] || \ + fail "Gitea OIDC start did not return exactly one non-empty Location header" + [[ "$location" != *'#'* ]] || \ + fail "Gitea OIDC Location unexpectedly contains a fragment" + case "$location" in + "${KEYCLOAK_AUTH_ENDPOINT}"\?*) + ;; + *) + fail "Gitea OIDC Location does not target the expected Keycloak authorization endpoint" + ;; + esac + + query="${location#*\?}" + IFS='&' read -r -a query_parameters <<<"$query" + for parameter in "${query_parameters[@]}"; do + [[ "$parameter" == *=* ]] || continue + parameter_name="${parameter%%=*}" + parameter_value="${parameter#*=}" + case "$parameter_name" in + client_id) + ((client_id_count += 1)) + [[ "$parameter_value" == "gitea" ]] || \ + fail "OIDC Location client_id is not gitea" + ;; + response_type) + ((response_type_count += 1)) + [[ "$parameter_value" == "code" ]] || \ + fail "OIDC Location response_type is not code" + ;; + redirect_uri) + ((redirect_uri_count += 1)) + [[ "$parameter_value" == "$GITEA_OIDC_CALLBACK_ENCODED" ]] || \ + fail "OIDC Location callback is not the exact public Gitea callback" + ;; + state) + ((state_count += 1)) + [[ -n "$parameter_value" ]] || fail "OIDC Location state is empty" + ;; + esac + done + + [[ "$client_id_count" == "1" ]] || fail "OIDC Location must contain one client_id" + [[ "$response_type_count" == "1" ]] || fail "OIDC Location must contain one response_type" + [[ "$redirect_uri_count" == "1" ]] || fail "OIDC Location must contain one redirect_uri" + [[ "$state_count" == "1" ]] || fail "OIDC Location must contain one non-empty state" + # Deliberately do not print $location or any parsed state value. +} + +check_branding_hashes() { + local remote_dir="$1" + local index + local source_path + local remote_path + local checksum_output + local local_sha256 + local remote_sha256 + local -a source_paths=( + "${REPOSITORY_ROOT}/services/gitea/branding/public/assets/css/hyeonworks.css" + "${REPOSITORY_ROOT}/services/gitea/branding/public/assets/img/logo.svg" + "${REPOSITORY_ROOT}/services/gitea/branding/public/assets/img/favicon.svg" + ) + local -a public_paths=( + "/assets/css/hyeonworks.css" + "/assets/img/logo.svg" + "/assets/img/favicon.svg" + ) + local -a labels=( + "hyeonworks.css" + "logo.svg" + "favicon.svg" + ) + + for index in "${!source_paths[@]}"; do + source_path="${source_paths[$index]}" + remote_path="${remote_dir}/${labels[$index]}" + [[ -f "$source_path" && ! -L "$source_path" ]] || \ + fail "branding source is missing or symlinked: ${source_path}" + + https_get \ + "$GITEA_HOST" \ + "${GITEA_ROOT_URL}${public_paths[$index]}" \ + "$remote_path" + + checksum_output="$(sha256sum -- "$source_path")" + local_sha256="${checksum_output%% *}" + checksum_output="$(sha256sum -- "$remote_path")" + remote_sha256="${checksum_output%% *}" + [[ "$remote_sha256" == "$local_sha256" ]] || \ + fail "public branding hash differs from local source: ${labels[$index]}" + printf '브랜딩 해시 일치: %-16s %s\n' \ + "${labels[$index]}" "$local_sha256" + done +} + +runtime_temp_dir="$(mktemp -d /tmp/gitea-oidc-apply.XXXXXX)" +chmod 0700 "$runtime_temp_dir" +render_temp_dir="$(mktemp -d /tmp/platform-phase1-apply.XXXXXX)" +chmod 0700 "$render_temp_dir" +readonly DISCOVERY_FILE="${runtime_temp_dir}/keycloak-discovery.json" +readonly HEALTH_FILE="${runtime_temp_dir}/gitea-health.json" +readonly AUTH_LIST_FILE="${runtime_temp_dir}/gitea-auth-list.txt" +readonly AUTH_ERROR_FILE="${runtime_temp_dir}/gitea-auth-list.err" +readonly APP_INI_ERROR_FILE="${runtime_temp_dir}/gitea-app-ini.err" +readonly LOGIN_HTML_FILE="${runtime_temp_dir}/gitea-login.html" +readonly SIGNUP_HTML_FILE="${runtime_temp_dir}/gitea-signup.html" +readonly SIGNUP_HEADERS_FILE="${runtime_temp_dir}/gitea-signup-headers" +readonly OIDC_HEADERS_FILE="${runtime_temp_dir}/gitea-oidc-headers" +readonly BRANDING_REMOTE_DIR="${runtime_temp_dir}/branding-remote" +mkdir -m 0700 -- "$BRANDING_REMOTE_DIR" + +printf '[1/8] 현재 context, 노드, Gitea 상태 확인\n' +check_target_node +kubectl --namespace "$GITEA_NAMESPACE" get "deployment/${GITEA_DEPLOYMENT}" >/dev/null +check_gitea_health "$HEALTH_FILE" + +printf '[2/8] Host Nginx 로컬 SNI 경로의 Keycloak discovery JSON 확인\n' +check_public_discovery "$DISCOVERY_FILE" + +printf '[3/8] Gitea OIDC Secret의 type과 key 이름만 확인\n' +check_oidc_secret_contract + +printf '[4/8] 고정 Chart SHA 검증 후 Phase 1 manifest 렌더링\n' +cd -- "$REPOSITORY_ROOT" +PLATFORM_HELM_BIN="$HELM_BIN" \ + bash scripts/validate/render-phase1.sh \ + --verified-output-dir "$render_temp_dir" + +verified_entry_count="$( + find "$render_temp_dir" -mindepth 1 -maxdepth 1 -type f \ + -name '*.yaml' | wc -l | tr -d '[:space:]' +)" +[[ "$verified_entry_count" == "${#VERIFIED_MANIFEST_NAMES[@]}" ]] || \ + fail "verified handoff must contain exactly ${#VERIFIED_MANIFEST_NAMES[@]} YAML manifests" +for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do + manifest_path="${render_temp_dir}/${manifest_name}.yaml" + [[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \ + fail "verified manifest is missing or unsafe: ${manifest_name}.yaml" + [[ "$(stat --format='%a' -- "$manifest_path")" == "600" ]] || \ + fail "verified manifest must have mode 0600: ${manifest_name}.yaml" +done +checksum_output="$(sha256sum -- "${render_temp_dir}/gitea-oidc.yaml")" +gitea_manifest_sha256="${checksum_output%% *}" +readonly gitea_manifest_sha256 + +rg --quiet --fixed-strings 'gitea-keycloak-oidc' "${render_temp_dir}/gitea-oidc.yaml" || \ + fail "rendered Gitea manifest does not reference the OIDC Secret" +rg --quiet --fixed-strings "$KEYCLOAK_DISCOVERY_URL" "${render_temp_dir}/gitea-oidc.yaml" || \ + fail "rendered Gitea manifest does not contain the exact discovery URL" + +printf '\nKubernetes context: %s\n' "$CURRENT_CONTEXT" +printf 'API server: %s\n' "$API_SERVER" +printf 'Target node: %s\n' "$TARGET_NODE" +printf 'gitea-oidc.yaml SHA-256: %s\n' "$gitea_manifest_sha256" +printf '적용 범위: 검증된 gitea-oidc.yaml 하나\n' +printf 'Type APPLY %s GITEA-OIDC %s to continue: ' \ + "$CURRENT_CONTEXT" "$gitea_manifest_sha256" +read -r confirmation +[[ "$confirmation" == "APPLY ${CURRENT_CONTEXT} GITEA-OIDC ${gitea_manifest_sha256}" ]] || \ + fail "cancelled" + +[[ "$(kubectl config current-context)" == "$CURRENT_CONTEXT" ]] || \ + fail "kubectl context changed after confirmation" +[[ "$( + kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}' +)" == "$API_SERVER" ]] || \ + fail "Kubernetes API server changed after confirmation" +check_target_node +check_public_discovery "$DISCOVERY_FILE" +check_oidc_secret_contract +check_gitea_health "$HEALTH_FILE" +verify_rendered_manifest_unchanged + +printf '\n[5/8] 검증된 gitea-oidc.yaml 하나만 적용\n' +apply_started=1 +kubectl apply --filename="${render_temp_dir}/gitea-oidc.yaml" +verify_rendered_manifest_unchanged + +printf '[6/8] Deployment rollout과 ready EndpointSlice 대기\n' +kubectl --namespace "$GITEA_NAMESPACE" rollout status \ + "deployment/${GITEA_DEPLOYMENT}" --timeout=10m +kubectl --namespace "$GITEA_NAMESPACE" wait \ + --for=jsonpath='{.endpoints[0].conditions.ready}'=true \ + endpointslice \ + --selector="kubernetes.io/service-name=${GITEA_SERVICE}" \ + --timeout=2m +check_gitea_health "$HEALTH_FILE" + +printf '[7/8] 활성 OAuth2 source, app.ini 정책, 로그인/OIDC 흐름 확인\n' +check_auth_source "$AUTH_LIST_FILE" "$AUTH_ERROR_FILE" +check_app_ini_policy "$APP_INI_ERROR_FILE" +check_login_html "$LOGIN_HTML_FILE" "$SIGNUP_HTML_FILE" "$SIGNUP_HEADERS_FILE" +check_oidc_redirect "$OIDC_HEADERS_FILE" +check_public_discovery "$DISCOVERY_FILE" + +printf '[8/8] 공개 브랜딩 자산과 로컬 소스 SHA-256 비교\n' +check_branding_hashes "$BRANDING_REMOTE_DIR" + +printf '\nGITEA OIDC APPLY SUCCESS\n' +printf '적용 manifest SHA-256: %s\n' "$gitea_manifest_sha256" +printf 'Keycloak discovery, 활성 OAuth2 source, 외부 인증 전용 가입 정책, OIDC redirect를 확인했습니다.\n' +printf 'OIDC Secret 값, 토큰, 전체 Location/state는 출력하지 않았습니다.\n' +printf '실제 realm 사용자 login/callback/logout은 별도 수동 수용 시험으로 남습니다.\n' diff --git a/scripts/bootstrap/apply-host-nginx-admin.sh b/scripts/bootstrap/apply-host-nginx-admin.sh new file mode 100755 index 0000000..23e7030 --- /dev/null +++ b/scripts/bootstrap/apply-host-nginx-admin.sh @@ -0,0 +1,332 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)" +readonly CANDIDATE="$ROOT/infrastructure/networking/host-nginx/learn-services-admin.conf" +readonly ACTIVE="/etc/nginx/sites-available/learn-services" +readonly CERT_NAME="storage-admin.learn.hyeonworks.com" +readonly CERT_DIR="/etc/letsencrypt/live/$CERT_NAME" +readonly CREDENTIALS="/home/donghyeon/.secrets/certbot/cloudflare.ini" +readonly STORAGE_HOST="storage-admin.learn.hyeonworks.com" +readonly DB_HOST="db-admin.learn.hyeonworks.com" +readonly LAN_IP="192.168.0.107" +readonly TAIL_IP="100.92.240.34" +execute=false +certificate_only=false +certbot_email="" +rollback_armed=false +backup="" + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'USAGE' +사용법: + bash scripts/bootstrap/apply-host-nginx-admin.sh + bash scripts/bootstrap/apply-host-nginx-admin.sh \ + --execute --certificate-only --certbot-email you@example.com + bash scripts/bootstrap/apply-host-nginx-admin.sh --execute + +dry-run은 공개 DNS, 후보 hash, Ingress endpoint, Cloudflare credential 상태를 +검사합니다. --execute는 필요하면 공식 Cloudflare snap plugin과 SAN 인증서를 +발급하고 Nginx를 timestamp backup 뒤 교체합니다. +USAGE +} + +while (( $# > 0 )); do + case "$1" in + --certificate-only) + certificate_only=true + shift + ;; + --execute) + execute=true + shift + ;; + --certbot-email) + (( $# >= 2 )) || fail "--certbot-email 값이 필요합니다" + certbot_email="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage >&2 + fail "지원하지 않는 인자: $1" + ;; + esac +done + +for cmd in awk curl dig install jq kubectl openssl rg sha256sum sleep stat sudo; do + command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다" +done +[[ -f "$CANDIDATE" && ! -L "$CANDIDATE" ]] || fail "Nginx 후보가 없습니다" +[[ -f "$ACTIVE" && ! -L "$ACTIVE" ]] || fail "활성 Nginx site가 없습니다" +[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요" + +for name in "$STORAGE_HOST" "$DB_HOST"; do + [[ -z "$(dig +short @1.1.1.1 A "$name" | tr -d '[:space:]')" ]] || fail "$name 공개 A가 있습니다" + [[ -z "$(dig +short @1.1.1.1 AAAA "$name" | tr -d '[:space:]')" ]] || fail "$name 공개 AAAA가 있습니다" +done + +kubectl -n object-storage get ingress minio-aistor-console >/dev/null 2>&1 || \ + printf 'WARNING: AIStor Console Ingress가 아직 적용되지 않았습니다.\n' >&2 +kubectl -n platform-admin get ingress pgadmin >/dev/null 2>&1 || \ + printf 'WARNING: pgAdmin Ingress가 아직 적용되지 않았습니다.\n' >&2 + +candidate_sha="$(sha256sum "$CANDIDATE" | awk '{print $1}')" +active_sha="$(sha256sum "$ACTIVE" | awk '{print $1}')" +printf 'Active SHA-256: %s\n' "$active_sha" +printf 'Candidate SHA-256: %s\n' "$candidate_sha" +printf 'Certificate SAN: %s, %s\n' "$STORAGE_HOST" "$DB_HOST" + +# Ubuntu의 sites-enabled/default는 미등록 HTTP Host에 welcome page(200)를 +# 반환할 수 있다. HTTP status만으로 관리 도메인의 부분 적용을 판정하지 않고, +# 실제 활성 site에 관리 server_name이 들어갔는지를 먼저 확인한다. +if [[ "$active_sha" != "$candidate_sha" ]] && \ + { rg -q -F "$STORAGE_HOST" "$ACTIVE" || rg -q -F "$DB_HOST" "$ACTIVE"; }; then + fail "활성 Nginx site에 후보와 다른 admin 도메인 설정이 부분 적용돼 있습니다" +fi + +check_cloudflare_credentials() { + sudo test -f "$CREDENTIALS" || \ + fail "Cloudflare 제한 토큰 파일이 없습니다: $CREDENTIALS" + + owner="$(sudo stat -c '%U:%G' "$CREDENTIALS")" + mode="$(sudo stat -c '%a' "$CREDENTIALS")" + size="$(sudo stat -c '%s' "$CREDENTIALS")" + [[ "$owner" == root:root && "$mode" == 600 ]] || \ + fail "$CREDENTIALS 소유권/권한은 root:root 0600이어야 합니다" + (( size > 0 )) || fail "$CREDENTIALS 파일이 비어 있습니다" + + contract="$( + sudo awk -F= ' + BEGIN { count=0; nonempty=0; unexpected=0 } + /^[[:space:]]*($|#)/ { next } + /^[[:space:]]*dns_cloudflare_api_token[[:space:]]*=/ { + count++ + value=$0 + sub(/^[^=]*=[[:space:]]*/, "", value) + sub(/[[:space:]]+$/, "", value) + if (length(value) >= 20) nonempty++ + next + } + { unexpected++ } + END { printf "%d:%d:%d", count, nonempty, unexpected } + ' "$CREDENTIALS" + )" + [[ "$contract" == "1:1:0" ]] || \ + fail "$CREDENTIALS에는 비어 있지 않은 dns_cloudflare_api_token 한 개만 있어야 합니다" + + printf 'Cloudflare credential: root:root 0600 및 key 형식 확인\n' +} + +if [[ "$execute" == false ]]; then + if sudo -n test -f "$CREDENTIALS" 2>/dev/null; then + check_cloudflare_credentials + else + printf 'Cloudflare credential: root 전용이므로 --execute에서 sudo로 확인\n' + fi + printf 'DRY RUN PASS: --execute를 지정하지 않아 인증서와 Nginx를 변경하지 않았습니다.\n' + exit 0 +fi + +[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다" +sudo -v +check_cloudflare_credentials +context="$(kubectl config current-context)" +printf 'Type APPLY to issue/renew the admin certificate and replace Host Nginx: ' +read -r answer +[[ "$answer" == APPLY ]] || fail "취소했습니다" + +if [[ ! -x /snap/bin/certbot ]]; then + fail "지원 대상으로 고정한 snap certbot이 없습니다" +fi +if ! /snap/bin/certbot plugins 2>/dev/null | rg -q 'dns-cloudflare'; then + sudo snap set certbot trust-plugin-with-root=ok + sudo snap install certbot-dns-cloudflare +fi + +if ! sudo test -s "$CERT_DIR/fullchain.pem" || ! sudo test -s "$CERT_DIR/privkey.pem"; then + [[ "$certbot_email" == *@* ]] || fail "최초 인증서 발급에는 --certbot-email이 필요합니다" + sudo /snap/bin/certbot certonly \ + --dns-cloudflare \ + --dns-cloudflare-credentials "$CREDENTIALS" \ + --dns-cloudflare-propagation-seconds 60 \ + --cert-name "$CERT_NAME" \ + --domains "$STORAGE_HOST" \ + --domains "$DB_HOST" \ + --non-interactive \ + --agree-tos \ + --email "$certbot_email" +fi + +cert_text="$(sudo openssl x509 -in "$CERT_DIR/fullchain.pem" -noout -text)" +printf '%s\n' "$cert_text" | rg -q "DNS:$STORAGE_HOST" || fail "인증서에 storage-admin SAN이 없습니다" +printf '%s\n' "$cert_text" | rg -q "DNS:$DB_HOST" || fail "인증서에 db-admin SAN이 없습니다" +san_count="$(printf '%s\n' "$cert_text" | rg -o 'DNS:[^,[:space:]]+' | sort -u | wc -l | tr -d '[:space:]')" +[[ "$san_count" == 2 ]] || fail "admin 인증서 SAN은 정확히 두 개여야 합니다" + +sudo install -d -o root -g root -m 0755 /etc/letsencrypt/renewal-hooks/deploy +sudo install -o root -g root -m 0755 \ + "$ROOT/infrastructure/networking/host-nginx/reload-nginx.sh" \ + /etc/letsencrypt/renewal-hooks/deploy/reload-nginx + +if [[ "$certificate_only" == true ]]; then + printf 'ADMIN CERTIFICATE READY\n' + printf 'Nginx cutover는 서비스 적용 뒤 --certificate-only 없이 다시 실행하세요.\n' + exit 0 +fi + +for host in "$STORAGE_HOST" "$DB_HOST"; do + code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --noproxy '*' --resolve "$host:80:127.0.0.1" -H "Host: $host" "http://$host/")" + [[ "$code" == 200 || "$code" == 404 || "$code" == 301 ]] || \ + fail "기존 HTTP 경계가 예상과 다릅니다: $host=$code" + printf '기존 HTTP fallback: %s=%s\n' "$host" "$code" + route_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --noproxy '*' -H "Host: $host" "http://127.0.0.1:30080/")" + [[ "$route_code" == 200 || "$route_code" == 302 || "$route_code" == 303 ]] || \ + fail "Traefik route가 준비되지 않았습니다: $host=$route_code" +done + +probe_https_code() { + host="$1" + target_ip="$2" + source_ip="${3:-}" + + interface_args=() + if [[ -n "$source_ip" ]]; then + interface_args=(--interface "$source_ip") + fi + + curl --disable --silent --output /dev/null --write-out '%{http_code}' \ + --connect-timeout 1 --max-time 3 --noproxy '*' \ + "${interface_args[@]}" --resolve "$host:443:$target_ip" "https://$host/" \ + 2>/dev/null || true +} + +wait_for_admin_proxy_state() { + consecutive_passes=0 + storage_lan=000 + db_lan=000 + storage_denied=000 + db_denied=000 + + for ((attempt = 1; attempt <= 15; attempt++)); do + storage_lan="$(probe_https_code "$STORAGE_HOST" "$LAN_IP" "$LAN_IP")" + db_lan="$(probe_https_code "$DB_HOST" "$LAN_IP" "$LAN_IP")" + storage_denied="$(probe_https_code "$STORAGE_HOST" 127.0.0.1)" + db_denied="$(probe_https_code "$DB_HOST" 127.0.0.1)" + + if [[ "$storage_lan" == 200 || "$storage_lan" == 302 || "$storage_lan" == 303 ]] && + [[ "$db_lan" == 200 || "$db_lan" == 302 || "$db_lan" == 303 ]] && + [[ "$storage_denied" == 403 && "$db_denied" == 403 ]]; then + consecutive_passes=$((consecutive_passes + 1)) + if (( consecutive_passes >= 2 )); then + printf 'Nginx admin proxy state stabilized after %d probes.\n' "$attempt" + return 0 + fi + else + consecutive_passes=0 + fi + + if (( attempt == 1 || attempt % 5 == 0 )); then + printf 'Nginx admin proxy 대기: probe=%d LAN=%s/%s denied=%s/%s\n' \ + "$attempt" "$storage_lan" "$db_lan" "$storage_denied" "$db_denied" + fi + (( attempt < 15 )) && sleep 1 + done + + printf 'ERROR: Nginx admin proxy가 안정화되지 않았습니다: LAN=%s/%s denied=%s/%s\n' \ + "$storage_lan" "$db_lan" "$storage_denied" "$db_denied" >&2 + return 1 +} + +wait_for_admin_tls_rejection() { + consecutive_passes=0 + + for ((attempt = 1; attempt <= 15; attempt++)); do + if ! curl --disable --insecure --silent --output /dev/null \ + --connect-timeout 1 --max-time 3 --noproxy '*' \ + --resolve "$STORAGE_HOST:443:127.0.0.1" "https://$STORAGE_HOST/" 2>/dev/null && + ! curl --disable --insecure --silent --output /dev/null \ + --connect-timeout 1 --max-time 3 --noproxy '*' \ + --resolve "$DB_HOST:443:127.0.0.1" "https://$DB_HOST/" 2>/dev/null; then + consecutive_passes=$((consecutive_passes + 1)) + if (( consecutive_passes >= 2 )); then + return 0 + fi + else + consecutive_passes=0 + fi + (( attempt < 15 )) && sleep 1 + done + + return 1 +} + +backup="$ACTIVE.before-admin-$(date +%Y%m%d%H%M%S)" +printf 'Planned backup: %s\n' "$backup" +sudo install -o root -g root -m 0644 "$ACTIVE" "$backup" + +rollback() { + set +e + printf '\nROLLBACK: %s 복원\n' "$backup" >&2 + sudo install -o root -g root -m 0644 "$backup" "$ACTIVE" + sudo /usr/sbin/nginx -t + sudo systemctl reload nginx + if ! wait_for_admin_tls_rejection; then + printf 'WARNING: rollback 뒤 admin SNI 거부 안정화를 확인하지 못했습니다.\n' >&2 + fi + rollback_armed=false + printf 'ROLLBACK complete.\n' >&2 +} + +finish() { + rc=$? + trap - EXIT INT TERM + if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then + rollback + fi + exit "$rc" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +rollback_armed=true +sudo install -o root -g root -m 0644 "$CANDIDATE" "$ACTIVE" +sudo systemctl daemon-reload +sudo /usr/sbin/nginx -t +sudo systemctl reload nginx +sudo systemctl is-active --quiet nginx + +wait_for_admin_proxy_state || fail "LAN HTTPS와 allowlist 상태가 수렴하지 않았습니다" + +for host in "$STORAGE_HOST" "$DB_HOST"; do + redirect_result="$( + curl --disable --silent --show-error --output /dev/null \ + --write-out $'%{http_code}\n%{redirect_url}' --max-time 5 --noproxy '*' \ + --resolve "$host:80:$LAN_IP" "http://$host/" + )" + [[ "$redirect_result" == $'301\nhttps://'"$host/" ]] || \ + fail "HTTP redirect 검증 실패: $host=$redirect_result" +done + +if curl --insecure --silent --output /dev/null --noproxy '*' \ + --resolve unconfigured.invalid:443:127.0.0.1 https://unconfigured.invalid/ 2>/dev/null; then + fail "unknown SNI가 거부되지 않았습니다" +fi + +sudo /snap/bin/certbot renew --dry-run +rollback_armed=false +printf 'ADMIN NGINX CUTOVER SUCCESS\n' +printf 'Backup: %s\n' "$backup" +printf 'Candidate SHA-256: %s\n' "$candidate_sha" diff --git a/scripts/bootstrap/apply-host-nginx-gitea.sh b/scripts/bootstrap/apply-host-nginx-gitea.sh new file mode 100755 index 0000000..7197d75 --- /dev/null +++ b/scripts/bootstrap/apply-host-nginx-gitea.sh @@ -0,0 +1,351 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +# Do not expose headers, cookies, or future sensitive values through caller xtrace. +set +x +umask 077 + +readonly EXPECTED_CANDIDATE_SHA256="de7ebd4f69cd7d2204ee633e074bf6a4370a6f5e3f3fac9067b099d5d75269b5" +readonly EXPECTED_PRE_CUTOVER_SHA256="5b5941519ab677f751568827aa4f315193dfbd9eeac5e8a4ad85d724aaefe16f" +readonly EXPECTED_PRE_CUTOVER_HEALTH_SHA256="6b683cb16987ff1f5ded22e9847ac0a45995947927c459b21f427523a41c7484" +readonly REPOSITORY_ROOT="/home/donghyeon/workspace/platform" +readonly CANDIDATE="${REPOSITORY_ROOT}/infrastructure/networking/host-nginx/learn-services.conf" +readonly ACTIVE="/etc/nginx/sites-available/learn-services" +readonly ENABLED="/etc/nginx/sites-enabled/learn-services" +readonly CURL_BIN="/usr/bin/curl" +readonly JQ_BIN="/usr/bin/jq" +readonly NGINX_BIN="/usr/sbin/nginx" +readonly SYSTEMCTL_BIN="/usr/bin/systemctl" +readonly INSTALL_BIN="/usr/bin/install" +readonly SHA256SUM_BIN="/usr/bin/sha256sum" +readonly STAT_BIN="/usr/bin/stat" +readonly READLINK_BIN="/usr/bin/readlink" +readonly MKTEMP_BIN="/usr/bin/mktemp" +readonly RM_BIN="/usr/bin/rm" +readonly DATE_BIN="/usr/bin/date" +readonly AWK_BIN="/usr/bin/awk" +readonly SLEEP_BIN="/usr/bin/sleep" + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +usage() { + printf '%s\n' \ + 'Usage: sudo bash scripts/bootstrap/apply-host-nginx-gitea.sh --execute' \ + '' \ + 'Backs up the active learn-services site, installs the reviewed Gitea proxy,' \ + 'tests and reloads Nginx, then runs local acceptance checks. Any failure after' \ + 'the active file changes triggers an automatic restore and reload.' +} + +[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || { + usage + exit 2 +} +[[ "$EUID" -eq 0 ]] || fail "run this script through sudo" +[[ -t 0 ]] || fail "an interactive terminal is required" + +for required_binary in \ + "$CURL_BIN" "$JQ_BIN" "$NGINX_BIN" "$SYSTEMCTL_BIN" "$INSTALL_BIN" \ + "$SHA256SUM_BIN" "$STAT_BIN" "$READLINK_BIN" "$MKTEMP_BIN" \ + "$RM_BIN" "$DATE_BIN" "$AWK_BIN" "$SLEEP_BIN"; do + [[ -x "$required_binary" ]] || fail "required executable is missing: ${required_binary}" +done + +[[ -f "$CANDIDATE" && ! -L "$CANDIDATE" ]] || fail "unsafe candidate: ${CANDIDATE}" +[[ -f "$ACTIVE" && ! -L "$ACTIVE" ]] || fail "unsafe active file: ${ACTIVE}" +[[ -L "$ENABLED" ]] || fail "enabled path is not a symlink: ${ENABLED}" +[[ "$("$READLINK_BIN" -f "$ENABLED")" == "$ACTIVE" ]] || fail "enabled symlink target changed" +[[ "$("$STAT_BIN" --format='%U:%G %a' "$ACTIVE")" == "root:root 644" ]] || \ + fail "active file owner or mode changed" +"$SYSTEMCTL_BIN" is-active --quiet nginx || fail "nginx is not active" + +readonly TEMP_DIR="$("$MKTEMP_BIN" -d /tmp/nginx-gitea-cutover.XXXXXX)" +readonly CANDIDATE_SNAPSHOT="${TEMP_DIR}/learn-services.candidate" +readonly DIRECT_HEALTH="${TEMP_DIR}/direct-health.json" +readonly NGINX_HEALTH="${TEMP_DIR}/nginx-health.json" +readonly LOGIN_HEADERS="${TEMP_DIR}/login-headers" +rollback_armed=0 +backup="" + +cleanup() { + case "$TEMP_DIR" in + /tmp/nginx-gitea-cutover.*) + "$RM_BIN" -rf -- "$TEMP_DIR" + ;; + *) + printf 'WARNING: refusing to remove unexpected temp path: %s\n' "$TEMP_DIR" >&2 + ;; + esac +} + +rollback() { + local restore_install_rc + local restore_test_rc + local restore_reload_rc + + set +e + printf '\nROLLBACK: restoring %s\n' "$backup" >&2 + if "$INSTALL_BIN" -o root -g root -m 0644 "$backup" "$ACTIVE" && + [[ "$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" == "$("$SHA256SUM_BIN" "$backup" | "$AWK_BIN" '{print $1}')" ]]; then + restore_install_rc=0 + else + restore_install_rc=1 + fi + "$NGINX_BIN" -t + restore_test_rc=$? + if (( restore_install_rc == 0 && restore_test_rc == 0 )); then + if "$SYSTEMCTL_BIN" reload nginx && + "$SYSTEMCTL_BIN" is-active --quiet nginx && + wait_for_rollback_state; then + restore_reload_rc=0 + else + restore_reload_rc=1 + fi + else + restore_reload_rc=1 + fi + + if (( restore_install_rc == 0 && restore_test_rc == 0 && restore_reload_rc == 0 )); then + rollback_armed=0 + printf 'ROLLBACK complete. Active config and response were restored.\n' >&2 + else + printf 'CRITICAL: automatic rollback failed; backup remains at %s\n' "$backup" >&2 + fi +} + +on_exit() { + local rc=$? + + trap - EXIT INT TERM + if (( rc != 0 && rollback_armed == 1 )); then + rollback + fi + cleanup + exit "$rc" +} + +trap on_exit EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +"$INSTALL_BIN" -o root -g root -m 0600 "$CANDIDATE" "$CANDIDATE_SNAPSHOT" +readonly CANDIDATE_SHA256="$("$SHA256SUM_BIN" "$CANDIDATE_SNAPSHOT" | "$AWK_BIN" '{print $1}')" +readonly ACTIVE_SHA256="$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" +[[ "$CANDIDATE_SHA256" == "$EXPECTED_CANDIDATE_SHA256" ]] || \ + fail "candidate digest changed: ${CANDIDATE_SHA256}" +if [[ "$ACTIVE_SHA256" != "$EXPECTED_PRE_CUTOVER_SHA256" && "$ACTIVE_SHA256" != "$CANDIDATE_SHA256" ]]; then + fail "active config has an unexpected digest: ${ACTIVE_SHA256}" +fi + +check_health_json() { + local path="$1" + + "$JQ_BIN" -e ' + .status == "pass" and + ([.checks["database:ping"][], .checks["cache:ping"][]] | + all(.status == "pass")) + ' "$path" >/dev/null +} + +pre_cutover_check() { + "$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \ + --header 'Host: git.learn.hyeonworks.com' \ + http://127.0.0.1:30080/api/healthz >"$DIRECT_HEALTH" + check_health_json "$DIRECT_HEALTH" +} + +wait_for_nginx_health() { + local attempt + local probe_metadata="" + local http_code="curl-error" + local content_type="unavailable" + local size_download="0" + local body_sha256="" + local classification="transport-error" + local consecutive_passes=0 + + for ((attempt = 1; attempt <= 10; attempt++)); do + if probe_metadata="$( + "$CURL_BIN" --noproxy '*' --silent --connect-timeout 1 --max-time 2 \ + --resolve git.learn.hyeonworks.com:443:127.0.0.1 \ + --output "$NGINX_HEALTH" \ + --write-out $'%{http_code}\t%{content_type}\t%{size_download}' \ + https://git.learn.hyeonworks.com/api/healthz + )"; then + IFS=$'\t' read -r http_code content_type size_download <<<"$probe_metadata" + body_sha256="$( + "$SHA256SUM_BIN" "$NGINX_HEALTH" | "$AWK_BIN" '{print $1}' + )" + + if [[ "$http_code" == "200" ]] && + check_health_json "$NGINX_HEALTH" 2>/dev/null; then + classification="healthy-json" + consecutive_passes=$((consecutive_passes + 1)) + if (( consecutive_passes >= 2 )); then + printf 'Nginx proxy health stabilized after %d probes.\n' "$attempt" + return 0 + fi + else + consecutive_passes=0 + if [[ "$body_sha256" == "$EXPECTED_PRE_CUTOVER_HEALTH_SHA256" ]]; then + classification="stale-old-generation" + elif [[ "$http_code" =~ ^(502|503|504)$ ]]; then + classification="transient-upstream" + elif "$JQ_BIN" -e ' + type == "object" and + has("status") and + (.checks | type == "object") + ' "$NGINX_HEALTH" >/dev/null 2>&1; then + classification="unhealthy-health-json" + else + printf 'ERROR: unexpected Nginx health response (status=%s, content-type=%s, bytes=%s, sha256=%s)\n' \ + "$http_code" "${content_type:-none}" "$size_download" "$body_sha256" >&2 + return 1 + fi + fi + else + http_code="curl-error" + content_type="unavailable" + size_download="0" + body_sha256="" + classification="transport-error" + consecutive_passes=0 + fi + + if (( attempt < 10 )); then + "$SLEEP_BIN" 1 + fi + done + + fail "Nginx health did not converge during the bounded retry window (classification=${classification}, status=${http_code}, content-type=${content_type:-none}, bytes=${size_download})" +} + +wait_for_rollback_state() { + local attempt + local probe_metadata="" + local http_code="curl-error" + local content_type="unavailable" + local size_download="0" + local body_sha256="" + + for ((attempt = 1; attempt <= 10; attempt++)); do + if probe_metadata="$( + "$CURL_BIN" --noproxy '*' --silent --connect-timeout 1 --max-time 2 \ + --resolve git.learn.hyeonworks.com:443:127.0.0.1 \ + --output "$NGINX_HEALTH" \ + --write-out $'%{http_code}\t%{content_type}\t%{size_download}' \ + https://git.learn.hyeonworks.com/api/healthz + )"; then + IFS=$'\t' read -r http_code content_type size_download <<<"$probe_metadata" + body_sha256="$( + "$SHA256SUM_BIN" "$NGINX_HEALTH" | "$AWK_BIN" '{print $1}' + )" + if [[ "$http_code" == "200" && + "$body_sha256" == "$EXPECTED_PRE_CUTOVER_HEALTH_SHA256" ]]; then + if (( attempt > 1 )); then + printf 'Rollback proxy state stabilized after %d probes.\n' "$attempt" >&2 + fi + return 0 + fi + else + http_code="curl-error" + content_type="unavailable" + size_download="0" + body_sha256="" + fi + + if (( attempt < 10 )); then + "$SLEEP_BIN" 1 + fi + done + + printf 'CRITICAL: rollback response did not converge (status=%s, content-type=%s, bytes=%s, sha256=%s)\n' \ + "$http_code" "${content_type:-none}" "$size_download" "${body_sha256:-none}" >&2 + return 1 +} + +post_cutover_checks() { + local id_body + local redirect_result + + wait_for_nginx_health + + id_body="$( + "$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \ + --resolve id.learn.hyeonworks.com:443:127.0.0.1 \ + https://id.learn.hyeonworks.com/ + )" + [[ "$id_body" == "Keycloak domain reached Nginx successfully" ]] || \ + fail "Keycloak hold response changed" + + redirect_result="$( + "$CURL_BIN" --noproxy '*' --silent --show-error --max-time 10 \ + --resolve git.learn.hyeonworks.com:80:127.0.0.1 \ + --output /dev/null --write-out $'%{http_code}\n%{redirect_url}' \ + http://git.learn.hyeonworks.com/api/healthz + )" + [[ "$redirect_result" == $'301\nhttps://git.learn.hyeonworks.com/api/healthz' ]] || \ + fail "HTTP redirect check failed: ${redirect_result}" + + if "$CURL_BIN" --noproxy '*' --insecure --silent --output /dev/null --max-time 5 \ + --resolve unconfigured.invalid:443:127.0.0.1 \ + https://unconfigured.invalid/ 2>/dev/null; then + fail "unknown TLS hostname was not rejected" + fi + + "$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \ + --resolve git.learn.hyeonworks.com:443:127.0.0.1 \ + --dump-header "$LOGIN_HEADERS" --output /dev/null \ + https://git.learn.hyeonworks.com/user/login + "$AWK_BIN" ' + BEGIN { found = 0; insecure = 0 } + tolower($0) ~ /^set-cookie:/ { + found++ + if (tolower($0) !~ /; secure([;[:space:]]|$)/) insecure = 1 + } + END { exit(found == 0 || insecure) } + ' "$LOGIN_HEADERS" || fail "login cookie Secure check failed" +} + +pre_cutover_check +"$NGINX_BIN" -t + +if [[ "$ACTIVE_SHA256" == "$CANDIDATE_SHA256" ]]; then + post_cutover_checks + printf 'The reviewed Host Nginx configuration is already active and healthy.\n' + exit 0 +fi + +backup="${ACTIVE}.before-gitea-$("$DATE_BIN" +%Y%m%d%H%M%S)" +[[ ! -e "$backup" && ! -L "$backup" ]] || fail "backup path already exists: ${backup}" + +printf '\nActive SHA-256: %s\n' "$ACTIVE_SHA256" +printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256" +printf 'Planned backup: %s\n' "$backup" +printf 'Type APPLY to replace the Host Nginx site: ' +read -r confirmation +[[ "$confirmation" == "APPLY" ]] || fail "cancelled" + +"$INSTALL_BIN" -o root -g root -m 0644 "$ACTIVE" "$backup" +[[ "$("$SHA256SUM_BIN" "$backup" | "$AWK_BIN" '{print $1}')" == "$ACTIVE_SHA256" ]] || \ + fail "backup digest mismatch" + +rollback_armed=1 +"$INSTALL_BIN" -o root -g root -m 0644 "$CANDIDATE_SNAPSHOT" "$ACTIVE" +[[ "$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" == "$CANDIDATE_SHA256" ]] || \ + fail "installed digest mismatch" +"$NGINX_BIN" -t +"$SYSTEMCTL_BIN" reload nginx +"$SYSTEMCTL_BIN" is-active --quiet nginx +post_cutover_checks + +rollback_armed=0 +printf '\nCUTOVER SUCCESS\n' +printf 'Backup: %s\n' "$backup" +printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256" +printf 'Local HTTPS health, redirect, Secure cookie, Keycloak hold, and unknown-host rejection: PASS\n' +printf 'Run the public HTTPS and Git clone/push checks from a separate client next.\n' + diff --git a/scripts/bootstrap/apply-host-nginx-keycloak.sh b/scripts/bootstrap/apply-host-nginx-keycloak.sh new file mode 100755 index 0000000..d4d94de --- /dev/null +++ b/scripts/bootstrap/apply-host-nginx-keycloak.sh @@ -0,0 +1,299 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +set +x +umask 077 + +readonly EXPECTED_CANDIDATE_SHA256="5c5cd74b4992f537fd27c50cf2209573a80a9904e0b19b58c3154717be6ff4a5" +readonly EXPECTED_PRE_CUTOVER_SHA256="de7ebd4f69cd7d2204ee633e074bf6a4370a6f5e3f3fac9067b099d5d75269b5" +readonly REPOSITORY_ROOT="/home/donghyeon/workspace/platform" +readonly CANDIDATE="${REPOSITORY_ROOT}/infrastructure/networking/host-nginx/learn-services-keycloak.conf" +readonly ACTIVE="/etc/nginx/sites-available/learn-services" +readonly ENABLED="/etc/nginx/sites-enabled/learn-services" +readonly GITEA_HOST="git.learn.hyeonworks.com" +readonly KEYCLOAK_HOST="id.learn.hyeonworks.com" +readonly KEYCLOAK_ISSUER="https://${KEYCLOAK_HOST}/realms/hyeonworks" +readonly CURL_BIN="/usr/bin/curl" +readonly JQ_BIN="/usr/bin/jq" +readonly NGINX_BIN="/usr/sbin/nginx" +readonly SYSTEMCTL_BIN="/usr/bin/systemctl" +readonly INSTALL_BIN="/usr/bin/install" +readonly SHA256SUM_BIN="/usr/bin/sha256sum" +readonly STAT_BIN="/usr/bin/stat" +readonly READLINK_BIN="/usr/bin/readlink" +readonly MKTEMP_BIN="/usr/bin/mktemp" +readonly RM_BIN="/usr/bin/rm" +readonly DATE_BIN="/usr/bin/date" +readonly AWK_BIN="/usr/bin/awk" +readonly SLEEP_BIN="/usr/bin/sleep" + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +usage() { + printf '%s\n' \ + 'Usage: sudo bash scripts/bootstrap/apply-host-nginx-keycloak.sh --execute' \ + '' \ + 'Replaces the static id.learn response with the reviewed Keycloak proxy.' \ + 'The active site is backed up first; a failed test or health check restores' \ + 'the prior Gitea-only configuration and reloads Nginx automatically.' +} + +[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || { + usage + exit 2 +} +[[ "$EUID" -eq 0 ]] || fail "run this script through sudo" +[[ -t 0 ]] || fail "an interactive terminal is required" + +for required_binary in \ + "$CURL_BIN" "$JQ_BIN" "$NGINX_BIN" "$SYSTEMCTL_BIN" "$INSTALL_BIN" \ + "$SHA256SUM_BIN" "$STAT_BIN" "$READLINK_BIN" "$MKTEMP_BIN" \ + "$RM_BIN" "$DATE_BIN" "$AWK_BIN" "$SLEEP_BIN"; do + [[ -x "$required_binary" ]] || fail "required executable is missing: ${required_binary}" +done + +[[ -f "$CANDIDATE" && ! -L "$CANDIDATE" ]] || fail "unsafe candidate: ${CANDIDATE}" +[[ -f "$ACTIVE" && ! -L "$ACTIVE" ]] || fail "unsafe active file: ${ACTIVE}" +[[ -L "$ENABLED" ]] || fail "enabled path is not a symlink: ${ENABLED}" +[[ "$("$READLINK_BIN" -f "$ENABLED")" == "$ACTIVE" ]] || \ + fail "enabled symlink target changed" +[[ "$("$STAT_BIN" --format='%U:%G %a' "$ACTIVE")" == "root:root 644" ]] || \ + fail "active file owner or mode changed" +"$SYSTEMCTL_BIN" is-active --quiet nginx || fail "nginx is not active" + +readonly TEMP_DIR="$("$MKTEMP_BIN" -d /tmp/nginx-keycloak-cutover.XXXXXX)" +readonly CANDIDATE_SNAPSHOT="${TEMP_DIR}/learn-services.candidate" +readonly GITEA_HEALTH="${TEMP_DIR}/gitea-health.json" +readonly KEYCLOAK_DISCOVERY="${TEMP_DIR}/keycloak-discovery.json" +readonly LOGIN_HEADERS="${TEMP_DIR}/gitea-login-headers" +rollback_armed=0 +backup="" + +cleanup() { + case "$TEMP_DIR" in + /tmp/nginx-keycloak-cutover.*) + "$RM_BIN" -rf -- "$TEMP_DIR" + ;; + *) + printf 'WARNING: refusing to remove unexpected temp path: %s\n' \ + "$TEMP_DIR" >&2 + ;; + esac +} + +check_gitea_health() { + "$JQ_BIN" -e ' + .status == "pass" and + ([.checks["database:ping"][], .checks["cache:ping"][]] | + all(.status == "pass")) + ' "$GITEA_HEALTH" >/dev/null +} + +check_keycloak_discovery() { + "$JQ_BIN" -e \ + --arg issuer "$KEYCLOAK_ISSUER" ' + .issuer == $issuer and + (.authorization_endpoint | startswith($issuer)) and + (.token_endpoint | startswith($issuer)) and + (.userinfo_endpoint | startswith($issuer)) and + (.jwks_uri | startswith($issuer)) + ' "$KEYCLOAK_DISCOVERY" >/dev/null +} + +pre_cutover_checks() { + "$CURL_BIN" --disable --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \ + --header "Host: ${GITEA_HOST}" \ + --header 'X-Forwarded-Proto: https' \ + http://127.0.0.1:30080/api/healthz >"$GITEA_HEALTH" + check_gitea_health + + "$CURL_BIN" --disable --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \ + --header "Host: ${KEYCLOAK_HOST}" \ + --header 'X-Forwarded-Host: id.learn.hyeonworks.com' \ + --header 'X-Forwarded-Proto: https' \ + --header 'X-Forwarded-Port: 443' \ + "http://127.0.0.1:30080/realms/hyeonworks/.well-known/openid-configuration" \ + >"$KEYCLOAK_DISCOVERY" + check_keycloak_discovery +} + +wait_for_proxy_state() { + local attempt + + for ((attempt = 1; attempt <= 12; attempt++)); do + if "$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \ + --connect-timeout 1 --max-time 3 \ + --resolve "${GITEA_HOST}:443:127.0.0.1" \ + "https://${GITEA_HOST}/api/healthz" >"$GITEA_HEALTH" 2>/dev/null && + check_gitea_health 2>/dev/null && + "$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \ + --connect-timeout 1 --max-time 3 \ + --resolve "${KEYCLOAK_HOST}:443:127.0.0.1" \ + "https://${KEYCLOAK_HOST}/realms/hyeonworks/.well-known/openid-configuration" \ + >"$KEYCLOAK_DISCOVERY" 2>/dev/null && + check_keycloak_discovery 2>/dev/null; then + if (( attempt > 1 )); then + printf 'Nginx proxy state stabilized after %d probes.\n' "$attempt" + fi + return 0 + fi + (( attempt < 12 )) && "$SLEEP_BIN" 1 + done + + return 1 +} + +wait_for_rollback_state() { + local attempt + local id_body + + for ((attempt = 1; attempt <= 12; attempt++)); do + id_body="$( + "$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \ + --connect-timeout 1 --max-time 3 \ + --resolve "${KEYCLOAK_HOST}:443:127.0.0.1" \ + "https://${KEYCLOAK_HOST}/" 2>/dev/null + )" || id_body="" + if [[ "$id_body" == "Keycloak domain reached Nginx successfully" ]] && + "$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \ + --connect-timeout 1 --max-time 3 \ + --resolve "${GITEA_HOST}:443:127.0.0.1" \ + "https://${GITEA_HOST}/api/healthz" >"$GITEA_HEALTH" 2>/dev/null && + check_gitea_health 2>/dev/null; then + return 0 + fi + (( attempt < 12 )) && "$SLEEP_BIN" 1 + done + + return 1 +} + +rollback() { + local restore_rc=0 + + set +e + printf '\nROLLBACK: restoring %s\n' "$backup" >&2 + "$INSTALL_BIN" -o root -g root -m 0644 "$backup" "$ACTIVE" || restore_rc=1 + "$NGINX_BIN" -t || restore_rc=1 + if (( restore_rc == 0 )); then + "$SYSTEMCTL_BIN" reload nginx || restore_rc=1 + "$SYSTEMCTL_BIN" is-active --quiet nginx || restore_rc=1 + wait_for_rollback_state || restore_rc=1 + fi + + if (( restore_rc == 0 )); then + rollback_armed=0 + printf 'ROLLBACK complete. The Gitea proxy and static Keycloak hold were restored.\n' >&2 + else + printf 'CRITICAL: automatic rollback failed; backup remains at %s\n' \ + "$backup" >&2 + fi +} + +on_exit() { + local rc=$? + + trap - EXIT INT TERM + if (( rc != 0 && rollback_armed == 1 )); then + rollback + fi + cleanup + exit "$rc" +} + +trap on_exit EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +"$INSTALL_BIN" -o root -g root -m 0600 "$CANDIDATE" "$CANDIDATE_SNAPSHOT" +readonly CANDIDATE_SHA256="$( + "$SHA256SUM_BIN" "$CANDIDATE_SNAPSHOT" | "$AWK_BIN" '{print $1}' +)" +readonly ACTIVE_SHA256="$( + "$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}' +)" +[[ "$CANDIDATE_SHA256" == "$EXPECTED_CANDIDATE_SHA256" ]] || \ + fail "candidate digest changed: ${CANDIDATE_SHA256}" +if [[ "$ACTIVE_SHA256" != "$EXPECTED_PRE_CUTOVER_SHA256" && + "$ACTIVE_SHA256" != "$CANDIDATE_SHA256" ]]; then + fail "active config has an unexpected digest: ${ACTIVE_SHA256}" +fi + +pre_cutover_checks +"$NGINX_BIN" -t + +post_cutover_checks() { + local host + local redirect_result + + wait_for_proxy_state || fail "Gitea and Keycloak HTTPS proxies did not converge" + + for host in "$GITEA_HOST" "$KEYCLOAK_HOST"; do + redirect_result="$( + "$CURL_BIN" --disable --noproxy '*' --silent --show-error --max-time 10 \ + --resolve "${host}:80:127.0.0.1" \ + --output /dev/null \ + --write-out $'%{http_code}\n%{redirect_url}' \ + "http://${host}/" + )" + [[ "$redirect_result" == $'301\nhttps://'"${host}/" ]] || \ + fail "${host} HTTP redirect check failed: ${redirect_result}" + done + + "$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error --max-time 10 \ + --resolve "${GITEA_HOST}:443:127.0.0.1" \ + --dump-header "$LOGIN_HEADERS" --output /dev/null \ + "https://${GITEA_HOST}/user/login" + "$AWK_BIN" ' + BEGIN { found = 0; insecure = 0 } + tolower($0) ~ /^set-cookie:/ { + found++ + if (tolower($0) !~ /; secure([;[:space:]]|$)/) insecure = 1 + } + END { exit(found == 0 || insecure) } + ' "$LOGIN_HEADERS" || fail "Gitea login cookie Secure check failed" + + if "$CURL_BIN" --disable --noproxy '*' --insecure --silent --output /dev/null --max-time 5 \ + --resolve unconfigured.invalid:443:127.0.0.1 \ + https://unconfigured.invalid/ 2>/dev/null; then + fail "unknown TLS hostname was not rejected" + fi +} + +if [[ "$ACTIVE_SHA256" == "$CANDIDATE_SHA256" ]]; then + post_cutover_checks + printf 'The reviewed Keycloak Host Nginx configuration is already active and healthy.\n' + exit 0 +fi + +backup="${ACTIVE}.before-keycloak-$("$DATE_BIN" +%Y%m%d%H%M%S)" +[[ ! -e "$backup" && ! -L "$backup" ]] || fail "backup path already exists: ${backup}" + +printf '\nActive SHA-256: %s\n' "$ACTIVE_SHA256" +printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256" +printf 'Planned backup: %s\n' "$backup" +printf 'Type APPLY to expose Keycloak through Host Nginx: ' +read -r confirmation +[[ "$confirmation" == "APPLY" ]] || fail "cancelled" + +"$INSTALL_BIN" -o root -g root -m 0644 "$ACTIVE" "$backup" +[[ "$("$SHA256SUM_BIN" "$backup" | "$AWK_BIN" '{print $1}')" == \ + "$ACTIVE_SHA256" ]] || fail "backup digest mismatch" + +rollback_armed=1 +"$INSTALL_BIN" -o root -g root -m 0644 "$CANDIDATE_SNAPSHOT" "$ACTIVE" +[[ "$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" == \ + "$CANDIDATE_SHA256" ]] || fail "installed digest mismatch" +"$NGINX_BIN" -t +"$SYSTEMCTL_BIN" reload nginx +"$SYSTEMCTL_BIN" is-active --quiet nginx +post_cutover_checks + +rollback_armed=0 +printf '\nKEYCLOAK CUTOVER SUCCESS\n' +printf 'Backup: %s\n' "$backup" +printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256" +printf 'Gitea health, Keycloak discovery issuer, HTTPS cookies, redirects, and unknown-host rejection: PASS\n' diff --git a/scripts/bootstrap/apply-host-nginx-observability.sh b/scripts/bootstrap/apply-host-nginx-observability.sh new file mode 100755 index 0000000..27f7abc --- /dev/null +++ b/scripts/bootstrap/apply-host-nginx-observability.sh @@ -0,0 +1,1132 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +set +x +umask 077 + +readonly HOST_NGINX_PRODUCTION_PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' +PATH=$HOST_NGINX_PRODUCTION_PATH +export PATH +hash -r + +readonly ROOT="$(cd -- "${BASH_SOURCE[0]%/*}/../.." && pwd -P)" +declare -Ar HOST_NGINX_PRODUCTION_COMMANDS=( + [awk]=/usr/bin/awk [bash]=/usr/bin/bash [cat]=/usr/bin/cat + [curl]=/usr/bin/curl [date]=/usr/bin/date [dig]=/usr/bin/dig + [dirname]=/usr/bin/dirname [grep]=/usr/bin/grep [head]=/usr/bin/head + [id]=/usr/bin/id [install]=/usr/bin/install [jq]=/usr/bin/jq + [kubectl]=/usr/local/bin/kubectl [mktemp]=/usr/bin/mktemp + [openssl]=/usr/bin/openssl [python3]=/usr/bin/python3 + [readlink]=/usr/bin/readlink [rm]=/usr/bin/rm [sed]=/usr/bin/sed + [sha256sum]=/usr/bin/sha256sum [sleep]=/usr/bin/sleep + [sort]=/usr/bin/sort [ss]=/usr/bin/ss [stat]=/usr/bin/stat + [tail]=/usr/bin/tail [test]=/usr/bin/test [tr]=/usr/bin/tr +) +readonly HOST_NGINX_SOURCE="$ROOT/infrastructure/networking/host-nginx" +readonly METRICS_CANDIDATE="$HOST_NGINX_SOURCE/learn-services-metrics-guard.conf" +readonly DENY_CANDIDATE="$HOST_NGINX_SOURCE/learn-services-grafana-deny-guard.conf" +readonly FULL_CANDIDATE="$HOST_NGINX_SOURCE/learn-services-observability.conf" +readonly ADMIN_CANDIDATE="$HOST_NGINX_SOURCE/learn-services-admin.conf" +readonly SOURCE_TEST="$ROOT/scripts/validate/test-host-nginx-observability-source.sh" +readonly ACTIVE="${PLATFORM_HOST_NGINX_ACTIVE:-/etc/nginx/sites-available/learn-services}" +readonly ENABLED="${PLATFORM_HOST_NGINX_ENABLED:-/etc/nginx/sites-enabled/learn-services}" +readonly ROLLBACK_BASE="${PLATFORM_HOST_NGINX_ROLLBACK_BASE:-/var/lib/hyeonworks/platform-rollbacks}" +readonly GRAFANA_HOST=grafana.learn.hyeonworks.com +readonly GITEA_HOST=git.learn.hyeonworks.com +readonly KEYCLOAK_HOST=id.learn.hyeonworks.com +readonly STORAGE_HOST=storage-admin.learn.hyeonworks.com +readonly DB_HOST=db-admin.learn.hyeonworks.com +readonly KEYCLOAK_ISSUER="https://$KEYCLOAK_HOST/realms/hyeonworks" +readonly LAN_IP=192.168.0.107 +readonly TAIL_IP=100.92.240.34 +readonly CERT_NAME="$GRAFANA_HOST" +readonly CERT_DIR="${PLATFORM_HOST_NGINX_CERT_DIR:-/etc/letsencrypt/live/$CERT_NAME}" +readonly CREDENTIALS="${PLATFORM_HOST_NGINX_CREDENTIALS:-/home/donghyeon/.secrets/certbot/cloudflare.ini}" +readonly NGINX_BIN="${PLATFORM_HOST_NGINX_NGINX_BIN:-/usr/sbin/nginx}" +readonly SYSTEMCTL_BIN="${PLATFORM_HOST_NGINX_SYSTEMCTL_BIN:-/usr/bin/systemctl}" +readonly SUDO_BIN="${PLATFORM_HOST_NGINX_SUDO_BIN:-/usr/bin/sudo}" +readonly TEST_BIN="${PLATFORM_HOST_NGINX_TEST_BIN:-/usr/bin/test}" +readonly CERTBOT_BIN="${PLATFORM_HOST_NGINX_CERTBOT_BIN:-/snap/bin/certbot}" +readonly PREFLIGHT_COMMAND="${PLATFORM_HOST_NGINX_PREFLIGHT_COMMAND:-}" +readonly POSTCHECK_COMMAND="${PLATFORM_HOST_NGINX_POSTCHECK_COMMAND:-}" +readonly TEST_MODE="${PLATFORM_HOST_NGINX_TEST_MODE:-0}" +readonly RULES_ACCEPTANCE_SCHEMA=platform-observability-rules-alerts-v2 +readonly SOURCE_PROOF_SCHEMA=platform-blackbox-source-v1 +readonly STAGE_LEDGER_HEADER=$'stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc' + +execute=false +mode=full +mode_count=0 +certbot_email='' +verified_output_dir='' +rollback_armed=false +rollback_payload='' +rollback_previous_sha='' +rollback_stage='' +timestamp_backup='' + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +usage() { + cat <<'USAGE' +Usage: + bash scripts/bootstrap/apply-host-nginx-observability.sh + bash scripts/bootstrap/apply-host-nginx-observability.sh --execute --metrics-guard-only + bash scripts/bootstrap/apply-host-nginx-observability.sh \ + --execute --certificate-only --certbot-email you@example.com + bash scripts/bootstrap/apply-host-nginx-observability.sh --execute --grafana-deny-guard-only + bash scripts/bootstrap/apply-host-nginx-observability.sh \ + --execute --verified-output-dir /tmp/platform-observability-metrics.XXXXXX + +Dry-run prints the exact stage hashes and does not call sudo, Nginx, systemd, +Certbot, Kubernetes, or network probes. Execute modes are interactive. The full +cutover additionally requires source proof, rules-alerts acceptance, and the two +accepted metric inventories for the same rollback ID. +USAGE +} + +sha_of() { + sha256sum -- "$1" | awk '{print $1}' +} + +is_sha256() { + [[ "$1" =~ ^[0-9a-f]{64}$ ]] +} + +require_regular_source() { + [[ -f "$1" && ! -L "$1" ]] || fail "unsafe or missing source: $1" +} + +root_cat() { + "$SUDO_BIN" -n cat -- "$1" +} + +root_stat() { + "$SUDO_BIN" -n stat "$@" +} + +expected_owner_mode() { + local path=$1 mode=$2 metadata + metadata="$(root_stat -c '%U:%G %a' -- "$path")" || return 1 + if [[ "$TEST_MODE" == 1 ]]; then + [[ "${metadata##* }" == "$mode" ]] + else + [[ "$metadata" == "root:root $mode" ]] + fi +} + +ensure_root_directory_no_follow() { + local path=$1 metadata type owner mode + if "$SUDO_BIN" -n "$TEST_BIN" -L "$path"; then + fail "root directory path is a symlink: $path" + return 1 + fi + if ! "$SUDO_BIN" -n "$TEST_BIN" -e "$path"; then + if ! "$SUDO_BIN" install -d -o root -g root -m 0700 "$path"; then + fail "cannot create root directory: $path" + return 1 + fi + fi + if ! metadata="$("$SUDO_BIN" -n stat -c '%F|%U:%G|%a' -- "$path")"; then + fail "cannot lstat root directory: $path" + return 1 + fi + IFS='|' read -r type owner mode <<<"$metadata" + if [[ "$type" != directory || "$mode" != 700 ]]; then + fail "root directory type or mode is unsafe: $path" + return 1 + fi + if [[ "$TEST_MODE" == 0 && "$owner" != root:root ]]; then + fail "root directory owner is unsafe: $path" + return 1 + fi + return 0 +} + +validate_root_file_no_follow() { + local path=$1 expected_mode=$2 metadata type owner mode links + if "$SUDO_BIN" -n "$TEST_BIN" -L "$path"; then + fail "root file path is a symlink: $path" + return 1 + fi + if ! metadata="$("$SUDO_BIN" -n stat -c '%F|%U:%G|%a|%h' -- "$path")"; then + fail "cannot lstat root file: $path" + return 1 + fi + IFS='|' read -r type owner mode links <<<"$metadata" + if [[ "$type" != 'regular file' || "$mode" != "$expected_mode" || "$links" != 1 ]]; then + fail "root file type, mode, or link count is unsafe: $path" + return 1 + fi + if [[ "$TEST_MODE" == 0 && "$owner" != root:root ]]; then + fail "root file owner is unsafe: $path" + return 1 + fi + return 0 +} + +validate_test_command_boundary() { + local label=$1 command_path=$2 fixture_bin=$3 resolved metadata type owner mode mode_value + if [[ "$command_path" == /usr/bin/sudo ]]; then + fail 'test command boundary is unsafe: real /usr/bin/sudo is forbidden' + return 1 + fi + if ! resolved="$(readlink -f -- "$command_path" 2>/dev/null)" || [[ "$resolved" != "$command_path" ]]; then + fail "test command boundary is unsafe: $label is not canonical" + return 1 + fi + if [[ "$(dirname -- "$resolved")" != "$fixture_bin" ]]; then + fail "test command boundary is unsafe: $label escapes the fixture bin" + return 1 + fi + if [[ ! -f "$command_path" || -L "$command_path" || ! -x "$command_path" ]]; then + fail "test command boundary is unsafe: $label is not a regular executable" + return 1 + fi + if ! metadata="$(stat -c '%F|%u|%a' -- "$command_path")"; then + fail "test command boundary is unsafe: cannot stat $label" + return 1 + fi + IFS='|' read -r type owner mode <<<"$metadata" + mode_value=$((8#$mode)) + if [[ "$type" != 'regular file' || "$owner" != "$(id -u)" || + $((mode_value & 0022)) != 0 || $((mode_value & 07000)) != 0 ]]; then + fail "test command boundary is unsafe: $label owner or mode" + return 1 + fi + return 0 +} + +validate_test_command_fixture_directory() { + local path=$1 label=$2 metadata type owner mode mode_value + if [[ ! -d "$path" || -L "$path" ]]; then + fail "test command fixture $label is not a regular directory" + return 1 + fi + if ! metadata="$(stat -c '%F|%u|%a' -- "$path")"; then + fail "cannot stat test command fixture $label" + return 1 + fi + IFS='|' read -r type owner mode <<<"$metadata" + mode_value=$((8#$mode)) + if [[ "$type" != directory || "$owner" != "$(id -u)" || + $((mode_value & 0022)) != 0 || $((mode_value & 07000)) != 0 ]]; then + fail "test command fixture $label owner or mode is unsafe" + return 1 + fi + return 0 +} + +production_executable_metadata_is_safe() { + local label=$1 lexical=$2 canonical=$3 type=$4 uid=$5 gid=$6 mode=$7 links=$8 policy=$9 + local mode_value + [[ "$type" == 'regular file' && "$uid" == 0 && "$gid" == 0 && "$links" == 1 ]] || return 1 + [[ "$mode" =~ ^[0-7]{3,4}$ ]] || return 1 + mode_value=$((8#$mode)) + (( (mode_value & 0111) != 0 )) || return 1 + (( (mode_value & 0022) == 0 )) || return 1 + case "$policy" in + sudo) + [[ "$label" == sudo && "$lexical" == /usr/bin/sudo && "$canonical" == /usr/bin/sudo ]] || return 1 + (( mode_value == 04755 )) || return 1 + ;; + ordinary) + (( (mode_value & 06000) == 0 )) || return 1 + ;; + *) return 1 ;; + esac +} + +validate_production_executable_boundary() { + local label=$1 lexical=$2 policy=$3 canonical metadata type uid gid mode links + [[ "$lexical" == /* ]] || { + fail "production command path is not absolute: $label" + return 1 + } + if ! canonical="$(/usr/bin/readlink -f -- "$lexical" 2>/dev/null)"; then + fail "production command target does not resolve: $label" + return 1 + fi + if ! metadata="$(/usr/bin/stat -Lc '%F|%u|%g|%a|%h' -- "$lexical" 2>/dev/null)"; then + fail "production command metadata is unreadable: $label" + return 1 + fi + IFS='|' read -r type uid gid mode links <<<"$metadata" + if ! production_executable_metadata_is_safe "$label" "$lexical" "$canonical" \ + "$type" "$uid" "$gid" "$mode" "$links" "$policy"; then + fail "production command boundary is unsafe: $label" + return 1 + fi +} + +validate_production_path_directory() { + local path=$1 canonical metadata type uid gid mode mode_value + if ! canonical="$(/usr/bin/readlink -f -- "$path" 2>/dev/null)" || + ! metadata="$(/usr/bin/stat -Lc '%F|%u|%g|%a' -- "$path" 2>/dev/null)"; then + fail "production PATH directory does not resolve: $path" + return 1 + fi + IFS='|' read -r type uid gid mode <<<"$metadata" + mode_value=$((8#$mode)) + if [[ "$canonical" != /* || "$type" != directory || "$uid" != 0 || "$gid" != 0 || + $((mode_value & 0022)) != 0 ]]; then + fail "production PATH directory is unsafe: $path" + return 1 + fi +} + +validate_production_command_boundaries() { + local name expected resolved path_directory + local -a production_path_directories=() + [[ "$PATH" == "$HOST_NGINX_PRODUCTION_PATH" ]] || { + fail 'production PATH is not exact' + return 1 + } + IFS=: read -r -a production_path_directories <<<"$HOST_NGINX_PRODUCTION_PATH" + for path_directory in "${production_path_directories[@]}"; do + validate_production_path_directory "$path_directory" || return 1 + done + for name in "${!HOST_NGINX_PRODUCTION_COMMANDS[@]}"; do + expected=${HOST_NGINX_PRODUCTION_COMMANDS[$name]} + if [[ "$name" == test ]]; then + resolved=$expected + elif ! resolved="$(command -v -- "$name")" || [[ "$resolved" != "$expected" ]]; then + fail "production command resolution drifted: $name" + return 1 + fi + validate_production_executable_boundary "$name" "$expected" ordinary || return 1 + done + validate_production_executable_boundary nginx "$NGINX_BIN" ordinary || return 1 + validate_production_executable_boundary systemctl "$SYSTEMCTL_BIN" ordinary || return 1 + validate_production_executable_boundary certbot "$CERTBOT_BIN" ordinary || return 1 + validate_production_executable_boundary sudo "$SUDO_BIN" sudo || return 1 +} + +validate_test_mode() { + local override_name active_fixture_root rollback_fixture_root fixture_bin fixture_root + if [[ "$TEST_MODE" != 0 && "$TEST_MODE" != 1 ]]; then + fail 'invalid test-mode flag' + return 1 + fi + if [[ "$TEST_MODE" == 0 ]]; then + for override_name in ${!PLATFORM_HOST_NGINX_@}; do + fail "production rejects Host Nginx override: $override_name" + return 1 + done + if [[ "$ACTIVE" != /etc/nginx/sites-available/learn-services || + "$ENABLED" != /etc/nginx/sites-enabled/learn-services || + "$ROLLBACK_BASE" != /var/lib/hyeonworks/platform-rollbacks || + "$CERT_DIR" != /etc/letsencrypt/live/grafana.learn.hyeonworks.com || + "$CREDENTIALS" != /home/donghyeon/.secrets/certbot/cloudflare.ini || + "$NGINX_BIN" != /usr/sbin/nginx || "$SYSTEMCTL_BIN" != /usr/bin/systemctl || + "$SUDO_BIN" != /usr/bin/sudo || "$TEST_BIN" != /usr/bin/test || "$CERTBOT_BIN" != /snap/bin/certbot || + -n "$PREFLIGHT_COMMAND" || -n "$POSTCHECK_COMMAND" ]]; then + fail 'production Host Nginx constants are not exact' + return 1 + fi + return 0 + fi + case "$ACTIVE:$ROLLBACK_BASE" in + /tmp/platform-host-nginx-test.*:/tmp/platform-host-nginx-test.*) ;; + *) + fail 'test mode is restricted to an isolated /tmp/platform-host-nginx-test.* tree' + return 1 + ;; + esac + if [[ "$SUDO_BIN" == /usr/bin/sudo ]]; then + fail 'test command boundary is unsafe: real /usr/bin/sudo is forbidden' + return 1 + fi + if ! active_fixture_root="$(readlink -f -- "$(dirname -- "$(dirname -- "$ACTIVE")")" 2>/dev/null)" || + ! rollback_fixture_root="$(readlink -f -- "$(dirname -- "$ROLLBACK_BASE")" 2>/dev/null)"; then + fail 'test fixture root does not resolve' + return 1 + fi + fixture_root=$active_fixture_root + fixture_bin="$fixture_root/bin" + if [[ "$fixture_root" != /tmp/platform-host-nginx-test.* || + "$ACTIVE" != "$fixture_root/active/learn-services" || + "$ENABLED" != "$fixture_root/enabled/learn-services" || + "$ROLLBACK_BASE" != "$fixture_root/rollbacks" || + "$rollback_fixture_root" != "$fixture_root" || + "$(readlink -f -- "$fixture_root" 2>/dev/null)" != "$fixture_root" ]]; then + fail 'test command fixture bin is not tied to the active and rollback fixture tree' + return 1 + fi + if ! validate_test_command_fixture_directory "$fixture_root" root || + ! validate_test_command_fixture_directory "$fixture_bin" bin; then + return 1 + fi + validate_test_command_boundary sudo "$SUDO_BIN" "$fixture_bin" || return 1 + validate_test_command_boundary test "$TEST_BIN" "$fixture_bin" || return 1 + validate_test_command_boundary nginx "$NGINX_BIN" "$fixture_bin" || return 1 + validate_test_command_boundary systemctl "$SYSTEMCTL_BIN" "$fixture_bin" || return 1 + validate_test_command_boundary certbot "$CERTBOT_BIN" "$fixture_bin" || return 1 + validate_test_command_boundary preflight "$PREFLIGHT_COMMAND" "$fixture_bin" || return 1 + validate_test_command_boundary postcheck "$POSTCHECK_COMMAND" "$fixture_bin" || return 1 + PATH="$fixture_bin:$HOST_NGINX_PRODUCTION_PATH" + export PATH + hash -r + return 0 +} + +parse_args() { + while (( $# > 0 )); do + case "$1" in + --execute) + execute=true + shift + ;; + --metrics-guard-only) + mode=metrics + mode_count=$((mode_count + 1)) + shift + ;; + --certificate-only) + mode=certificate + mode_count=$((mode_count + 1)) + shift + ;; + --grafana-deny-guard-only) + mode=deny + mode_count=$((mode_count + 1)) + shift + ;; + --certbot-email) + (( $# >= 2 )) || fail '--certbot-email requires a value' + certbot_email=$2 + shift 2 + ;; + --verified-output-dir) + (( $# >= 2 )) || fail '--verified-output-dir requires a value' + verified_output_dir=$2 + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage >&2 + fail "unsupported argument: $1" + ;; + esac + done + (( mode_count <= 1 )) || fail 'execute modes are mutually exclusive' + [[ -z "$certbot_email" || "$mode" == certificate ]] || fail '--certbot-email is certificate-only' + [[ -z "$verified_output_dir" || "$mode" == full ]] || fail '--verified-output-dir is full-mode only' +} + +source_contract() { + require_regular_source "$ADMIN_CANDIDATE" + require_regular_source "$METRICS_CANDIDATE" + require_regular_source "$DENY_CANDIDATE" + require_regular_source "$FULL_CANDIDATE" + require_regular_source "$SOURCE_TEST" + bash "$SOURCE_TEST" >/dev/null + python3 - "$ADMIN_CANDIDATE" "$METRICS_CANDIDATE" <<'PY' +import pathlib +import sys + +prior, guard = map(lambda value: pathlib.Path(value).read_text(), sys.argv[1:]) +needle = " location / {\n" +addition = " location = /metrics {\n return 404;\n }\n\n" +if prior.count(needle) < 1 or guard != prior.replace(needle, addition + needle, 1): + raise SystemExit("metrics guard differs from the admin source outside exact /metrics") +PY +} + +print_plan() { + local active_sha=unreadable + [[ -f "$ACTIVE" && ! -L "$ACTIVE" && -r "$ACTIVE" ]] && active_sha="$(sha_of "$ACTIVE")" + printf 'HOST_NGINX_ACTIVE_SHA256=%s\n' "$active_sha" + printf 'HOST_NGINX_ADMIN_SHA256=%s\n' "$(sha_of "$ADMIN_CANDIDATE")" + printf 'HOST_NGINX_METRICS_GUARD_SHA256=%s\n' "$(sha_of "$METRICS_CANDIDATE")" + printf 'HOST_NGINX_GRAFANA_DENY_GUARD_SHA256=%s\n' "$(sha_of "$DENY_CANDIDATE")" + printf 'HOST_NGINX_FULL_SHA256=%s\n' "$(sha_of "$FULL_CANDIDATE")" + printf 'HOST_NGINX_CERTIFICATE_EXPECTED_SAN=%s\n' "$GRAFANA_HOST" + printf 'HOST_NGINX_CERTIFICATE_SAN=NOT_CHECKED_DRY_RUN\n' + printf 'HOST_NGINX_GRAFANA_PUBLIC_DNS=NOT_CHECKED_DRY_RUN\n' + printf 'HOST_NGINX_OBSERVABILITY_DRY_RUN=PASS\n' +} + +validate_host_files() { + "$SUDO_BIN" -n "$TEST_BIN" -f "$ACTIVE" || fail "active site is missing: $ACTIVE" + ! "$SUDO_BIN" -n "$TEST_BIN" -L "$ACTIVE" || fail "active site is a symlink: $ACTIVE" + "$SUDO_BIN" -n "$TEST_BIN" -L "$ENABLED" || fail "enabled site is not a symlink: $ENABLED" + [[ "$("$SUDO_BIN" -n readlink -f -- "$ENABLED")" == "$("$SUDO_BIN" -n readlink -f -- "$ACTIVE")" ]] || + fail 'enabled site target changed' + expected_owner_mode "$ACTIVE" 644 || fail 'active site must be root:root 0644' +} + +validate_rollback_root() { + local id=${PLATFORM_OBSERVABILITY_ROLLBACK_ID:-} + [[ "$id" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || fail 'PLATFORM_OBSERVABILITY_ROLLBACK_ID format is invalid' + ROLLBACK_ROOT="$ROLLBACK_BASE/observability-$id" + "$SUDO_BIN" -n "$TEST_BIN" -d "$ROLLBACK_ROOT" || fail "rollback root is missing: $ROLLBACK_ROOT" + ! "$SUDO_BIN" -n "$TEST_BIN" -L "$ROLLBACK_ROOT" || fail "rollback root is a symlink: $ROLLBACK_ROOT" + expected_owner_mode "$ROLLBACK_ROOT" 700 || fail 'rollback root must be root:root 0700' + readonly ROLLBACK_ROOT +} + +declare -A PARSED_ENV=() +read_exact_env() { + local file=$1 + shift + local line key value allowed expected=$# count=0 + PARSED_ENV=() + if ! "$SUDO_BIN" -n "$TEST_BIN" -f "$file"; then + fail "evidence file is missing: $file" + return 1 + fi + if "$SUDO_BIN" -n "$TEST_BIN" -L "$file"; then + fail "evidence file is a symlink: $file" + return 1 + fi + if ! expected_owner_mode "$file" 600; then + fail "evidence file must be root:root 0600: $file" + return 1 + fi + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$line" != *=* || "$line" == *$'\t'* || "$line" == *$'\r'* ]]; then + fail "malformed evidence line: $file" + return 1 + fi + key=${line%%=*} + value=${line#*=} + if [[ -z "$key" || -z "$value" || -n "${PARSED_ENV[$key]+x}" ]]; then + fail "duplicate or empty evidence field: $file" + return 1 + fi + allowed=false + for expected_key in "$@"; do + [[ "$key" == "$expected_key" ]] && allowed=true + done + if [[ "$allowed" != true ]]; then + fail "unexpected evidence field $key: $file" + return 1 + fi + PARSED_ENV[$key]=$value + count=$((count + 1)) + done < <(root_cat "$file") + if (( count != expected )); then + fail "evidence field count is not exact: $file" + return 1 + fi + for expected_key in "$@"; do + if [[ -z "${PARSED_ENV[$expected_key]:-}" ]]; then + fail "missing evidence field $expected_key: $file" + return 1 + fi + done + return 0 +} + +validate_slack_acceptance_tuple() { + local tuple + tuple="${PARSED_ENV[slack_deployment_gate]}|${PARSED_ENV[slack_gate_approval_ref]}|${PARSED_ENV[slack_gate_accepted_by_uid]}" + case "$tuple" in + 'RECOVERY|strict-recovery-evidence-v1|not-applicable'|\ + 'RISK_ACCEPTED|2026-08-14-observability-slack-recovery-risk-acceptance-design|1000') + return 0 + ;; + *) + fail 'rules-alerts Slack deployment gate tuple mismatch' + return 1 + ;; + esac +} + +validate_recent_utc() { + local value=$1 label=$2 then now age + if [[ ! "$value" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then + fail "$label timestamp format is invalid" + return 1 + fi + if ! then="$(date -u -d "$value" +%s 2>/dev/null)"; then + fail "$label timestamp is invalid" + return 1 + fi + if ! now="$(date -u +%s)"; then + fail "cannot read current UTC time for $label" + return 1 + fi + age=$((now - then)) + if (( age < -300 || age > 86400 )); then + fail "$label evidence is stale or from the future" + return 1 + fi + return 0 +} + +validate_utc() { + local value=$1 label=$2 + if [[ ! "$value" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then + fail "$label timestamp format is invalid" + return 1 + fi + if ! date -u -d "$value" +%s >/dev/null 2>&1; then + fail "$label timestamp is invalid" + return 1 + fi + return 0 +} + +inventory_sha() { + local root=$1 phase=$2 directory checksum actual declared filename extra directory_mode json_mode checksum_mode + directory="$root/$phase" + checksum="$directory/inventory.sha256" + if [[ ! -d "$directory" || -L "$directory" ]]; then + fail "inventory phase is missing or unsafe: $phase" + return 1 + fi + if [[ ! -f "$directory/inventory.json" || -L "$directory/inventory.json" ]]; then + fail "inventory JSON is unsafe: $phase" + return 1 + fi + if [[ ! -f "$checksum" || -L "$checksum" ]]; then + fail "inventory checksum is unsafe: $phase" + return 1 + fi + if ! directory_mode="$(stat -c %a -- "$directory")"; then + fail "cannot stat inventory directory: $phase" + return 1 + fi + if [[ "$directory_mode" != 700 ]]; then + fail "inventory directory mode is not 0700: $phase" + return 1 + fi + if ! json_mode="$(stat -c %a -- "$directory/inventory.json")" || + ! checksum_mode="$(stat -c %a -- "$checksum")"; then + fail "cannot stat inventory files: $phase" + return 1 + fi + if [[ "$json_mode" != 600 || "$checksum_mode" != 600 ]]; then + fail "inventory file mode is not 0600: $phase" + return 1 + fi + if ! read -r declared filename extra <"$checksum"; then + fail "cannot read inventory checksum: $phase" + return 1 + fi + if [[ "$filename" != inventory.json || -n "${extra:-}" ]]; then + fail "inventory checksum shape is invalid: $phase" + return 1 + fi + if ! is_sha256 "$declared"; then + fail "inventory checksum is invalid: $phase" + return 1 + fi + if ! actual="$(sha_of "$directory/inventory.json")"; then + fail "cannot hash inventory JSON: $phase" + return 1 + fi + if [[ "$actual" != "$declared" ]]; then + fail "inventory checksum mismatch: $phase" + return 1 + fi + printf '%s' "$actual" || return 1 + return 0 +} + +validate_full_evidence() { + local deny_sha=$1 acceptance proof initial_sha post_sha canonical_metric_root + local metric_root_mode + if [[ -z "$verified_output_dir" ]]; then + fail 'full execute requires --verified-output-dir' + return 1 + fi + if ! canonical_metric_root="$(readlink -f -- "$verified_output_dir" 2>/dev/null)"; then + fail 'verified output dir does not resolve' + return 1 + fi + if [[ "$canonical_metric_root" != "$verified_output_dir" || ! -d "$verified_output_dir" || -L "$verified_output_dir" ]]; then + fail 'verified output dir must be a canonical regular directory' + return 1 + fi + if ! metric_root_mode="$(stat -c %a -- "$verified_output_dir")"; then + fail 'cannot stat verified output root' + return 1 + fi + if [[ "$metric_root_mode" != 700 ]]; then + fail 'verified output root mode must be 0700' + return 1 + fi + if ! initial_sha="$(inventory_sha "$verified_output_dir" target-initial)"; then + return 1 + fi + if ! post_sha="$(inventory_sha "$verified_output_dir" post-substrate)"; then + return 1 + fi + + proof="$ROLLBACK_ROOT/blackbox-source-proof.env" + if ! read_exact_env "$proof" schema rollback_id nginx_sha256 tested_at_utc \ + grafana_remote_addr grafana_status storage_admin_remote_addr storage_admin_status \ + db_admin_remote_addr db_admin_status; then + return 1 + fi + if [[ "${PARSED_ENV[schema]}" != "$SOURCE_PROOF_SCHEMA" ]]; then + fail 'source-proof schema mismatch' + return 1 + fi + if [[ "${PARSED_ENV[rollback_id]}" != "$PLATFORM_OBSERVABILITY_ROLLBACK_ID" ]]; then + fail 'source-proof rollback ID mismatch' + return 1 + fi + if [[ "${PARSED_ENV[nginx_sha256]}" != "$deny_sha" ]]; then + fail 'source-proof Nginx hash mismatch' + return 1 + fi + if [[ "${PARSED_ENV[grafana_status]}" != 403 || "${PARSED_ENV[storage_admin_status]}" != 403 || + "${PARSED_ENV[db_admin_status]}" != 403 ]]; then + fail 'source-proof statuses are not exact 403' + return 1 + fi + if ! python3 - "${PARSED_ENV[grafana_remote_addr]}" "${PARSED_ENV[storage_admin_remote_addr]}" \ + "${PARSED_ENV[db_admin_remote_addr]}" <<'PY' +import ipaddress +import sys +for value in sys.argv[1:]: + ipaddress.ip_address(value) +PY + then + fail 'source-proof remote address is invalid' + return 1 + fi + if ! validate_recent_utc "${PARSED_ENV[tested_at_utc]}" source-proof; then + return 1 + fi + + acceptance="$ROLLBACK_ROOT/access-rules-alerts/acceptance.env" + if ! read_exact_env "$acceptance" \ + schema rollback_id target_initial_sha256 post_substrate_sha256 \ + slack_deployment_gate slack_gate_approval_ref slack_gate_accepted_by_uid \ + accepted_at_utc state; then + return 1 + fi + if [[ "${PARSED_ENV[schema]}" != "$RULES_ACCEPTANCE_SCHEMA" ]]; then + fail 'rules-alerts acceptance schema mismatch' + return 1 + fi + if [[ "${PARSED_ENV[rollback_id]}" != "$PLATFORM_OBSERVABILITY_ROLLBACK_ID" ]]; then + fail 'rules-alerts rollback ID mismatch' + return 1 + fi + if [[ "${PARSED_ENV[target_initial_sha256]}" != "$initial_sha" ]]; then + fail 'target-initial acceptance hash mismatch' + return 1 + fi + if [[ "${PARSED_ENV[post_substrate_sha256]}" != "$post_sha" ]]; then + fail 'post-substrate acceptance hash mismatch' + return 1 + fi + if ! validate_slack_acceptance_tuple; then + return 1 + fi + if [[ "${PARSED_ENV[state]}" != accepted ]]; then + fail 'rules-alerts state is not accepted' + return 1 + fi + if ! validate_utc "${PARSED_ENV[accepted_at_utc]}" rules-alerts; then + return 1 + fi + return 0 +} + +validate_exact_certificate() { + local san_output san_set + if ! "$SUDO_BIN" -n "$TEST_BIN" -s "$CERT_DIR/fullchain.pem"; then + fail 'Grafana certificate is missing' + return 1 + fi + if ! "$SUDO_BIN" -n "$TEST_BIN" -s "$CERT_DIR/privkey.pem"; then + fail 'Grafana private key is missing' + return 1 + fi + if ! "$SUDO_BIN" -n openssl x509 -in "$CERT_DIR/fullchain.pem" -noout -checkend 0 >/dev/null; then + fail 'Grafana certificate is expired' + return 1 + fi + if ! san_output="$("$SUDO_BIN" -n openssl x509 -in "$CERT_DIR/fullchain.pem" -noout -ext subjectAltName)"; then + fail 'cannot read Grafana certificate SAN' + return 1 + fi + if ! san_set="$(printf '%s\n' "$san_output" | grep -oE 'DNS:[^,[:space:]]+' | sed 's/^DNS://' | sort -u)"; then + fail 'cannot parse Grafana certificate SAN' + return 1 + fi + if [[ "$san_set" != "$GRAFANA_HOST" ]]; then + fail 'Grafana certificate SAN set is not exact' + return 1 + fi + return 0 +} + +validate_cloudflare_credentials() { + local contract + "$SUDO_BIN" -n "$TEST_BIN" -f "$CREDENTIALS" || fail "Cloudflare credential is missing: $CREDENTIALS" + expected_owner_mode "$CREDENTIALS" 600 || fail 'Cloudflare credential must be root:root 0600' + contract="$("$SUDO_BIN" -n awk -F= ' + BEGIN { count=0; valid=0; unexpected=0 } + /^[[:space:]]*($|#)/ { next } + /^[[:space:]]*dns_cloudflare_api_token[[:space:]]*=/ { + count++; value=$0; sub(/^[^=]*=[[:space:]]*/, "", value); sub(/[[:space:]]+$/, "", value) + if (length(value) >= 20) valid++; next + } + { unexpected++ } + END { printf "%d:%d:%d", count, valid, unexpected } + ' "$CREDENTIALS")" + [[ "$contract" == 1:1:0 ]] || fail 'Cloudflare credential must contain one nonempty API token key only' +} + +certificate_mode() { + if validate_exact_certificate 2>/dev/null; then + printf 'HOST_NGINX_GRAFANA_CERTIFICATE=READY\n' + return 0 + fi + [[ "$certbot_email" == *@* ]] || fail 'missing exact certificate; --certbot-email is required for issuance' + validate_cloudflare_credentials + [[ -x "$CERTBOT_BIN" ]] || fail "supported Certbot is missing: $CERTBOT_BIN" + "$CERTBOT_BIN" plugins 2>/dev/null | grep -q dns-cloudflare || fail 'Certbot dns-cloudflare plugin is missing' + confirm "issue the exact Grafana certificate" + "$SUDO_BIN" "$CERTBOT_BIN" certonly --dns-cloudflare \ + --dns-cloudflare-credentials "$CREDENTIALS" --dns-cloudflare-propagation-seconds 60 \ + --cert-name "$CERT_NAME" --domains "$GRAFANA_HOST" --non-interactive --agree-tos \ + --email "$certbot_email" + validate_exact_certificate + printf 'HOST_NGINX_GRAFANA_CERTIFICATE=READY\n' +} + +status_code() { + local code + if code="$(curl --disable --noproxy '*' --silent --show-error --output /dev/null \ + --connect-timeout 2 --max-time 5 --write-out '%{http_code}' "$@" 2>/dev/null)"; then + printf '%s' "$code" + else + printf 000 + fi +} + +external_preflight() { + local stage=$1 code host + "$SYSTEMCTL_BIN" is-active --quiet nginx || fail 'Nginx is not active' + "$SUDO_BIN" -n "$NGINX_BIN" -t + if [[ "$TEST_MODE" == 1 ]]; then + "$PREFLIGHT_COMMAND" "$stage" + return + fi + case "$stage" in + metrics) return 0 ;; + deny) + validate_exact_certificate + [[ -z "$(ss -H -ltn '( sport = :1 )')" ]] || fail 'TCP 127.0.0.1:1 guard listener is not absent' + return 0 + ;; + full) ;; + *) fail "unknown transition stage: $stage" ;; + esac + validate_exact_certificate + [[ "$(dig +time=2 +tries=1 +short "@$LAN_IP" A "$GRAFANA_HOST" | tail -n1)" == "$LAN_IP" ]] || fail 'LAN private DNS answer mismatch' + [[ "$(dig +time=2 +tries=1 +short "@$TAIL_IP" A "$GRAFANA_HOST" | tail -n1)" == "$TAIL_IP" ]] || fail 'Tailscale private DNS answer mismatch' + [[ -z "$(dig +time=2 +tries=1 +short @1.1.1.1 A "$GRAFANA_HOST" | tr -d '[:space:]')" ]] || fail 'Grafana public A exists' + [[ -z "$(dig +time=2 +tries=1 +short @1.1.1.1 AAAA "$GRAFANA_HOST" | tr -d '[:space:]')" ]] || fail 'Grafana public AAAA exists' + kubectl -n observability exec deployment/grafana -c grafana --request-timeout=10s -- \ + getent ahostsv4 "$GRAFANA_HOST" | awk -v ip="$LAN_IP" '$1 == ip { found=1 } END { exit !found }' || + fail 'Grafana Pod private DNS answer mismatch' + kubectl -n observability get ingress grafana -o json --request-timeout=10s | jq -e --arg host "$GRAFANA_HOST" ' + .spec.ingressClassName == "traefik" and (.spec.tls // [] | length == 0) and + ([.spec.rules[] | select(.host == $host) | .http.paths[] | + select(.path == "/" and .pathType == "Prefix" and + .backend.service.name == "grafana" and .backend.service.port.number == 80)] | length == 1) + ' >/dev/null || fail 'Grafana Ingress contract is not ready' + kubectl -n observability get endpointslice -l kubernetes.io/service-name=grafana -o json --request-timeout=10s | + jq -e '[.items[].endpoints[] | select(.conditions.ready == true) | .addresses[]] | length > 0' >/dev/null || + fail 'Grafana EndpointSlice has no ready address' + [[ "$(status_code --header "Host: $GRAFANA_HOST" http://127.0.0.1:30080/api/health)" == 200 ]] || fail 'Grafana internal health is not 200' + [[ "$(status_code --resolve "$GITEA_HOST:443:127.0.0.1" "https://$GITEA_HOST/api/healthz")" == 200 ]] || fail 'Gitea regression failed' + [[ "$(status_code --resolve "$KEYCLOAK_HOST:443:127.0.0.1" "https://$KEYCLOAK_HOST/realms/hyeonworks/.well-known/openid-configuration")" == 200 ]] || fail 'Keycloak regression failed' + for host in "$STORAGE_HOST" "$DB_HOST"; do + code="$(status_code --interface "$LAN_IP" --resolve "$host:443:$LAN_IP" "https://$host/")" + [[ "$code" == 200 || "$code" == 302 || "$code" == 303 ]] || fail "admin endpoint regression failed: $host=$code" + done + [[ "$(status_code http://$LAN_IP:30080/)" == 000 ]] || fail 'LAN NodePort 30080 is reachable' + [[ "$(status_code --insecure https://$LAN_IP:30443/)" == 000 ]] || fail 'LAN NodePort 30443 is reachable' +} + +stage_postcheck_once() { + local stage=$1 + local code host + [[ "$(status_code --resolve "$GITEA_HOST:443:127.0.0.1" "https://$GITEA_HOST/metrics")" == 404 ]] || return 1 + [[ "$(status_code --resolve "$GITEA_HOST:443:127.0.0.1" "https://$GITEA_HOST/api/healthz")" == 200 ]] || return 1 + [[ "$(status_code --resolve "$KEYCLOAK_HOST:443:127.0.0.1" "https://$KEYCLOAK_HOST/realms/hyeonworks/.well-known/openid-configuration")" == 200 ]] || return 1 + for host in "$STORAGE_HOST" "$DB_HOST"; do + code="$(status_code --interface "$LAN_IP" --resolve "$host:443:$LAN_IP" "https://$host/")" + [[ "$code" == 200 || "$code" == 302 || "$code" == 303 ]] || return 1 + [[ "$(status_code --resolve "$host:443:127.0.0.1" "https://$host/")" == 403 ]] || return 1 + done + [[ "$(status_code http://$LAN_IP:30080/)" == 000 ]] || return 1 + [[ "$(status_code --insecure https://$LAN_IP:30443/)" == 000 ]] || return 1 + [[ "$stage" == metrics ]] && return 0 + + [[ "$(status_code --resolve "$GRAFANA_HOST:443:127.0.0.1" "https://$GRAFANA_HOST/")" == 403 ]] || return 1 + [[ "$(status_code --resolve "$GRAFANA_HOST:443:127.0.0.1" "https://$GRAFANA_HOST/metrics")" == 404 ]] || return 1 + code="$(status_code --interface "$LAN_IP" --resolve "$GRAFANA_HOST:443:$LAN_IP" "https://$GRAFANA_HOST/")" + if [[ "$stage" == deny ]]; then + [[ "$code" == 502 ]] || return 1 + else + [[ "$stage" == full && ( "$code" == 200 || "$code" == 302 ) ]] || return 1 + fi +} + +external_postcheck() { + local stage=$1 stable=0 attempt + "$SYSTEMCTL_BIN" is-active --quiet nginx || return 1 + for ((attempt=1; attempt<=15; attempt++)); do + if { [[ "$TEST_MODE" == 1 ]] && "$POSTCHECK_COMMAND" "$stage"; } || + { [[ "$TEST_MODE" == 0 ]] && stage_postcheck_once "$stage"; }; then + stable=$((stable + 1)) + (( stable >= 3 )) && break + else + stable=0 + fi + [[ "$TEST_MODE" == 0 && attempt -lt 15 ]] && sleep 1 + done + (( stable >= 3 )) || return 1 + [[ "$TEST_MODE" == 1 ]] && return 0 + if curl --disable --noproxy '*' --insecure --silent --output /dev/null --connect-timeout 2 --max-time 5 \ + --resolve unconfigured.invalid:443:127.0.0.1 https://unconfigured.invalid/ 2>/dev/null; then + return 1 + fi +} + +confirm() { + local description=$1 answer + printf 'Type APPLY to %s: ' "$description" + if [[ "$TEST_MODE" == 1 ]]; then + answer=${PLATFORM_HOST_NGINX_CONFIRMATION:-} + printf '%s\n' "$answer" + else + [[ -t 0 ]] || fail '--execute requires an interactive terminal' + read -r answer + fi + [[ "$answer" == APPLY ]] || fail 'cancelled' +} + +validate_full_first_cutover_residue_absent() { + local host_root child + host_root="$ROLLBACK_ROOT/host-nginx" + if ! "$SUDO_BIN" -n "$TEST_BIN" ! -e "$host_root" || + ! "$SUDO_BIN" -n "$TEST_BIN" ! -L "$host_root"; then + fail 'full first-cutover Host Nginx evidence root already exists or is unsafe' + return 1 + fi + for child in \ + "$host_root/stages.tsv" \ + "$host_root/payloads" \ + "$host_root/payloads/full-prior-0001.conf"; do + if ! "$SUDO_BIN" -n "$TEST_BIN" ! -e "$child" || + ! "$SUDO_BIN" -n "$TEST_BIN" ! -L "$child"; then + fail 'full first-cutover Host Nginx evidence residue exists or is unsafe' + return 1 + fi + done + if ! "$SUDO_BIN" -n "$TEST_BIN" ! -e "$host_root" || + ! "$SUDO_BIN" -n "$TEST_BIN" ! -L "$host_root"; then + fail 'full first-cutover Host Nginx evidence root changed during validation' + return 1 + fi +} + +prepare_stage_ledger() { + local stage=$1 previous_sha=$2 candidate_sha=$3 host_root payload_root ledger attempt payload_rel payload tmp + host_root="$ROLLBACK_ROOT/host-nginx" + payload_root="$host_root/payloads" + ledger="$host_root/stages.tsv" + if ! ensure_root_directory_no_follow "$host_root"; then + return 1 + fi + if ! ensure_root_directory_no_follow "$payload_root"; then + return 1 + fi + if "$SUDO_BIN" -n "$TEST_BIN" -L "$ledger"; then + fail 'Host Nginx ledger must not be a symlink' + return 1 + elif "$SUDO_BIN" -n "$TEST_BIN" -e "$ledger"; then + if ! validate_root_file_no_follow "$ledger" 600; then + return 1 + fi + if [[ "$(root_cat "$ledger" | head -n1)" != "$STAGE_LEDGER_HEADER" ]]; then + fail 'Host Nginx ledger header mismatch' + return 1 + fi + else + tmp="$(mktemp /tmp/platform-host-nginx-ledger.XXXXXX)" + printf '%s\n' "$STAGE_LEDGER_HEADER" >"$tmp" + if ! "$SUDO_BIN" install -o root -g root -m 0600 "$tmp" "$ledger"; then + rm -f -- "$tmp" + return 1 + fi + rm -f -- "$tmp" || return 1 + if ! validate_root_file_no_follow "$ledger" 600; then + return 1 + fi + fi + attempt=$(( $(root_cat "$ledger" | awk -F '\t' -v stage="$stage" 'NR > 1 && $1 == stage { count++ } END { print count+1 }') )) + printf -v payload_rel 'payloads/%s-prior-%04d.conf' "$stage" "$attempt" + payload="$host_root/$payload_rel" + if "$SUDO_BIN" -n "$TEST_BIN" -L "$payload" || "$SUDO_BIN" -n "$TEST_BIN" -e "$payload"; then + fail "ledger payload already exists or is a symlink: $payload_rel" + return 1 + fi + "$SUDO_BIN" install -o root -g root -m 0600 "$ACTIVE" "$payload" + if ! validate_root_file_no_follow "$payload" 600; then + return 1 + fi + payload_sha="$(root_cat "$payload" | sha256sum | awk '{print $1}')" + [[ "$payload_sha" == "$previous_sha" ]] || fail 'ledger payload hash mismatch' + tmp="$(mktemp /tmp/platform-host-nginx-ledger.XXXXXX)" + root_cat "$ledger" >"$tmp" + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$stage" "$previous_sha" "$candidate_sha" \ + "$payload_rel" "$payload_sha" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >>"$tmp" + if ! "$SUDO_BIN" install -o root -g root -m 0600 "$tmp" "$ledger"; then + rm -f -- "$tmp" + return 1 + fi + rm -f -- "$tmp" || return 1 + rollback_payload=$payload + rollback_previous_sha=$previous_sha + rollback_stage=$stage + return 0 +} + +rollback() { + local rc=0 actual + set +e + "$SUDO_BIN" -n "$TEST_BIN" -f "$rollback_payload" || rc=1 + "$SUDO_BIN" -n "$TEST_BIN" -L "$rollback_payload" && rc=1 + expected_owner_mode "$rollback_payload" 600 || rc=1 + actual="$(root_cat "$rollback_payload" 2>/dev/null | sha256sum | awk '{print $1}')" + [[ "$actual" == "$rollback_previous_sha" ]] || rc=1 + if (( rc == 0 )); then + "$SUDO_BIN" install -o root -g root -m 0644 "$rollback_payload" "$ACTIVE" || rc=1 + [[ "$(sha_of "$ACTIVE")" == "$rollback_previous_sha" ]] || rc=1 + "$SUDO_BIN" "$NGINX_BIN" -t || rc=1 + "$SUDO_BIN" "$SYSTEMCTL_BIN" reload nginx || rc=1 + "$SUDO_BIN" "$SYSTEMCTL_BIN" is-active --quiet nginx || rc=1 + fi + if (( rc == 0 )); then + rollback_armed=false + printf 'HOST_NGINX_OBSERVABILITY_ROLLBACK=PASS\n' >&2 + else + printf 'HOST_NGINX_OBSERVABILITY_ROLLBACK=FAIL\nMANUAL_RECOVERY_REQUIRED=YES\n' >&2 + fi + set -e + return "$rc" +} + +on_exit() { + local rc=$? + trap - EXIT INT TERM + if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then + rollback || true + fi + exit "$rc" +} + +transition() { + local stage=$1 expected=$2 candidate=$3 description=$4 previous_sha candidate_sha active_sha + previous_sha="$(sha_of "$expected")" + candidate_sha="$(sha_of "$candidate")" + active_sha="$(sha_of "$ACTIVE")" + if [[ "$stage" == full ]]; then + if ! validate_full_evidence "$(sha_of "$DENY_CANDIDATE")"; then + return 1 + fi + fi + if [[ "$stage" == full && "$active_sha" == "$candidate_sha" ]]; then + fail 'full cutover is already active without first-cutover provenance' + return 1 + fi + if [[ "$active_sha" == "$candidate_sha" ]]; then + external_preflight "$stage" + external_postcheck "$stage" || fail "already-active $stage stage is unhealthy" + printf 'HOST_NGINX_%s_STAGE=ALREADY_ACTIVE\n' "${stage^^}" + return 0 + fi + [[ "$active_sha" == "$previous_sha" ]] || fail "$stage active hash gate failed: $active_sha" + if [[ "$stage" == full ]]; then + validate_full_first_cutover_residue_absent || return 1 + fi + external_preflight "$stage" + [[ "$(sha_of "$ACTIVE")" == "$previous_sha" ]] || fail "$stage active config changed during preflight" + printf 'HOST_NGINX_STAGE=%s\nHOST_NGINX_PREVIOUS_SHA256=%s\nHOST_NGINX_CANDIDATE_SHA256=%s\n' \ + "$stage" "$previous_sha" "$candidate_sha" + confirm "$description" + [[ "$(sha_of "$ACTIVE")" == "$previous_sha" ]] || fail "$stage active config changed before commit" + [[ "$(sha_of "$candidate")" == "$candidate_sha" ]] || fail "$stage candidate changed before commit" + if ! prepare_stage_ledger "$stage" "$previous_sha" "$candidate_sha"; then + fail 'fresh rollback ID + source proof + complete Task 6 required' + return 1 + fi + timestamp_backup="${ACTIVE}.before-observability-${stage}-$(date -u +%Y%m%dT%H%M%SZ)" + if "$SUDO_BIN" -n "$TEST_BIN" -e "$timestamp_backup" || "$SUDO_BIN" -n "$TEST_BIN" -L "$timestamp_backup"; then + fail "timestamp backup already exists or is unsafe: $timestamp_backup" + fail 'fresh rollback ID + source proof + complete Task 6 required' + return 1 + fi + if ! "$SUDO_BIN" install -o root -g root -m 0644 "$ACTIVE" "$timestamp_backup"; then + fail 'fresh rollback ID + source proof + complete Task 6 required' + return 1 + fi + if [[ "$(sha_of "$timestamp_backup")" != "$previous_sha" ]]; then + fail 'timestamp backup hash mismatch' + fail 'fresh rollback ID + source proof + complete Task 6 required' + return 1 + fi + rollback_armed=true + "$SUDO_BIN" install -o root -g root -m 0644 "$candidate" "$ACTIVE" + [[ "$(sha_of "$ACTIVE")" == "$candidate_sha" ]] || fail 'installed active hash mismatch' + "$SUDO_BIN" "$SYSTEMCTL_BIN" daemon-reload + "$SUDO_BIN" "$NGINX_BIN" -t + "$SUDO_BIN" "$SYSTEMCTL_BIN" reload nginx + "$SUDO_BIN" "$SYSTEMCTL_BIN" is-active --quiet nginx + external_postcheck "$stage" || fail "$stage bounded postcheck failed" + rollback_armed=false + printf 'HOST_NGINX_%s_STAGE=PASS\nHOST_NGINX_TIMESTAMP_BACKUP=%s\n' "${stage^^}" "$timestamp_backup" +} + +main() { + parse_args "$@" + validate_test_mode + if [[ "$TEST_MODE" == 0 ]]; then + validate_production_command_boundaries + fi + source_contract + if [[ "$execute" == false ]]; then + print_plan + return 0 + fi + for cmd in awk cat curl date dig grep head install jq kubectl mktemp openssl python3 readlink sed sha256sum sort ss stat; do + command -v "$cmd" >/dev/null 2>&1 || fail "required command is missing: $cmd" + done + [[ -x "$NGINX_BIN" && -x "$SYSTEMCTL_BIN" && -x "$SUDO_BIN" ]] || fail 'required host command boundary is missing' + "$SUDO_BIN" -v + validate_host_files + validate_rollback_root + trap on_exit EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + case "$mode" in + metrics) transition metrics "$ADMIN_CANDIDATE" "$METRICS_CANDIDATE" 'install only the Gitea metrics guard' ;; + certificate) certificate_mode ;; + deny) transition deny "$METRICS_CANDIDATE" "$DENY_CANDIDATE" 'install only the Grafana deny guard' ;; + full) transition full "$DENY_CANDIDATE" "$FULL_CANDIDATE" 'replace only the Grafana deny upstream with Traefik' ;; + *) fail "invalid mode: $mode" ;; + esac +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/scripts/bootstrap/apply-k3s-secret-encryption.sh b/scripts/bootstrap/apply-k3s-secret-encryption.sh new file mode 100644 index 0000000..e8e9749 --- /dev/null +++ b/scripts/bootstrap/apply-k3s-secret-encryption.sh @@ -0,0 +1,1624 @@ +#!/usr/bin/env bash + +# Fail-stop bootstrap for enabling k3s Secret encryption. This file is +# source-safe so tests can replace host-operation functions without exposing a +# production environment switch for fake execution. +set -Eeuo pipefail + +if [[ "${BASH_SOURCE[0]}" == */* ]]; then + readonly KSEB_SCRIPT_DIR="${BASH_SOURCE[0]%/*}" +else + readonly KSEB_SCRIPT_DIR='.' +fi +readonly KSEB_ROOT="$(cd -- "${KSEB_SCRIPT_DIR}/../.." && pwd -P)" +readonly KSEB_LIBRARY="${KSEB_ROOT}/scripts/lib/k3s-secret-encryption.sh" +readonly KSEB_LOCAL_RECOVERY_LIBRARY="${KSEB_ROOT}/scripts/lib/k3s-local-recovery.sh" +readonly KSEB_VALIDATOR="${KSEB_ROOT}/scripts/validate/k3s-secret-encryption.sh" +readonly KSEB_LOCAL_RECOVERY_VALIDATOR="${KSEB_ROOT}/scripts/validate/k3s-local-recovery.sh" +readonly KSEB_LOCAL_RECOVERY_CONTRACT="${KSEB_ROOT}/infrastructure/security/k3s/local-recovery.env" +readonly KSEB_DROPIN_SOURCE="${KSEB_ROOT}/infrastructure/security/k3s/40-secrets-encryption.yaml" +readonly KSEB_DATA_DIR='/var/lib/rancher/k3s' +readonly KSEB_CONFIG_DIR='/etc/rancher/k3s' +readonly KSEB_POST_MARKER='/etc/rancher/k3s/platform-post-bundle.env' +readonly KSEB_RESTORE_EVIDENCE='/etc/rancher/k3s/platform-restore-evidence.env' +readonly KSEB_RECOVERY_DOCUMENT='bootstrap/manual/k3s-secret-encryption.md' +readonly KSEB_MINIMUM_FREE_BYTES=10737418240 +readonly KSEB_SECURE_PATH='/usr/sbin:/usr/bin:/sbin:/bin' +readonly KSEB_API_RECOVERY_BUDGET=430 +readonly KSEB_NODE_RECOVERY_BUDGET=130 +readonly KSEB_WORKLOAD_RECOVERY_BUDGET=30 +readonly KSEB_ESTIMATE_METADATA_BYTES=16777216 +readonly KSEB_SIGNED_MAX=9223372036854775807 +KSEB_BACKUP_ROOT_IDENTITY='' +KSEB_LAST_SAFE_STATE='unknown' + +# shellcheck source=/dev/null +source "$KSEB_LIBRARY" +# shellcheck source=/dev/null +source "$KSEB_LOCAL_RECOVERY_LIBRARY" + +_kseb_fail() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +_kseb_terminal_failure() { + local state="$1" + case "$state" in + unknown|initial_unsafe|datastore_unsafe|backup_root_unsafe|disabled_state_drift|enabled_start_drift|enabled_finished_drift|\ + reencrypt_stage_start|reencrypt_stage_unexpected|reencrypt_active_timeout|hash_mismatch|\ + expectation_failed|final_validation_failed|pre_backup_failed|post_backup_failed|\ + service_recovery_failed|marker_install_failed|partial_enable_failed|partial_dropin_failed|\ + partial_initial_restart_failed|partial_final_restart_failed) ;; + *) state=unknown ;; + esac + printf 'ERROR: current-state=%s recovery=%s\n' "$state" "$KSEB_RECOVERY_DOCUMENT" >&2 + return 1 +} + +_kseb_run_quiet() { + "$@" >/dev/null 2>&1 +} + +_kseb_privileged_exec() { + /usr/bin/sudo "$@" >/dev/null 2>&1 +} + +_kseb_usage() { + cat <<'USAGE' +사용법: + bash scripts/bootstrap/apply-k3s-secret-encryption.sh + bash scripts/bootstrap/apply-k3s-secret-encryption.sh \ + --execute --backup-root "$K3S_RECOVERY_ROOT" \ + --recovery-policy encrypted-off-host + bash scripts/bootstrap/apply-k3s-secret-encryption.sh \ + --execute --rotate-existing --backup-root "$K3S_RECOVERY_ROOT" \ + --recovery-policy encrypted-off-host + bash scripts/bootstrap/apply-k3s-secret-encryption.sh \ + --execute --backup-root /srv/recovery/k3s \ + --recovery-policy local-separate-disk-luks + bash scripts/bootstrap/apply-k3s-secret-encryption.sh \ + --estimate-recovery-bytes + +인자 없이 실행하면 읽기 전용 상태와 필요한 운영자 확인만 출력합니다. +실행에는 명시적인 recovery policy와 해당 정책의 암호화 recovery mount 검증이 필요합니다. +USAGE +} + +_kseb_safe_remove_handoff() { + local directory="$1" + case "$directory" in + /tmp/platform-k3s-encryption.[A-Za-z0-9]*) rm -rf -- "$directory" ;; + *) return 1 ;; + esac +} + +_kseb_inventory_value() { + local file="$1" key="$2" value count + count="$(awk -F= -v key="$key" '$1 == key { count++ } END { print count + 0 }' "$file")" || return 1 + [[ "$count" == 1 ]] || return 1 + value="$(awk -F= -v key="$key" '$1 == key { sub(/^[^=]*=/, ""); print }' "$file")" || return 1 + [[ "$value" =~ ^[A-Za-z0-9_.+-]+$ ]] || return 1 + printf '%s\n' "$value" +} + +_kseb_read_status_json() { + /usr/bin/timeout --signal=TERM --kill-after=1s 9s \ + /usr/bin/sudo --non-interactive -- /usr/local/bin/k3s \ + secrets-encrypt status --output json 2>/dev/null +} + +_kseb_load_inventory() { + local handoff inventory encryption rotation datastore integrity server_hashes + local status_class provider=none owner=none status_json + umask 077 + handoff="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" || return 1 + chmod 0700 "$handoff" || { _kseb_safe_remove_handoff "$handoff"; return 1; } + if ! bash "$KSEB_VALIDATOR" --verified-output-dir "$handoff" >/dev/null; then + status_json="$(_kseb_read_status_json)" || { _kseb_safe_remove_handoff "$handoff"; return 1; } + status_class="$(classify_encryption_status "$status_json")" + _kseb_safe_remove_handoff "$handoff" + if [[ "$status_class" == hash_mismatch ]]; then + printf 'hash_mismatch|unsafe|ambiguous|invalid|ambiguous|mismatch|mismatch\n' + return 0 + fi + return 1 + fi + inventory="$handoff/inventory.env" + encryption="$(_kseb_inventory_value "$inventory" encryption)" || { _kseb_safe_remove_handoff "$handoff"; return 1; } + rotation="$(_kseb_inventory_value "$inventory" rotation)" || { _kseb_safe_remove_handoff "$handoff"; return 1; } + datastore="$(_kseb_inventory_value "$inventory" datastore)" || { _kseb_safe_remove_handoff "$handoff"; return 1; } + integrity="$(_kseb_inventory_value "$inventory" integrity)" || { _kseb_safe_remove_handoff "$handoff"; return 1; } + server_hashes="$(_kseb_inventory_value "$inventory" server_hashes)" || { _kseb_safe_remove_handoff "$handoff"; return 1; } + _kseb_safe_remove_handoff "$handoff" || return 1 + + case "${encryption}/${rotation}" in + Disabled/none) status_class=disabled_no_config ;; + Disabled/start) status_class=transition_start ;; + Enabled/start|Enabled/reencrypt_finished) status_class=enabled_stable ;; + *) status_class=invalid ;; + esac + if [[ "$status_class" == enabled_stable ]]; then + status_json="$(_kseb_read_status_json)" || return 1 + provider="$(classify_encryption_provider "$status_json")" + owner="$(detect_effective_encryption_config_owner)" || owner=ambiguous + fi + printf '%s|%s|%s|%s|%s|%s|%s\n' \ + "$status_class" "$rotation" "$datastore" "$provider" "$owner" "$integrity" "$server_hashes" +} + +_kseb_require_exact_enabled_start() { + local expected_backend="$1" inventory status rotation backend provider owner integrity server_hashes + inventory="$(_kseb_load_inventory)" || return 1 + IFS='|' read -r status rotation backend provider owner integrity server_hashes <<<"$inventory" + [[ "$status" == enabled_stable && "$rotation" == start && "$backend" == "$expected_backend" && + "$provider" == aescbc && "$owner" == aescbc/* && "$owner" != aescbc/ambiguous && + "$integrity" == match && "$server_hashes" == match ]] +} + +_kseb_require_exact_disabled() { + local expected_backend="$1" inventory status rotation backend provider owner integrity server_hashes + inventory="$(_kseb_load_inventory)" || return 1 + IFS='|' read -r status rotation backend provider owner integrity server_hashes <<<"$inventory" + [[ "$status" == disabled_no_config && "$rotation" == none && + "$backend" == "$expected_backend" && ( "$backend" == sqlite || "$backend" == embedded-etcd ) && + "$provider" == none && "$owner" == none ]] +} + +_kseb_require_exact_finished() { + local expected_backend="$1" inventory status rotation backend provider owner integrity server_hashes + inventory="$(_kseb_load_inventory)" || return 1 + IFS='|' read -r status rotation backend provider owner integrity server_hashes <<<"$inventory" + [[ "$status" == enabled_stable && "$rotation" == reencrypt_finished && + "$backend" == "$expected_backend" && "$provider" == aescbc && + "$owner" == aescbc/* && "$owner" != aescbc/ambiguous && + "$integrity" == match && "$server_hashes" == match ]] +} + +_kseb_dry_run() { + local inventory="$1" status rotation backend ignored + IFS='|' read -r status rotation backend ignored <<<"$inventory" + printf '현재 상태: %s (rotation=%s, datastore=%s)\n' "$status" "$rotation" "$backend" + printf '실행 시 --recovery-policy와 현재 context의 정책별 확인이 필요합니다.\n' + printf 'DRY RUN PASS: 변경하지 않았습니다.\n' +} + +_kseb_current_context() { + /usr/bin/timeout --signal=TERM --kill-after=1s 9s \ + /usr/bin/sudo --non-interactive -- /usr/local/bin/k3s \ + kubectl config current-context 2>/dev/null +} + +_kseb_prepare_execute_context() { + local context + [[ -t 0 ]] || { _kseb_fail '--execute는 대화형 터미널이 필요합니다'; return 1; } + /usr/bin/sudo -v || return 1 + context="$(_kseb_current_context)" || return 1 + [[ -n "$context" && "$context" != *$'\n'* ]] || return 1 + printf '%s\n' "$context" +} + +_kseb_confirm_apply() { + local context="$1" answer + [[ -n "$context" && "$context" != *$'\n'* ]] || return 1 + printf 'Type APPLY %s to continue: ' "$context" + read -r answer + [[ "$answer" == "APPLY $context" ]] || { _kseb_fail '취소했습니다'; return 1; } +} + +_kseb_local_recovery_contract() { + printf '%s\n' "$KSEB_LOCAL_RECOVERY_CONTRACT" +} + +_kseb_local_recovery_root() { + local contract root + [[ "$-" != *x* ]] || return 1 + contract="$(_kseb_local_recovery_contract)" || return 1 + [[ -n "$contract" && "$contract" != *$'\n'* ]] || return 1 + root="$(_k3slr_config_value "$contract" K3SLR_INNER_MOUNT)" || return 1 + [[ -n "$root" && "$root" != *$'\n'* ]] || return 1 + printf '%s\n' "$root" +} + +_kseb_local_validator_command() { + "$@" +} + +_kseb_safe_remove_local_validator_handoff() { + local directory="$1" path cleanup_rc=0 + case "$directory" in + /tmp/platform-k3s-local-validator.[A-Za-z0-9]*) + [[ -d "$directory" && ! -L "$directory" ]] || return 1 + for path in "$directory/stdout" "$directory/stderr"; do + if [[ -e "$path" || -L "$path" ]]; then + [[ -f "$path" && ! -L "$path" ]] || return 1 + /usr/bin/rm -- "$path" >/dev/null 2>&1 || cleanup_rc=1 + fi + done + /usr/bin/rmdir -- "$directory" >/dev/null 2>&1 || cleanup_rc=1 + return "$cleanup_rc" + ;; + *) return 1 ;; + esac +} + +_kseb_local_validator_result_is_exact() { + local command_rc="$1" stdout_file="$2" stderr_file="$3" output='' expected stdout_size stderr_size + [[ "$command_rc" =~ ^(0|[1-9][0-9]{0,2})$ ]] || return 1 + (( command_rc == 0 )) || return 1 + [[ -f "$stdout_file" && ! -L "$stdout_file" && -f "$stderr_file" && ! -L "$stderr_file" ]] || return 1 + stdout_size="$(/usr/bin/stat --format='%s' -- "$stdout_file")" || return 1 + stderr_size="$(/usr/bin/stat --format='%s' -- "$stderr_file")" || return 1 + [[ "$stdout_size" == 86 && "$stderr_size" == 0 ]] || return 1 + if IFS= read -r -d '' output <"$stdout_file"; then + return 1 + fi + printf -v expected '%s\n' \ + 'Recovery device: match' \ + 'Recovery state: open' \ + 'Lineage: match' \ + 'Latest bundle: not_checked' + [[ "$output" == "$expected" ]] +} + +_kseb_run_local_recovery_validator() ( + local handoff='' stdout_file stderr_file command_rc=0 result_rc=1 cleanup_rc=0 + [[ "$-" != *x* ]] || return 1 + umask 077 + handoff="$(/usr/bin/mktemp -d /tmp/platform-k3s-local-validator.XXXXXX)" || return 1 + /usr/bin/chmod 0700 -- "$handoff" || { + _kseb_safe_remove_local_validator_handoff "$handoff" >/dev/null 2>&1 || true + return 1 + } + trap '_kseb_safe_remove_local_validator_handoff "$handoff" >/dev/null 2>&1 || true' EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + stdout_file="$handoff/stdout" + stderr_file="$handoff/stderr" + : >"$stdout_file" + : >"$stderr_file" + /usr/bin/chmod 0600 -- "$stdout_file" "$stderr_file" || return 1 + ulimit -f 8 || return 1 + set +e + _kseb_local_validator_command \ + /usr/bin/env -i \ + PATH="$KSEB_SECURE_PATH" \ + LC_ALL=C \ + /usr/bin/timeout --signal=TERM --kill-after=1s 60s \ + /usr/bin/bash --noprofile --norc -- \ + "$KSEB_LOCAL_RECOVERY_VALIDATOR" --expect-open \ + >"$stdout_file" 2>"$stderr_file" + command_rc=$? + set -e + if _kseb_local_validator_result_is_exact "$command_rc" "$stdout_file" "$stderr_file"; then + result_rc=0 + fi + _kseb_safe_remove_local_validator_handoff "$handoff" >/dev/null 2>&1 || cleanup_rc=1 + if (( cleanup_rc == 0 )); then + trap - EXIT + else + result_rc=1 + fi + trap - HUP INT TERM + return "$result_rc" +) + +_kseb_validate_recovery_policy() { + local policy="$1" root="$2" expected_root confirmed_root + [[ -n "$root" ]] || return 1 + case "$policy" in + encrypted-off-host) return 0 ;; + local-separate-disk-luks) + expected_root="$(_kseb_local_recovery_root)" || return 1 + [[ -n "$expected_root" && "$expected_root" != *$'\n'* && "$root" == "$expected_root" ]] || return 1 + _kseb_run_local_recovery_validator >/dev/null 2>&1 || return 1 + confirmed_root="$(_kseb_local_recovery_root)" || return 1 + [[ "$confirmed_root" == "$expected_root" ]] || return 1 + _kseb_revalidate_backup_root "$root" + ;; + *) return 1 ;; + esac +} + +_kseb_confirm_recovery() { + local context="$1" policy="$2" answer check + [[ -n "$context" && "$context" != *$'\n'* ]] || return 1 + printf 'Type RECOVERY %s after testing escrow decryption access now: ' "$context" + read -r answer + [[ "$answer" == "RECOVERY $context" ]] || { _kseb_fail 'recovery 접근 확인이 없습니다'; return 1; } + if [[ "$policy" == encrypted-off-host ]]; then + printf 'Type ENCRYPTED %s to attest the mount is encrypted off-host media: ' "$context" + elif [[ "$policy" == local-separate-disk-luks ]]; then + printf 'Type ENCRYPTED %s to attest the mounted local recovery volume is LUKS2 encrypted: ' "$context" + else + return 1 + fi + read -r answer + [[ "$answer" == "ENCRYPTED $context" ]] || { _kseb_fail '암호화 recovery media 확인이 없습니다'; return 1; } + if [[ "$policy" == local-separate-disk-luks ]]; then + printf 'Type LOCAL_RISK_ACCEPTED %s to accept same-host local disk failure risk: ' "$context" + read -r answer + [[ "$answer" == "LOCAL_RISK_ACCEPTED $context" ]] || { + _kseb_fail 'local recovery disk 위험 승인이 없습니다' + return 1 + } + fi + check="$(_kseb_current_context)" || return 1 + [[ "$check" == "$context" ]] || { _kseb_fail 'context가 바뀌었습니다'; return 1; } +} + +_kseb_validate_backup_root() { + local root="$1" kind physical metadata backup_source data_source free_bytes + [[ "$root" =~ ^/[A-Za-z0-9._/-]+$ && "$root" != *'//'* + && "$root" != */../* && "$root" != */./* ]] || return 1 + kind="$(_kseb_privileged_root_kind "$root")" || return 1 + [[ "$kind" == directory ]] || return 1 + _kseb_privileged_backup_root_chain_safe "$root" || return 1 + physical="$(_kseb_privileged_realpath "$root")" || return 1 + [[ "$physical" == "$root" ]] || return 1 + metadata="$(_kseb_privileged_root_metadata "$root")" || return 1 + [[ "$metadata" == '0:0:700:directory' ]] || return 1 + backup_source="$(_kseb_privileged_mount_source "$root")" || return 1 + data_source="$(_kseb_privileged_mount_source "$KSEB_DATA_DIR")" || return 1 + [[ -n "$backup_source" && -n "$data_source" && "$backup_source" != "$data_source" ]] || return 1 + free_bytes="$(_kseb_privileged_free_bytes "$root")" || return 1 + [[ "$free_bytes" =~ ^[0-9]+$ ]] && (( free_bytes >= KSEB_MINIMUM_FREE_BYTES )) || return 1 + KSEB_BACKUP_ROOT_IDENTITY="$(_kseb_privileged_root_identity "$root")" || return 1 + [[ "$KSEB_BACKUP_ROOT_IDENTITY" =~ ^[0-9]+:[0-9]+$ ]] +} + +_kseb_privileged_root_kind() { + local path="$1" + if /usr/bin/sudo /usr/bin/test -L "$path"; then printf 'symlink\n' + elif /usr/bin/sudo /usr/bin/test -d "$path"; then printf 'directory\n' + elif /usr/bin/sudo /usr/bin/test -e "$path"; then printf 'other\n' + else printf 'absent\n'; fi +} + +_kseb_privileged_realpath() { + /usr/bin/sudo /usr/bin/realpath --canonicalize-existing -- "$1" 2>/dev/null +} + +_kseb_privileged_root_metadata() { + /usr/bin/sudo /usr/bin/stat --format='%u:%g:%a:%F' -- "$1" 2>/dev/null +} + +_kseb_privileged_mount_source() { + /usr/bin/sudo /usr/bin/findmnt --noheadings --output SOURCE --target "$1" 2>/dev/null | + awk 'NF == 1 { print }' +} + +_kseb_privileged_free_bytes() { + /usr/bin/sudo /usr/bin/df --block-size=1 --output=avail "$1" 2>/dev/null | + awk 'NR == 2 && $1 ~ /^[0-9]+$/ { print $1 }' +} + +_kseb_privileged_root_identity() { + /usr/bin/sudo /usr/bin/stat --format='%d:%i' -- "$1" 2>/dev/null +} + +_kseb_privileged_path_lstat() { + /usr/bin/sudo /usr/bin/stat --format='%u:%g:%a:%F' -- "$1" 2>/dev/null +} + +_kseb_privileged_backup_root_chain_safe() { + _kseb_backup_root_chain_safe "$1" _kseb_privileged_path_lstat +} + +_kseb_path_lstat() { + /usr/bin/stat --format='%u:%g:%a:%F' -- "$1" 2>/dev/null +} + +_kseb_backup_root_chain_safe() { + local root="$1" reader="${2:-_kseb_path_lstat}" current='/' component metadata uid gid mode file_type + local -a components=() + [[ "$root" =~ ^/[A-Za-z0-9._/-]+$ && "$root" != *'//'* + && "$root" != */../* && "$root" != */./* ]] || return 1 + IFS='/' read -r -a components <<<"${root#/}" + for current in /; do + metadata="$("$reader" "$current")" || return 1 + IFS=: read -r uid gid mode file_type <<<"$metadata" + [[ "$uid" == 0 && "$gid" == 0 && "$mode" =~ ^[0-7]{3,4}$ && "$file_type" == directory ]] || return 1 + (( (8#$mode & 8#022) == 0 )) || return 1 + done + current='' + for component in "${components[@]}"; do + [[ -n "$component" ]] || return 1 + current+="/$component" + metadata="$("$reader" "$current")" || return 1 + IFS=: read -r uid gid mode file_type <<<"$metadata" + [[ "$uid" == 0 && "$gid" == 0 && "$mode" =~ ^[0-7]{3,4}$ && "$file_type" == directory ]] || return 1 + (( (8#$mode & 8#022) == 0 )) || return 1 + done +} + +_kseb_backup_root_identity() { + /usr/bin/stat --format='%d:%i' -- "$1" 2>/dev/null +} + +_kseb_backup_root_identity_matches() { + [[ "$1" =~ ^[0-9]+:[0-9]+$ && "$1" == "$2" ]] +} + +_kseb_revalidate_backup_root() { + local root="$1" identity + [[ -n "$KSEB_BACKUP_ROOT_IDENTITY" ]] || return 1 + _kseb_privileged_backup_root_chain_safe "$root" || return 1 + identity="$(_kseb_privileged_root_identity "$root")" || return 1 + _kseb_backup_root_identity_matches "$KSEB_BACKUP_ROOT_IDENTITY" "$identity" +} + +_kseb_privileged_relative_components_safe() { + local root="$1" relative="$2" current="$root" component + [[ "$relative" =~ ^[A-Za-z0-9._/-]+$ && "$relative" != /* && + "$relative" != *'..'* && "$relative" != *'//'* ]] || return 1 + IFS='/' read -r -a _kseb_relative_parts <<<"$relative" + for component in "${_kseb_relative_parts[@]}"; do + [[ -n "$component" ]] || return 1 + current+="/$component" + if /usr/bin/sudo /usr/bin/test -L "$current"; then return 1; fi + /usr/bin/sudo /usr/bin/test -e "$current" || break + done +} + +_kseb_run_privileged_shell() { + /usr/bin/sudo /usr/bin/bash "$@" +} + +_kseb_transaction_data_dir() { printf '%s\n' "$KSEB_DATA_DIR"; } +_kseb_transaction_config_dir() { printf '%s\n' "$KSEB_CONFIG_DIR"; } +_kseb_transaction_k3s_binary() { printf '/usr/local/bin/k3s\n'; } +_kseb_transaction_systemctl_binary() { printf '/usr/bin/systemctl\n'; } +_kseb_transaction_post_marker() { printf '%s\n' "$KSEB_POST_MARKER"; } +_kseb_transaction_restore_evidence() { printf '%s\n' "$KSEB_RESTORE_EVIDENCE"; } +_kseb_transaction_systemd_paths() { + printf '%s\n' \ + /etc/systemd/system/k3s.service \ + /etc/systemd/system/k3s.service.env \ + /etc/systemd/system/k3s.service.d \ + /usr/lib/systemd/system/k3s.service \ + /lib/systemd/system/k3s.service +} + +_kseb_normalize_signed_uint() { + local value="$1" + [[ "$value" =~ ^[0-9]+$ ]] || return 1 + while [[ "$value" == 0* && "$value" != 0 ]]; do value="${value#0}"; done + if (( ${#value} > ${#KSEB_SIGNED_MAX} )); then return 1; fi + if (( ${#value} == ${#KSEB_SIGNED_MAX} )) && [[ "$value" > "$KSEB_SIGNED_MAX" ]]; then return 1; fi + printf '%s\n' "$value" +} + +_kseb_checked_estimate_add() { + local left right sum + left="$(_kseb_normalize_signed_uint "$1")" || return 1 + right="$(_kseb_normalize_signed_uint "$2")" || return 1 + (( right <= KSEB_SIGNED_MAX - left )) || return 1 + sum=$((left + right)) + printf '%s\n' "$sum" +} + +_kseb_required_recovery_bytes() { + local phase="$1" phase_bytes quarter remainder uplift padded required + case "$phase" in pre|post) ;; *) return 1 ;; esac + phase_bytes="$(_kseb_normalize_signed_uint "$2")" || return 1 + quarter=$((phase_bytes / 4)) + remainder=$((phase_bytes % 4)) + uplift="$quarter" + if (( remainder != 0 )); then + uplift="$(_kseb_checked_estimate_add "$uplift" 1)" || return 1 + fi + padded="$(_kseb_checked_estimate_add "$phase_bytes" "$uplift")" || return 1 + if [[ "$phase" == pre ]]; then + required="$(_kseb_checked_estimate_add "$padded" "$padded")" || return 1 + else + required="$padded" + fi + _kseb_checked_estimate_add "$required" "$KSEB_MINIMUM_FREE_BYTES" +} + +_kseb_require_phase_capacity() { + local phase="$1" backend="$2" root="$3" phase_bytes required free_bytes + phase_bytes="$(_kseb_estimate_phase_bytes "$backend")" || return 1 + phase_bytes="$(_kseb_normalize_signed_uint "$phase_bytes")" || return 1 + required="$(_kseb_required_recovery_bytes "$phase" "$phase_bytes")" || return 1 + free_bytes="$(_kseb_privileged_free_bytes "$root")" || return 1 + free_bytes="$(_kseb_normalize_signed_uint "$free_bytes")" || return 1 + (( free_bytes >= required )) +} + +_kseb_estimator_source_allowed() { + local backend="$1" path="$2" + case "$backend:$path" in + sqlite:/var/lib/rancher/k3s/server/db|\ + embedded-etcd:/var/lib/rancher/k3s/server/db/etcd|\ + sqlite:/var/lib/rancher/k3s/server/token|\ + embedded-etcd:/var/lib/rancher/k3s/server/token|\ + sqlite:/etc/rancher/k3s|\ + embedded-etcd:/etc/rancher/k3s|\ + sqlite:/etc/systemd/system/k3s.service|\ + embedded-etcd:/etc/systemd/system/k3s.service|\ + sqlite:/etc/systemd/system/k3s.service.env|\ + embedded-etcd:/etc/systemd/system/k3s.service.env|\ + sqlite:/etc/systemd/system/k3s.service.d|\ + embedded-etcd:/etc/systemd/system/k3s.service.d|\ + sqlite:/usr/lib/systemd/system/k3s.service|\ + embedded-etcd:/usr/lib/systemd/system/k3s.service|\ + sqlite:/lib/systemd/system/k3s.service|\ + embedded-etcd:/lib/systemd/system/k3s.service|\ + sqlite:/var/lib/rancher/k3s/server/cred/encryption-config.json|\ + embedded-etcd:/var/lib/rancher/k3s/server/cred/encryption-config.json) return 0 ;; + *) return 1 ;; + esac +} + +_kseb_privileged_estimator_source_kind() { + local path="$1" + if /usr/bin/sudo --non-interactive -- /usr/bin/test -L "$path" 2>/dev/null; then + printf 'symlink\n' + elif /usr/bin/sudo --non-interactive -- /usr/bin/test -e "$path" 2>/dev/null; then + printf 'present\n' + elif /usr/bin/sudo --non-interactive -- /usr/bin/test ! -e "$path" 2>/dev/null; then + printf 'absent\n' + else + return 1 + fi +} + +_kseb_privileged_estimator_capture() { + /usr/bin/sudo --non-interactive -- "$@" 2>/dev/null +} + +_kseb_privileged_estimator_lstat() { + _kseb_privileged_estimator_capture /usr/bin/stat \ + --format='%d:%i|%f' -- "$1" +} + +_kseb_privileged_estimator_du() { + /usr/bin/sudo --non-interactive -- /usr/bin/du \ + --summarize --bytes -- "$1" 2>/dev/null +} + +_kseb_estimator_source_identity() { + local path="$1" current='' metadata identity mode_hex mode_type component + local -a components=() + [[ "$path" == /* && "$path" != *'//'* && "$path" != */../* && "$path" != */./* ]] || return 1 + metadata="$(_kseb_privileged_estimator_lstat /)" || return 1 + [[ "$metadata" != *$'\n'* ]] || return 1 + identity="${metadata%%|*}"; mode_hex="${metadata#*|}" + [[ "$identity" =~ ^[0-9]+:[0-9]+$ && "$mode_hex" =~ ^[0-9a-fA-F]{1,8}$ ]] || return 1 + mode_type=$((16#$mode_hex & 8#170000)) + (( mode_type == 8#40000 )) || return 1 + IFS='/' read -r -a components <<<"${path#/}" + for component in "${components[@]}"; do + [[ -n "$component" ]] || return 1 + current+="/$component" + metadata="$(_kseb_privileged_estimator_lstat "$current")" || return 1 + [[ "$metadata" != *$'\n'* ]] || return 1 + identity="${metadata%%|*}"; mode_hex="${metadata#*|}" + [[ "$identity" =~ ^[0-9]+:[0-9]+$ && "$mode_hex" =~ ^[0-9a-fA-F]{1,8}$ ]] || return 1 + mode_type=$((16#$mode_hex & 8#170000)) + if [[ "$current" == "$path" ]]; then + (( mode_type == 8#40000 || mode_type == 8#100000 )) || return 1 + else + (( mode_type == 8#40000 )) || return 1 + fi + done + printf '%s\n' "$identity" +} + +_kseb_estimator_source_probe() { + local path="$1" requirement="$2" kind identity + [[ "$requirement" == required || "$requirement" == optional ]] || return 1 + kind="$(_kseb_privileged_estimator_source_kind "$path")" || return 1 + case "$kind" in + absent) [[ "$requirement" == optional ]] || return 1; return 0 ;; + present) ;; + *) return 1 ;; + esac + identity="$(_kseb_estimator_source_identity "$path")" || return 1 + [[ "$identity" =~ ^[0-9]+:[0-9]+$ ]] || return 1 + printf '%s\n' "$identity" +} + +_kseb_estimate_source_bytes() { + local backend="$1" path="$2" requirement="$3" expected_identity="${4:-}" + local before after raw bytes reported du_rc sentinel=$'\036' + _kseb_estimator_source_allowed "$backend" "$path" || return 1 + before="$(_kseb_estimator_source_probe "$path" "$requirement")" || return 1 + [[ -n "$before" ]] || { printf '\n'; return 0; } + [[ -z "$expected_identity" || "$before" == "$expected_identity" ]] || return 1 + raw="$( + _kseb_privileged_estimator_du "$path" + du_rc=$? + printf '%s' "$sentinel" + exit "$du_rc" + )" || return 1 + [[ "$raw" == *"$sentinel" ]] || return 1 + raw="${raw%"$sentinel"}" + [[ "$raw" == *$'\n' ]] || return 1 + raw="${raw%$'\n'}" + [[ "$raw" != *$'\n'* && "$raw" == *$'\t'* ]] || return 1 + bytes="${raw%%$'\t'*}"; reported="${raw#*$'\t'}" + [[ "$reported" == "$path" ]] || return 1 + bytes="$(_kseb_normalize_signed_uint "$bytes")" || return 1 + after="$(_kseb_estimator_source_identity "$path")" || return 1 + [[ "$after" == "$before" ]] || return 1 + printf '%s\n' "$bytes" +} + +_kseb_estimator_source_records() { + local backend="$1" path + case "$backend" in + sqlite) printf 'required|/var/lib/rancher/k3s/server/db\n' ;; + embedded-etcd) printf 'required|/var/lib/rancher/k3s/server/db/etcd\n' ;; + *) return 1 ;; + esac + printf 'required|/var/lib/rancher/k3s/server/token\n' + printf 'optional|/etc/rancher/k3s\n' + while IFS= read -r path; do + [[ -n "$path" ]] || continue + printf 'optional|%s\n' "$path" + done < <(_kseb_transaction_systemd_paths) + printf 'optional|/var/lib/rancher/k3s/server/cred/encryption-config.json\n' +} + +_kseb_estimate_phase_bytes() { + local backend="$1" records record requirement path extra identity bytes seen + local total="$KSEB_ESTIMATE_METADATA_BYTES" + local -a identities=() + case "$backend" in sqlite|embedded-etcd) ;; *) return 1 ;; esac + records="$(_kseb_estimator_source_records "$backend")" || return 1 + [[ -n "$records" ]] || return 1 + while IFS= read -r record; do + IFS='|' read -r requirement path extra <<<"$record" + [[ -n "$requirement" && -n "$path" && -z "$extra" ]] || return 1 + _kseb_estimator_source_allowed "$backend" "$path" || return 1 + identity="$(_kseb_estimator_source_probe "$path" "$requirement")" || return 1 + [[ -n "$identity" ]] || continue + for seen in "${identities[@]}"; do + [[ "$seen" == "$identity" ]] && continue 2 + done + bytes="$(_kseb_estimate_source_bytes "$backend" "$path" "$requirement" "$identity")" || return 1 + [[ -n "$bytes" ]] || return 1 + total="$(_kseb_checked_estimate_add "$total" "$bytes")" || return 1 + identities+=("$identity") + done <<<"$records" + printf '%s\n' "$total" +} + +_kseb_privileged_pinned_dispatch() { + local root="$1" expected_identity="$2" relative="$3" operation="$4" + shift 4 + _kseb_run_privileged_shell -c ' + set -Eeuo pipefail + export LC_ALL=C + root=$1; expected=$2; relative=$3; operation=$4 + shift 4 + [[ "$relative" =~ ^[A-Za-z0-9._/-]+$ && "$relative" != /* && + "$relative" != *".."* && "$relative" != *"//"* ]] + exec {root_fd}<"$root" + root_handle="/proc/self/fd/${root_fd}" + actual=$(/usr/bin/stat --dereference --format="%d:%i" -- "$root_handle") + [[ "$actual" == "$expected" ]] + owner_uid=$(/usr/bin/id -u); owner_gid=$(/usr/bin/id -g) + root_metadata=$(/usr/bin/stat --dereference --format="%u:%g:%a:%F" -- "$root_handle") + [[ "$root_metadata" == "$owner_uid:$owner_gid:700:directory" ]] + + if [[ "$operation" == inspect ]]; then + exec {phase_fd}<"$root_handle/$relative" + phase_metadata=$(/usr/bin/stat --dereference --format="%u:%g:%a:%F" -- "/proc/self/fd/${phase_fd}") + printf "%s|%s\n" "$actual" "$phase_metadata" + exit 0 + fi + [[ "$operation" == transaction && $# == 14 ]] + + data_dir=$1; config_dir=$2; k3s_binary=$3; systemctl_binary=$4 + post_marker=$5; restore_evidence=$6; systemd_paths=$7 + backend=$8; bundle_id=$9; phase=${10}; utc_stamp=${11} + secret_count=${12}; version=${13}; requested_phase=${14} + [[ "$phase" == "$requested_phase" && ( "$phase" == pre || "$phase" == post ) ]] + [[ "$backend" == sqlite || "$backend" == embedded-etcd ]] + [[ "$bundle_id" =~ ^[A-Za-z0-9._-]+$ && "$utc_stamp" =~ ^[A-Za-z0-9._-]+$ ]] + [[ "$secret_count" =~ ^[0-9]+$ && "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+$ ]] + [[ "$utc_stamp" =~ ^([0-9]{4})([0-9]{2})([0-9]{2})T([0-9]{2})([0-9]{2})([0-9]{2})Z$ ]] + created_at_utc="${BASH_REMATCH[1]}-${BASH_REMATCH[2]}-${BASH_REMATCH[3]}T${BASH_REMATCH[4]}:${BASH_REMATCH[5]}:${BASH_REMATCH[6]}Z" + normalized_created_at=$(/usr/bin/date -u -d "$created_at_utc" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null) + [[ "$normalized_created_at" == "$created_at_utc" ]] + bundle_relative=${relative%/*}; relative_phase=${relative##*/} + [[ "$bundle_relative" != "$relative" && "$relative_phase" == "$phase" && + "$bundle_relative" =~ ^[A-Za-z0-9._/-]+$ ]] + + bundle_handle="$root_handle/$bundle_relative" + phase_handle="$root_handle/$relative" + phase_relative_handle="$phase_handle" + bundle_created=false; phase_created=false; transaction_committed=false + stop_attempted=false; start_dispatched=false; pending_signal=0 + marker_tmp="" + + cleanup_transaction() { + local rc=$? + trap "" INT TERM + if "$stop_attempted" && ! "$start_dispatched"; then + start_dispatched=true + "$systemctl_binary" start k3s >/dev/null 2>&1 || rc=75 + fi + if [[ -n "$marker_tmp" ]]; then + /bin/rm -f -- "$marker_tmp" >/dev/null 2>&1 || : + fi + if "$phase_created" && ! "$transaction_committed"; then + if cd -- "$phase_handle" >/dev/null 2>&1; then + /usr/bin/find . -mindepth 1 -depth -delete >/dev/null 2>&1 || : + fi + fi + exit "$rc" + } + trap cleanup_transaction EXIT + trap "pending_signal=130; exit 130" INT + trap "pending_signal=143; exit 143" TERM + + if [[ -L "$bundle_handle" || -L "$phase_handle" || -e "$phase_handle" ]]; then exit 1; fi + if [[ -e "$bundle_handle" ]]; then + bundle_metadata=$(/usr/bin/stat --dereference --format="%u:%g:%a:%F" -- "$bundle_handle") + [[ "$bundle_metadata" == "$owner_uid:$owner_gid:700:directory" ]] + if [[ "$phase" == post ]]; then + [[ -d "$bundle_handle/pre" && ! -L "$bundle_handle/pre" ]] + else + exit 1 + fi + else + /usr/bin/install -d -o "$owner_uid" -g "$owner_gid" -m 0700 -- "$bundle_handle" + bundle_created=true + fi + /usr/bin/install -d -o "$owner_uid" -g "$owner_gid" -m 0700 -- "$phase_handle" + phase_created=true + exec {phase_fd}<"$phase_handle" + phase_handle="/proc/self/fd/${phase_fd}" + phase_identity=$(/usr/bin/stat --dereference --format="%d:%i" -- "$phase_handle") + [[ "$phase_identity" =~ ^[0-9]+:[0-9]+$ ]] + phase_metadata=$(/usr/bin/stat --dereference --format="%u:%g:%a:%F" -- "$phase_handle") + [[ "$phase_metadata" == "$owner_uid:$owner_gid:700:directory" ]] + + copy_common_recovery() { + local target relative_path + local -a archive_paths=() + /usr/bin/install -o "$owner_uid" -g "$owner_gid" -m 0600 -- \ + "$data_dir/server/token" "$phase_handle/server-token" + if [[ -d "$config_dir" ]]; then + /usr/bin/tar --exclude="./platform-post-bundle.env" \ + --exclude="./platform-restore-evidence.env" -C "$config_dir" \ + -cpf "$phase_handle/host-config.tar" . >/dev/null 2>&1 + /usr/bin/chown "$owner_uid:$owner_gid" "$phase_handle/host-config.tar" + /usr/bin/chmod 0600 "$phase_handle/host-config.tar" + fi + while IFS= read -r target; do + [[ -n "$target" ]] || continue + if [[ -e "$target" ]]; then + relative_path=${target#/}; archive_paths+=("$relative_path") + fi + done <<<"$systemd_paths" + if (( ${#archive_paths[@]} > 0 )); then + /usr/bin/tar -C / -cpf "$phase_handle/systemd-recovery.tar" \ + "${archive_paths[@]}" >/dev/null 2>&1 + /usr/bin/chown "$owner_uid:$owner_gid" "$phase_handle/systemd-recovery.tar" + /usr/bin/chmod 0600 "$phase_handle/systemd-recovery.tar" + fi + if [[ -f "$data_dir/server/cred/encryption-config.json" ]]; then + /usr/bin/install -o "$owner_uid" -g "$owner_gid" -m 0600 -- \ + "$data_dir/server/cred/encryption-config.json" \ + "$phase_handle/generated-encryption-config.json" + fi + } + + write_manifest() { + local file hash manifest_tmp list_tmp + cd -- "$phase_handle" + umask 077 + manifest_tmp=".verification.manifest.tmp.$$" + list_tmp=".verification.files.tmp.$$" + /bin/rm -f -- "$manifest_tmp" "$list_tmp" + : >"$manifest_tmp"; : >"$list_tmp" + /usr/bin/find . -type f ! -name verification.manifest \ + ! -name "$manifest_tmp" ! -name "$list_tmp" -print0 >"$list_tmp" + while IFS= read -r -d "" file; do + [[ "$file" =~ ^\./[A-Za-z0-9._/-]+$ && "$file" != *".."* ]] + hash=$(/usr/bin/sha256sum -- "$file"); hash=${hash%% *} + [[ "$hash" =~ ^[0-9a-f]{64}$ ]] + printf "%s %s\n" "$hash" "$file" >>"$manifest_tmp" + done <"$list_tmp" + /bin/rm -f -- "$list_tmp" + /usr/bin/chown "$owner_uid:$owner_gid" "$manifest_tmp" + /usr/bin/chmod 0600 "$manifest_tmp" + /bin/mv -f -- "$manifest_tmp" verification.manifest + /usr/bin/sha256sum --check verification.manifest >/dev/null + } + + workloads_ready() { + local workloads=$1 + jq -e '\'' + [.items[] | select(.status.phase != "Succeeded" and .status.phase != "Failed") | + select(.status.phase != "Running" or + ((.spec.containers // []) | length) == 0 or + ((.status.containerStatuses // []) | length) != ((.spec.containers // []) | length) or + any((.status.containerStatuses // [])[]; .ready != true) or + ((.status.initContainerStatuses // []) | length) != ((.spec.initContainers // []) | length) or + any((.status.initContainerStatuses // [])[]; (.state.terminated.exitCode? // -1) != 0))] | + length == 0'\'' >/dev/null 2>&1 <<<"$workloads" + } + + recovery_check() { + local workloads started deadline now remaining command_timeout delay + /usr/bin/timeout --signal=TERM --kill-after=1s 430s \ + "$k3s_binary" kubectl get --raw=/readyz >/dev/null 2>&1 || return 1 + /usr/bin/timeout --signal=TERM --kill-after=1s 130s \ + "$k3s_binary" kubectl wait --for=condition=Ready nodes --all \ + --timeout=120s >/dev/null 2>&1 || return 1 + started=$(/usr/bin/date +%s) || return 1 + [[ "$started" =~ ^[0-9]+$ ]] || return 1 + deadline=$((started + 30)) + while :; do + now=$(/usr/bin/date +%s) || return 1 + [[ "$now" =~ ^[0-9]+$ ]] || return 1 + remaining=$((deadline - now)) + (( remaining > 1 )) || break + command_timeout=9 + (( command_timeout < remaining )) || command_timeout=$((remaining - 1)) + if workloads=$(/usr/bin/timeout --signal=TERM --kill-after=1s "${command_timeout}s" \ + "$k3s_binary" kubectl get pods --all-namespaces -o json 2>/dev/null); then + workloads_ready "$workloads" && return 0 + fi + now=$(/usr/bin/date +%s) || return 1 + [[ "$now" =~ ^[0-9]+$ ]] || return 1 + remaining=$((deadline - now)) + (( remaining > 0 )) || break + delay=5 + (( delay <= remaining )) || delay=$remaining + /usr/bin/sleep "$delay" || return 1 + done + return 1 + } + + if [[ "$backend" == sqlite ]]; then + stop_attempted=true + "$systemctl_binary" stop k3s >/dev/null 2>&1 + /bin/cp -a -- "$data_dir/server/db" "$phase_handle/datastore" + else + "$k3s_binary" etcd-snapshot save \ + --name="${phase}-secrets-encryption-${utc_stamp}" \ + --etcd-snapshot-compress --dir="$phase_handle" >/dev/null 2>&1 + "$k3s_binary" etcd-snapshot list --dir="$phase_handle" >/dev/null 2>&1 + fi + copy_common_recovery + if [[ "$backend" == sqlite ]]; then + /usr/bin/diff --no-dereference --recursive --brief \ + "$data_dir/server/db" "$phase_handle/datastore" >/dev/null 2>&1 + fi + umask 077 + printf "schema=platform-k3s-bundle-v1\nbundle_id=%s\nphase=%s\nk3s_version=%s\ndatastore=%s\ncreated_at_utc=%s\nsecret_count=%s\n" \ + "$bundle_id" "$phase" "$version" "$backend" "$created_at_utc" "$secret_count" \ + >"$phase_handle/bundle.env" + /usr/bin/chown "$owner_uid:$owner_gid" "$phase_handle/bundle.env" + /usr/bin/chmod 0600 "$phase_handle/bundle.env" + write_manifest + total_bytes=$(/usr/bin/du --summarize --bytes "$phase_handle" | /usr/bin/awk "{print \$1}") + [[ "$total_bytes" =~ ^[0-9]+$ ]] + + if [[ "$backend" == sqlite ]]; then + trap "pending_signal=130" INT + trap "pending_signal=143" TERM + start_dispatched=true + "$systemctl_binary" start k3s >/dev/null 2>&1 || exit 75 + trap "pending_signal=130; exit 130" INT + trap "pending_signal=143; exit 143" TERM + (( pending_signal == 0 )) || exit "$pending_signal" + fi + recovery_check || exit 75 + commit_identity=$(/usr/bin/stat --dereference --format="%d:%i" -- "$phase_relative_handle") || exit 1 + [[ "$commit_identity" == "$phase_identity" ]] || exit 1 + if [[ "$phase" == post ]]; then + marker_tmp="${post_marker}.new.$$" + /usr/bin/install -o "$owner_uid" -g "$owner_gid" -m 0600 -- \ + "$phase_handle/bundle.env" "$marker_tmp" || exit 76 + /bin/rm -f -- "$restore_evidence" || exit 76 + /bin/mv -f -- "$marker_tmp" "$post_marker" || exit 76 + marker_tmp="" + fi + transaction_committed=true + trap - EXIT INT TERM + printf "%s\n" "$total_bytes" + ' kseb-pinned "$root" "$expected_identity" "$relative" "$operation" "$@" +} + +_kseb_pinned_phase_transaction() { + local root="$1" relative="$2" backend="$3" bundle_id="$4" phase="$5" + local utc_stamp="$6" secret_count="$7" version="$8" + local data_dir config_dir k3s_binary systemctl_binary post_marker restore_evidence systemd_paths + [[ -n "$KSEB_BACKUP_ROOT_IDENTITY" ]] || return 1 + data_dir="$(_kseb_transaction_data_dir)" || return 1 + config_dir="$(_kseb_transaction_config_dir)" || return 1 + k3s_binary="$(_kseb_transaction_k3s_binary)" || return 1 + systemctl_binary="$(_kseb_transaction_systemctl_binary)" || return 1 + post_marker="$(_kseb_transaction_post_marker)" || return 1 + restore_evidence="$(_kseb_transaction_restore_evidence)" || return 1 + systemd_paths="$(_kseb_transaction_systemd_paths)" || return 1 + _kseb_privileged_pinned_dispatch "$root" "$KSEB_BACKUP_ROOT_IDENTITY" "$relative" transaction \ + "$data_dir" "$config_dir" "$k3s_binary" "$systemctl_binary" \ + "$post_marker" "$restore_evidence" "$systemd_paths" "$backend" "$bundle_id" \ + "$phase" "$utc_stamp" "$secret_count" "$version" "$phase" 2>/dev/null +} + +_kseb_pinned_root_operation() { + local root="$1" relative="$2" operation="$3" identity + [[ -n "$KSEB_BACKUP_ROOT_IDENTITY" ]] || return 1 + _kseb_privileged_backup_root_chain_safe "$root" || return 1 + identity="$(_kseb_privileged_root_identity "$root")" || return 1 + _kseb_backup_root_identity_matches "$KSEB_BACKUP_ROOT_IDENTITY" "$identity" || return 1 + _kseb_privileged_relative_components_safe "$root" "$relative" || return 1 + _kseb_privileged_pinned_dispatch "$root" "$KSEB_BACKUP_ROOT_IDENTITY" "$relative" "$operation" +} + +_kseb_validate_expectation() { + bash "$KSEB_VALIDATOR" "$1" >/dev/null +} + +_kseb_enable() { + /usr/bin/sudo /usr/local/bin/k3s secrets-encrypt enable >/dev/null 2>&1 +} + +_kseb_install_dropin() { + /usr/bin/sudo /usr/bin/install -o root -g root -m 0644 \ + "$KSEB_DROPIN_SOURCE" \ + /etc/rancher/k3s/config.yaml.d/40-secrets-encryption.yaml >/dev/null 2>&1 +} + +_kseb_restart() { + /usr/bin/sudo /usr/bin/systemctl restart k3s >/dev/null 2>&1 +} + +_kseb_rotate_keys() { + /usr/bin/sudo /usr/local/bin/k3s secrets-encrypt rotate-keys >/dev/null 2>&1 +} + +_kseb_wait_for_reencrypt() { + local rotate_rc="$1" started now elapsed deadline stage status_json status_class + [[ "$rotate_rc" =~ ^[0-9]+$ ]] || return 1 + KSEB_LAST_SAFE_STATE=unknown + started="$SECONDS" + deadline=$((started + 600)) + while (( SECONDS < deadline )); do + status_json="$(_kseb_read_status_json)" || { KSEB_LAST_SAFE_STATE=unknown; return 1; } + status_class="$(classify_encryption_status "$status_json")" + if [[ "$status_class" == hash_mismatch ]]; then KSEB_LAST_SAFE_STATE=hash_mismatch; return 1; fi + if [[ "$status_class" == invalid ]]; then KSEB_LAST_SAFE_STATE=reencrypt_stage_unexpected; return 1; fi + stage="$(jq -er '.stage' <<<"$status_json" 2>/dev/null)" || { KSEB_LAST_SAFE_STATE=reencrypt_stage_unexpected; return 1; } + case "$stage" in + start|reencrypt_active|reencrypt_finished) ;; + *) KSEB_LAST_SAFE_STATE=reencrypt_stage_unexpected; return 1 ;; + esac + now="$SECONDS"; elapsed=$((now - started)) + printf '재암호화 상태: stage=%s elapsed=%ss\n' "$stage" "$elapsed" + [[ "$stage" == reencrypt_finished ]] && return 0 + if [[ "$stage" == start ]]; then KSEB_LAST_SAFE_STATE=reencrypt_stage_start; return 1; fi + [[ "$stage" == reencrypt_active ]] || { KSEB_LAST_SAFE_STATE=reencrypt_stage_unexpected; return 1; } + KSEB_LAST_SAFE_STATE=reencrypt_active_timeout + (( SECONDS + 30 < deadline )) || break + sleep 30 + done + return 1 +} + +_kseb_secret_count() { + /usr/bin/timeout --signal=TERM --kill-after=1s 30s \ + /usr/bin/sudo --non-interactive -- /usr/local/bin/k3s kubectl \ + get secrets --all-namespaces -o json 2>/dev/null | + jq -er '.items | length' 2>/dev/null +} + +_kseb_version() { + local raw + raw="$(/usr/bin/sudo --non-interactive -- /usr/local/bin/k3s --version 2>/dev/null)" || return 1 + sed -n -E 's/^k3s version (v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+)([[:space:]].*)?$/\1/p' <<<"$raw" +} + +_kseb_privileged_path_kind() { + local path="$1" + if /usr/bin/sudo /usr/bin/test -L "$path"; then printf 'symlink\n' + elif /usr/bin/sudo /usr/bin/test -d "$path"; then printf 'directory\n' + elif /usr/bin/sudo /usr/bin/test -e "$path"; then printf 'other\n' + else printf 'absent\n'; fi +} + +_kseb_privileged_directory_metadata() { + /usr/bin/sudo /usr/bin/stat --format='%u:%g:%a:%F' -- "$1" 2>/dev/null +} + +_kseb_phase_layout_allowed() { + local phase="$1" bundle_kind="$2" bundle_metadata="$3" pre_kind="$4" + case "$phase/$bundle_kind" in + pre/absent|post/absent) return 0 ;; + post/directory) + [[ "$bundle_metadata" == '0:0:700:directory' && "$pre_kind" == directory ]] + ;; + *) return 1 ;; + esac +} + +_kseb_prepare_phase_dir() { + local bundle_dir="$1" phase_dir="$2" phase="$3" + local bundle_kind phase_kind pre_kind=absent bundle_metadata='' + bundle_kind="$(_kseb_privileged_path_kind "$bundle_dir")" || return 1 + phase_kind="$(_kseb_privileged_path_kind "$phase_dir")" || return 1 + [[ "$phase_kind" == absent ]] || return 1 + if [[ "$bundle_kind" == directory ]]; then + bundle_metadata="$(_kseb_privileged_directory_metadata "$bundle_dir")" || return 1 + pre_kind="$(_kseb_privileged_path_kind "$bundle_dir/pre")" || return 1 + fi + _kseb_phase_layout_allowed "$phase" "$bundle_kind" "$bundle_metadata" "$pre_kind" || return 1 + /usr/bin/sudo /usr/bin/install -d -o root -g root -m 0700 "$bundle_dir" "$phase_dir" 2>/dev/null +} + +_kseb_copy_common_recovery() { + local phase_dir="$1" target relative + local -a systemd_paths=() + /usr/bin/sudo /usr/bin/install -o root -g root -m 0600 \ + "$KSEB_DATA_DIR/server/token" "$phase_dir/server-token" || return 1 + + if /usr/bin/sudo /usr/bin/test -d "$KSEB_CONFIG_DIR"; then + _kseb_archive_host_config "$phase_dir" || return 1 + /usr/bin/sudo /usr/bin/chown root:root "$phase_dir/host-config.tar" || return 1 + /usr/bin/sudo /usr/bin/chmod 0600 "$phase_dir/host-config.tar" || return 1 + fi + + for target in \ + /etc/systemd/system/k3s.service \ + /etc/systemd/system/k3s.service.env \ + /etc/systemd/system/k3s.service.d \ + /usr/lib/systemd/system/k3s.service \ + /lib/systemd/system/k3s.service; do + if /usr/bin/sudo /usr/bin/test -e "$target"; then + relative="${target#/}" + [[ " ${systemd_paths[*]} " == *" $relative "* ]] || systemd_paths+=("$relative") + fi + done + if (( ${#systemd_paths[@]} > 0 )); then + /usr/bin/sudo /usr/bin/tar -C / -cpf "$phase_dir/systemd-recovery.tar" \ + "${systemd_paths[@]}" || return 1 + /usr/bin/sudo /usr/bin/chown root:root "$phase_dir/systemd-recovery.tar" || return 1 + /usr/bin/sudo /usr/bin/chmod 0600 "$phase_dir/systemd-recovery.tar" || return 1 + fi + + target="$KSEB_DATA_DIR/server/cred/encryption-config.json" + if /usr/bin/sudo /usr/bin/test -f "$target"; then + /usr/bin/sudo /usr/bin/install -o root -g root -m 0600 \ + "$target" "$phase_dir/generated-encryption-config.json" || return 1 + fi +} + +_kseb_archive_host_config() { + local phase_dir="$1" + _kseb_privileged_exec /usr/bin/tar \ + --exclude='./platform-post-bundle.env' \ + --exclude='./platform-restore-evidence.env' \ + -C "$KSEB_CONFIG_DIR" -cpf "$phase_dir/host-config.tar" . +} + +_kseb_render_bundle_metadata() { + local bundle_id="$1" phase="$2" datastore="$3" created_at_utc="$4" + local secret_count="$5" k3s_version="$6" + [[ "$bundle_id" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ ]] || return 1 + [[ "$phase" == pre || "$phase" == post ]] || return 1 + [[ "$datastore" == sqlite || "$datastore" == embedded-etcd ]] || return 1 + [[ "$created_at_utc" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || return 1 + [[ "$secret_count" =~ ^[0-9]+$ ]] || return 1 + [[ "$k3s_version" == v1.36.2+k3s1 ]] || return 1 + printf 'schema=platform-k3s-bundle-v1\nbundle_id=%s\nphase=%s\nk3s_version=%s\ndatastore=%s\ncreated_at_utc=%s\nsecret_count=%s\n' \ + "$bundle_id" "$phase" "$k3s_version" "$datastore" "$created_at_utc" "$secret_count" +} + +_kseb_rfc3339_from_stamp() { + local stamp="$1" + if [[ "$stamp" =~ ^([0-9]{4})([0-9]{2})([0-9]{2})T([0-9]{2})([0-9]{2})([0-9]{2})Z$ ]]; then + printf '%s-%s-%sT%s:%s:%sZ\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" \ + "${BASH_REMATCH[3]}" "${BASH_REMATCH[4]}" "${BASH_REMATCH[5]}" "${BASH_REMATCH[6]}" + else + return 1 + fi +} + +_kseb_write_bundle_metadata() { + local phase_dir="$1" bundle_id="$2" phase="$3" backend="$4" utc_stamp="$5" + local secret_count="$6" version="$7" temporary created_at_utc + [[ "$secret_count" =~ ^[0-9]+$ ]] || return 1 + [[ "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+$ ]] || return 1 + created_at_utc="$(_kseb_rfc3339_from_stamp "$utc_stamp")" || return 1 + temporary="$(mktemp /tmp/k3s-secrets-encryption-metadata.XXXXXX)" || return 1 + chmod 0600 "$temporary" || { rm -f -- "$temporary"; return 1; } + _kseb_render_bundle_metadata "$bundle_id" "$phase" "$backend" "$created_at_utc" \ + "$secret_count" "$version" >"$temporary" || { rm -f -- "$temporary"; return 1; } + /usr/bin/sudo /usr/bin/install -o root -g root -m 0600 \ + "$temporary" "$phase_dir/bundle.env" || { rm -f -- "$temporary"; return 1; } + rm -f -- "$temporary" +} + +_kseb_manifest_list_files() { + /usr/bin/sudo /usr/bin/find "$1" -type f ! -name verification.manifest -print0 2>/dev/null +} + +_kseb_manifest_hash_file() { + local line hash + line="$(/usr/bin/sudo /usr/bin/sha256sum -- "$1" 2>/dev/null)" || return 1 + hash="${line%% *}" + [[ "$hash" =~ ^[0-9a-f]{64}$ ]] || return 1 + printf '%s\n' "$hash" +} + +_kseb_build_relative_manifest() { + local phase_dir="$1" output="$2" file relative hash list_file rc=0 + : >"$output" || return 1 + chmod 0600 "$output" || return 1 + list_file="$(mktemp /tmp/k3s-secrets-encryption-files.XXXXXX)" || return 1 + chmod 0600 "$list_file" || { rm -f -- "$list_file"; return 1; } + if ! _kseb_manifest_list_files "$phase_dir" >"$list_file"; then + rm -f -- "$list_file" + return 1 + fi + while IFS= read -r -d '' file; do + [[ "$file" == "$phase_dir/"* ]] || { rc=1; break; } + relative="${file#"$phase_dir/"}" + [[ -n "$relative" && "$relative" != /* && "$relative" != *'..'* && + "$relative" =~ ^[A-Za-z0-9._/-]+$ ]] || { rc=1; break; } + hash="$(_kseb_manifest_hash_file "$file")" || { rc=1; break; } + [[ "$hash" =~ ^[0-9a-f]{64}$ ]] || { rc=1; break; } + printf '%s ./%s\n' "$hash" "$relative" >>"$output" || { rc=1; break; } + done <"$list_file" + rm -f -- "$list_file" + return "$rc" +} + +_kseb_manifest_install() { + /usr/bin/sudo /usr/bin/install -o root -g root -m 0600 \ + "$1" "$2/verification.manifest" >/dev/null 2>&1 +} + +_kseb_manifest_check() { + /usr/bin/sudo /usr/bin/env --chdir="$1" \ + /usr/bin/sha256sum --check verification.manifest >/dev/null 2>&1 +} + +_kseb_write_verification_manifest() { + _kseb_privileged_manifest_lifecycle "$1" +} + +_kseb_privileged_manifest_lifecycle() { + local phase_dir="$1" + /usr/bin/sudo /usr/bin/bash -c ' + set -Eeuo pipefail + phase=$1 + exec {phase_fd}<"$phase" + metadata=$(/usr/bin/stat --format="%u:%g:%a:%F" -- "/proc/self/fd/${phase_fd}") + [[ "$metadata" == "0:0:700:directory" ]] + cd -- "/proc/self/fd/${phase_fd}" + umask 077 + manifest_tmp=".verification.manifest.tmp.$$" + list_tmp=".verification.files.tmp.$$" + cleanup() { /bin/rm -f -- "$manifest_tmp" "$list_tmp"; } + trap cleanup EXIT INT TERM + : >"$manifest_tmp" + : >"$list_tmp" + /usr/bin/find . -type f \ + ! -name verification.manifest \ + ! -name "$manifest_tmp" ! -name "$list_tmp" -print0 >"$list_tmp" + while IFS= read -r -d "" file; do + [[ "$file" =~ ^\./[A-Za-z0-9._/-]+$ && "$file" != *".."* ]] + hash=$(/usr/bin/sha256sum -- "$file") + hash=${hash%% *} + [[ "$hash" =~ ^[0-9a-f]{64}$ ]] + printf "%s %s\n" "$hash" "$file" >>"$manifest_tmp" + done <"$list_tmp" + /bin/rm -f -- "$list_tmp" + /usr/bin/chown root:root "$manifest_tmp" + /usr/bin/chmod 0600 "$manifest_tmp" + /bin/mv -f -- "$manifest_tmp" verification.manifest + /usr/bin/sha256sum --check verification.manifest >/dev/null + trap - EXIT INT TERM + ' kseb-manifest "$phase_dir" >/dev/null 2>&1 +} + +_kseb_api_ready_once() { + local command_timeout="${1:-9}" + [[ "$command_timeout" =~ ^[1-9]$ ]] || return 1 + /usr/bin/timeout --signal=TERM --kill-after=1s "${command_timeout}s" \ + /usr/bin/sudo --non-interactive -- /usr/local/bin/k3s kubectl \ + get --raw=/readyz >/dev/null 2>&1 +} + +_kseb_now_seconds() { + date +%s +} + +_kseb_wait_for_api() { + local started deadline now remaining command_timeout delay + started="$(_kseb_now_seconds)" || return 1 + [[ "$started" =~ ^[0-9]+$ ]] || return 1 + deadline=$((started + KSEB_API_RECOVERY_BUDGET)) + while :; do + now="$(_kseb_now_seconds)" || return 1 + [[ "$now" =~ ^[0-9]+$ ]] || return 1 + remaining=$((deadline - now)) + (( remaining > 1 )) || break + command_timeout=9 + (( command_timeout < remaining )) || command_timeout=$((remaining - 1)) + _kseb_api_ready_once "$command_timeout" && return 0 + now="$(_kseb_now_seconds)" || return 1 + remaining=$((deadline - now)) + (( remaining > 0 )) || break + delay=10 + (( delay <= remaining )) || delay="$remaining" + sleep "$delay" + done + return 1 +} + +_kseb_node_recovery_check() { + /usr/bin/timeout --signal=TERM --kill-after=1s 130s \ + /usr/bin/sudo --non-interactive -- /usr/local/bin/k3s kubectl \ + wait --for=condition=Ready nodes --all --timeout=120s >/dev/null 2>&1 +} + +_kseb_read_workloads_json() { + local command_timeout="${1:-30}" + [[ "$command_timeout" =~ ^([1-9]|[12][0-9]|30)$ ]] || return 1 + /usr/bin/timeout --signal=TERM --kill-after=1s "${command_timeout}s" \ + /usr/bin/sudo --non-interactive -- /usr/local/bin/k3s kubectl \ + get pods --all-namespaces -o json 2>/dev/null +} + +_kseb_workloads_ready_json() { + jq -e ' + [.items[] | + select(.status.phase != "Succeeded" and .status.phase != "Failed") | + select( + .status.phase != "Running" or + ((.spec.containers // []) | length) == 0 or + ((.status.containerStatuses // []) | length) != ((.spec.containers // []) | length) or + any((.status.containerStatuses // [])[]; .ready != true) or + ((.status.initContainerStatuses // []) | length) != ((.spec.initContainers // []) | length) or + any((.status.initContainerStatuses // [])[]; (.state.terminated.exitCode? // -1) != 0) + )] | length == 0 + ' >/dev/null 2>&1 <<<"$1" +} + +_kseb_recovery_budget_valid() { + local api="$1" node="$2" workload="$3" + [[ "$api" =~ ^[0-9]+$ && "$node" =~ ^[0-9]+$ && "$workload" =~ ^[0-9]+$ ]] || return 1 + (( api + node + workload <= 600 )) +} + +_kseb_wait_for_workloads() { + local started deadline now remaining command_timeout delay workloads + started="$(_kseb_now_seconds)" || return 1 + [[ "$started" =~ ^[0-9]+$ ]] || return 1 + deadline=$((started + KSEB_WORKLOAD_RECOVERY_BUDGET)) + while :; do + now="$(_kseb_now_seconds)" || return 1 + [[ "$now" =~ ^[0-9]+$ ]] || return 1 + remaining=$((deadline - now)) + (( remaining > 1 )) || break + command_timeout=9 + (( command_timeout < remaining )) || command_timeout=$((remaining - 1)) + workloads="$(_kseb_read_workloads_json "$command_timeout")" || workloads='' + if [[ -n "$workloads" ]] && _kseb_workloads_ready_json "$workloads"; then + return 0 + fi + now="$(_kseb_now_seconds)" || return 1 + [[ "$now" =~ ^[0-9]+$ ]] || return 1 + remaining=$((deadline - now)) + (( remaining > 0 )) || break + delay=5 + (( delay <= remaining )) || delay="$remaining" + sleep "$delay" + done + return 1 +} + +_kseb_recovery_check() { + _kseb_recovery_budget_valid "$KSEB_API_RECOVERY_BUDGET" \ + "$KSEB_NODE_RECOVERY_BUDGET" "$KSEB_WORKLOAD_RECOVERY_BUDGET" || return 1 + _kseb_wait_for_api || return 1 + _kseb_node_recovery_check || return 1 + _kseb_wait_for_workloads +} + +_kseb_sqlite_copy_and_verify() { + local phase_dir="$1" bundle_id="$2" phase="$3" utc_stamp="$4" secret_count="$5" version="$6" + _kseb_copy_sqlite_datastore "$phase_dir" || return 1 + _kseb_copy_common_recovery "$phase_dir" || return 1 + _kseb_compare_sqlite_datastore "$phase_dir" || return 1 + _kseb_write_bundle_metadata "$phase_dir" "$bundle_id" "$phase" sqlite "$utc_stamp" \ + "$secret_count" "$version" || return 1 + _kseb_write_verification_manifest "$phase_dir" +} + +_kseb_copy_sqlite_datastore() { + _kseb_privileged_exec /bin/cp -a -- "$KSEB_DATA_DIR/server/db" "$1/datastore" +} + +_kseb_compare_sqlite_datastore() { + _kseb_privileged_exec /usr/bin/diff --no-dereference --recursive --brief \ + "$KSEB_DATA_DIR/server/db" "$1/datastore" +} + +_kseb_stop_k3s() { + /usr/bin/sudo /usr/bin/systemctl stop k3s +} + +_kseb_start_k3s() { + /usr/bin/sudo /usr/bin/systemctl start k3s +} + +_kseb_backup_sqlite_guarded() ( + local phase_dir="$1" bundle_id="$2" phase="$3" utc_stamp="$4" secret_count="$5" version="$6" + local stop_attempted=false start_dispatched=false service_recovered=false pending_signal=0 + _kseb_sqlite_exit_recovery() { + local rc=$? + trap - EXIT INT TERM + if "$stop_attempted" && ! "$start_dispatched"; then + start_dispatched=true + _kseb_start_k3s >/dev/null 2>&1 || rc=75 + fi + exit "$rc" + } + trap _kseb_sqlite_exit_recovery EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + stop_attempted=true + _kseb_stop_k3s || return 1 + _kseb_sqlite_copy_and_verify "$phase_dir" "$bundle_id" "$phase" "$utc_stamp" \ + "$secret_count" "$version" || return 1 + _kseb_before_start_dispatch + trap 'pending_signal=130' INT + trap 'pending_signal=143' TERM + start_dispatched=true + _kseb_start_k3s || return 75 + service_recovered=true + trap 'exit 130' INT + trap 'exit 143' TERM + (( pending_signal == 0 )) || return "$pending_signal" + "$service_recovered" || return 1 + _kseb_recovery_check +) + +_kseb_before_start_dispatch() { + : +} + +_kseb_backup_sqlite() { + _kseb_backup_sqlite_guarded "$@" +} + +_kseb_backup_etcd() { + local phase_dir="$1" bundle_id="$2" phase="$3" utc_stamp="$4" secret_count="$5" version="$6" + _kseb_etcd_snapshot_save "$phase" "$utc_stamp" "$phase_dir" || return 1 + _kseb_etcd_snapshot_list "$phase_dir" || return 1 + _kseb_copy_common_recovery "$phase_dir" || return 1 + _kseb_write_bundle_metadata "$phase_dir" "$bundle_id" "$phase" embedded-etcd "$utc_stamp" \ + "$secret_count" "$version" || return 1 + _kseb_write_verification_manifest "$phase_dir" || return 1 + _kseb_recovery_check +} + +_kseb_etcd_snapshot_save() { + local phase="$1" utc_stamp="$2" phase_dir="$3" + _kseb_privileged_exec /usr/local/bin/k3s etcd-snapshot save \ + --name="${phase}-secrets-encryption-${utc_stamp}" \ + --etcd-snapshot-compress --dir="$phase_dir" +} + +_kseb_etcd_snapshot_list() { + _kseb_privileged_exec /usr/local/bin/k3s etcd-snapshot list --dir="$1" +} + +_kseb_backup_phase() { + local phase="$1" backend="$2" backup_root="$3" bundle_dir="$4" bundle_id="$5" utc_stamp="$6" + local bundle_name="${bundle_dir##*/}" relative_phase total_bytes secret_count version transaction_rc + [[ "$phase" == pre || "$phase" == post ]] || return 1 + _kseb_revalidate_backup_root "$backup_root" || return 1 + _kseb_require_phase_capacity "$phase" "$backend" "$backup_root" || return 1 + secret_count="$(_kseb_secret_count)" || return 1 + version="$(_kseb_version)" || return 1 + relative_phase="$bundle_name/$phase" + if total_bytes="$(_kseb_pinned_phase_transaction "$backup_root" "$relative_phase" "$backend" \ + "$bundle_id" "$phase" "$utc_stamp" "$secret_count" "$version")"; then + : + else + transaction_rc=$? + return "$transaction_rc" + fi + [[ "$total_bytes" =~ ^[0-9]+$ ]] || return 1 + printf 'Recovery bundle %s 검증 완료: bundle-id=%s, total-bytes=%s\n' \ + "$phase" "$bundle_id" "$total_bytes" +} + +_kseb_rotate_wait_restart_validate() { + local rotate_rc + set +e + _kseb_rotate_keys + rotate_rc=$? + set -e + if ! _kseb_wait_for_reencrypt "$rotate_rc"; then + _kseb_terminal_failure "$KSEB_LAST_SAFE_STATE" + return 1 + fi + _kseb_restart || { _kseb_terminal_failure partial_final_restart_failed; return 1; } + _kseb_validate_expectation --expect-reencrypted || _kseb_terminal_failure final_validation_failed +} + +apply_k3s_secret_encryption_main() { + local execute=false rotate_existing=false backup_root='' backup_seen=false estimate_recovery=false + local recovery_policy='' recovery_policy_seen=false context + local inventory status rotation backend provider owner integrity server_hashes + local utc_stamp bundle_id bundle_dir backup_rc phase_bytes argument estimate_argument_count=0 + + for argument in "$@"; do + if [[ "$argument" == --estimate-recovery-bytes ]]; then + estimate_argument_count=$((estimate_argument_count + 1)) + fi + done + if (( estimate_argument_count > 0 )) && + { (( $# != 1 )) || [[ "${1:-}" != --estimate-recovery-bytes ]]; }; then + _kseb_usage >&2 + return 2 + fi + + while (( $# > 0 )); do + case "$1" in + --execute) "$execute" && { _kseb_usage >&2; return 2; }; execute=true ;; + --rotate-existing) "$rotate_existing" && { _kseb_usage >&2; return 2; }; rotate_existing=true ;; + --estimate-recovery-bytes) + "$estimate_recovery" && { _kseb_usage >&2; return 2; } + estimate_recovery=true + ;; + --backup-root) + (( $# >= 2 )) && ! "$backup_seen" && [[ "$2" == /* ]] || { _kseb_usage >&2; return 2; } + backup_root="$2"; backup_seen=true; shift + ;; + --recovery-policy) + (( $# >= 2 )) && ! "$recovery_policy_seen" || { _kseb_usage >&2; return 2; } + case "$2" in + encrypted-off-host|local-separate-disk-luks) recovery_policy="$2" ;; + *) _kseb_usage >&2; return 2 ;; + esac + recovery_policy_seen=true; shift + ;; + --help|-h) _kseb_usage; return 0 ;; + *) _kseb_usage >&2; return 2 ;; + esac + shift + done + if "$estimate_recovery"; then + ! "$execute" && ! "$rotate_existing" && ! "$backup_seen" && ! "$recovery_policy_seen" || { + _kseb_usage >&2 + return 2 + } + elif ! "$execute"; then + ! "$rotate_existing" && ! "$backup_seen" && ! "$recovery_policy_seen" || { + _kseb_usage >&2 + return 2 + } + else + "$backup_seen" && "$recovery_policy_seen" || { _kseb_usage >&2; return 2; } + fi + + inventory="$(_kseb_load_inventory)" || { + _kseb_terminal_failure initial_unsafe + return 1 + } + IFS='|' read -r status rotation backend provider owner integrity server_hashes <<<"$inventory" + if [[ "$status" == hash_mismatch ]]; then _kseb_terminal_failure hash_mismatch; return 1; fi + case "$backend" in sqlite|embedded-etcd) ;; *) _kseb_terminal_failure datastore_unsafe; return 1 ;; esac + + if [[ "$estimate_recovery" == true ]]; then + phase_bytes="$(_kseb_estimate_phase_bytes "$backend")" || { + _kseb_fail 'recovery phase 용량을 안전하게 계산하지 못했습니다' + return 1 + } + phase_bytes="$(_kseb_normalize_signed_uint "$phase_bytes")" || { + _kseb_fail 'recovery phase 용량을 안전하게 계산하지 못했습니다' + return 1 + } + printf 'phase_bytes=%s\n' "$phase_bytes" + return 0 + fi + + if [[ "$execute" == false ]]; then + _kseb_dry_run "$inventory" + return 0 + fi + + case "$status/$rotation" in + disabled_no_config/none) ;; + enabled_stable/reencrypt_finished) + [[ "$provider" == aescbc && "$owner" == aescbc/* && "$owner" != aescbc/ambiguous && "$integrity" == match ]] || { + _kseb_fail '기존 enabled 상태의 provider owner 또는 local integrity가 안전하지 않습니다' + return 1 + } + ;; + enabled_stable/start) + [[ "$provider" == aescbc && "$owner" == aescbc/* && "$owner" != aescbc/ambiguous && "$integrity" == match ]] || { + _kseb_fail '기존 enabled/start 상태의 provider owner 또는 local integrity가 안전하지 않습니다' + return 1 + } + "$rotate_existing" || { + _kseb_fail 'enabled/start는 --rotate-existing 명시 승인 없이는 진행할 수 없습니다' + return 1 + } + ;; + *) _kseb_terminal_failure initial_unsafe; return 1 ;; + esac + + context="$(_kseb_prepare_execute_context)" || return 1 + _kseb_confirm_apply "$context" || return 1 + _kseb_validate_backup_root "$backup_root" || { _kseb_terminal_failure backup_root_unsafe; return 1; } + _kseb_validate_recovery_policy "$recovery_policy" "$backup_root" || { + _kseb_terminal_failure backup_root_unsafe + return 1 + } + _kseb_confirm_recovery "$context" "$recovery_policy" || return 1 + utc_stamp="$(date -u +%Y%m%dT%H%M%SZ)" || return 1 + bundle_id="$(tr -d '\n' &2 + fi +} + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + report_retained_state + exit 1 +} + +on_error() { + local status="$1" + local line="$2" + + trap - ERR + set +e + printf 'ERROR: command failed at line %s (exit %s).\n' "$line" "$status" >&2 + report_retained_state + exit "$status" +} + +on_signal() { + local status="$1" + + trap - INT TERM + set +e + printf 'INTERRUPTED: stopping without deleting cluster state.\n' >&2 + report_retained_state + exit "$status" +} + +usage() { + cat <<'USAGE' +Usage: bash scripts/bootstrap/apply-keycloak.sh --execute + +Renders, verifies, and applies only the Keycloak path in this order: + Keycloak Namespace + two Keycloak DB Secrets + CloudNativePG DatabaseRole, Database, and NetworkPolicy + Keycloak Operator + Keycloak custom resource and HTTP Ingress + +AIStor namespaces, Secrets, storage, Operator, and ObjectStore are not required +or applied. Host Nginx and Gitea OIDC configuration are separate cutovers. +USAGE +} + +[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || { + usage + exit 2 +} + +for command_name in kubectl curl find mktemp rg sed sha256sum stat wc; do + command -v "$command_name" >/dev/null 2>&1 || \ + fail "${command_name} is required" +done + +kustomize_version="$( + kubectl version --client --output=yaml | + sed -n 's/^kustomizeVersion: //p' +)" +[[ "$kustomize_version" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \ + fail "expected Kustomize ${EXPECTED_KUSTOMIZE_VERSION}, found ${kustomize_version:-unknown}" + +umask 077 +render_temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-keycloak-apply.XXXXXX")" + +cleanup() { + case "$render_temp_dir" in + /tmp/platform-keycloak-apply.*|"${TMPDIR:-/tmp}"/platform-keycloak-apply.*) + rm -rf -- "$render_temp_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected render directory: %s\n' \ + "$render_temp_dir" >&2 + ;; + esac +} +trap cleanup EXIT +trap 'on_error "$?" "$LINENO"' ERR +trap 'on_signal 130' INT +trap 'on_signal 143' TERM + +assert_regex_count() { + local file="$1" + local pattern="$2" + local expected="$3" + local description="$4" + local actual + + actual="$(rg --count --no-filename -- "$pattern" "$file" || true)" + actual="${actual:-0}" + [[ "$actual" == "$expected" ]] || \ + fail "${description}: expected ${expected}, found ${actual}" +} + +render_kustomization() { + local name="$1" + local relative_path="$2" + local output="${render_temp_dir}/${name}.yaml" + + kubectl kustomize "${REPOSITORY_ROOT}/${relative_path}" >"$output" + [[ -s "$output" ]] || fail "${name} rendered an empty manifest" + printf 'Rendered %-30s %8s bytes\n' \ + "$name" "$(wc -c <"$output" | tr -d '[:space:]')" +} + +cd -- "$REPOSITORY_ROOT" + +kubectl create \ + --dry-run=client \ + --filename=infrastructure/namespaces/phase2/keycloak.yaml \ + --output=yaml >"${render_temp_dir}/keycloak-namespace.yaml" +render_kustomization \ + platform-postgres-keycloak services/platform-postgres-keycloak +render_kustomization \ + keycloak-operator infrastructure/controllers/keycloak-operator +render_kustomization keycloak services/keycloak + +if rg --line-number \ + '^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' \ + "${render_temp_dir}"/*.yaml; then + fail "a rendered Keycloak manifest unexpectedly contains a Secret" +fi + +assert_regex_count \ + "${render_temp_dir}/keycloak-namespace.yaml" \ + '^kind:[[:space:]]Namespace$' 1 \ + "Keycloak namespace resource count" +assert_regex_count \ + "${render_temp_dir}/keycloak-namespace.yaml" \ + '^[[:space:]]*name:[[:space:]]keycloak$' 1 \ + "Keycloak namespace name" + +assert_regex_count \ + "${render_temp_dir}/platform-postgres-keycloak.yaml" \ + '^kind:[[:space:]]DatabaseRole$' 1 \ + "Keycloak DatabaseRole count" +assert_regex_count \ + "${render_temp_dir}/platform-postgres-keycloak.yaml" \ + '^kind:[[:space:]]Database$' 1 \ + "Keycloak Database count" +assert_regex_count \ + "${render_temp_dir}/platform-postgres-keycloak.yaml" \ + '^kind:[[:space:]]NetworkPolicy$' 1 \ + "Keycloak PostgreSQL NetworkPolicy count" +assert_regex_count \ + "${render_temp_dir}/platform-postgres-keycloak.yaml" \ + '^[[:space:]]*namespace:[[:space:]]platform-data$' 3 \ + "Keycloak PostgreSQL resource namespace count" +assert_regex_count \ + "${render_temp_dir}/platform-postgres-keycloak.yaml" \ + '^[[:space:]]*(databaseRoleReclaimPolicy|databaseReclaimPolicy):[[:space:]]retain$' 2 \ + "Keycloak database Retain policy count" +assert_regex_count \ + "${render_temp_dir}/platform-postgres-keycloak.yaml" \ + '^[[:space:]]*name:[[:space:]]keycloak-db-credentials$' 1 \ + "Keycloak DatabaseRole Secret reference" + +assert_regex_count \ + "${REPOSITORY_ROOT}/infrastructure/controllers/keycloak-operator/kustomization.yaml" \ + 'github\.com/keycloak/keycloak-k8s-resources/kubernetes\?ref=26\.7\.0' 1 \ + "Keycloak Operator 26.7.0 source pin" +assert_regex_count \ + "${render_temp_dir}/keycloak-operator.yaml" \ + '^kind:[[:space:]]CustomResourceDefinition$' 4 \ + "Keycloak Operator CRD count" +assert_regex_count \ + "${render_temp_dir}/keycloak-operator.yaml" \ + '^kind:[[:space:]]Deployment$' 1 \ + "Keycloak Operator Deployment count" +assert_regex_count \ + "${render_temp_dir}/keycloak-operator.yaml" \ + '^[[:space:]]*image:[[:space:]]quay\.io/keycloak/keycloak-operator:26\.7\.0$' 1 \ + "Keycloak Operator image" + +assert_regex_count \ + "${render_temp_dir}/keycloak.yaml" \ + '^kind:[[:space:]]Keycloak$' 1 \ + "Keycloak custom resource count" +assert_regex_count \ + "${render_temp_dir}/keycloak.yaml" \ + '^kind:[[:space:]]Ingress$' 1 \ + "Keycloak Ingress count" +assert_regex_count \ + "${render_temp_dir}/keycloak.yaml" \ + '^[[:space:]]*namespace:[[:space:]]keycloak$' 2 \ + "Keycloak service resource namespace count" +assert_regex_count \ + "${render_temp_dir}/keycloak.yaml" \ + '^[[:space:]]*instances:[[:space:]]1$' 1 \ + "Keycloak instance count" +assert_regex_count \ + "${render_temp_dir}/keycloak.yaml" \ + '^[[:space:]]*hostname:[[:space:]]https://id\.learn\.hyeonworks\.com$' 1 \ + "Keycloak external hostname" +assert_regex_count \ + "${render_temp_dir}/keycloak.yaml" \ + '^[[:space:]]*headers:[[:space:]]xforwarded$' 1 \ + "Keycloak forwarded header mode" +assert_regex_count \ + "${render_temp_dir}/keycloak.yaml" \ + '^[[:space:]]*ingressClassName:[[:space:]]traefik$' 1 \ + "Keycloak Ingress class" +assert_regex_count \ + "${render_temp_dir}/keycloak.yaml" \ + '^[[:space:]]*name:[[:space:]]keycloak-service$' 1 \ + "Keycloak Ingress backend" +assert_regex_count \ + "${render_temp_dir}/keycloak.yaml" \ + '^[[:space:]]*number:[[:space:]]8080$' 1 \ + "Keycloak Ingress backend port" + +if rg --quiet \ + '^[[:space:]]*namespace:[[:space:]](aistor|object-storage)[[:space:]]*$' \ + "${render_temp_dir}"/*.yaml; then + fail "the Keycloak-only render contains an AIStor namespace" +fi +if rg --quiet \ + '^[[:space:]]*(serviceType|type):[[:space:]]*(NodePort|LoadBalancer)[[:space:]]*$' \ + "${render_temp_dir}/keycloak.yaml"; then + fail "Keycloak must not render NodePort or LoadBalancer exposure" +fi + +declare -A verified_manifest_sha256=() +for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do + manifest_path="${render_temp_dir}/${manifest_name}.yaml" + [[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \ + fail "verified manifest is missing or unsafe: ${manifest_path}" + [[ "$(stat --format='%a' -- "$manifest_path")" == "600" ]] || \ + fail "verified manifest must have mode 0600: ${manifest_path}" + checksum_output="$(sha256sum -- "$manifest_path")" + verified_manifest_sha256["$manifest_name"]="${checksum_output%% *}" +done + +verified_entry_count="$( + find "$render_temp_dir" -mindepth 1 -maxdepth 1 | wc -l | tr -d '[:space:]' +)" +[[ "$verified_entry_count" == "${#VERIFIED_MANIFEST_NAMES[@]}" ]] || \ + fail "verified handoff must contain exactly four manifest files" + +verify_manifest_unchanged() { + local manifest_name="$1" + local manifest_path="${render_temp_dir}/${manifest_name}.yaml" + local checksum_output + local actual_sha256 + + [[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \ + fail "verified manifest became missing or unsafe: ${manifest_path}" + checksum_output="$(sha256sum -- "$manifest_path")" + actual_sha256="${checksum_output%% *}" + [[ "$actual_sha256" == "${verified_manifest_sha256[$manifest_name]}" ]] || \ + fail "verified manifest changed before apply: ${manifest_name}.yaml" +} + +current_context="$(kubectl config current-context)" +api_server="$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" +kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \ + fail "target node is missing from the selected cluster: ${TARGET_NODE}" +kubectl get namespace platform-data >/dev/null 2>&1 || \ + fail "Phase 1 namespace platform-data is missing" +kubectl get customresourcedefinition \ + clusters.postgresql.cnpg.io \ + databaseroles.postgresql.cnpg.io \ + databases.postgresql.cnpg.io >/dev/null +postgres_ready="$( + kubectl --namespace platform-data get cluster platform-postgres \ + --output='go-template={{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}true{{end}}{{end}}' +)" +[[ "$postgres_ready" == "true" ]] || \ + fail "platform-data/platform-postgres is not Ready" + +printf '\nKubernetes context: %s\nAPI server: %s\nTarget node: %s\n' \ + "$current_context" "$api_server" "$TARGET_NODE" +printf '%s\n' \ + 'Scope: Keycloak namespace, two DB Secrets, DB Role/Database/NetworkPolicy, Operator, Keycloak, and Ingress.' \ + 'Excluded: AIStor resources, Host Nginx, Gitea OIDC source, and all data deletion.' \ + 'Rollback boundary: applied state is retained on failure; rerunning is the recovery path.' +[[ -t 0 ]] || fail "an interactive terminal is required" +printf 'Type APPLY to start the Keycloak cluster mutation: ' +read -r confirmation +[[ "$confirmation" == "APPLY" ]] || fail "cancelled" + +assert_cluster_identity() { + [[ "$(kubectl config current-context)" == "$current_context" ]] || \ + fail "kubectl context changed after confirmation" + [[ "$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" == "$api_server" ]] || \ + fail "Kubernetes API server changed after confirmation" + kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \ + fail "target node disappeared after confirmation: ${TARGET_NODE}" +} + +assert_cluster_identity +for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do + verify_manifest_unchanged "$manifest_name" +done + +mutation_started=true + +current_step="[1/6] Keycloak namespace" +printf '\n%s\n' "$current_step" +assert_cluster_identity +verify_manifest_unchanged keycloak-namespace +kubectl apply --dry-run=server \ + --filename="${render_temp_dir}/keycloak-namespace.yaml" >/dev/null +kubectl apply --filename="${render_temp_dir}/keycloak-namespace.yaml" + +current_step="[2/6] Keycloak database Secret contracts" +printf '\n%s\n' "$current_step" +assert_cluster_identity +bash scripts/bootstrap/create-keycloak-secrets.sh --execute + +current_step="[3/6] Keycloak DatabaseRole, Database, and NetworkPolicy" +printf '\n%s\n' "$current_step" +assert_cluster_identity +verify_manifest_unchanged platform-postgres-keycloak +kubectl apply --server-side --dry-run=server \ + --filename="${render_temp_dir}/platform-postgres-keycloak.yaml" >/dev/null +kubectl apply --server-side \ + --filename="${render_temp_dir}/platform-postgres-keycloak.yaml" +kubectl --namespace platform-data wait \ + --for=jsonpath='{.status.applied}'=true \ + databaserole/platform-postgres-keycloak \ + database/platform-postgres-keycloak \ + --timeout=3m + +current_step="[4/6] Keycloak Operator and CRDs" +printf '\n%s\n' "$current_step" +assert_cluster_identity +verify_manifest_unchanged keycloak-operator +kubectl apply --server-side --dry-run=server \ + --filename="${render_temp_dir}/keycloak-operator.yaml" >/dev/null +kubectl apply --server-side \ + --filename="${render_temp_dir}/keycloak-operator.yaml" +kubectl wait --for=condition=Established \ + "${KEYCLOAK_CRDS[@]}" \ + --timeout=3m +kubectl --namespace keycloak rollout status \ + deployment/keycloak-operator --timeout=5m + +current_step="[5/6] Keycloak instance and HTTP Ingress" +printf '\n%s\n' "$current_step" +assert_cluster_identity +verify_manifest_unchanged keycloak +kubectl apply --server-side --dry-run=server \ + --filename="${render_temp_dir}/keycloak.yaml" >/dev/null +kubectl apply --server-side \ + --filename="${render_temp_dir}/keycloak.yaml" +kubectl --namespace keycloak wait \ + --for=condition=Ready keycloak/keycloak --timeout=15m +kubectl --namespace keycloak wait \ + --for=jsonpath='{.endpoints[0].conditions.ready}'=true \ + endpointslice \ + --selector=kubernetes.io/service-name=keycloak-service \ + --timeout=2m + +current_step="[6/6] OIDC discovery through Traefik HTTP NodePort" +printf '\n%s\n' "$current_step" +discovery_file="${render_temp_dir}/keycloak-discovery.json" +curl --fail --silent --show-error \ + --retry 24 \ + --retry-all-errors \ + --retry-connrefused \ + --retry-delay 5 \ + --retry-max-time 120 \ + --max-time 10 \ + --header 'Host: id.learn.hyeonworks.com' \ + --output "$discovery_file" \ + http://127.0.0.1:30080/realms/master/.well-known/openid-configuration +rg --quiet \ + '"issuer"[[:space:]]*:[[:space:]]*"https://id\.learn\.hyeonworks\.com/realms/master"' \ + "$discovery_file" || \ + fail "OIDC discovery issuer does not match the public Keycloak hostname" + +mutation_started=false +printf '\nKeycloak cluster resources and internal OIDC discovery are ready.\n' +printf 'No AIStor resource, Host Nginx configuration, or Gitea OIDC source was changed.\n' diff --git a/scripts/bootstrap/apply-observability-access.sh b/scripts/bootstrap/apply-observability-access.sh new file mode 100644 index 0000000..5f6ae66 --- /dev/null +++ b/scripts/bootstrap/apply-observability-access.sh @@ -0,0 +1,1419 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +set +x +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly EXPECTED_CONTEXT=default +readonly EXPECTED_API_SERVER=https://127.0.0.1:6443 +readonly EXPECTED_NODE=donghyeon-system-product-name +readonly PRODUCTION_INITIAL_SHA=79688d017d38eec9a6f100f8d0f784a5474e79802046ef1c2c11b30d170b0b0c +readonly PRODUCTION_POST_SHA=b1c3049206a1a88165ee672ae9aceac7945673a3bb9c3cf3670b7f0d56c3f291 +readonly PRODUCTION_INITIAL_TARGET_COUNT=21 +readonly PRODUCTION_POST_TARGET_COUNT=30 +readonly LEDGER_HEADER=$'phase\towner\tapiVersion\tkind\tnamespace\tname\texisted\trestore-mode\tpayload-file\tpayload-sha256' +readonly MUTATION_HEADER=$'ordinal\tapiVersion\tkind\tnamespace\tname\tuid' +readonly ACCEPTANCE_SCHEMA=platform-observability-rules-alerts-v2 +readonly TEST_MODE="${PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE:-0}" +if [[ "$TEST_MODE" == 1 ]]; then + readonly KUBECTL_BIN="${PLATFORM_OBSERVABILITY_ACCESS_KUBECTL_BIN:-}" + readonly SUDO_BIN="${PLATFORM_OBSERVABILITY_ACCESS_SUDO_BIN:-}" + readonly ENCRYPTION_SCRIPT="${PLATFORM_OBSERVABILITY_ACCESS_ENCRYPTION_SCRIPT:-}" + readonly RESTORE_SCRIPT="${PLATFORM_OBSERVABILITY_ACCESS_RESTORE_SCRIPT:-}" + readonly RECOVERY_SCRIPT="${PLATFORM_OBSERVABILITY_ACCESS_RECOVERY_SCRIPT:-}" + readonly ROLLBACK_BASE="${PLATFORM_OBSERVABILITY_ACCESS_ROLLBACK_BASE:-}" +else + readonly KUBECTL_BIN=/usr/local/bin/kubectl + readonly SUDO_BIN=/usr/bin/sudo + readonly ENCRYPTION_SCRIPT="$ROOT/scripts/validate/k3s-secret-encryption.sh" + readonly RESTORE_SCRIPT="$ROOT/scripts/validate/k3s-secret-encryption-restore-evidence.sh" + readonly RECOVERY_SCRIPT="$ROOT/scripts/bootstrap/create-observability-secrets.sh" + readonly ROLLBACK_BASE=/var/lib/hyeonworks/platform-rollbacks +fi + +# Source-only assertions bind the handoff to the same exact resource and policy +# contract used by the renderer. The renderer's executable main is guarded. +# shellcheck source=../validate/render-observability-access.sh +source "$ROOT/scripts/validate/render-observability-access.sh" + +readonly -a RESOURCE_ROWS=( + 'platform-rules|v1|ConfigMap|observability|grafana-dashboard-https-endpoints|dashboards.yaml' + 'platform-rules|v1|ConfigMap|observability|grafana-dashboard-kubernetes-node|dashboards.yaml' + 'platform-rules|v1|ConfigMap|observability|grafana-dashboard-observability-backends|dashboards.yaml' + 'platform-rules|v1|ConfigMap|observability|grafana-dashboard-platform-services|dashboards.yaml' + 'platform-rules|v1|ConfigMap|observability|grafana-dashboard-workload-health|dashboards.yaml' + 'platform-rules|monitoring.coreos.com/v1|PrometheusRule|observability|platform-aistor-storage-quota|rules.yaml' + 'platform-rules|monitoring.coreos.com/v1|PrometheusRule|observability|platform-certificate-probes|rules.yaml' + 'platform-rules|monitoring.coreos.com/v1|PrometheusRule|observability|platform-observability-core|rules.yaml' + 'platform-rules|monitoring.coreos.com/v1|PrometheusRule|observability|platform-verified-services|rules.yaml' + 'platform-rules|monitoring.coreos.com/v1alpha1|AlertmanagerConfig|observability|platform-alertmanager|alertmanager.yaml' + 'kube-prometheus-stack|monitoring.coreos.com/v1|Alertmanager|observability|observability-core-kube-pr-alertmanager|alertmanager.yaml' + 'kube-prometheus-stack|networking.k8s.io/v1|NetworkPolicy|observability|observability-allow-alertmanager-public-https|alertmanager.yaml' +) + +execute=false +rules_alerts=false +substrate=false +mode_count=0 +verified_output_dir='' +verified_output_count=0 +rollback_root='' +ledger_root='' +work='' +snapshot='' +mutation_started=false +rollback_running=false +failure_stage=preflight +rollback_ambiguous=false +api_proxy_pid='' +api_proxy_socket='' +expected_initial_sha=$PRODUCTION_INITIAL_SHA +expected_post_sha=$PRODUCTION_POST_SHA +target_initial_sha='' +post_substrate_sha='' +node_uid='' +slack_deployment_gate='' +slack_deployment_gate_first='' +slack_deployment_gate_value='' +slack_gate_approval_ref='' +slack_gate_accepted_by_uid='' +slack_gate_temp_root='' + +declare -A HANDOFF_SHA=() +declare -A PRIOR_EXISTED=() +declare -A PRIOR_UID=() +declare -A SPLIT_FILE=() + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_STAGE=%s\n' "$failure_stage" >&2 + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_RC=1\n' >&2 + exit 1 +} + +soft_fail() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +usage() { + cat <<'USAGE' +Usage: + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-observability-access.sh + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-observability-access.sh --rules-alerts + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-observability-access.sh \ + --execute --rules-alerts \ + --verified-output-dir /tmp/platform-observability-metrics.XXXXXX + +Dry-run is source-only and performs no sudo, Kubernetes, network, encryption, +or recovery-evidence command. Execute accepts exactly the rules-alerts mode and +an already verified renderer handoff containing only the two inventories plus +dashboards.yaml, rules.yaml, and alertmanager.yaml. Substrate execution remains +outside this source-only implementation. +USAGE +} + +parse_args() { + while (( $# > 0 )); do + case "$1" in + --execute) + [[ "$execute" == false ]] || return 2 + execute=true + shift + ;; + --rules-alerts) + rules_alerts=true + mode_count=$((mode_count + 1)) + shift + ;; + --substrate) + substrate=true + mode_count=$((mode_count + 1)) + shift + ;; + --verified-output-dir) + (( $# >= 2 )) || return 2 + verified_output_count=$((verified_output_count + 1)) + verified_output_dir=$2 + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + return 2 + ;; + esac + done + (( mode_count <= 1 && verified_output_count <= 1 )) || return 2 + if [[ "$execute" == true ]]; then + [[ "$rules_alerts" == true && "$substrate" == false && $mode_count == 1 && + $verified_output_count == 1 && -n "$verified_output_dir" ]] || return 2 + else + [[ $verified_output_count == 0 ]] || return 2 + fi +} + +print_dry_run() { + printf 'OBSERVABILITY_ACCESS_EXECUTE_MODE=RULES_ALERTS_ONLY\n' + printf 'OBSERVABILITY_ACCESS_TARGET_INITIAL_SHA256=%s\n' "$PRODUCTION_INITIAL_SHA" + printf 'OBSERVABILITY_ACCESS_POST_SUBSTRATE_SHA256=%s\n' "$PRODUCTION_POST_SHA" + printf 'OBSERVABILITY_ACCESS_HANDOFF=target-initial,post-substrate,dashboards.yaml,rules.yaml,alertmanager.yaml\n' + printf 'OBSERVABILITY_ACCESS_MUTATION_SCOPE=ConfigMap,PrometheusRule,AlertmanagerConfig,Alertmanager,NetworkPolicy\n' + printf 'OBSERVABILITY_ACCESS_FORBIDDEN_SCOPE=Secret,PVC,Grafana,Blackbox,Probe,Ingress,Host-Nginx\n' + printf 'OBSERVABILITY_ACCESS_RUNBOOK_URL=REQUIRED_HTTPS_INPUT\n' + printf 'OBSERVABILITY_ACCESS_SLACK_SECRET=REQUIRED_EXACT_SCHEMA_ONLY\n' + printf 'OBSERVABILITY_ACCESS_DRY_RUN=PASS\n' + # Backward-compatible concise names used by the operator checkpoint. + printf 'TARGET_INITIAL_SHA256=%s\n' "$PRODUCTION_INITIAL_SHA" + printf 'POST_SUBSTRATE_SHA256=%s\n' "$PRODUCTION_POST_SHA" +} + +is_sha256() { + [[ "$1" =~ ^[0-9a-f]{64}$ ]] +} + +validate_test_mode() { + local variable fixture_root fixture_parent fixture_bin command_path label expected_path + [[ "$TEST_MODE" == 0 || "$TEST_MODE" == 1 ]] || fail 'invalid test-mode flag' + (( EUID != 0 )) || fail 'whole-script root execution is forbidden; use narrow sudo from the invoking user' + if [[ "$TEST_MODE" == 1 ]]; then + [[ "$SUDO_BIN" != /usr/bin/sudo ]] || fail 'test command boundary is unsafe: real /usr/bin/sudo is forbidden' + fixture_root="$(dirname -- "$ROLLBACK_BASE")" + fixture_parent="$(dirname -- "$fixture_root")" + fixture_bin="$fixture_root/bin" + [[ "$fixture_parent" == /tmp/platform-observability-access-test.?????? && + "$fixture_root" == "$fixture_parent"/* && "$fixture_root" != "$fixture_parent"/ && + "$ROLLBACK_BASE" == "$fixture_root/rollbacks" && + "$(readlink -f -- "$fixture_parent" 2>/dev/null)" == "$fixture_parent" && + "$(readlink -f -- "$fixture_root" 2>/dev/null)" == "$fixture_root" && + "$(readlink -f -- "$ROLLBACK_BASE" 2>/dev/null)" == "$ROLLBACK_BASE" ]] || + fail 'test command fixture root is not exact and canonical' + local directory metadata mode_value + for directory in "$fixture_parent" "$fixture_root" "$fixture_bin" "$ROLLBACK_BASE"; do + [[ -d "$directory" && ! -L "$directory" ]] || fail 'test command fixture directory is unsafe' + metadata="$(stat -c '%F|%u|%a' -- "$directory")" || fail 'cannot stat test command fixture directory' + IFS='|' read -r type owner mode <<<"$metadata" + mode_value=$((8#$mode)) + [[ "$type" == directory && "$owner" == "$(id -u)" && + $((mode_value & 0022)) == 0 && $((mode_value & 07000)) == 0 ]] || + fail 'test command fixture directory owner or mode is unsafe' + done + local -a boundary_names=(kubectl sudo encryption restore recovery) + local -a boundary_paths=("$KUBECTL_BIN" "$SUDO_BIN" "$ENCRYPTION_SCRIPT" "$RESTORE_SCRIPT" \ + "$RECOVERY_SCRIPT") + local index resolved type owner mode + for index in "${!boundary_names[@]}"; do + label=${boundary_names[$index]} + command_path=${boundary_paths[$index]} + expected_path="$fixture_bin/$label" + resolved="$(readlink -f -- "$command_path" 2>/dev/null)" || + fail "test command boundary is unsafe: $label does not resolve" + [[ "$command_path" == "$expected_path" && "$resolved" == "$expected_path" && + -f "$command_path" && ! -L "$command_path" && -x "$command_path" ]] || + fail "test command boundary is unsafe: $label escapes the exact fixture bin" + metadata="$(stat -c '%F|%u|%a' -- "$command_path")" || + fail "test command boundary is unsafe: cannot stat $label" + IFS='|' read -r type owner mode <<<"$metadata" + mode_value=$((8#$mode)) + [[ "$type" == 'regular file' && "$owner" == "$(id -u)" && + $((mode_value & 0022)) == 0 && $((mode_value & 07000)) == 0 ]] || + fail "test command boundary is unsafe: $label owner or mode" + done + if [[ -n "$verified_output_dir" ]]; then + [[ "$verified_output_dir" == "$fixture_root/metrics" && + "$(readlink -f -- "$verified_output_dir" 2>/dev/null)" == "$verified_output_dir" ]] || + fail 'test verified output escapes the exact fixture root' + expected_initial_sha="${PLATFORM_TEST_EXPECTED_INITIAL_SHA:-}" + expected_post_sha="${PLATFORM_TEST_EXPECTED_POST_SHA:-}" + is_sha256 "$expected_initial_sha" && is_sha256 "$expected_post_sha" || + fail 'test inventory hashes are malformed' + fi + else + for variable in ${!PLATFORM_OBSERVABILITY_ACCESS_@}; do + fail "production rejects observability access override: $variable" + done + [[ "$KUBECTL_BIN" == /usr/local/bin/kubectl && "$SUDO_BIN" == /usr/bin/sudo && + "$ENCRYPTION_SCRIPT" == "$ROOT/scripts/validate/k3s-secret-encryption.sh" && + "$RESTORE_SCRIPT" == "$ROOT/scripts/validate/k3s-secret-encryption-restore-evidence.sh" && + "$RECOVERY_SCRIPT" == "$ROOT/scripts/bootstrap/create-observability-secrets.sh" && + "$ROLLBACK_BASE" == /var/lib/hyeonworks/platform-rollbacks ]] || + fail 'production observability access constants are not exact' + [[ -f "$KUBECTL_BIN" && ! -L "$KUBECTL_BIN" && -x "$KUBECTL_BIN" && + -f "$SUDO_BIN" && ! -L "$SUDO_BIN" && -x "$SUDO_BIN" ]] || + fail 'production command boundary is unsafe' + for command_path in "$ENCRYPTION_SCRIPT" "$RESTORE_SCRIPT" "$RECOVERY_SCRIPT"; do + [[ -f "$command_path" && ! -L "$command_path" ]] || + fail "required production gate script is missing or unsafe: $command_path" + done + local required + for required in base64 curl date find gzip jq mktemp python3 sha256sum stat tac; do + command -v "$required" >/dev/null 2>&1 || fail "required command is unavailable: $required" + done + fi +} + +kubectl_bounded() { + "$KUBECTL_BIN" --request-timeout=10s "$@" +} + +run_encryption_gates() { + /usr/bin/bash "$ENCRYPTION_SCRIPT" --expect-reencrypted || fail 'Secret encryption evidence gate failed' + /usr/bin/bash "$RESTORE_SCRIPT" --check || fail 'Secret restore evidence gate failed' +} + +validate_context() { + local context api node_json + context="$(kubectl_bounded config current-context)" || return 1 + [[ "$context" == "$EXPECTED_CONTEXT" ]] || fail "Kubernetes context changed: $context" + api="$(kubectl_bounded config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" || return 1 + [[ "$api" == "$EXPECTED_API_SERVER" ]] || fail "Kubernetes API server changed: $api" + node_json="$(kubectl_bounded get node "$EXPECTED_NODE" --output=json)" || return 1 + node_uid="$(python3 -c ' +import json, sys +item=json.load(sys.stdin) +ready=any(x.get("type")=="Ready" and x.get("status")=="True" for x in item.get("status",{}).get("conditions",[])) +meta=item.get("metadata",{}) +if meta.get("name") != sys.argv[1] or not meta.get("uid") or not ready: + raise SystemExit(1) +print(meta["uid"]) +' "$EXPECTED_NODE" <<<"$node_json")" || fail 'target node identity or Ready condition changed' +} + +validate_inventory_phase() { + local phase=$1 expected_sha=$2 expected_target_count=$3 directory checksum_line declared filename extra actual entries current_uid captured captured_epoch now_epoch age + [[ "$expected_target_count" =~ ^[1-9][0-9]*$ ]] || + fail "inventory expected target count is invalid: $phase" + current_uid="$(id -u)" || return 1 + directory="$verified_output_dir/$phase" + [[ -d "$directory" && ! -L "$directory" && "$(readlink -f -- "$directory")" == "$directory" ]] || + fail "inventory directory is unsafe: $phase" + [[ "$(stat -c '%u:%a:%h' -- "$directory")" == "$current_uid:700:2" ]] || + fail "inventory directory metadata changed: $phase" + entries="$(find "$directory" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" || return 1 + [[ "$entries" == $'inventory.json\ninventory.sha256' ]] || fail "inventory entry set changed: $phase" + local file + for file in inventory.json inventory.sha256; do + [[ -f "$directory/$file" && ! -L "$directory/$file" && + "$(stat -c '%u:%a:%h' -- "$directory/$file")" == "$current_uid:600:1" ]] || + fail "inventory file metadata changed: $phase/$file" + done + IFS=' ' read -r declared filename extra <"$directory/inventory.sha256" + [[ "$declared" =~ ^[0-9a-f]{64}$ && "$filename" == inventory.json && -z "${extra:-}" ]] || + fail "inventory checksum schema changed: $phase" + actual="$(sha256sum -- "$directory/inventory.json" | awk '{print $1}')" || return 1 + [[ "$actual" == "$declared" && "$actual" == "$expected_sha" ]] || + fail "inventory hash changed: $phase" + jq -e --arg phase "$phase" --argjson expected_target_count "$expected_target_count" ' + .schema == "platform-observability-metric-inventory/v1" and + .phase == $phase and + .context == "default" and .api_server == "https://127.0.0.1:6443" and + (.captured_at_utc | type == "string") and + (.targets | type == "array" and length > 0 and length == $expected_target_count) and + all(.targets[]; + .health == "up" and .last_error == "" and + (.metrics | type == "array" and length > 0) and + all(.metrics[]; + (.name | type == "string" and length > 0) and + (.label_names | type == "array") and + all(.label_names[]; type == "string"))) + ' "$directory/inventory.json" >/dev/null || fail "inventory semantic contract changed: $phase" + captured="$(jq -r '.captured_at_utc' "$directory/inventory.json")" || return 1 + [[ "$captured" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || + fail "inventory capture timestamp is malformed: $phase" + captured_epoch="$(date -u -d "$captured" +%s 2>/dev/null)" || + fail "inventory capture timestamp is invalid: $phase" + now_epoch="$(date -u +%s)" || return 1 + age=$((now_epoch - captured_epoch)) + (( age >= -300 )) || fail "inventory capture timestamp is too far in the future: $phase" + printf '%s' "$actual" +} + +validate_handoff_root() { + local current_uid entries file validation_reason expected_test_root + current_uid="$(id -u)" || return 1 + if [[ "$TEST_MODE" == 1 ]]; then + expected_test_root="$(dirname -- "$ROLLBACK_BASE")/metrics" + [[ "$verified_output_dir" == "$expected_test_root" ]] || + fail 'test verified output is not confined to the exact fixture root' + else + [[ "$verified_output_dir" =~ ^/tmp/platform-observability-metrics\.[A-Za-z0-9]{6}$ ]] || + fail 'verified output path is not an exact /tmp metric handoff' + [[ -d /tmp && ! -L /tmp && "$(readlink -f -- /tmp)" == /tmp && + "$(stat -c '%u:%a' -- /tmp)" == 0:1777 ]] || fail '/tmp lineage is unsafe' + fi + [[ -d "$verified_output_dir" && ! -L "$verified_output_dir" && + "$(readlink -f -- "$verified_output_dir")" == "$verified_output_dir" && + "$(stat -c '%u:%a:%h' -- "$verified_output_dir")" == "$current_uid:700:4" ]] || + fail 'verified output root metadata changed' + entries="$(find "$verified_output_dir" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" || return 1 + [[ "$entries" == $'alertmanager.yaml\ndashboards.yaml\npost-substrate\nrules.yaml\ntarget-initial' ]] || + fail 'verified renderer handoff must contain only two inventories and three rules-alerts manifests' + target_initial_sha="$(validate_inventory_phase target-initial "$expected_initial_sha" \ + "$PRODUCTION_INITIAL_TARGET_COUNT")" || return 1 + post_substrate_sha="$(validate_inventory_phase post-substrate "$expected_post_sha" \ + "$PRODUCTION_POST_TARGET_COUNT")" || return 1 + for file in dashboards.yaml rules.yaml alertmanager.yaml; do + [[ -f "$verified_output_dir/$file" && ! -L "$verified_output_dir/$file" && + "$(stat -c '%u:%a:%h' -- "$verified_output_dir/$file")" == "$current_uid:600:1" ]] || + fail "renderer handoff file metadata changed: $file" + HANDOFF_SHA[$file]="$(sha256sum -- "$verified_output_dir/$file" | awk '{print $1}')" || return 1 + done + if ! validation_reason="$(python3 - "$verified_output_dir" <<'PY' +import pathlib +import re +import sys +import yaml + +root=pathlib.Path(sys.argv[1]) +def load(name): + with (root/name).open(encoding="utf-8") as stream: + return [item for item in yaml.safe_load_all(stream) if item is not None] +def identity(item): + meta=item.get("metadata") or {} + return item.get("apiVersion"), item.get("kind"), meta.get("namespace"), meta.get("name") +expected_dashboards={ + ("v1","ConfigMap","observability",name) for name in ( + "grafana-dashboard-https-endpoints", "grafana-dashboard-kubernetes-node", + "grafana-dashboard-observability-backends", "grafana-dashboard-platform-services", + "grafana-dashboard-workload-health", + ) +} +expected_rules={ + ("monitoring.coreos.com/v1","PrometheusRule","observability",name) for name in ( + "platform-aistor-storage-quota", "platform-certificate-probes", + "platform-observability-core", "platform-verified-services", + ) +} +expected_alertmanager={ + ("monitoring.coreos.com/v1","Alertmanager","observability","observability-core-kube-pr-alertmanager"), + ("monitoring.coreos.com/v1alpha1","AlertmanagerConfig","observability","platform-alertmanager"), + ("networking.k8s.io/v1","NetworkPolicy","observability","observability-allow-alertmanager-public-https"), +} +dashboards, rules, alertmanager = load("dashboards.yaml"), load("rules.yaml"), load("alertmanager.yaml") +groups=((dashboards, expected_dashboards), (rules, expected_rules), (alertmanager, expected_alertmanager)) +for items, expected in groups: + actual=[identity(item) for item in items] + if len(actual) != len(set(actual)) or set(actual) != expected: + print("INVALID_RESOURCE_SET") + raise SystemExit(1) +for item in dashboards: + if ((item.get("metadata") or {}).get("labels") or {}).get("grafana_dashboard") != "1": + print("INVALID_DASHBOARD_CONTRACT") + raise SystemExit(1) +runbook=re.compile(r"^https://[^/\s]+/.+", re.ASCII) +for item in rules: + for group in (item.get("spec") or {}).get("groups") or []: + for rule in group.get("rules") or []: + if "alert" in rule: + value=(rule.get("annotations") or {}).get("runbook_url") + if not isinstance(value,str) or not runbook.fullmatch(value) or re.search(r"placeholder|example|todo|tbd",value,re.I): + print("BLOCKED_RUNBOOK_URL") + raise SystemExit(1) +am=next(item for item in alertmanager if item.get("kind")=="Alertmanager") +if (((am.get("spec") or {}).get("alertmanagerConfiguration") or {}).get("name")) != "platform-alertmanager": + print("INVALID_ALERTMANAGER_REFERENCE") + raise SystemExit(1) +config=next(item for item in alertmanager if item.get("kind")=="AlertmanagerConfig") +receivers=(config.get("spec") or {}).get("receivers") or [] +slack_receiver=next((receiver for receiver in receivers if receiver.get("name")=="platform-slack"), {}) +slacks=(slack_receiver.get("slackConfigs") or []) +if len(slacks)!=1 or slacks[0].get("apiURL") != {"name":"alertmanager-slack-webhook","key":"url"} or slacks[0].get("sendResolved") is not True: + print("INVALID_SLACK_REFERENCE") + raise SystemExit(1) +print("PASS") +PY +)"; then + if [[ "$validation_reason" == BLOCKED_RUNBOOK_URL ]]; then + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_READINESS=BLOCKED_RUNBOOK_URL\n' >&2 + fail 'every alert requires the operator-supplied HTTPS runbook URL; guessed URLs are forbidden' + else + fail "renderer handoff contract failed: ${validation_reason:-unknown}" + fi + fi + [[ "$validation_reason" == PASS ]] || fail 'renderer handoff validation did not complete' + assert_access_rules_alerts_contract \ + "$verified_output_dir/dashboards.yaml" \ + "$verified_output_dir/rules.yaml" \ + "$verified_output_dir/alertmanager.yaml" || + fail 'renderer rules-alerts assertion library rejected the handoff' + assert_access_scope_contract "$verified_output_dir/rules.yaml" || + fail 'renderer scope assertion library rejected the handoff' + assert_no_credentials "$verified_output_dir/dashboards.yaml" \ + "$verified_output_dir/rules.yaml" "$verified_output_dir/alertmanager.yaml" || + fail 'renderer handoff contains a credential payload' +} + +verify_handoff_unchanged() { + local file actual + validate_inventory_phase target-initial "$target_initial_sha" \ + "$PRODUCTION_INITIAL_TARGET_COUNT" >/dev/null || return 1 + validate_inventory_phase post-substrate "$post_substrate_sha" \ + "$PRODUCTION_POST_TARGET_COUNT" >/dev/null || return 1 + for file in dashboards.yaml rules.yaml alertmanager.yaml; do + [[ -f "$verified_output_dir/$file" && ! -L "$verified_output_dir/$file" ]] || return 1 + actual="$(sha256sum -- "$verified_output_dir/$file" | awk '{print $1}')" || return 1 + [[ "$actual" == "${HANDOFF_SHA[$file]}" ]] || fail "verified renderer handoff changed: $file" + done +} + +validate_slack_secret_schema() { + local secret_json + secret_json="$(kubectl_bounded get secret alertmanager-slack-webhook \ + --namespace observability --output=json)" || fail 'Slack Secret is absent or unreadable' + python3 -c ' +import json, sys +item=json.load(sys.stdin) +meta=item.get("metadata") or {} +if item.get("apiVersion")!="v1" or item.get("kind")!="Secret" or item.get("type")!="Opaque": + raise SystemExit(1) +if meta.get("namespace")!="observability" or meta.get("name")!="alertmanager-slack-webhook": + raise SystemExit(1) +if sorted((item.get("data") or {}).keys()) != ["url"]: + raise SystemExit(1) +' <<<"$secret_json" || fail 'Slack Secret schema or key set is not exact' +} + +capture_slack_deployment_gate() { + local out_var=$1 label=$2 output producer_rc verdict + [[ "$out_var" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ && "$label" =~ ^[a-z0-9-]+$ ]] || + fail 'Slack deployment gate capture arguments are invalid' + if [[ -z "$slack_gate_temp_root" ]]; then + slack_gate_temp_root="$(mktemp -d /tmp/platform-observability-slack-gate.XXXXXX)" || + fail 'cannot create Slack deployment gate temporary root' + chmod 0700 "$slack_gate_temp_root" || fail 'cannot secure Slack deployment gate temporary root' + fi + [[ "$slack_gate_temp_root" =~ ^/tmp/platform-observability-slack-gate\.[A-Za-z0-9]{6}$ && + -d "$slack_gate_temp_root" && ! -L "$slack_gate_temp_root" && + "$(readlink -f -- "$slack_gate_temp_root")" == "$slack_gate_temp_root" && + "$(stat -c '%u:%a' -- "$slack_gate_temp_root")" == "$(id -u):700" ]] || + fail 'Slack deployment gate temporary root is unsafe' + output="$slack_gate_temp_root/$label.stdout" + [[ ! -e "$output" && ! -L "$output" ]] || fail 'Slack deployment gate output already exists' + umask 077 + if /usr/bin/bash "$RECOVERY_SCRIPT" --check-slack-deployment-evidence >"$output"; then + producer_rc=0 + else + producer_rc=$? + fi + chmod 0600 "$output" || fail 'cannot secure Slack deployment gate output' + (( producer_rc == 0 )) || fail 'Slack deployment gate is unavailable' + verdict="$(python3 - "$output" <<'PY' +import pathlib, sys +raw=pathlib.Path(sys.argv[1]).read_bytes() +accepted={ + b"SLACK_DEPLOYMENT_GATE=RECOVERY\n": "SLACK_DEPLOYMENT_GATE=RECOVERY", + b"SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n": "SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED", +} +value=accepted.get(raw) +if value is None: + raise SystemExit(1) +print(value) +PY +)" || fail 'Slack deployment gate verdict is malformed' + printf -v "$out_var" '%s' "$verdict" +} + +map_slack_deployment_gate_acceptance() { + case "$slack_deployment_gate" in + SLACK_DEPLOYMENT_GATE=RECOVERY) + slack_deployment_gate_value=RECOVERY + slack_gate_approval_ref=strict-recovery-evidence-v1 + slack_gate_accepted_by_uid=not-applicable + ;; + SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED) + slack_deployment_gate_value=RISK_ACCEPTED + slack_gate_approval_ref=2026-08-14-observability-slack-recovery-risk-acceptance-design + slack_gate_accepted_by_uid=1000 + ;; + *) + return 1 + ;; + esac +} + +require_matching_slack_deployment_gates() { + [[ "$slack_deployment_gate" == "$slack_deployment_gate_first" ]] || + fail 'Slack deployment gate verdict changed before first apply' +} + +run_production_preflight() { + kubectl_bounded wait --namespace observability --for=condition=Available \ + deployment/grafana deployment/blackbox-exporter --timeout=60s >/dev/null || return 1 + kubectl_bounded wait --namespace observability --for=condition=Available \ + prometheus/observability-core-kube-pr-prometheus \ + alertmanager/observability-core-kube-pr-alertmanager --timeout=60s >/dev/null || return 1 + kubectl_bounded rollout status --namespace observability \ + statefulset/prometheus-observability-core-kube-pr-prometheus --timeout=60s >/dev/null || return 1 + local probes + probes="$(kubectl_bounded get probe --namespace observability --output=json)" || return 1 + jq -e ' + [.items[].metadata.name] | sort == + ["platform-private-edge","platform-private-internal","platform-public-edge"] + ' <<<"$probes" >/dev/null || return 1 + kubectl_bounded get servicemonitor aistor-bucket-usage --namespace object-storage --output=name >/dev/null || return 1 + kubectl_bounded get podmonitor platform-postgres --namespace platform-data --output=name >/dev/null || return 1 +} + +run_preflight() { + run_production_preflight +} + +server_dry_run_handoff() { + local file + for file in dashboards.yaml rules.yaml alertmanager.yaml; do + kubectl_bounded apply --dry-run=server -f "$verified_output_dir/$file" >/dev/null || + fail "server dry-run rejected renderer handoff: $file" + done +} + +validate_root_metadata() { + local path=$1 expected_mode=$2 expected_type=$3 metadata type owner mode + "$SUDO_BIN" -n test ! -L "$path" || return 1 + metadata="$($SUDO_BIN -n stat -c '%F|%U:%G|%a' -- "$path")" || return 1 + IFS='|' read -r type owner mode <<<"$metadata" + [[ "$type" == "$expected_type" && "$mode" == "$expected_mode" ]] || return 1 + [[ "$TEST_MODE" == 1 || "$owner" == root:root ]] || return 1 +} + +validate_rollback_root() { + local id=${PLATFORM_OBSERVABILITY_ROLLBACK_ID:-} + [[ "$id" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || fail 'PLATFORM_OBSERVABILITY_ROLLBACK_ID format is invalid' + rollback_root="$ROLLBACK_BASE/observability-$id" + "$SUDO_BIN" -n test -d "$rollback_root" || fail 'rollback root is absent' + [[ "$($SUDO_BIN -n readlink -f -- "$rollback_root")" == "$rollback_root" ]] || + fail 'rollback root lineage contains a symlink' + validate_root_metadata "$rollback_root" 700 directory || fail 'rollback root metadata is unsafe' + ledger_root="$rollback_root/access-rules-alerts" +} + +confirm_apply() { + local answer='' + printf 'Type APPLY default to apply only observability rules-alerts: ' + if [[ "$TEST_MODE" == 1 ]]; then + answer=${PLATFORM_OBSERVABILITY_ACCESS_CONFIRMATION:-} + printf '%s\n' "$answer" + else + [[ -t 0 ]] || fail '--execute requires an interactive terminal' + read -r answer + fi + [[ "$answer" == APPLY ]] || fail 'cancelled' +} + +resource_for_kind() { + case "$1" in + ConfigMap) printf 'configmap' ;; + PrometheusRule) printf 'prometheusrule' ;; + AlertmanagerConfig) printf 'alertmanagerconfig' ;; + Alertmanager) printf 'alertmanager' ;; + NetworkPolicy) printf 'networkpolicy' ;; + *) return 1 ;; + esac +} + +object_key() { + printf '%s|%s|%s' "$1" "$2" "$3" +} + +get_object_json() { + local kind=$1 namespace=$2 name=$3 resource output rc + resource="$(resource_for_kind "$kind")" || return 2 + if output="$(kubectl_bounded get "$resource" "$name" --namespace "$namespace" \ + --ignore-not-found --output=json)"; then + rc=0 + else + rc=$? + fi + (( rc == 0 )) || return 2 + printf '%s' "$output" +} + +object_uid_from_json() { + python3 -c ' +import json, sys +item=json.load(sys.stdin) +value=(item.get("metadata") or {}).get("uid") +if not isinstance(value,str) or not value: + raise SystemExit(1) +print(value) +' +} + +prepare_split_files() { + local ordinal=0 row owner api kind namespace name source key output + mkdir -m 0700 "$work/apply" + for row in "${RESOURCE_ROWS[@]}"; do + ordinal=$((ordinal + 1)) + IFS='|' read -r owner api kind namespace name source <<<"$row" + output="$work/apply/$(printf '%02d' "$ordinal")-$kind-$name.yaml" + python3 - "$verified_output_dir/$source" "$api" "$kind" "$namespace" "$name" "$output" <<'PY' +import pathlib, sys, yaml +source, api, kind, namespace, name, output=sys.argv[1:] +with open(source,encoding="utf-8") as stream: + matches=[item for item in yaml.safe_load_all(stream) if item and item.get("apiVersion")==api and item.get("kind")==kind and (item.get("metadata") or {}).get("namespace")==namespace and (item.get("metadata") or {}).get("name")==name] +if len(matches)!=1: + raise SystemExit(1) +with open(output,"x",encoding="utf-8") as stream: + yaml.safe_dump(matches[0],stream,explicit_start=True,sort_keys=False) +PY + chmod 0600 "$output" + key="$(object_key "$kind" "$namespace" "$name")" + SPLIT_FILE[$key]=$output + done +} + +sanitize_prior_object() { + local input=$1 output=$2 + python3 - "$input" "$output" <<'PY' +import json, sys +item=json.load(open(sys.argv[1],encoding="utf-8")) +item.pop("status",None) +meta=item.setdefault("metadata",{}) +for key in ("managedFields","resourceVersion","uid","generation","creationTimestamp","deletionTimestamp","deletionGracePeriodSeconds","selfLink"): + meta.pop(key,None) +annotations=meta.get("annotations") or {} +annotations.pop("kubectl.kubernetes.io/last-applied-configuration",None) +if annotations: + meta["annotations"]=annotations +else: + meta.pop("annotations",None) +with open(sys.argv[2],"x",encoding="utf-8") as stream: + json.dump(item,stream,separators=(",",":"),sort_keys=True) + stream.write("\n") +PY + chmod 0600 "$output" +} + +validate_local_ledger() { + local ledger=$1 payload_root=$2 line_count expected_count ordinal=0 row + [[ -f "$ledger" && ! -L "$ledger" && "$(stat -c %a "$ledger")" == 600 ]] || return 1 + [[ "$(head -n1 "$ledger")" == "$LEDGER_HEADER" ]] || return 1 + line_count="$(wc -l <"$ledger" | tr -d ' ')" + expected_count=$((${#RESOURCE_ROWS[@]} + 1)) + [[ "$line_count" == "$expected_count" ]] || return 1 + while IFS=$'\t' read -r phase owner api kind namespace name existed restore payload_file payload_sha extra; do + (( ordinal += 1 )) + row=${RESOURCE_ROWS[$((ordinal - 1))]} + local expected_owner expected_api expected_kind expected_namespace expected_name source + IFS='|' read -r expected_owner expected_api expected_kind expected_namespace expected_name source <<<"$row" + [[ "$phase" == access-rules-alerts && "$owner" == "$expected_owner" && "$api" == "$expected_api" && + "$kind" == "$expected_kind" && "$namespace" == "$expected_namespace" && "$name" == "$expected_name" && + "$restore" == apply && -z "${extra:-}" ]] || return 1 + [[ "$existed" == true || "$existed" == false ]] || return 1 + if [[ "$existed" == true ]]; then + [[ "$payload_file" =~ ^payloads/prior-[0-9]{2}\.json$ && "$payload_sha" =~ ^[0-9a-f]{64}$ ]] || return 1 + [[ -f "$payload_root/${payload_file#payloads/}" && ! -L "$payload_root/${payload_file#payloads/}" ]] || return 1 + [[ "$(sha256sum "$payload_root/${payload_file#payloads/}" | awk '{print $1}')" == "$payload_sha" ]] || return 1 + else + [[ "$payload_file" == - && "$payload_sha" == - ]] || return 1 + fi + done < <(tail -n +2 "$ledger") + (( ordinal == ${#RESOURCE_ROWS[@]} )) +} + +capture_snapshot() { + local ledger="$snapshot/objects.tsv" payload_root="$snapshot/payloads" + local ordinal=0 row owner api kind namespace name source key live live_file uid payload_rel payload sha + mkdir -m 0700 "$payload_root" + printf '%s\n' "$LEDGER_HEADER" >"$ledger" + chmod 0600 "$ledger" + for row in "${RESOURCE_ROWS[@]}"; do + ordinal=$((ordinal + 1)) + IFS='|' read -r owner api kind namespace name source <<<"$row" + key="$(object_key "$kind" "$namespace" "$name")" + live="$(get_object_json "$kind" "$namespace" "$name")" || fail "cannot capture current object: $kind/$name" + if [[ -n "$live" ]]; then + live_file="$snapshot/live-$(printf '%02d' "$ordinal").json" + printf '%s\n' "$live" >"$live_file" + chmod 0600 "$live_file" + python3 - "$live_file" "$api" "$kind" "$namespace" "$name" <<'PY' +import json,sys +item=json.load(open(sys.argv[1])) +meta=item.get("metadata") or {} +if (item.get("apiVersion"),item.get("kind"),meta.get("namespace"),meta.get("name")) != tuple(sys.argv[2:]): + raise SystemExit(1) +if not meta.get("uid"): + raise SystemExit(1) +PY + uid="$(object_uid_from_json <"$live_file")" || return 1 + PRIOR_EXISTED[$key]=true + PRIOR_UID[$key]=$uid + payload_rel="payloads/prior-$(printf '%02d' "$ordinal").json" + payload="$snapshot/$payload_rel" + sanitize_prior_object "$live_file" "$payload" + sha="$(sha256sum -- "$payload" | awk '{print $1}')" + printf 'access-rules-alerts\t%s\t%s\t%s\t%s\t%s\ttrue\tapply\t%s\t%s\n' \ + "$owner" "$api" "$kind" "$namespace" "$name" "$payload_rel" "$sha" >>"$ledger" + else + PRIOR_EXISTED[$key]=false + PRIOR_UID[$key]='-' + printf 'access-rules-alerts\t%s\t%s\t%s\t%s\t%s\tfalse\tapply\t-\t-\n' \ + "$owner" "$api" "$kind" "$namespace" "$name" >>"$ledger" + fi + done + validate_local_ledger "$ledger" "$payload_root" || fail 'captured rollback ledger is invalid' +} + +ensure_root_dir() { + local path=$1 + if "$SUDO_BIN" -n test -e "$path"; then + validate_root_metadata "$path" 700 directory || return 1 + else + "$SUDO_BIN" -n install -d -o root -g root -m 0700 "$path" || return 1 + validate_root_metadata "$path" 700 directory || return 1 + fi +} + +install_snapshot() { + local root_payload="$ledger_root/payloads" payload + ensure_root_dir "$ledger_root" || fail 'rules-alerts ledger directory is unsafe' + ensure_root_dir "$root_payload" || fail 'rules-alerts payload directory is unsafe' + for payload in objects.tsv mutations.tsv acceptance.env; do + ! "$SUDO_BIN" -n test -e "$ledger_root/$payload" || fail "rules-alerts ledger already exists: $payload" + done + [[ -z "$($SUDO_BIN -n find "$root_payload" -mindepth 1 -maxdepth 1 -print -quit)" ]] || + fail 'rules-alerts payload directory is not empty' + for payload in "$snapshot"/payloads/*.json; do + [[ -e "$payload" ]] || continue + root_atomic_install "$payload" "$root_payload/${payload##*/}" create || return 1 + done + root_atomic_install "$snapshot/objects.tsv" "$ledger_root/objects.tsv" create || return 1 + printf '%s\n' "$MUTATION_HEADER" >"$snapshot/mutations.tsv" + chmod 0600 "$snapshot/mutations.tsv" + root_atomic_install "$snapshot/mutations.tsv" "$ledger_root/mutations.tsv" create || return 1 + validate_root_metadata "$ledger_root/objects.tsv" 600 'regular file' || return 1 + validate_root_metadata "$ledger_root/mutations.tsv" 600 'regular file' || return 1 +} + +root_atomic_install() { + local source=$1 destination=$2 mode=$3 temporary + temporary="${destination}.next-$$" + [[ "$mode" == create || "$mode" == replace ]] || return 2 + ! "$SUDO_BIN" -n test -e "$temporary" || return 1 + if [[ "$mode" == create ]]; then + ! "$SUDO_BIN" -n test -e "$destination" || return 1 + else + "$SUDO_BIN" -n test -f "$destination" || return 1 + "$SUDO_BIN" -n test ! -L "$destination" || return 1 + fi + "$SUDO_BIN" -n install -o root -g root -m 0600 "$source" "$temporary" || return 1 + validate_root_metadata "$temporary" 600 'regular file' || return 1 + if [[ "$mode" == create ]]; then + ! "$SUDO_BIN" -n test -e "$destination" || return 1 + fi + "$SUDO_BIN" -n mv -T -- "$temporary" "$destination" || return 1 + validate_root_metadata "$destination" 600 'regular file' +} + +verify_live_preconditions() { + local row owner api kind namespace name source key live uid + for row in "${RESOURCE_ROWS[@]}"; do + IFS='|' read -r owner api kind namespace name source <<<"$row" + key="$(object_key "$kind" "$namespace" "$name")" + live="$(get_object_json "$kind" "$namespace" "$name")" || return 1 + if [[ "${PRIOR_EXISTED[$key]}" == true ]]; then + [[ -n "$live" ]] || fail "existing object disappeared after snapshot: $kind/$name" + uid="$(object_uid_from_json <<<"$live")" || return 1 + [[ "$uid" == "${PRIOR_UID[$key]}" ]] || fail "existing object UID changed after snapshot: $kind/$name" + else + [[ -z "$live" ]] || fail "new object appeared after snapshot: $kind/$name" + fi + done +} + +record_mutation() { + local ordinal=$1 api=$2 kind=$3 namespace=$4 name=$5 uid=$6 + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$ordinal" "$api" "$kind" "$namespace" "$name" "$uid" \ + >>"$snapshot/mutations.tsv" + root_atomic_install "$snapshot/mutations.tsv" "$ledger_root/mutations.tsv" replace +} + +apply_resources() { + local ordinal=0 row owner api kind namespace name source key live uid + for row in "${RESOURCE_ROWS[@]}"; do + ordinal=$((ordinal + 1)) + IFS='|' read -r owner api kind namespace name source <<<"$row" + key="$(object_key "$kind" "$namespace" "$name")" + live="$(get_object_json "$kind" "$namespace" "$name")" || return 1 + if [[ "${PRIOR_EXISTED[$key]}" == true ]]; then + [[ -n "$live" && "$(object_uid_from_json <<<"$live")" == "${PRIOR_UID[$key]}" ]] || { + soft_fail "forward ownership precondition changed: $kind/$name" + return 1 + } + else + [[ -z "$live" ]] || { + soft_fail "forward absence precondition changed: $kind/$name" + return 1 + } + fi + failure_stage="apply-$kind-$name" + # A non-zero client result cannot prove that the API server made no change. + # Arm rollback before the request and retain ambiguity unless the resulting + # UID is observed and durably recorded below. + mutation_started=true + rollback_ambiguous=true + if ! kubectl_bounded apply -f "${SPLIT_FILE[$key]}" >/dev/null; then + return 1 + fi + live="$(get_object_json "$kind" "$namespace" "$name")" || { + rollback_ambiguous=true + return 1 + } + [[ -n "$live" ]] || { + rollback_ambiguous=true + return 1 + } + uid="$(object_uid_from_json <<<"$live")" || { + rollback_ambiguous=true + return 1 + } + if [[ "${PRIOR_EXISTED[$key]}" == true && "$uid" != "${PRIOR_UID[$key]}" ]]; then + rollback_ambiguous=true + return 1 + fi + record_mutation "$ordinal" "$api" "$kind" "$namespace" "$name" "$uid" || { + rollback_ambiguous=true + return 1 + } + rollback_ambiguous=false + done +} + +validate_authoritative_ledger_for_rollback() { + local copy="$work/root-ledger.tsv" payload_copy_root="$work/root-payloads" line payload_rel payload_sha + mkdir -m 0700 "$payload_copy_root" || return 1 + validate_root_metadata "$ledger_root" 700 directory || return 1 + validate_root_metadata "$ledger_root/payloads" 700 directory || return 1 + validate_root_metadata "$ledger_root/objects.tsv" 600 'regular file' || return 1 + validate_root_metadata "$ledger_root/mutations.tsv" 600 'regular file' || return 1 + "$SUDO_BIN" -n cat -- "$ledger_root/objects.tsv" >"$copy" || return 1 + chmod 0600 "$copy" || return 1 + while IFS=$'\t' read -r phase owner api kind namespace name existed restore payload_rel payload_sha; do + [[ "$existed" == true ]] || continue + validate_root_metadata "$ledger_root/$payload_rel" 600 'regular file' || return 1 + "$SUDO_BIN" -n cat -- "$ledger_root/$payload_rel" >"$payload_copy_root/${payload_rel##*/}" || return 1 + chmod 0600 "$payload_copy_root/${payload_rel##*/}" || return 1 + done < <(tail -n +2 "$copy") + validate_local_ledger "$copy" "$payload_copy_root" || return 1 + printf '%s' "$copy" +} + +load_mutation_uids() { + local copy=$1 + declare -gA MUTATED_UID=() + local ordinal api kind namespace name uid extra key + [[ "$(head -n1 "$copy")" == "$MUTATION_HEADER" ]] || return 1 + while IFS=$'\t' read -r ordinal api kind namespace name uid extra; do + [[ "$ordinal" =~ ^[0-9]+$ && "$api" != '' && "$kind" != '' && "$namespace" == observability && + "$name" != '' && "$uid" != '' && -z "${extra:-}" ]] || return 1 + key="$(object_key "$kind" "$namespace" "$name")" + [[ -z "${MUTATED_UID[$key]+x}" ]] || return 1 + MUTATED_UID[$key]=$uid + done < <(tail -n +2 "$copy") +} + +restore_owned_object() { + local kind=$1 namespace=$2 name=$3 expected_uid=$4 live_json=$5 prior_payload=$6 + local conditional="$work/conditional-restore-$kind-$name.json" + python3 - "$live_json" "$prior_payload" "$expected_uid" "$conditional" <<'PY' +import json, sys +live=json.loads(sys.argv[1]) +prior=json.load(open(sys.argv[2],encoding="utf-8")) +expected=sys.argv[3] +output=sys.argv[4] +live_meta=live.get("metadata") or {} +if live_meta.get("uid") != expected or not live_meta.get("resourceVersion"): + raise SystemExit(1) +meta=prior.setdefault("metadata",{}) +meta["uid"]=expected +meta["resourceVersion"]=live_meta["resourceVersion"] +with open(output,"x",encoding="utf-8") as stream: + json.dump(prior,stream,separators=(",",":"),sort_keys=True) + stream.write("\n") +PY + chmod 0600 "$conditional" || return 1 + kubectl_bounded replace -f "$conditional" >/dev/null +} + +delete_api_path() { + local kind=$1 namespace=$2 name=$3 + [[ "$namespace" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ && + "$name" =~ ^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$ ]] || return 1 + case "$kind" in + ConfigMap) printf '/api/v1/namespaces/%s/configmaps/%s' "$namespace" "$name" ;; + PrometheusRule) printf '/apis/monitoring.coreos.com/v1/namespaces/%s/prometheusrules/%s' "$namespace" "$name" ;; + AlertmanagerConfig) printf '/apis/monitoring.coreos.com/v1alpha1/namespaces/%s/alertmanagerconfigs/%s' "$namespace" "$name" ;; + Alertmanager) printf '/apis/monitoring.coreos.com/v1/namespaces/%s/alertmanagers/%s' "$namespace" "$name" ;; + NetworkPolicy) printf '/apis/networking.k8s.io/v1/namespaces/%s/networkpolicies/%s' "$namespace" "$name" ;; + *) return 1 ;; + esac +} + +ensure_api_proxy() { + if [[ -n "$api_proxy_pid" ]] && kill -0 "$api_proxy_pid" 2>/dev/null && [[ -S "$api_proxy_socket" ]]; then + return 0 + fi + api_proxy_socket="$work/kube-api-proxy.sock" + "$KUBECTL_BIN" --request-timeout=10s proxy --unix-socket="$api_proxy_socket" \ + --accept-paths='^/(api|apis)/.*$' \ + --reject-methods='^(POST|PUT|PATCH)$' \ + >"$work/kube-api-proxy.log" 2>&1 & + api_proxy_pid=$! + local attempt + for ((attempt=1; attempt<=50; attempt++)); do + [[ -S "$api_proxy_socket" ]] && return 0 + kill -0 "$api_proxy_pid" 2>/dev/null || return 1 + sleep 0.1 + done + return 1 +} + +delete_owned_object() { + local kind=$1 namespace=$2 name=$3 expected_uid=$4 api_path body response status live uid attempt + ensure_api_proxy || return 1 + api_path="$(delete_api_path "$kind" "$namespace" "$name")" || return 1 + body="$work/delete-options-$kind-$name.json" + response="$work/delete-response-$kind-$name.json" + python3 - "$expected_uid" "$body" <<'PY' +import json,sys +uid=sys.argv[1] +if not uid or any(character.isspace() for character in uid): + raise SystemExit(1) +with open(sys.argv[2],"x",encoding="utf-8") as stream: + json.dump({"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":uid},"propagationPolicy":"Background"},stream,separators=(",",":")) + stream.write("\n") +PY + chmod 0600 "$body" || return 1 + if ! status="$(curl --silent --show-error --connect-timeout 1 --max-time 1 \ + --output "$response" --write-out '%{http_code}' \ + --unix-socket "$api_proxy_socket" \ + --request DELETE --header 'Content-Type: application/json' \ + --data-binary "@$body" "http://localhost$api_path")"; then + return 1 + fi + [[ "$status" == 200 || "$status" == 202 ]] || return 1 + chmod 0600 "$response" || return 1 + for ((attempt=1; attempt<=30; attempt++)); do + live="$(get_object_json "$kind" "$namespace" "$name")" || return 1 + [[ -z "$live" ]] && return 0 + uid="$(object_uid_from_json <<<"$live")" || return 1 + [[ "$uid" == "$expected_uid" ]] || return 1 + (( attempt < 30 )) && sleep 1 + done + return 1 +} + +rollback_resources() { + rollback_running=true + local authority mutation_copy payload_copy_root row phase owner api kind namespace name existed restore payload_rel payload_sha + local key live uid resource payload action_failed=false + authority="$(validate_authoritative_ledger_for_rollback)" || { + rollback_ambiguous=true + return 1 + } + mutation_copy="$work/root-mutations.tsv" + "$SUDO_BIN" -n cat -- "$ledger_root/mutations.tsv" >"$mutation_copy" || { + rollback_ambiguous=true + return 1 + } + chmod 0600 "$mutation_copy" || { + rollback_ambiguous=true + return 1 + } + load_mutation_uids "$mutation_copy" || { + rollback_ambiguous=true + return 1 + } + payload_copy_root="$work/root-payloads" + mapfile -t rollback_rows < <(tail -n +2 "$authority" | tac) || { + rollback_ambiguous=true + return 1 + } + for row in "${rollback_rows[@]}"; do + IFS=$'\t' read -r phase owner api kind namespace name existed restore payload_rel payload_sha <<<"$row" + key="$(object_key "$kind" "$namespace" "$name")" + [[ -n "${MUTATED_UID[$key]+x}" ]] || continue + live="$(get_object_json "$kind" "$namespace" "$name")" || { + rollback_ambiguous=true + action_failed=true + continue + } + if [[ -z "$live" ]]; then + if [[ "$existed" == true ]]; then + rollback_ambiguous=true + action_failed=true + fi + continue + fi + uid="$(object_uid_from_json <<<"$live")" || { + rollback_ambiguous=true + action_failed=true + continue + } + if [[ "$uid" != "${MUTATED_UID[$key]}" ]]; then + rollback_ambiguous=true + action_failed=true + continue + fi + if [[ "$existed" == false ]]; then + if ! delete_owned_object "$kind" "$namespace" "$name" "${MUTATED_UID[$key]}"; then + rollback_ambiguous=true + action_failed=true + continue + fi + live="$(get_object_json "$kind" "$namespace" "$name")" || { + rollback_ambiguous=true + action_failed=true + continue + } + [[ -z "$live" ]] || { + rollback_ambiguous=true + action_failed=true + } + else + payload="$payload_copy_root/${payload_rel##*/}" + if ! restore_owned_object "$kind" "$namespace" "$name" \ + "${MUTATED_UID[$key]}" "$live" "$payload"; then + rollback_ambiguous=true + action_failed=true + continue + fi + live="$(get_object_json "$kind" "$namespace" "$name")" || { + rollback_ambiguous=true + action_failed=true + continue + } + uid="$(object_uid_from_json <<<"$live")" || { + rollback_ambiguous=true + action_failed=true + continue + } + [[ "$uid" == "${PRIOR_UID[$key]}" ]] || { + rollback_ambiguous=true + action_failed=true + } + fi + done + [[ "$action_failed" == false && "$rollback_ambiguous" == false ]] +} + +run_production_postcheck_once() { + local prometheus_ready alertmanager_ready rules receivers dashboard_hashes + local rules_file="$work/prometheus-rules-api.json" + local receivers_file="$work/alertmanager-receivers-api.json" + local dashboard_hashes_file="$work/grafana-dashboard-sidecar.sha256" + kubectl_bounded wait --namespace observability --for=condition=Available \ + prometheus/observability-core-kube-pr-prometheus \ + alertmanager/observability-core-kube-pr-alertmanager --timeout=10s >/dev/null || return 1 + kubectl_bounded rollout status --namespace observability \ + statefulset/prometheus-observability-core-kube-pr-prometheus --timeout=10s >/dev/null || return 1 + kubectl_bounded rollout status --namespace observability \ + statefulset/alertmanager-observability-core-kube-pr-alertmanager --timeout=10s >/dev/null || return 1 + prometheus_ready="$(kubectl_bounded get \ + --raw=/api/v1/namespaces/observability/services/http:observability-core-kube-pr-prometheus:9090/proxy/-/ready)" || return 1 + [[ "$prometheus_ready" == *'Prometheus Server is Ready.'* ]] || return 1 + alertmanager_ready="$(kubectl_bounded get \ + --raw=/api/v1/namespaces/observability/services/http:observability-core-kube-pr-alertmanager:9093/proxy/-/ready)" || return 1 + [[ "$alertmanager_ready" == *'OK'* ]] || return 1 + rules="$(kubectl_bounded get \ + --raw=/api/v1/namespaces/observability/services/http:observability-core-kube-pr-prometheus:9090/proxy/api/v1/rules)" || return 1 + receivers="$(kubectl_bounded get \ + --raw=/api/v1/namespaces/observability/services/http:observability-core-kube-pr-alertmanager:9093/proxy/api/v2/receivers)" || return 1 + dashboard_hashes="$(kubectl_bounded exec --namespace observability deployment/grafana --container grafana -- \ + /bin/sh -ceu 'for file in https-endpoints.json kubernetes-node.json observability-backends.json platform-services.json workload-health.json; do test -f "/tmp/dashboards/$file"; sha256sum "/tmp/dashboards/$file"; done')" || return 1 + printf '%s\n' "$rules" >"$rules_file" + printf '%s\n' "$receivers" >"$receivers_file" + printf '%s\n' "$dashboard_hashes" >"$dashboard_hashes_file" + chmod 0600 "$rules_file" "$receivers_file" "$dashboard_hashes_file" + python3 - "$verified_output_dir/rules.yaml" "$rules_file" \ + "$verified_output_dir/dashboards.yaml" "$dashboard_hashes_file" "$receivers_file" <<'PY' +import collections +import hashlib +import json +import pathlib +import re +import sys +import yaml + +rules_yaml, rules_api, dashboards_yaml, dashboard_hashes, receivers_api = map(pathlib.Path, sys.argv[1:]) + +def documents(path): + with path.open(encoding="utf-8") as stream: + return [item for item in yaml.safe_load_all(stream) if item] + +desired_rules = collections.Counter() +desired_groups = set() +for item in documents(rules_yaml): + for group in (item.get("spec") or {}).get("groups") or []: + group_name = group.get("name") + if not isinstance(group_name, str) or not group_name: + raise SystemExit(1) + desired_groups.add(group_name) + for rule in group.get("rules") or []: + alert = rule.get("alert") + record = rule.get("record") + if bool(alert) == bool(record): + raise SystemExit(1) + name = alert if alert else record + if not isinstance(name, str) or not name: + raise SystemExit(1) + desired_rules[(group_name, name, "alerting" if alert else "recording")] += 1 +if not desired_rules: + raise SystemExit(1) + +payload = json.loads(rules_api.read_text(encoding="utf-8")) +if payload.get("status") != "success": + raise SystemExit(1) +accepted = collections.Counter() +for group in (payload.get("data") or {}).get("groups") or []: + group_name = group.get("name") + if group_name not in desired_groups: + continue + for rule in group.get("rules") or []: + if rule.get("health") != "ok" or rule.get("lastError") not in (None, ""): + raise SystemExit(1) + accepted[(group_name, rule.get("name"), rule.get("type"))] += 1 +if accepted != desired_rules: + raise SystemExit(1) + +expected_dashboard_names = { + "grafana-dashboard-https-endpoints": "https-endpoints.json", + "grafana-dashboard-kubernetes-node": "kubernetes-node.json", + "grafana-dashboard-observability-backends": "observability-backends.json", + "grafana-dashboard-platform-services": "platform-services.json", + "grafana-dashboard-workload-health": "workload-health.json", +} +expected_hashes = {} +dashboard_items = documents(dashboards_yaml) +if len(dashboard_items) != len(expected_dashboard_names): + raise SystemExit(1) +for item in dashboard_items: + metadata = item.get("metadata") or {} + name = metadata.get("name") + filename = expected_dashboard_names.get(name) + data = item.get("data") or {} + if not filename or set(data) != {filename} or not isinstance(data[filename], str): + raise SystemExit(1) + expected_hashes[filename] = hashlib.sha256(data[filename].encode()).hexdigest() + +actual_hashes = {} +line_pattern = re.compile(r"^([0-9a-f]{64})\s+/tmp/dashboards/([^/\s]+)$") +for line in dashboard_hashes.read_text(encoding="utf-8").splitlines(): + match = line_pattern.fullmatch(line) + if not match or match.group(2) in actual_hashes: + raise SystemExit(1) + actual_hashes[match.group(2)] = match.group(1) +if actual_hashes != expected_hashes: + raise SystemExit(1) + +receiver_payload = json.loads(receivers_api.read_text(encoding="utf-8")) +expected_receivers = { + "observability/platform-alertmanager/platform-null", + "observability/platform-alertmanager/platform-slack", +} +if ( + not isinstance(receiver_payload, list) + or len(receiver_payload) != 2 + or any( + not isinstance(item, dict) + or set(item) != {"labels", "name"} + or item.get("labels") != {"name": item.get("name")} + for item in receiver_payload + ) + or {item["name"] for item in receiver_payload} != expected_receivers +): + raise SystemExit(1) +PY +} + +run_postcheck() { + local attempt max_attempts=30 delay=2 + if [[ "$TEST_MODE" == 1 ]]; then + max_attempts=1 + delay=0 + fi + for ((attempt=1; attempt<=max_attempts; attempt++)); do + if run_production_postcheck_once; then + return 0 + fi + (( attempt < max_attempts )) && sleep "$delay" + done + return 1 +} + +validate_acceptance() { + local path=$1 expected=$2 + "$SUDO_BIN" -n cmp -s -- "$expected" "$path" +} + +write_acceptance() { + local accepted_at marker="$snapshot/acceptance.env" + accepted_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" || return 1 + cat >"$marker" </dev/null; then + kill "$api_proxy_pid" 2>/dev/null || true + wait "$api_proxy_pid" 2>/dev/null || true + fi + case "$work" in + /tmp/platform-observability-access-apply.??????) rm -rf -- "$work" ;; + esac + case "$slack_gate_temp_root" in + /tmp/platform-observability-slack-gate.??????) + [[ -d "$slack_gate_temp_root" && ! -L "$slack_gate_temp_root" && + "$(readlink -f -- "$slack_gate_temp_root")" == "$slack_gate_temp_root" && + "$(stat -c '%u:%a' -- "$slack_gate_temp_root")" == "$(id -u):700" ]] && + rm -rf -- "$slack_gate_temp_root" + ;; + esac + exit "$rc" +} + +on_signal() { + local rc=$1 + trap - HUP INT TERM + set +e + if [[ "$mutation_started" == true && "$rollback_running" == false ]]; then + rollback_resources + if [[ $? == 0 ]]; then + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS\n' >&2 + else + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS\nMANUAL_RECOVERY_REQUIRED=YES\n' >&2 + fi + fi + exit "$rc" +} + +on_error() { + local rc=$1 + trap - ERR + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_STAGE=%s\n' "$failure_stage" >&2 + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_RC=%s\n' "$rc" >&2 + exit "$rc" +} + +main() { + if ! parse_args "$@"; then + usage >&2 + return 2 + fi + validate_test_mode + if [[ "$execute" == false ]]; then + print_dry_run + return 0 + fi + trap cleanup EXIT + trap 'on_signal 129' HUP + trap 'on_signal 130' INT + trap 'on_signal 143' TERM + + # Refresh sudo before validators that perform narrow root evidence reads. + # This is authorization only; no object, file, or host state is mutated. + "$SUDO_BIN" -v + "$SUDO_BIN" -n true + failure_stage=encryption-pre + run_encryption_gates + failure_stage=context + validate_context + failure_stage=handoff + validate_handoff_root + failure_stage=slack-secret + validate_slack_secret_schema + failure_stage=slack-deployment-gate + capture_slack_deployment_gate slack_deployment_gate_first pre-prompt + failure_stage=substrate-target-health + run_preflight || fail 'substrate or target health preflight failed' + failure_stage=server-dry-run + server_dry_run_handoff + + validate_rollback_root + confirm_apply + + work="$(mktemp -d /tmp/platform-observability-access-apply.XXXXXX)" + chmod 0700 "$work" + snapshot="$work/snapshot" + mkdir -m 0700 "$snapshot" + trap cleanup EXIT + trap 'on_signal 129' HUP + trap 'on_signal 130' INT + trap 'on_signal 143' TERM + + failure_stage=renderer-split + prepare_split_files + failure_stage=rollback-ledger + capture_snapshot + install_snapshot + failure_stage=encryption-last + run_encryption_gates + failure_stage=last-gate + verify_handoff_unchanged + verify_live_preconditions + failure_stage=slack-secret-last + validate_slack_secret_schema + failure_stage=slack-deployment-gate-last + capture_slack_deployment_gate slack_deployment_gate last + require_matching_slack_deployment_gates + map_slack_deployment_gate_acceptance || fail 'Slack deployment gate verdict is invalid' + + if ! apply_resources; then + if [[ "$mutation_started" == true ]]; then + if rollback_resources; then + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS\n' >&2 + else + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS\n' >&2 + printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2 + fi + fi + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_FAILED_AT=%s\n' "$failure_stage" >&2 + return 1 + fi + + failure_stage=reconcile + if ! run_postcheck; then + if rollback_resources; then + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS\n' >&2 + else + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS\n' >&2 + printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2 + fi + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_FAILED_AT=%s\n' "$failure_stage" >&2 + return 1 + fi + + failure_stage=acceptance + write_acceptance || { + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_STATE=APPLIED_ACCEPTANCE_UNRECORDED\n' >&2 + printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2 + return 1 + } + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_TARGET_INITIAL_SHA256=%s\n' "$target_initial_sha" + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS_POST_SUBSTRATE_SHA256=%s\n' "$post_substrate_sha" + printf 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS\n' +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + trap 'on_error "$?"' ERR + main "$@" +fi diff --git a/scripts/bootstrap/apply-phase1-gitea.sh b/scripts/bootstrap/apply-phase1-gitea.sh new file mode 100755 index 0000000..303dd2f --- /dev/null +++ b/scripts/bootstrap/apply-phase1-gitea.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +# Do not allow a caller's `bash -x` setting to expose prompts or future secrets. +set +x + +readonly EXPECTED_HELM_VERSION="v3.19.4" +readonly TARGET_NODE="donghyeon-system-product-name" +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly -a VERIFIED_MANIFEST_NAMES=( + namespaces + ssd-local-pv + cnpg-operator + platform-postgres + gitea + gitea-oidc +) + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +check_create_only_state() { + if kubectl --namespace gitea get deployment/gitea >/dev/null 2>&1; then + fail "Phase 1 is create-only and gitea/gitea already exists; use a dedicated Gitea lifecycle script" + fi + if kubectl --namespace gitea get secret/gitea-keycloak-oidc >/dev/null 2>&1; then + fail "Phase 1 baseline is blocked because gitea/gitea-keycloak-oidc already exists" + fi +} + +usage() { + cat <<'USAGE' +Usage: bash scripts/bootstrap/apply-phase1-gitea.sh --execute + +Applies the create-only Gitea baseline in dependency order. It does not modify +Host Nginx, configure Keycloak OIDC, or install Argo CD. If a Gitea Deployment +or OIDC Secret already exists, use the dedicated lifecycle scripts instead. +USAGE +} + +[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || { + usage + exit 2 +} + +for command_name in kubectl curl rg sha256sum stat; do + command -v "$command_name" >/dev/null 2>&1 || fail "${command_name} is required" +done + +if [[ -n "${PLATFORM_HELM_BIN:-}" ]]; then + [[ "$PLATFORM_HELM_BIN" == /* ]] || fail "PLATFORM_HELM_BIN must be an absolute path" + [[ -f "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \ + fail "PLATFORM_HELM_BIN is not an executable file: ${PLATFORM_HELM_BIN}" + readonly HELM_BIN="$PLATFORM_HELM_BIN" +else + HELM_BIN="$(command -v helm 2>/dev/null)" || \ + fail "Helm ${EXPECTED_HELM_VERSION} is required" + readonly HELM_BIN +fi + +[[ "$("$HELM_BIN" version --template '{{.Version}}')" == "$EXPECTED_HELM_VERSION" ]] || \ + fail "Helm must be exactly ${EXPECTED_HELM_VERSION}" + +render_temp_dir="$(mktemp -d /tmp/platform-phase1-apply.XXXXXX)" +cleanup() { + case "$render_temp_dir" in + /tmp/platform-phase1-apply.*) + rm -rf -- "$render_temp_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected render directory: %s\n' \ + "$render_temp_dir" >&2 + ;; + esac +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +cd -- "$REPOSITORY_ROOT" +PLATFORM_HELM_BIN="$HELM_BIN" bash scripts/validate/render-phase1.sh \ + --verified-output-dir "$render_temp_dir" + +declare -A verified_manifest_sha256=() +for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do + manifest_path="${render_temp_dir}/${manifest_name}.yaml" + [[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \ + fail "verified manifest is missing or unsafe: ${manifest_path}" + [[ "$(stat --format='%a' -- "$manifest_path")" == "600" ]] || \ + fail "verified manifest must have mode 0600: ${manifest_path}" + checksum_output="$(sha256sum -- "$manifest_path")" + verified_manifest_sha256["$manifest_name"]="${checksum_output%% *}" +done +verified_entry_count="$(find "$render_temp_dir" -mindepth 1 -maxdepth 1 | wc -l | tr -d '[:space:]')" +[[ "$verified_entry_count" == "${#VERIFIED_MANIFEST_NAMES[@]}" ]] || \ + fail "verified handoff must contain exactly ${#VERIFIED_MANIFEST_NAMES[@]} manifest files" + +verify_manifest_unchanged() { + local manifest_name="$1" + local manifest_path="${render_temp_dir}/${manifest_name}.yaml" + local checksum_output + local actual_sha256 + + [[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \ + fail "verified manifest became missing or unsafe: ${manifest_path}" + checksum_output="$(sha256sum -- "$manifest_path")" + actual_sha256="${checksum_output%% *}" + [[ "$actual_sha256" == "${verified_manifest_sha256[$manifest_name]}" ]] || \ + fail "verified manifest changed before apply: ${manifest_name}.yaml" +} + +current_context="$(kubectl config current-context)" +api_server="$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" +kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \ + fail "target node is missing from the selected cluster: ${TARGET_NODE}" +check_create_only_state + +printf '\nKubernetes context: %s\nAPI server: %s\nTarget node: %s\n' \ + "$current_context" "$api_server" "$TARGET_NODE" +printf 'Type APPLY %s to start the cluster mutation: ' "$current_context" +read -r confirmation +[[ "$confirmation" == "APPLY ${current_context}" ]] || fail "cancelled" + +[[ "$(kubectl config current-context)" == "$current_context" ]] || \ + fail "kubectl context changed after confirmation" +confirmed_api_server="$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" +[[ "$confirmed_api_server" == "$api_server" ]] || \ + fail "Kubernetes API server changed after confirmation" +kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \ + fail "target node disappeared after confirmation: ${TARGET_NODE}" +check_create_only_state + +for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do + verify_manifest_unchanged "$manifest_name" +done + +printf '\n[1/8] Preparing exact SSD Local PV directories\n' +bash scripts/bootstrap/prepare-ssd-local-paths.sh + +printf '\n[2/8] Applying Phase 1 namespaces\n' +verify_manifest_unchanged namespaces +kubectl apply --filename="${render_temp_dir}/namespaces.yaml" + +printf '\n[3/8] Applying static SSD StorageClasses and Local PVs\n' +verify_manifest_unchanged ssd-local-pv +kubectl apply --filename="${render_temp_dir}/ssd-local-pv.yaml" + +printf '\n[4/8] Installing CloudNativePG CRDs and operator\n' +verify_manifest_unchanged cnpg-operator +kubectl apply --server-side --filename="${render_temp_dir}/cnpg-operator.yaml" +kubectl wait --for=condition=Established \ + customresourcedefinition/clusters.postgresql.cnpg.io \ + customresourcedefinition/databaseroles.postgresql.cnpg.io \ + customresourcedefinition/databases.postgresql.cnpg.io \ + --timeout=3m +kubectl --namespace cnpg-system wait --for=condition=Available deployment \ + --selector=app.kubernetes.io/name=cloudnative-pg --timeout=5m + +printf '\n[5/8] Ensuring or reusing the three Secret contracts\n' +bash scripts/bootstrap/create-phase1-secrets.sh --execute + +printf '\n[6/8] Applying the shared platform PostgreSQL resources\n' +verify_manifest_unchanged platform-postgres +kubectl apply --server-side --filename="${render_temp_dir}/platform-postgres.yaml" +kubectl --namespace platform-data wait --for=condition=Ready \ + cluster/platform-postgres --timeout=10m +kubectl --namespace platform-data wait \ + --for=jsonpath='{.status.applied}'=true \ + database/platform-postgres-gitea --timeout=3m + +printf '\n[7/8] Applying the Gitea baseline, PVC, policies, and HTTP Ingress\n' +verify_manifest_unchanged gitea +kubectl apply --filename="${render_temp_dir}/gitea.yaml" +kubectl --namespace gitea rollout status deployment/gitea --timeout=10m +kubectl --namespace gitea wait \ + --for=jsonpath='{.endpoints[0].conditions.ready}'=true \ + endpointslice \ + --selector=kubernetes.io/service-name=gitea-http \ + --timeout=2m + +printf '\n[8/8] Checking Host-based routing through Traefik HTTP NodePort\n' +curl --fail-with-body --show-error \ + --retry 24 \ + --retry-all-errors \ + --retry-connrefused \ + --retry-delay 5 \ + --retry-max-time 120 \ + --max-time 10 \ + --header 'Host: git.learn.hyeonworks.com' \ + http://127.0.0.1:30080/api/healthz + +printf '\nPhase 1 cluster resources are ready. Host Nginx was not changed.\n' +printf 'Review infrastructure/networking/host-nginx/README.md for the final cutover.\n' diff --git a/scripts/bootstrap/apply-private-dns.sh b/scripts/bootstrap/apply-private-dns.sh new file mode 100755 index 0000000..f00fad2 --- /dev/null +++ b/scripts/bootstrap/apply-private-dns.sh @@ -0,0 +1,287 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)" +readonly HOST_SOURCE="$ROOT/infrastructure/networking/private-dns/host" +readonly K8S_SOURCE="$ROOT/infrastructure/networking/private-dns/kubernetes" +readonly LAN_IP="192.168.0.107" +readonly TAIL_IP="100.92.240.34" +readonly TARGET_NODE="donghyeon-system-product-name" +readonly BUSYBOX_IMAGE="docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0" +readonly -a PRIVATE_HOSTS=(git.learn.hyeonworks.com id.learn.hyeonworks.com storage-admin.learn.hyeonworks.com db-admin.learn.hyeonworks.com grafana.learn.hyeonworks.com) +readonly -a PUBLIC_PRIVATE_HOSTS=(storage-admin.learn.hyeonworks.com db-admin.learn.hyeonworks.com grafana.learn.hyeonworks.com) +execute=false +mutation=false +rollback_armed=false +temp="" +lan_was_active=false +tail_was_active=false +lan_was_enabled=false +tail_was_enabled=false + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'USAGE' +사용법: + bash scripts/bootstrap/apply-private-dns.sh + bash scripts/bootstrap/apply-private-dns.sh --execute + +인자 없이 실행하면 설정, 주소, 공개 DNS, manifest hash만 검사합니다. +--execute는 두 dnsmasq 인스턴스와 coredns-custom을 적용합니다. +공유기 DHCP DNS와 Tailscale 관리 화면은 변경하지 않습니다. +USAGE +} + +if (( $# == 0 )); then + : +elif (( $# == 1 )) && [[ "$1" == "-h" || "$1" == "--help" ]]; then + usage + exit 0 +elif (( $# == 1 )) && [[ "$1" == "--execute" ]]; then + execute=true +else + usage >&2 + exit 2 +fi + +for cmd in awk dig find ip install journalctl kubectl rg sha256sum sleep ss stat systemctl systemd-analyze; do + command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다" +done +[[ -x /usr/sbin/dnsmasq ]] || fail "/usr/sbin/dnsmasq가 없습니다" +[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요" + +for file in dnsmasq-lan.conf dnsmasq-tailscale.conf \ + hyeonworks-dnsmasq-lan.service hyeonworks-dnsmasq-tailscale.service; do + [[ -f "$HOST_SOURCE/$file" && ! -L "$HOST_SOURCE/$file" ]] || fail "후보 파일이 없습니다: $file" +done + +/usr/sbin/dnsmasq --test --conf-file="$HOST_SOURCE/dnsmasq-lan.conf" +/usr/sbin/dnsmasq --test --conf-file="$HOST_SOURCE/dnsmasq-tailscale.conf" +systemd-analyze verify "$HOST_SOURCE/hyeonworks-dnsmasq-lan.service" \ + "$HOST_SOURCE/hyeonworks-dnsmasq-tailscale.service" +kubectl kustomize "$K8S_SOURCE" >"/tmp/private-dns-render.$$" +trap 'rm -f -- "/tmp/private-dns-render.$$"' EXIT +manifest_sha="$(sha256sum "/tmp/private-dns-render.$$" | awk '{print $1}')" + +ip -4 address show | rg -q -F "$LAN_IP/" || fail "LAN 주소 $LAN_IP가 호스트에 없습니다" +ip -4 address show | rg -q -F "$TAIL_IP/" || fail "Tailscale 주소 $TAIL_IP가 호스트에 없습니다" +[[ "$(kubectl get node "$TARGET_NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')" == True ]] || \ + fail "대상 노드가 Ready가 아닙니다" + +for name in "${PUBLIC_PRIVATE_HOSTS[@]}"; do + public_a="$(dig +short @1.1.1.1 A "$name" | tr -d '[:space:]')" + public_aaaa="$(dig +short @1.1.1.1 AAAA "$name" | tr -d '[:space:]')" + [[ -z "$public_a" && -z "$public_aaaa" ]] || fail "$name 공개 A/AAAA가 존재합니다" +done + +printf 'Current context: %s\n' "$(kubectl config current-context)" +printf 'LAN listener: %s:53\n' "$LAN_IP" +printf 'Tail listener: %s:53\n' "$TAIL_IP" +printf 'CoreDNS SHA-256: %s\n' "$manifest_sha" +printf '공개 private-service A/AAAA: 없음\n' + +if [[ "$execute" == false ]]; then + printf 'DRY RUN PASS: --execute를 지정하지 않아 변경하지 않았습니다.\n' + exit 0 +fi + +[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다" +context="$(kubectl config current-context)" +api="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" +printf 'Type APPLY %s to install private DNS: ' "$context" +read -r answer +[[ "$answer" == "APPLY $context" ]] || fail "취소했습니다" +[[ "$(kubectl config current-context)" == "$context" ]] || fail "context가 바뀌었습니다" +[[ "$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" == "$api" ]] || \ + fail "API server가 바뀌었습니다" + +sudo -v + +managed_listeners_are_exact() { + local lan_pid tail_pid line protocol local_address + local lan_tcp=0 lan_udp=0 tail_tcp=0 tail_udp=0 + local -a listener_lines=() + + sudo systemctl is-active --quiet hyeonworks-dnsmasq-lan.service || return 1 + sudo systemctl is-active --quiet hyeonworks-dnsmasq-tailscale.service || return 1 + lan_pid="$(sudo systemctl show hyeonworks-dnsmasq-lan.service --property MainPID --value)" || return 1 + tail_pid="$(sudo systemctl show hyeonworks-dnsmasq-tailscale.service --property MainPID --value)" || return 1 + [[ "$lan_pid" =~ ^[1-9][0-9]*$ && "$tail_pid" =~ ^[1-9][0-9]*$ ]] || return 1 + + sudo cmp -s "$HOST_SOURCE/dnsmasq-lan.conf" /etc/dnsmasq-hyeonworks/lan.conf || return 1 + sudo cmp -s "$HOST_SOURCE/dnsmasq-tailscale.conf" /etc/dnsmasq-hyeonworks/tailscale.conf || return 1 + sudo cmp -s "$HOST_SOURCE/hyeonworks-dnsmasq-lan.service" \ + /etc/systemd/system/hyeonworks-dnsmasq-lan.service || return 1 + sudo cmp -s "$HOST_SOURCE/hyeonworks-dnsmasq-tailscale.service" \ + /etc/systemd/system/hyeonworks-dnsmasq-tailscale.service || return 1 + + mapfile -t listener_lines < <(sudo ss -H -lnupt '( sport = :53 )') || return 1 + for line in "${listener_lines[@]}"; do + read -r protocol _ _ _ local_address _ <<<"$line" + case "$local_address" in + "$LAN_IP:53") + [[ "$line" == *"pid=$lan_pid,"* ]] || return 1 + [[ "$protocol" == tcp ]] && lan_tcp=$((lan_tcp + 1)) + [[ "$protocol" == udp ]] && lan_udp=$((lan_udp + 1)) + ;; + "$TAIL_IP:53") + [[ "$line" == *"pid=$tail_pid,"* ]] || return 1 + [[ "$protocol" == tcp ]] && tail_tcp=$((tail_tcp + 1)) + [[ "$protocol" == udp ]] && tail_udp=$((tail_udp + 1)) + ;; + esac + done + (( lan_tcp == 1 && lan_udp == 1 && tail_tcp == 1 && tail_udp == 1 )) +} + +if sudo ss -H -lntu '( sport = :53 )' | awk -v lan="$LAN_IP:53" -v tail="$TAIL_IP:53" \ + '$5 == lan || $5 == tail {found=1} END {exit found ? 0 : 1}'; then + if managed_listeners_are_exact; then + printf '기존 exact managed DNS listener를 안전한 재적용 대상으로 확인했습니다.\n' + else + fail "대상 LAN/Tailscale 주소의 53번 포트를 exact managed listener가 아닌 프로세스가 사용 중입니다" + fi +fi + +umask 077 +temp="$(mktemp -d /tmp/platform-private-dns.XXXXXX)" +cm_existed=false +sudo systemctl is-active --quiet hyeonworks-dnsmasq-lan.service && lan_was_active=true +sudo systemctl is-active --quiet hyeonworks-dnsmasq-tailscale.service && tail_was_active=true +[[ "$(sudo systemctl is-enabled hyeonworks-dnsmasq-lan.service 2>/dev/null || true)" == enabled ]] && lan_was_enabled=true +[[ "$(sudo systemctl is-enabled hyeonworks-dnsmasq-tailscale.service 2>/dev/null || true)" == enabled ]] && tail_was_enabled=true +[[ -e /etc/dnsmasq-hyeonworks ]] && sudo cp -a /etc/dnsmasq-hyeonworks "$temp/etc-dnsmasq" +[[ -e /etc/systemd/system/hyeonworks-dnsmasq-lan.service ]] && \ + sudo cp -a /etc/systemd/system/hyeonworks-dnsmasq-lan.service "$temp/lan.service" +[[ -e /etc/systemd/system/hyeonworks-dnsmasq-tailscale.service ]] && \ + sudo cp -a /etc/systemd/system/hyeonworks-dnsmasq-tailscale.service "$temp/tailscale.service" +if kubectl -n kube-system get configmap coredns-custom -o yaml >"$temp/coredns-custom.yaml" 2>/dev/null; then + cm_existed=true +fi + +restore_service_state() { + local unit=$1 was_active=$2 was_enabled=$3 + if [[ "$was_enabled" == true ]]; then + sudo systemctl enable "$unit" >/dev/null 2>&1 + else + sudo systemctl disable "$unit" >/dev/null 2>&1 || true + fi + if [[ "$was_active" == true ]]; then + sudo systemctl start "$unit" >/dev/null 2>&1 + else + sudo systemctl stop "$unit" >/dev/null 2>&1 || true + fi +} + +rollback() { + set +e + printf '\nROLLBACK: private DNS 이전 상태를 복원합니다.\n' >&2 + sudo systemctl disable --now hyeonworks-dnsmasq-lan.service \ + hyeonworks-dnsmasq-tailscale.service >/dev/null 2>&1 + if [[ -d "$temp/etc-dnsmasq" ]]; then + sudo rm -rf -- /etc/dnsmasq-hyeonworks + sudo cp -a "$temp/etc-dnsmasq" /etc/dnsmasq-hyeonworks + else + sudo rm -rf -- /etc/dnsmasq-hyeonworks + fi + for unit in lan tailscale; do + target="/etc/systemd/system/hyeonworks-dnsmasq-$unit.service" + if [[ -f "$temp/$unit.service" ]]; then + sudo install -o root -g root -m 0644 "$temp/$unit.service" "$target" + else + sudo rm -f -- "$target" + fi + done + sudo systemctl daemon-reload + restore_service_state hyeonworks-dnsmasq-lan.service "$lan_was_active" "$lan_was_enabled" + restore_service_state hyeonworks-dnsmasq-tailscale.service "$tail_was_active" "$tail_was_enabled" + if [[ "$cm_existed" == true ]]; then + kubectl apply -f "$temp/coredns-custom.yaml" >/dev/null + else + kubectl -n kube-system delete configmap coredns-custom --ignore-not-found >/dev/null + fi + kubectl -n kube-system rollout restart deployment/coredns >/dev/null + kubectl -n kube-system rollout status deployment/coredns --timeout=90s >/dev/null + rollback_armed=false + printf 'ROLLBACK complete.\n' >&2 +} + +finish() { + rc=$? + trap - EXIT INT TERM + if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then + rollback + fi + if [[ -n "$temp" ]]; then + case "$temp" in + /tmp/platform-private-dns.*) rm -rf -- "$temp" ;; + esac + fi + rm -f -- "/tmp/private-dns-render.$$" + exit "$rc" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +rollback_armed=true +mutation=true +sudo install -d -o root -g root -m 0755 /etc/dnsmasq-hyeonworks +sudo install -o root -g root -m 0644 "$HOST_SOURCE/dnsmasq-lan.conf" /etc/dnsmasq-hyeonworks/lan.conf +sudo install -o root -g root -m 0644 "$HOST_SOURCE/dnsmasq-tailscale.conf" /etc/dnsmasq-hyeonworks/tailscale.conf +sudo install -o root -g root -m 0644 "$HOST_SOURCE/hyeonworks-dnsmasq-lan.service" \ + /etc/systemd/system/hyeonworks-dnsmasq-lan.service +sudo install -o root -g root -m 0644 "$HOST_SOURCE/hyeonworks-dnsmasq-tailscale.service" \ + /etc/systemd/system/hyeonworks-dnsmasq-tailscale.service +sudo systemctl daemon-reload +sudo systemctl enable --now hyeonworks-dnsmasq-lan.service hyeonworks-dnsmasq-tailscale.service + +stable=0 +for (( attempt=1; attempt<=20; attempt++ )); do + if sudo systemctl is-active --quiet hyeonworks-dnsmasq-lan.service \ + hyeonworks-dnsmasq-tailscale.service \ + && [[ "$(dig +time=1 +tries=1 +short "@$LAN_IP" git.learn.hyeonworks.com A | tail -n1)" == "$LAN_IP" ]] \ + && [[ "$(dig +time=1 +tries=1 +short "@$TAIL_IP" git.learn.hyeonworks.com A | tail -n1)" == "$TAIL_IP" ]]; then + stable=$((stable + 1)) + if (( stable >= 3 )); then + break + fi + else + stable=0 + fi + sleep 0.5 +done +if (( stable < 3 )); then + for unit in hyeonworks-dnsmasq-lan.service hyeonworks-dnsmasq-tailscale.service; do + sudo systemctl status "$unit" --no-pager -n 20 >&2 || true + sudo journalctl -u "$unit" --no-pager -n 20 >&2 || true + done + fail "private DNS listener가 안정화되지 않았습니다" +fi + +kubectl apply -f "/tmp/private-dns-render.$$" +kubectl -n kube-system rollout restart deployment/coredns +kubectl -n kube-system rollout status deployment/coredns --timeout=90s + +for resolver in "$LAN_IP" "$TAIL_IP"; do + for name in "${PRIVATE_HOSTS[@]}"; do + [[ "$(dig +short "@$resolver" A "$name" | tail -n1)" == "$resolver" ]] || + fail "$name private DNS 검증 실패" + done +done + +pod="private-dns-smoke-$(date +%H%M%S)" +kubectl -n default run "$pod" --restart=Never --image="$BUSYBOX_IMAGE" \ + --labels=platform.hyeonworks.com/transient=true \ + --command -- sh -c 'nslookup git.learn.hyeonworks.com >/dev/null && nslookup id.learn.hyeonworks.com >/dev/null && nslookup storage-admin.learn.hyeonworks.com >/dev/null && nslookup db-admin.learn.hyeonworks.com >/dev/null && nslookup grafana.learn.hyeonworks.com >/dev/null' +kubectl -n default wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$pod" --timeout=60s +kubectl -n default delete "pod/$pod" --wait=true >/dev/null + +rollback_armed=false +printf 'PRIVATE DNS APPLY SUCCESS\n' +printf '공유기 DHCP DNS와 Tailscale split DNS는 문서에 따라 별도로 등록하세요.\n' diff --git a/scripts/bootstrap/backup-slack-webhook-recovery.sh b/scripts/bootstrap/backup-slack-webhook-recovery.sh new file mode 100755 index 0000000..a5466e5 --- /dev/null +++ b/scripts/bootstrap/backup-slack-webhook-recovery.sh @@ -0,0 +1,22 @@ +#!/usr/bin/bash + +set -Eeuo pipefail +set +x +umask 077 +ulimit -c 0 + +if [[ "${BASH_SOURCE[0]}" == */* ]]; then + _swr_wrapper_directory="${BASH_SOURCE[0]%/*}" +else + _swr_wrapper_directory='.' +fi +readonly SWR_REPOSITORY_ROOT="$(cd -- "${_swr_wrapper_directory}/../.." && pwd -P)" +readonly SWR_RECOVERY_CONTRACT="${SWR_REPOSITORY_ROOT}/infrastructure/security/k3s/local-recovery.env" +unset _swr_wrapper_directory + +# shellcheck source=/dev/null +source "${SWR_REPOSITORY_ROOT}/scripts/lib/slack-webhook-recovery.sh" + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + _swr_main "$@" +fi diff --git a/scripts/bootstrap/close-k3s-local-recovery.sh b/scripts/bootstrap/close-k3s-local-recovery.sh new file mode 100644 index 0000000..bb18f68 --- /dev/null +++ b/scripts/bootstrap/close-k3s-local-recovery.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +PATH='/usr/sbin:/usr/bin:/sbin:/bin' +export PATH +LC_ALL=C +export LC_ALL +umask 077 + +_k3slr_wrapper_initial_guard() { + local effective_uid="${1-}" shell_options="${2-}" + [[ "$effective_uid" =~ ^[0-9]+$ && "$effective_uid" != 0 ]] || return 1 + [[ "$shell_options" != *x* ]] +} + +if ! _k3slr_wrapper_initial_guard "${EUID:-}" "$-"; then + printf 'ERROR: lifecycle wrapper refuses root or xtrace execution\n' >&2 + return 1 2>/dev/null || exit 1 +fi + +if [[ "${BASH_SOURCE[0]}" == */* ]]; then + K3SLR_WRAPPER_DIRECTORY="${BASH_SOURCE[0]%/*}" +else + K3SLR_WRAPPER_DIRECTORY='.' +fi +K3SLR_WRAPPER_ROOT="$(cd -- "${K3SLR_WRAPPER_DIRECTORY}/../.." && pwd -P)" || { + return 1 2>/dev/null || exit 1 +} +K3SLR_WRAPPER_CONTRACT="${K3SLR_WRAPPER_ROOT}/infrastructure/security/k3s/local-recovery.env" +# shellcheck source=/dev/null +source "${K3SLR_WRAPPER_ROOT}/scripts/lib/k3s-local-recovery.sh" || { + return 1 2>/dev/null || exit 1 +} + +_k3slr_close_usage() { + printf 'Usage: bash scripts/bootstrap/close-k3s-local-recovery.sh [--execute]\n' >&2 +} + +_k3slr_close_main() { + local execution_mode + if ! _k3slr_parse_lifecycle_cli execution_mode "$@"; then + _k3slr_close_usage + return 2 + fi + _k3slr_lifecycle_main close "$execution_mode" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + _k3slr_close_main "$@" +fi diff --git a/scripts/bootstrap/configure-keycloak-admin-oidc.sh b/scripts/bootstrap/configure-keycloak-admin-oidc.sh new file mode 100755 index 0000000..1883b31 --- /dev/null +++ b/scripts/bootstrap/configure-keycloak-admin-oidc.sh @@ -0,0 +1,400 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +set +x +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)" +readonly KC_NS=keycloak +readonly KC_NAME=keycloak +readonly KC_SERVICE=keycloak-service +readonly KC_HOST=id.learn.hyeonworks.com +readonly REALM=hyeonworks +readonly PORT=18081 +readonly BASE=http://127.0.0.1:$PORT +execute=false +object_admin="" +db_admin="" +temp="" +pf="" +rollback_armed=false +ai_client_created=false +pg_client_created=false +ai_group_created=false +pg_group_created=false +ai_member_added=false +pg_member_added=false +ai_client_uuid="" +pg_client_uuid="" +ai_group_uuid="" +pg_group_uuid="" +ai_user_uuid="" +pg_user_uuid="" + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'USAGE' +사용법: + bash scripts/bootstrap/configure-keycloak-admin-oidc.sh + bash scripts/bootstrap/configure-keycloak-admin-oidc.sh --execute \ + --object-admin donghyeon.kang --db-admin donghyeon.kang + +두 confidential client, 두 관리자 그룹, claim mapper와 다음 Secret을 만듭니다. +- object-storage/aistor-keycloak-oidc: client-id, client-secret +- platform-admin/pgadmin-keycloak-oidc: client-id, client-secret +USAGE +} + +while (( $# > 0 )); do + case "$1" in + --execute) execute=true; shift ;; + --object-admin) + (( $# >= 2 )) || fail "--object-admin 값이 필요합니다" + object_admin="$2"; shift 2 ;; + --db-admin) + (( $# >= 2 )) || fail "--db-admin 값이 필요합니다" + db_admin="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; fail "지원하지 않는 인자: $1" ;; + esac +done + +[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요" +for cmd in base64 chmod curl jq kill kubectl mktemp rg rm seq sleep sort tr; do + command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다" +done +context="$(kubectl config current-context)" +api="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" +kubectl -n "$KC_NS" wait --for=condition=Ready "keycloak.k8s.keycloak.org/$KC_NAME" --timeout=30s >/dev/null +kubectl -n "$KC_NS" get "secret/$KC_NAME-initial-admin" >/dev/null + +printf 'Current context: %s\nRealm: %s\n' "$context" "$REALM" +printf 'Clients: aistor-console, pgadmin\n' +printf 'Groups: /platform-object-admins, /platform-db-admins\n' +[[ -n "$object_admin" ]] && printf 'Object admin: %s\n' "$object_admin" +[[ -n "$db_admin" ]] && printf 'DB admin: %s\n' "$db_admin" +if [[ "$execute" == false ]]; then + printf 'DRY RUN PASS: --execute를 지정하지 않아 변경하지 않았습니다.\n' + exit 0 +fi + +[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다" +printf 'Type APPLY %s to configure admin OIDC: ' "$context" +read -r answer +[[ "$answer" == "APPLY $context" ]] || fail "취소했습니다" +[[ "$(kubectl config current-context)" == "$context" ]] || fail "context가 바뀌었습니다" +[[ "$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" == "$api" ]] || fail "API server가 바뀌었습니다" + +kubectl apply -k "$ROOT/infrastructure/namespaces/admin-tools" >/dev/null +temp="$(mktemp -d /tmp/keycloak-admin-oidc.XXXXXX)" +admin_user="$temp/admin-user" +admin_password="$temp/admin-password" +token_json="$temp/token.json" +token="$temp/token" +auth="$temp/auth.conf" +response="$temp/response.json" +early_cleanup() { + rc=$? + trap - EXIT INT TERM + if [[ -n "$pf" ]] && kill -0 "$pf" 2>/dev/null; then + kill "$pf" 2>/dev/null || true + wait "$pf" 2>/dev/null || true + fi + case "$temp" in /tmp/keycloak-admin-oidc.*) rm -rf -- "$temp" ;; esac + exit "$rc" +} +trap early_cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +kubectl -n "$KC_NS" get "secret/$KC_NAME-initial-admin" \ + -o jsonpath='{.data.username}' | base64 -d >"$admin_user" +kubectl -n "$KC_NS" get "secret/$KC_NAME-initial-admin" \ + -o jsonpath='{.data.password}' | base64 -d >"$admin_password" +chmod 0600 "$admin_user" "$admin_password" +kubectl -n "$KC_NS" port-forward --address=127.0.0.1 "service/$KC_SERVICE" \ + "$PORT:8080" >"$temp/port-forward.log" 2>&1 & +pf=$! +ready=false +for _ in $(seq 1 30); do + kill -0 "$pf" 2>/dev/null || fail "Keycloak port-forward가 종료됐습니다" + if rg -q -F "Forwarding from 127.0.0.1:$PORT -> 8080" "$temp/port-forward.log"; then + ready=true + break + fi + sleep 1 +done +[[ "$ready" == true ]] || fail "Keycloak port-forward가 준비되지 않았습니다" + +kc_curl() { + curl --disable --silent --show-error --noproxy '*' \ + --connect-timeout 3 --max-time 20 \ + --header "Host: $KC_HOST" \ + --header 'X-Forwarded-Proto: https' \ + --header 'X-Forwarded-Port: 443' "$@" +} + +kc_curl --fail-with-body --output "$token_json" \ + --data-urlencode grant_type=password --data-urlencode client_id=admin-cli \ + --data-urlencode "username@$admin_user" --data-urlencode "password@$admin_password" \ + "$BASE/realms/master/protocol/openid-connect/token" >/dev/null +jq -er '.access_token | select(type == "string" and length > 0)' "$token_json" >"$token" +{ + printf 'header = "Authorization: Bearer ' + tr -d '\r\n' <"$token" + printf '"\n' +} >"$auth" +chmod 0600 "$token" "$auth" + +request() { + kc_curl --config "$auth" "$@" +} + +backup_secret() { + if kubectl -n "$1" get "secret/$2" -o yaml >"$3" 2>/dev/null; then + printf true + else + printf false + fi +} +ai_secret_existed="$(backup_secret object-storage aistor-keycloak-oidc "$temp/ai-secret.yaml")" +pg_secret_existed="$(backup_secret platform-admin pgadmin-keycloak-oidc "$temp/pg-secret.yaml")" + +upsert_group() { + group_name="$1" + policy="$2" + before="$3" + body="$temp/group.json" + request --get --data-urlencode "search=$group_name" --data-urlencode exact=true \ + -o "$response" "$BASE/admin/realms/$REALM/groups" + group_id="$(jq -r --arg path "/$group_name" '[.[] | select(.path == $path)][0].id // empty' "$response")" + if [[ -n "$policy" ]]; then + jq -n --arg name "$group_name" --arg policy "$policy" \ + '{name:$name,attributes:{policy:[$policy]}}' >"$body" + else + jq -n --arg name "$group_name" '{name:$name}' >"$body" + fi + if [[ -z "$group_id" ]]; then + code="$(request -X POST -H 'Content-Type: application/json' --data-binary "@$body" \ + -o "$response" -w '%{http_code}' "$BASE/admin/realms/$REALM/groups")" + [[ "$code" == 201 ]] || fail "그룹 생성 실패: $group_name HTTP $code" + request --get --data-urlencode "search=$group_name" --data-urlencode exact=true \ + -o "$response" "$BASE/admin/realms/$REALM/groups" + group_id="$(jq -er --arg path "/$group_name" '[.[] | select(.path == $path)][0].id' "$response")" + last_created=true + else + request -o "$before" "$BASE/admin/realms/$REALM/groups/$group_id" + code="$(request -X PUT -H 'Content-Type: application/json' --data-binary "@$body" \ + -o "$response" -w '%{http_code}' "$BASE/admin/realms/$REALM/groups/$group_id")" + [[ "$code" == 204 ]] || fail "그룹 갱신 실패: $group_name HTTP $code" + last_created=false + fi + last_id="$group_id" +} + +upsert_client() { + client="$1" + display="$2" + root_url="$3" + redirect="$4" + before="$5" + body="$temp/client.json" + jq -n --arg client "$client" --arg display "$display" --arg root "$root_url" \ + --arg redirect "$redirect" '{ + clientId:$client,name:$display,enabled:true,protocol:"openid-connect", + clientAuthenticatorType:"client-secret",publicClient:false, + standardFlowEnabled:true,implicitFlowEnabled:false,directAccessGrantsEnabled:false, + serviceAccountsEnabled:false,authorizationServicesEnabled:false, + consentRequired:false,fullScopeAllowed:false,rootUrl:$root,baseUrl:($root+"/"), + redirectUris:[$redirect],webOrigins:[$root], + attributes:{"oauth2.device.authorization.grant.enabled":"false", + "oidc.ciba.grant.enabled":"false","post.logout.redirect.uris":($root+"/*")} + }' >"$body" + request --get --data-urlencode "clientId=$client" --data-urlencode max=2 \ + -o "$response" "$BASE/admin/realms/$REALM/clients" + count="$(jq 'length' "$response")" + [[ "$count" == 0 || "$count" == 1 ]] || fail "$client clientId가 중복됐습니다" + if [[ "$count" == 0 ]]; then + code="$(request -X POST -H 'Content-Type: application/json' --data-binary "@$body" \ + -o "$response" -w '%{http_code}' "$BASE/admin/realms/$REALM/clients")" + [[ "$code" == 201 ]] || fail "client 생성 실패: $client HTTP $code" + last_created=true + else + client_id="$(jq -er '.[0].id' "$response")" + request -o "$before" "$BASE/admin/realms/$REALM/clients/$client_id" + code="$(request -X PUT -H 'Content-Type: application/json' --data-binary "@$body" \ + -o "$response" -w '%{http_code}' "$BASE/admin/realms/$REALM/clients/$client_id")" + [[ "$code" == 204 ]] || fail "client 갱신 실패: $client HTTP $code" + last_created=false + fi + request --get --data-urlencode "clientId=$client" --data-urlencode max=2 \ + -o "$response" "$BASE/admin/realms/$REALM/clients" + last_id="$(jq -er 'select(length == 1) | .[0].id' "$response")" +} + +upsert_mapper() { + client_id="$1" + mapper_name="$2" + mapper_type="$3" + body="$temp/mapper.json" + if [[ "$mapper_type" == group ]]; then + jq -n --arg name "$mapper_name" '{ + name:$name,protocol:"openid-connect",protocolMapper:"oidc-group-membership-mapper", + consentRequired:false,config:{"claim.name":"groups","full.path":"true", + "id.token.claim":"true","access.token.claim":"true", + "userinfo.token.claim":"true","introspection.token.claim":"true"} + }' >"$body" + else + jq -n --arg name "$mapper_name" '{ + name:$name,protocol:"openid-connect",protocolMapper:"oidc-usermodel-attribute-mapper", + consentRequired:false,config:{"user.attribute":"policy","claim.name":"policy", + "jsonType.label":"String","multivalued":"true","aggregate.attrs":"true", + "id.token.claim":"true","access.token.claim":"true", + "userinfo.token.claim":"true","introspection.token.claim":"true"} + }' >"$body" + fi + request -o "$response" "$BASE/admin/realms/$REALM/clients/$client_id/protocol-mappers/models" + mapper_id="$(jq -r --arg name "$mapper_name" '[.[] | select(.name == $name)][0].id // empty' "$response")" + if [[ -z "$mapper_id" ]]; then + code="$(request -X POST -H 'Content-Type: application/json' --data-binary "@$body" \ + -o "$response" -w '%{http_code}' "$BASE/admin/realms/$REALM/clients/$client_id/protocol-mappers/models")" + [[ "$code" == 201 ]] || fail "mapper 생성 실패: $mapper_name HTTP $code" + else + code="$(request -X PUT -H 'Content-Type: application/json' --data-binary "@$body" \ + -o "$response" -w '%{http_code}' \ + "$BASE/admin/realms/$REALM/clients/$client_id/protocol-mappers/models/$mapper_id")" + [[ "$code" == 204 ]] || fail "mapper 갱신 실패: $mapper_name HTTP $code" + fi +} + +assign_member() { + username="$1" + group_id="$2" + request --get --data-urlencode "username=$username" --data-urlencode exact=true \ + -o "$response" "$BASE/admin/realms/$REALM/users" + [[ "$(jq 'length' "$response")" == 1 ]] || fail "사용자를 정확히 찾지 못했습니다: $username" + user_id="$(jq -er '.[0].id' "$response")" + request -o "$response" "$BASE/admin/realms/$REALM/users/$user_id/groups" + if jq -e --arg id "$group_id" 'any(.[]; .id == $id)' "$response" >/dev/null; then + last_member_added=false + else + code="$(request -X PUT -o "$response" -w '%{http_code}' \ + "$BASE/admin/realms/$REALM/users/$user_id/groups/$group_id")" + [[ "$code" == 204 ]] || fail "그룹 구성원 추가 실패: $username HTTP $code" + last_member_added=true + fi + last_user_id="$user_id" +} + +restore_secret() { + if [[ "$1" == true ]]; then + kubectl apply -f "$2" >/dev/null 2>&1 || true + else + kubectl -n "$3" delete "secret/$4" --ignore-not-found >/dev/null 2>&1 + fi +} + +rollback() { + set +e + printf '\nROLLBACK: Keycloak 및 OIDC Secret 이전 상태 복원\n' >&2 + [[ "$ai_member_added" == true ]] && \ + request -X DELETE "$BASE/admin/realms/$REALM/users/$ai_user_uuid/groups/$ai_group_uuid" >/dev/null + [[ "$pg_member_added" == true ]] && \ + request -X DELETE "$BASE/admin/realms/$REALM/users/$pg_user_uuid/groups/$pg_group_uuid" >/dev/null + for item in "ai $ai_client_uuid $ai_client_created" "pg $pg_client_uuid $pg_client_created"; do + set -- $item + if [[ "$3" == true ]]; then + request -X DELETE "$BASE/admin/realms/$REALM/clients/$2" >/dev/null + elif [[ -n "$2" ]]; then + request -X PUT -H 'Content-Type: application/json' \ + --data-binary "@$temp/$1-client-before.json" \ + "$BASE/admin/realms/$REALM/clients/$2" >/dev/null + fi + done + for item in "ai $ai_group_uuid $ai_group_created" "pg $pg_group_uuid $pg_group_created"; do + set -- $item + if [[ "$3" == true ]]; then + request -X DELETE "$BASE/admin/realms/$REALM/groups/$2" >/dev/null + elif [[ -n "$2" ]]; then + request -X PUT -H 'Content-Type: application/json' \ + --data-binary "@$temp/$1-group-before.json" \ + "$BASE/admin/realms/$REALM/groups/$2" >/dev/null + fi + done + restore_secret "$ai_secret_existed" "$temp/ai-secret.yaml" object-storage aistor-keycloak-oidc + restore_secret "$pg_secret_existed" "$temp/pg-secret.yaml" platform-admin pgadmin-keycloak-oidc + rollback_armed=false + printf 'ROLLBACK complete.\n' >&2 +} + +cleanup() { + rc=$? + trap - EXIT INT TERM + if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then rollback; fi + if [[ -n "$pf" ]] && kill -0 "$pf" 2>/dev/null; then + kill "$pf" 2>/dev/null || true + wait "$pf" 2>/dev/null || true + fi + case "$temp" in /tmp/keycloak-admin-oidc.*) rm -rf -- "$temp" ;; esac + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +rollback_armed=true +upsert_group platform-object-admins consoleAdmin "$temp/ai-group-before.json" +ai_group_uuid="$last_id"; ai_group_created="$last_created" +upsert_group platform-db-admins "" "$temp/pg-group-before.json" +pg_group_uuid="$last_id"; pg_group_created="$last_created" + +upsert_client aistor-console "AIStor Console" https://storage-admin.learn.hyeonworks.com \ + https://storage-admin.learn.hyeonworks.com/oauth_callback "$temp/ai-client-before.json" +ai_client_uuid="$last_id"; ai_client_created="$last_created" +upsert_mapper "$ai_client_uuid" aistor-policy-claim policy + +upsert_client pgadmin pgAdmin https://db-admin.learn.hyeonworks.com \ + https://db-admin.learn.hyeonworks.com/oauth2/authorize "$temp/pg-client-before.json" +pg_client_uuid="$last_id"; pg_client_created="$last_created" +upsert_mapper "$pg_client_uuid" pgadmin-groups group + +for item in "object-storage aistor-keycloak-oidc aistor-console $ai_client_uuid ai" \ + "platform-admin pgadmin-keycloak-oidc pgadmin $pg_client_uuid pg"; do + set -- $item + secret_file="$temp/$5-client-secret" + request -o "$response" "$BASE/admin/realms/$REALM/clients/$4/client-secret" + jq --exit-status --join-output --raw-output '.value | select(type == "string" and length >= 16)' "$response" >"$secret_file" + chmod 0600 "$secret_file" + kubectl -n "$1" create secret generic "$2" --from-literal="client-id=$3" \ + --from-file="client-secret=$secret_file" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + kubectl -n "$1" label "secret/$2" app.kubernetes.io/component=oidc-client \ + app.kubernetes.io/part-of=platform app.kubernetes.io/managed-by=bootstrap-script \ + --overwrite >/dev/null +done + +if [[ -n "$object_admin" ]]; then + assign_member "$object_admin" "$ai_group_uuid" + ai_member_added="$last_member_added"; ai_user_uuid="$last_user_id" +fi +if [[ -n "$db_admin" ]]; then + assign_member "$db_admin" "$pg_group_uuid" + pg_member_added="$last_member_added"; pg_user_uuid="$last_user_id" +fi + +for contract in "object-storage aistor-keycloak-oidc" "platform-admin pgadmin-keycloak-oidc"; do + set -- $contract + keys="$(kubectl -n "$1" get "secret/$2" \ + -o go-template='{{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}' | LC_ALL=C sort)" + [[ "$keys" == $'client-id\nclient-secret' ]] || fail "$1/$2 Secret 계약이 다릅니다" +done +request -o "$response" "$BASE/realms/$REALM/.well-known/openid-configuration" +jq -e --arg issuer "https://$KC_HOST/realms/$REALM" '.issuer == $issuer' "$response" >/dev/null || \ + fail "OIDC discovery issuer가 다릅니다" + +rollback_armed=false +printf 'KEYCLOAK ADMIN OIDC CONFIG SUCCESS\n' +printf 'Secret payload, 관리자 암호, token은 출력하지 않았습니다.\n' diff --git a/scripts/bootstrap/configure-keycloak-gitea-oidc.sh b/scripts/bootstrap/configure-keycloak-gitea-oidc.sh new file mode 100755 index 0000000..3a12370 --- /dev/null +++ b/scripts/bootstrap/configure-keycloak-gitea-oidc.sh @@ -0,0 +1,445 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +# Never inherit caller xtrace: this script handles bootstrap credentials, +# bearer tokens, and the Gitea OIDC client secret. +set +x +umask 077 + +readonly REPOSITORY_ROOT="/home/donghyeon/workspace/platform" +readonly KEYCLOAK_NAMESPACE="keycloak" +readonly KEYCLOAK_NAME="keycloak" +readonly KEYCLOAK_SERVICE="keycloak-service" +readonly KEYCLOAK_HOST="id.learn.hyeonworks.com" +readonly KEYCLOAK_REALM="hyeonworks" +readonly GITEA_NAMESPACE="gitea" +readonly GITEA_OIDC_SECRET="gitea-keycloak-oidc" +readonly GITEA_CLIENT_ID="gitea" +readonly GITEA_ROOT_URL="https://git.learn.hyeonworks.com" +readonly GITEA_REDIRECT_URI="${GITEA_ROOT_URL}/user/oauth2/keycloak/callback" +readonly LOCAL_PORT="${KEYCLOAK_LOCAL_PORT:-18080}" +readonly LOCAL_BASE_URL="http://127.0.0.1:${LOCAL_PORT}" + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + printf '%s\n' \ + 'Usage: bash scripts/bootstrap/configure-keycloak-gitea-oidc.sh --execute' \ + '' \ + 'Uses the Operator-generated temporary Keycloak administrator through a' \ + 'loopback-only kubectl port-forward. It creates or updates the hyeonworks' \ + 'realm and confidential Gitea client, then writes only the generated client' \ + 'credential to gitea/gitea-keycloak-oidc.' \ + '' \ + 'No credential, token, or Secret payload is printed or written to Git.' +} + +[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || { + usage + exit 2 +} +[[ "$LOCAL_PORT" =~ ^[0-9]+$ ]] || fail "KEYCLOAK_LOCAL_PORT must be numeric" +(( LOCAL_PORT >= 1024 && LOCAL_PORT <= 65535 )) || \ + fail "KEYCLOAK_LOCAL_PORT must be between 1024 and 65535" +[[ "$PWD" == "$REPOSITORY_ROOT" ]] || \ + fail "run from ${REPOSITORY_ROOT}" +[[ -t 0 ]] || fail "an interactive terminal is required" + +for required_binary in \ + kubectl curl jq base64 mktemp chmod kill sleep seq rg sed tr sort rm; do + command -v "$required_binary" >/dev/null 2>&1 || \ + fail "${required_binary} is required" +done + +readonly CURRENT_CONTEXT="$(kubectl config current-context)" +readonly API_SERVER="$( + kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}' +)" +readonly TARGET_NODE="$( + kubectl get nodes \ + --selector='node-role.kubernetes.io/control-plane' \ + --output=jsonpath='{.items[0].metadata.name}' +)" +[[ -n "$CURRENT_CONTEXT" ]] || fail "kubectl current-context is empty" +[[ -n "$API_SERVER" ]] || fail "the selected Kubernetes API server is empty" +[[ -n "$TARGET_NODE" ]] || fail "the control-plane node was not found" + +kubectl get namespace "$KEYCLOAK_NAMESPACE" >/dev/null +kubectl get namespace "$GITEA_NAMESPACE" >/dev/null +kubectl --namespace "$KEYCLOAK_NAMESPACE" get \ + "keycloak.k8s.keycloak.org/${KEYCLOAK_NAME}" >/dev/null +kubectl --namespace "$KEYCLOAK_NAMESPACE" wait \ + --for=condition=Ready \ + "keycloak.k8s.keycloak.org/${KEYCLOAK_NAME}" \ + --timeout=30s >/dev/null +kubectl --namespace "$KEYCLOAK_NAMESPACE" get \ + "service/${KEYCLOAK_SERVICE}" >/dev/null +kubectl --namespace "$KEYCLOAK_NAMESPACE" get \ + "secret/${KEYCLOAK_NAME}-initial-admin" >/dev/null + +printf 'Current context: %s\n' "$CURRENT_CONTEXT" +printf 'API server: %s\n' "$API_SERVER" +printf 'Target node: %s\n' "$TARGET_NODE" +printf 'Realm: %s\n' "$KEYCLOAK_REALM" +printf 'OIDC client: %s\n' "$GITEA_CLIENT_ID" +printf 'Redirect URI: %s\n' "$GITEA_REDIRECT_URI" +printf 'Type APPLY %s to configure Keycloak and create the Gitea OIDC Secret: ' \ + "$CURRENT_CONTEXT" +read -r confirmation +[[ "$confirmation" == "APPLY ${CURRENT_CONTEXT}" ]] || fail "cancelled" +[[ "$(kubectl config current-context)" == "$CURRENT_CONTEXT" ]] || \ + fail "kubectl context changed after confirmation" +[[ "$( + kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}' +)" == "$API_SERVER" ]] || fail "Kubernetes API server changed after confirmation" +[[ "$( + kubectl get nodes \ + --selector='node-role.kubernetes.io/control-plane' \ + --output=jsonpath='{.items[0].metadata.name}' +)" == "$TARGET_NODE" ]] || fail "target control-plane node changed after confirmation" +kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \ + fail "target node disappeared after confirmation: ${TARGET_NODE}" + +readonly TEMP_DIR="$(mktemp -d /tmp/keycloak-gitea-oidc.XXXXXX)" +readonly PORT_FORWARD_LOG="${TEMP_DIR}/port-forward.log" +readonly ADMIN_USERNAME_FILE="${TEMP_DIR}/admin-username" +readonly ADMIN_PASSWORD_FILE="${TEMP_DIR}/admin-password" +readonly TOKEN_RESPONSE_FILE="${TEMP_DIR}/token-response.json" +readonly ACCESS_TOKEN_FILE="${TEMP_DIR}/access-token" +readonly AUTH_CONFIG_FILE="${TEMP_DIR}/curl-auth.conf" +readonly REALM_FILE="${TEMP_DIR}/realm.json" +readonly CLIENT_FILE="${TEMP_DIR}/client.json" +readonly RESPONSE_FILE="${TEMP_DIR}/response.json" +readonly CLIENT_SECRET_RESPONSE_FILE="${TEMP_DIR}/client-secret.json" +readonly CLIENT_SECRET_FILE="${TEMP_DIR}/client-secret" +port_forward_pid="" + +cleanup() { + local cleanup_rc=$? + + if [[ -n "$port_forward_pid" ]] && kill -0 "$port_forward_pid" 2>/dev/null; then + kill "$port_forward_pid" 2>/dev/null || true + wait "$port_forward_pid" 2>/dev/null || true + fi + + case "$TEMP_DIR" in + /tmp/keycloak-gitea-oidc.*) + rm -rf -- "$TEMP_DIR" + ;; + *) + printf 'WARNING: refusing to remove unexpected temp path: %s\n' \ + "$TEMP_DIR" >&2 + ;; + esac + + exit "$cleanup_rc" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +kubectl --namespace "$KEYCLOAK_NAMESPACE" get \ + "secret/${KEYCLOAK_NAME}-initial-admin" \ + --output=jsonpath='{.data.username}' \ + | base64 --decode >"$ADMIN_USERNAME_FILE" +kubectl --namespace "$KEYCLOAK_NAMESPACE" get \ + "secret/${KEYCLOAK_NAME}-initial-admin" \ + --output=jsonpath='{.data.password}' \ + | base64 --decode >"$ADMIN_PASSWORD_FILE" +chmod 0600 "$ADMIN_USERNAME_FILE" "$ADMIN_PASSWORD_FILE" +[[ -s "$ADMIN_USERNAME_FILE" ]] || fail "temporary admin username is empty" +[[ -s "$ADMIN_PASSWORD_FILE" ]] || fail "temporary admin password is empty" + +kubectl --namespace "$KEYCLOAK_NAMESPACE" port-forward \ + --address=127.0.0.1 \ + "service/${KEYCLOAK_SERVICE}" \ + "${LOCAL_PORT}:8080" >"$PORT_FORWARD_LOG" 2>&1 & +port_forward_pid=$! + +port_forward_ready=0 +for _ in $(seq 1 30); do + kill -0 "$port_forward_pid" 2>/dev/null || { + printf 'Port-forward failed; non-sensitive log follows:\n' >&2 + sed -n '1,20p' "$PORT_FORWARD_LOG" >&2 + fail "Keycloak port-forward exited" + } + if rg --quiet --fixed-strings \ + "Forwarding from 127.0.0.1:${LOCAL_PORT} -> 8080" \ + "$PORT_FORWARD_LOG"; then + port_forward_ready=1 + break + fi + sleep 1 +done +(( port_forward_ready == 1 )) || fail "Keycloak port-forward did not become ready" + +curl_common=( + --disable + --silent + --show-error + --noproxy '*' + --connect-timeout 3 + --max-time 20 + --header "Host: ${KEYCLOAK_HOST}" + --header 'X-Forwarded-Proto: https' + --header 'X-Forwarded-Port: 443' +) + +curl "${curl_common[@]}" \ + --fail-with-body \ + --output "$TOKEN_RESPONSE_FILE" \ + --data-urlencode 'grant_type=password' \ + --data-urlencode 'client_id=admin-cli' \ + --data-urlencode "username@${ADMIN_USERNAME_FILE}" \ + --data-urlencode "password@${ADMIN_PASSWORD_FILE}" \ + "${LOCAL_BASE_URL}/realms/master/protocol/openid-connect/token" \ + >/dev/null || fail "temporary Keycloak administrator authentication failed" + +jq --exit-status --raw-output \ + '.access_token | select(type == "string" and length > 0)' \ + "$TOKEN_RESPONSE_FILE" >"$ACCESS_TOKEN_FILE" || \ + fail "Keycloak token response did not contain an access token" +chmod 0600 "$ACCESS_TOKEN_FILE" + +{ + printf 'header = "Authorization: Bearer ' + tr -d '\r\n' <"$ACCESS_TOKEN_FILE" + printf '"\n' +} >"$AUTH_CONFIG_FILE" +chmod 0600 "$AUTH_CONFIG_FILE" + +cat >"$REALM_FILE" <<'JSON' +{ + "realm": "hyeonworks", + "displayName": "Hyeonworks", + "enabled": true, + "sslRequired": "external", + "registrationAllowed": false, + "registrationEmailAsUsername": false, + "rememberMe": true, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": true, + "editUsernameAllowed": false, + "bruteForceProtected": true, + "permanentLockout": false, + "maxFailureWaitSeconds": 900, + "minimumQuickLoginWaitSeconds": 60, + "waitIncrementSeconds": 60, + "quickLoginCheckMilliSeconds": 1000, + "maxDeltaTimeSeconds": 43200, + "failureFactor": 5, + "internationalizationEnabled": true, + "supportedLocales": ["ko", "en"], + "defaultLocale": "ko" +} +JSON + +cat >"$CLIENT_FILE" <<'JSON' +{ + "clientId": "gitea", + "name": "Hyeonworks Gitea", + "description": "Gitea confidential OIDC client managed by the platform bootstrap", + "enabled": true, + "protocol": "openid-connect", + "clientAuthenticatorType": "client-secret", + "publicClient": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "authorizationServicesEnabled": false, + "consentRequired": false, + "fullScopeAllowed": false, + "rootUrl": "https://git.learn.hyeonworks.com", + "baseUrl": "https://git.learn.hyeonworks.com/", + "redirectUris": [ + "https://git.learn.hyeonworks.com/user/oauth2/keycloak/callback" + ], + "webOrigins": [ + "https://git.learn.hyeonworks.com" + ], + "attributes": { + "post.logout.redirect.uris": "https://git.learn.hyeonworks.com/*", + "oauth2.device.authorization.grant.enabled": "false", + "oidc.ciba.grant.enabled": "false" + } +} +JSON + +admin_request() { + curl "${curl_common[@]}" \ + --config "$AUTH_CONFIG_FILE" \ + "$@" +} + +realm_status="$( + admin_request \ + --output "$RESPONSE_FILE" \ + --write-out '%{http_code}' \ + "${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}" +)" +case "$realm_status" in + 200) + update_status="$( + admin_request \ + --request PUT \ + --header 'Content-Type: application/json' \ + --data-binary "@${REALM_FILE}" \ + --output "$RESPONSE_FILE" \ + --write-out '%{http_code}' \ + "${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}" + )" + [[ "$update_status" == "204" ]] || \ + fail "Keycloak realm update failed with HTTP ${update_status}" + printf 'Updated Keycloak realm %s.\n' "$KEYCLOAK_REALM" + ;; + 404) + create_status="$( + admin_request \ + --request POST \ + --header 'Content-Type: application/json' \ + --data-binary "@${REALM_FILE}" \ + --output "$RESPONSE_FILE" \ + --write-out '%{http_code}' \ + "${LOCAL_BASE_URL}/admin/realms" + )" + [[ "$create_status" == "201" ]] || \ + fail "Keycloak realm creation failed with HTTP ${create_status}" + printf 'Created Keycloak realm %s.\n' "$KEYCLOAK_REALM" + ;; + *) + fail "Keycloak realm lookup failed with HTTP ${realm_status}" + ;; +esac + +client_lookup_status="$( + admin_request \ + --get \ + --data-urlencode "clientId=${GITEA_CLIENT_ID}" \ + --data-urlencode 'max=2' \ + --output "$RESPONSE_FILE" \ + --write-out '%{http_code}' \ + "${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}/clients" +)" +[[ "$client_lookup_status" == "200" ]] || \ + fail "Keycloak client lookup failed with HTTP ${client_lookup_status}" +client_count="$(jq 'length' "$RESPONSE_FILE")" +[[ "$client_count" == "0" || "$client_count" == "1" ]] || \ + fail "more than one Keycloak client uses clientId=${GITEA_CLIENT_ID}" + +if [[ "$client_count" == "0" ]]; then + client_create_status="$( + admin_request \ + --request POST \ + --header 'Content-Type: application/json' \ + --data-binary "@${CLIENT_FILE}" \ + --output "$RESPONSE_FILE" \ + --write-out '%{http_code}' \ + "${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}/clients" + )" + [[ "$client_create_status" == "201" ]] || \ + fail "Keycloak Gitea client creation failed with HTTP ${client_create_status}" + printf 'Created confidential Keycloak client %s.\n' "$GITEA_CLIENT_ID" +else + client_uuid="$(jq --exit-status --raw-output '.[0].id' "$RESPONSE_FILE")" + [[ -n "$client_uuid" ]] || fail "existing Gitea client has no internal id" + client_update_status="$( + admin_request \ + --request PUT \ + --header 'Content-Type: application/json' \ + --data-binary "@${CLIENT_FILE}" \ + --output "$RESPONSE_FILE" \ + --write-out '%{http_code}' \ + "${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}/clients/${client_uuid}" + )" + [[ "$client_update_status" == "204" ]] || \ + fail "Keycloak Gitea client update failed with HTTP ${client_update_status}" + printf 'Updated confidential Keycloak client %s.\n' "$GITEA_CLIENT_ID" +fi + +client_lookup_status="$( + admin_request \ + --get \ + --data-urlencode "clientId=${GITEA_CLIENT_ID}" \ + --data-urlencode 'max=2' \ + --output "$RESPONSE_FILE" \ + --write-out '%{http_code}' \ + "${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}/clients" +)" +[[ "$client_lookup_status" == "200" ]] || \ + fail "post-update Keycloak client lookup failed with HTTP ${client_lookup_status}" +[[ "$(jq 'length' "$RESPONSE_FILE")" == "1" ]] || \ + fail "post-update Gitea client lookup did not return exactly one client" +client_uuid="$(jq --exit-status --raw-output '.[0].id' "$RESPONSE_FILE")" + +client_secret_status="$( + admin_request \ + --output "$CLIENT_SECRET_RESPONSE_FILE" \ + --write-out '%{http_code}' \ + "${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}/clients/${client_uuid}/client-secret" +)" +[[ "$client_secret_status" == "200" ]] || \ + fail "Keycloak client-secret retrieval failed with HTTP ${client_secret_status}" +jq --exit-status --raw-output --join-output \ + '.value | select(type == "string" and length >= 16)' \ + "$CLIENT_SECRET_RESPONSE_FILE" >"$CLIENT_SECRET_FILE" || \ + fail "Keycloak returned an invalid Gitea client secret" +chmod 0600 "$CLIENT_SECRET_FILE" + +kubectl --namespace "$GITEA_NAMESPACE" create secret generic \ + "$GITEA_OIDC_SECRET" \ + --from-literal="key=${GITEA_CLIENT_ID}" \ + --from-file="secret=${CLIENT_SECRET_FILE}" \ + --dry-run=client \ + --output=yaml \ + | kubectl apply --filename=- >/dev/null +kubectl --namespace "$GITEA_NAMESPACE" label secret "$GITEA_OIDC_SECRET" \ + app.kubernetes.io/name=gitea \ + app.kubernetes.io/instance=gitea \ + app.kubernetes.io/component=oidc-client \ + app.kubernetes.io/part-of=platform \ + app.kubernetes.io/managed-by=bootstrap-script \ + --overwrite >/dev/null + +secret_type="$( + kubectl --namespace "$GITEA_NAMESPACE" get secret "$GITEA_OIDC_SECRET" \ + --output=jsonpath='{.type}' +)" +secret_keys="$( + kubectl --namespace "$GITEA_NAMESPACE" get secret "$GITEA_OIDC_SECRET" \ + --output=go-template='{{range $key, $value := .data}}{{$key}}{{"\n"}}{{end}}' \ + | LC_ALL=C sort +)" +[[ "$secret_type" == "Opaque" ]] || fail "Gitea OIDC Secret type is not Opaque" +[[ "$secret_keys" == $'key\nsecret' ]] || \ + fail "Gitea OIDC Secret key contract is invalid" + +discovery_status="$( + curl "${curl_common[@]}" \ + --output "$RESPONSE_FILE" \ + --write-out '%{http_code}' \ + "${LOCAL_BASE_URL}/realms/${KEYCLOAK_REALM}/.well-known/openid-configuration" +)" +[[ "$discovery_status" == "200" ]] || \ + fail "Keycloak OIDC discovery failed with HTTP ${discovery_status}" +jq --exit-status \ + --arg issuer "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}" \ + '.issuer == $issuer and + (.authorization_endpoint | startswith($issuer)) and + (.token_endpoint | startswith($issuer))' \ + "$RESPONSE_FILE" >/dev/null || \ + fail "Keycloak discovery metadata contains an unexpected public issuer" + +printf '\nKEYCLOAK OIDC BOOTSTRAP SUCCESS\n' +printf 'Realm: %s\n' "$KEYCLOAK_REALM" +printf 'Client: %s (confidential, authorization code flow)\n' "$GITEA_CLIENT_ID" +printf 'Gitea Secret: %s/%s\n' "$GITEA_NAMESPACE" "$GITEA_OIDC_SECRET" +printf 'Secret and token payloads were not printed and the temporary files were removed.\n' +printf 'Next: run scripts/bootstrap/apply-host-nginx-keycloak.sh, then apply-gitea-oidc.sh.\n' +printf 'Security follow-up: replace the temporary Keycloak administrator with a named administrator and MFA.\n' diff --git a/scripts/bootstrap/configure-keycloak-grafana-oidc.sh b/scripts/bootstrap/configure-keycloak-grafana-oidc.sh new file mode 100755 index 0000000..d682fc9 --- /dev/null +++ b/scripts/bootstrap/configure-keycloak-grafana-oidc.sh @@ -0,0 +1,1928 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +# Never inherit xtrace across credential, bearer-token, or client-secret handling. +set +x +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly EXPECTED_CONTEXT='default' +readonly EXPECTED_API_SERVER='https://127.0.0.1:6443' +readonly EXPECTED_NODE='donghyeon-system-product-name' +readonly KEYCLOAK_NAMESPACE='keycloak' +readonly KEYCLOAK_NAME='keycloak' +readonly KEYCLOAK_SERVICE='keycloak-service' +readonly KEYCLOAK_ADMIN_SECRET='keycloak-initial-admin' +readonly KEYCLOAK_HOST='id.learn.hyeonworks.com' +readonly KEYCLOAK_REALM='hyeonworks' +readonly OBSERVABILITY_NAMESPACE='observability' +readonly OIDC_SECRET='grafana-keycloak-oidc' +readonly CLIENT_ID='grafana' +readonly GRAFANA_URL='https://grafana.learn.hyeonworks.com' +readonly REDIRECT_URI='https://grafana.learn.hyeonworks.com/login/generic_oauth' +readonly POST_LOGOUT_URI='https://grafana.learn.hyeonworks.com/*' +readonly ADMIN_GROUP_NAME='platform-observability-admins' +readonly VIEWER_GROUP_NAME='platform-observability-viewers' +readonly ADMIN_GROUP_PATH='/platform-observability-admins' +readonly VIEWER_GROUP_PATH='/platform-observability-viewers' +readonly MAPPER_NAME='grafana-groups' +readonly EVIDENCE_SCHEMA='platform-observability-recovery-evidence-v1' +readonly EVIDENCE_RESOURCE='keycloak/hyeonworks/client/grafana' +readonly EVIDENCE_MAX_AGE_SECONDS=2592000 +readonly PRODUCTION_EVIDENCE_DIR='/etc/hyeonworks/platform/recovery-evidence' +readonly PRODUCTION_ENCRYPTION_SCRIPT="$ROOT/scripts/validate/k3s-secret-encryption.sh" +readonly PRODUCTION_RESTORE_SCRIPT="$ROOT/scripts/validate/k3s-secret-encryption-restore-evidence.sh" +readonly VALIDATOR_PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' +readonly VALIDATOR_HOME='/home/donghyeon' +readonly KUBECTL_REQUEST_TIMEOUT='5s' + +TEST_MODE=false +KUBECTL_BIN='/usr/local/bin/kubectl' +SUDO_BIN='/usr/bin/sudo' +ENCRYPTION_SCRIPT="$PRODUCTION_ENCRYPTION_SCRIPT" +RESTORE_SCRIPT="$PRODUCTION_RESTORE_SCRIPT" +EVIDENCE_DIR="$PRODUCTION_EVIDENCE_DIR" +KUBECTL_PROCESS_TIMEOUT='10s' +PORT_FORWARD_PROCESS_TIMEOUT='120s' +PROXY_PROCESS_TIMEOUT='120s' +CURL_PROCESS_TIMEOUT='12s' +CURL_MAX_TIME='8' +CURL_CONNECT_TIMEOUT='2' +EVIDENCE_EXPECTED_UID=0 +EVIDENCE_EXPECTED_GID=0 +EVIDENCE_DIR_MODE=700 +TEST_CONFIRMATIONS='' +TEST_NOW_UTC='' + +execute_requested=false +check_requested=false +admin_username='' +viewer_username='' +temporary_dir='' +port_forward_pid='' +port_forward_start='' +proxy_pid='' +proxy_start='' +local_port='' +local_base='' +kube_socket='' +confirmation_index=0 +declare -a confirmation_answers=() + +transaction_active=false +rollback_in_progress=false +exit_handler_running=false +manual_recovery_required=false +pending_kind='' +pending_before='' +pending_desired='' +pending_id='' +pending_token='' +pending_user_id='' +pending_group_id='' +pending_status_file='' +transaction_token='' + +client_state='absent' +client_id='' +client_before='' +client_desired='' +client_after='' +client_restore='' +client_secret_file='' +client_mutation='none' +admin_group_state='absent' +admin_group_id='' +admin_group_before='' +admin_group_after='' +admin_group_mutation='none' +viewer_group_state='absent' +viewer_group_id='' +viewer_group_before='' +viewer_group_after='' +viewer_group_mutation='none' +mapper_state='absent' +mapper_id='' +mapper_before='' +mapper_desired='' +mapper_after='' +mapper_mutation='none' +secret_state='absent' +secret_before='' +secret_uid='' +secret_resource_version='' +secret_created_uid='' +secret_created_resource_version='' +secret_mutation='none' +admin_user_id='' +viewer_user_id='' +admin_membership_before=false +viewer_membership_before=false +admin_membership_added=false +viewer_membership_added=false +marker_had_prior=false +marker_prior='' +marker_desired='' +marker_written=false + +last_http_status='' +last_body='' +last_status_file='' +last_curl_rc=0 +request_counter=0 + +usage() { + cat <<'USAGE' +Usage: + bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh + bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh --execute + bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh \ + --execute --admin USER --viewer USER + bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh \ + --check-recovery-evidence + +No arguments prints the fixed, payload-free plan and performs no external reads. +Execute reconciles only the exact Grafana client, two groups, full-path groups +mapper, OIDC Secret, and explicitly requested existing-user memberships. +USAGE +} + +emit_manual_recovery() { + [[ "$manual_recovery_required" == false ]] || return 0 + manual_recovery_required=true + printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2 +} + +fail() { + local message=$1 + if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then + transaction_fail "$message" + fi + printf 'ERROR: %s\n' "$message" >&2 + exit 1 +} + +reject_production_overrides() { + local variable + for variable in ${!PLATFORM_GRAFANA_OIDC_@}; do + printf 'ERROR: production boundary override is forbidden: %s\n' "$variable" >&2 + exit 1 + done +} + +parse_arguments() { + local seen_execute=false seen_check=false seen_admin=false seen_viewer=false + while (( $# > 0 )); do + case "$1" in + --execute) + [[ "$seen_execute" == false ]] || fail 'duplicate --execute' + seen_execute=true; execute_requested=true; shift + ;; + --check-recovery-evidence) + [[ "$seen_check" == false ]] || fail 'duplicate --check-recovery-evidence' + seen_check=true; check_requested=true; shift + ;; + --admin) + [[ "$seen_admin" == false && $# -ge 2 ]] || fail 'invalid --admin' + seen_admin=true; admin_username=$2; shift 2 + ;; + --viewer) + [[ "$seen_viewer" == false && $# -ge 2 ]] || fail 'invalid --viewer' + seen_viewer=true; viewer_username=$2; shift 2 + ;; + -h|--help) usage; exit 0 ;; + *) fail "unsupported argument: $1" ;; + esac + done + if [[ "$check_requested" == true ]]; then + [[ "$execute_requested" == false && "$seen_admin" == false && "$seen_viewer" == false ]] || + fail 'recovery evidence mode cannot be combined with execute or memberships' + return + fi + if [[ "$execute_requested" == false ]]; then + [[ "$seen_admin" == false && "$seen_viewer" == false ]] || + fail 'membership selection requires --execute' + return + fi + local username + for username in "$admin_username" "$viewer_username"; do + [[ -z "$username" || "$username" =~ ^[A-Za-z0-9][A-Za-z0-9._@+-]{0,127}$ ]] || + fail 'membership usernames must use the bounded printable Keycloak username form' + done +} + +print_dry_run() { + printf '%s\n' \ + 'GRAFANA_OIDC_DRY_RUN=PASS' \ + 'CONTEXT=default' \ + 'KEYCLOAK_CLIENT=grafana' \ + 'KEYCLOAK_GROUPS=/platform-observability-admins,/platform-observability-viewers' \ + 'KEYCLOAK_MAPPER=grafana-groups:groups:full-path:id-token,access-token,userinfo' \ + 'KUBERNETES_SECRET=observability/grafana-keycloak-oidc:Opaque:client-id,client-secret' \ + 'MEMBERSHIPS=EXPLICIT_ONLY' \ + 'MUTATION=NOT_REQUESTED' +} + +attest_test_executable() { + local path=$1 metadata + [[ -f "$path" && ! -L "$path" && -O "$path" && -x "$path" ]] || return 1 + metadata="$(stat -c '%u:%a:%h:%F' -- "$path")" || return 1 + [[ "$metadata" == "${EUID}:755:1:regular file" ]] +} + +configure_test_boundaries() { + local fixture=$1 resolved metadata + [[ "$fixture" =~ ^/tmp/platform-grafana-oidc-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+$ ]] || + fail 'test fixture root has an invalid shape' + [[ -d "$fixture" && ! -L "$fixture" && -O "$fixture" ]] || fail 'test fixture root is unsafe' + resolved="$(cd -- "$fixture" && pwd -P)" || fail 'cannot resolve test fixture root' + [[ "$resolved" == "$fixture" ]] || fail 'test fixture contains a symlink boundary' + metadata="$(stat -c '%u:%a:%F' -- "$fixture")" || fail 'cannot stat test fixture root' + [[ "$metadata" == "${EUID}:700:directory" ]] || fail 'test fixture metadata is unsafe' + [[ "$(cd -- "$fixture/bin" && pwd -P)" == "$fixture/bin" && + "$(cd -- "$fixture/evidence-parent" && pwd -P)" == "$fixture/evidence-parent" ]] || + fail 'test fixture child has a symlink boundary' + KUBECTL_BIN="$fixture/bin/kubectl" + SUDO_BIN="$fixture/bin/sudo" + ENCRYPTION_SCRIPT="$fixture/bin/encryption" + RESTORE_SCRIPT="$fixture/bin/restore" + EVIDENCE_DIR="$fixture/evidence-parent/recovery-evidence" + EVIDENCE_EXPECTED_UID=$EUID + EVIDENCE_EXPECTED_GID="$(id -g)" + EVIDENCE_DIR_MODE=700 + KUBECTL_PROCESS_TIMEOUT='3s' + PORT_FORWARD_PROCESS_TIMEOUT='30s' + PROXY_PROCESS_TIMEOUT='30s' + CURL_PROCESS_TIMEOUT='4s' + CURL_MAX_TIME='2' + CURL_CONNECT_TIMEOUT='1' + TEST_CONFIRMATIONS="${PLATFORM_GRAFANA_OIDC_CONFIRMATIONS:-}" + TEST_NOW_UTC="${PLATFORM_GRAFANA_OIDC_NOW_UTC:-}" + mapfile -t confirmation_answers <<<"$TEST_CONFIRMATIONS" + [[ "$TEST_NOW_UTC" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || + fail 'test clock is invalid' + attest_test_executable "$KUBECTL_BIN" || fail 'test kubectl boundary is unsafe' + attest_test_executable "$SUDO_BIN" || fail 'test sudo boundary is unsafe' + attest_test_executable "$ENCRYPTION_SCRIPT" || fail 'test encryption boundary is unsafe' + attest_test_executable "$RESTORE_SCRIPT" || fail 'test restore boundary is unsafe' + [[ "$SUDO_BIN" != /usr/bin/sudo ]] || fail 'real sudo is forbidden in the fixture entrypoint' + export PLATFORM_TEST_TARGET_PID=$BASHPID +} + +require_commands() { + local command_name + for command_name in base64 chmod cmp cp curl date grep id jq kill mktemp python3 rm sort stat timeout tr wc; do + command -v "$command_name" >/dev/null 2>&1 || fail "$command_name is required" + done + [[ -x "$KUBECTL_BIN" && ! -L "$KUBECTL_BIN" ]] || fail 'kubectl boundary is unavailable or unsafe' + [[ -x "$SUDO_BIN" && ! -L "$SUDO_BIN" ]] || fail 'sudo boundary is unavailable or unsafe' + [[ -f "$ENCRYPTION_SCRIPT" && ! -L "$ENCRYPTION_SCRIPT" && -r "$ENCRYPTION_SCRIPT" ]] || + fail 'k3s encryption validator is unavailable' + [[ -f "$RESTORE_SCRIPT" && ! -L "$RESTORE_SCRIPT" && -r "$RESTORE_SCRIPT" ]] || + fail 'k3s restore validator is unavailable' +} + +make_temporary_dir() { + temporary_dir="$(mktemp -d /tmp/platform-grafana-oidc.XXXXXX)" || fail 'cannot create private temporary directory' + [[ "$temporary_dir" == /tmp/platform-grafana-oidc.?????? && ! -L "$temporary_dir" ]] || + fail 'private temporary directory shape is unsafe' + chmod 0700 -- "$temporary_dir" + [[ "$(stat -c '%u:%a:%h:%F' -- "$temporary_dir")" == "${EUID}:700:2:directory" ]] || + fail 'private temporary directory metadata is unsafe' +} + +private_file_ok() { + local path=$1 allow_empty=${2:-false} metadata + [[ "$path" == "$temporary_dir"/* && -f "$path" && ! -L "$path" && -O "$path" ]] || return 1 + metadata="$(stat -c '%u:%a:%h' -- "$path")" || return 1 + [[ "$metadata" == "${EUID}:600:1" ]] || return 1 + [[ "$allow_empty" == true || -s "$path" ]] +} + +process_start_time() { + local pid=$1 + python3 -I -S - "$pid" <<'PY' +import pathlib,sys +raw=pathlib.Path('/proc',sys.argv[1],'stat').read_text() +end=raw.rfind(') ') +if end < 0: raise SystemExit(1) +fields=raw[end+2:].split() +if len(fields)<20 or not fields[19].isdigit(): raise SystemExit(1) +print(fields[19],end='') +PY +} + +stop_owned_process() { + local pid=$1 expected_start=$2 attempt current + [[ -n "$pid" && -n "$expected_start" && "$pid" =~ ^[0-9]+$ ]] || return 0 + current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; } + [[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; } + kill -TERM "$pid" 2>/dev/null || true + for ((attempt=0; attempt<30; attempt++)); do + if ! kill -0 "$pid" 2>/dev/null; then wait "$pid" 2>/dev/null || true; return 0; fi + /usr/bin/sleep 0.05 + done + current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; } + [[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; } + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true +} + +safe_temporary_path() { + [[ -n "$temporary_dir" && "$temporary_dir" == /tmp/platform-grafana-oidc.?????? && + ( "$1" == "$temporary_dir" || "$1" == "$temporary_dir"/* ) ]] +} + +cleanup() { + stop_owned_process "$proxy_pid" "$proxy_start" || true + stop_owned_process "$port_forward_pid" "$port_forward_start" || true + if [[ -n "$temporary_dir" ]] && safe_temporary_path "$temporary_dir"; then rm -rf -- "$temporary_dir"; fi +} + +kubectl_bounded() { + /usr/bin/timeout --signal=TERM --kill-after=1s "$KUBECTL_PROCESS_TIMEOUT" \ + "$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" "$@" +} + +run_common_gates() { + local -a safe_environment=(/usr/bin/env -i "PATH=$VALIDATOR_PATH" "HOME=$VALIDATOR_HOME") + if [[ "$TEST_MODE" == true ]]; then + safe_environment+=( + "PLATFORM_TEST_COMMAND_LOG=${PLATFORM_TEST_COMMAND_LOG:-}" + "PLATFORM_TEST_STATE=${PLATFORM_TEST_STATE:-}" + "PLATFORM_TEST_ENCRYPTION_COUNT=${PLATFORM_TEST_ENCRYPTION_COUNT:-}" + "PLATFORM_TEST_RESTORE_COUNT=${PLATFORM_TEST_RESTORE_COUNT:-}" + "PLATFORM_TEST_ENCRYPTION_FAIL_AT=${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" + "PLATFORM_TEST_RESTORE_FAIL_AT=${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" + "PLATFORM_TEST_VALIDATOR_ENV_LOG=${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" + ) + fi + "${safe_environment[@]}" /usr/bin/bash "$ENCRYPTION_SCRIPT" --expect-reencrypted >/dev/null 2>&1 || + fail 'k3s Secret encryption is not fully re-encrypted' + "${safe_environment[@]}" /usr/bin/bash "$RESTORE_SCRIPT" --check >/dev/null 2>&1 || + fail 'current k3s restore evidence is unavailable' +} + +read_confirmation() { + local prompt=$1 expected=$2 answer='' + printf '%s' "$prompt" >&2 + if [[ "$TEST_MODE" == true ]]; then + (( confirmation_index < ${#confirmation_answers[@]} )) || fail 'confirmation input is missing' + answer="${confirmation_answers[$confirmation_index]}"; confirmation_index=$((confirmation_index + 1)) + else + [[ -t 0 ]] || fail 'execute mode requires an interactive terminal' + IFS= read -r answer + fi + [[ "$answer" == "$expected" ]] || fail 'confirmation was not exact' +} + +now_utc() { + if [[ "$TEST_MODE" == true ]]; then printf '%s\n' "$TEST_NOW_UTC"; else date -u +%Y-%m-%dT%H:%M:%SZ; fi +} + +timestamp_epoch() { + local timestamp=$1 normalized + normalized="$(date -u -d "$timestamp" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" || return 1 + [[ "$normalized" == "$timestamp" ]] || return 1 + date -u -d "$timestamp" +%s +} + +now_epoch() { timestamp_epoch "$(now_utc)"; } + +json_equal() { + python3 -I -S - "$1" "$2" <<'PY' +import json,pathlib,sys +try: a=json.loads(pathlib.Path(sys.argv[1]).read_text()); b=json.loads(pathlib.Path(sys.argv[2]).read_text()) +except Exception: raise SystemExit(1) +raise SystemExit(0 if a==b else 1) +PY +} + +new_transaction_token() { python3 -I -S -c 'import secrets; print(secrets.token_hex(16))'; } + +validate_context_and_authority() { + local context api node_file namespace result + context="$(kubectl_bounded config current-context)" || fail 'cannot read Kubernetes context' + [[ "$context" == "$EXPECTED_CONTEXT" ]] || fail 'current Kubernetes context is not default' + api="$(kubectl_bounded config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" || + fail 'cannot read Kubernetes API server' + [[ "$api" == "$EXPECTED_API_SERVER" ]] || fail 'Kubernetes API server is not the exact local endpoint' + node_file="$temporary_dir/node.json" + kubectl_bounded get node "$EXPECTED_NODE" --output=json >"$node_file" || fail 'cannot read exact target node' + chmod 0600 -- "$node_file" + private_file_ok "$node_file" || fail 'target node response is unsafe' + jq -e --arg name "$EXPECTED_NODE" ' + .apiVersion == "v1" and .kind == "Node" and .metadata.name == $name and + (.metadata.uid | type == "string" and length > 0) and + any(.status.conditions[]?; .type == "Ready" and .status == "True") + ' "$node_file" >/dev/null || fail 'target node identity or readiness changed' + + for namespace in "$KEYCLOAK_NAMESPACE" "$OBSERVABILITY_NAMESPACE"; do + result="$(kubectl_bounded get namespace "$namespace" --output=name)" || fail "cannot read namespace $namespace" + [[ "$result" == "namespace/$namespace" ]] || fail "namespace identity changed: $namespace" + done + for permission in \ + 'get keycloak.k8s.keycloak.org/keycloak keycloak' \ + 'get service/keycloak-service keycloak' \ + 'get secret/keycloak-initial-admin keycloak' \ + 'get secret/grafana-keycloak-oidc observability' \ + 'create secret observability' \ + 'delete secret/grafana-keycloak-oidc observability' \ + 'get nodes -' \ + 'get namespaces -' \ + 'get pods keycloak' \ + 'list pods keycloak' \ + 'create pods/portforward keycloak'; do + set -- $permission + if [[ "$3" == - ]]; then + result="$(kubectl_bounded auth can-i "$1" "$2")" || fail 'cannot verify exact Kubernetes authority' + else + result="$(kubectl_bounded auth can-i "$1" "$2" --namespace="$3")" || fail 'cannot verify exact Kubernetes authority' + fi + [[ "$result" == yes ]] || fail "missing exact Kubernetes authority: $permission" + done +} + +snapshot_kubernetes_objects() { + local keycloak_file service_file admin_file + keycloak_file="$temporary_dir/keycloak-resource.json" + service_file="$temporary_dir/keycloak-service.json" + admin_file="$temporary_dir/keycloak-admin-secret.json" + kubectl_bounded --namespace "$KEYCLOAK_NAMESPACE" get "keycloak.k8s.keycloak.org/$KEYCLOAK_NAME" --output=json >"$keycloak_file" || + fail 'cannot read exact Keycloak resource' + kubectl_bounded --namespace "$KEYCLOAK_NAMESPACE" get "service/$KEYCLOAK_SERVICE" --output=json >"$service_file" || + fail 'cannot read exact Keycloak Service' + kubectl_bounded --namespace "$KEYCLOAK_NAMESPACE" get "secret/$KEYCLOAK_ADMIN_SECRET" --output=json >"$admin_file" || + fail 'cannot read exact initial-admin Secret' + chmod 0600 -- "$keycloak_file" "$service_file" "$admin_file" + private_file_ok "$keycloak_file" && private_file_ok "$service_file" && private_file_ok "$admin_file" || + fail 'Kubernetes bootstrap snapshots are unsafe' + jq -e --arg name "$KEYCLOAK_NAME" --arg namespace "$KEYCLOAK_NAMESPACE" ' + .apiVersion == "k8s.keycloak.org/v2beta1" and .kind == "Keycloak" and + .metadata.name == $name and .metadata.namespace == $namespace and + (.metadata.uid | type == "string" and length > 0) and + any(.status.conditions[]?; .type == "Ready" and .status == "True") + ' "$keycloak_file" >/dev/null || fail 'Keycloak resource identity or readiness changed' + jq -e --arg name "$KEYCLOAK_SERVICE" --arg namespace "$KEYCLOAK_NAMESPACE" ' + .apiVersion == "v1" and .kind == "Service" and .metadata.name == $name and + .metadata.namespace == $namespace and (.metadata.uid | type == "string" and length > 0) and + ([.spec.ports[]? | select(.port == 8080)] | length) == 1 + ' "$service_file" >/dev/null || fail 'Keycloak Service contract changed' + + local admin_username_file admin_password_file + admin_username_file="$temporary_dir/admin-username" + admin_password_file="$temporary_dir/admin-password" + python3 -I -S - "$admin_file" "$admin_username_file" "$admin_password_file" <<'PY' || +import base64,binascii,json,os,pathlib,sys +source,user_path,password_path=sys.argv[1:] +try: item=json.loads(pathlib.Path(source).read_text()) +except Exception: raise SystemExit(1) +meta=item.get('metadata') or {}; data=item.get('data') +if item.get('apiVersion')!='v1' or item.get('kind')!='Secret' or item.get('type')!='kubernetes.io/basic-auth' or meta.get('namespace')!='keycloak' or meta.get('name')!='keycloak-initial-admin' or not meta.get('uid') or sorted(data or {})!=['password','username']: + raise SystemExit(1) +for key,destination in [('username',user_path),('password',password_path)]: + try: payload=base64.b64decode(data[key],validate=True) + except (binascii.Error,ValueError): raise SystemExit(1) + if not payload or b'\x00' in payload or b'\r' in payload or b'\n' in payload: raise SystemExit(1) + fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600) + try: os.write(fd,payload) + finally: os.close(fd) +PY + fail 'initial-admin Secret contract is invalid' + private_file_ok "$admin_username_file" && private_file_ok "$admin_password_file" || + fail 'private initial-admin credential snapshot is unsafe' + rm -f -- "$admin_file" +} + +allocate_loopback_port() { + local reservation_file="$temporary_dir/port-reservation" + python3 -I -S - "$reservation_file" <<'PY' || return 1 +import os,socket,sys +s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) +s.bind(('127.0.0.1',0)); port=s.getsockname()[1] +fd=os.open(sys.argv[1],os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600) +try: os.write(fd,str(port).encode()) +finally: os.close(fd); s.close() +PY + private_file_ok "$reservation_file" || return 1 + IFS= read -r local_port <"$reservation_file" + [[ "$local_port" =~ ^[0-9]+$ ]] || return 1 + (( local_port >= 1024 && local_port <= 65535 )) || return 1 + local_base="http://127.0.0.1:$local_port" +} + +loopback_port_open() { + python3 -I -S - "$local_port" <<'PY' +import socket,sys +port=int(sys.argv[1]) +with socket.create_connection(('127.0.0.1',port),timeout=0.2): pass +PY +} + +start_port_forward() { + local log="$temporary_dir/port-forward.log" ready=false attempt + allocate_loopback_port || fail 'cannot allocate a validated loopback port' + /usr/bin/timeout --signal=TERM --kill-after=1s "$PORT_FORWARD_PROCESS_TIMEOUT" \ + "$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" \ + port-forward --address=127.0.0.1 --namespace="$KEYCLOAK_NAMESPACE" \ + "service/$KEYCLOAK_SERVICE" "$local_port:8080" >"$log" 2>&1 & + port_forward_pid=$! + port_forward_start="$(process_start_time "$port_forward_pid")" || fail 'cannot pin port-forward process identity' + for ((attempt=0; attempt<60; attempt++)); do + kill -0 "$port_forward_pid" 2>/dev/null || fail 'Keycloak port-forward exited before readiness' + if grep -Fqx "Forwarding from 127.0.0.1:$local_port -> 8080" "$log" && + loopback_port_open >/dev/null 2>&1; then ready=true; break; fi + /usr/bin/sleep 0.05 + done + [[ "$ready" == true ]] || fail 'Keycloak port-forward did not become ready in time' + [[ "$(process_start_time "$port_forward_pid")" == "$port_forward_start" ]] || + fail 'Keycloak port-forward process identity changed' +} + +write_curl_config() { + local destination=$1 token_file=${2:-} + { + printf '%s\n' \ + 'silent' \ + 'show-error' \ + 'noproxy = "*"' \ + "connect-timeout = \"$CURL_CONNECT_TIMEOUT\"" \ + "max-time = \"$CURL_MAX_TIME\"" \ + "header = \"Host: $KEYCLOAK_HOST\"" \ + 'header = "X-Forwarded-Proto: https"' \ + 'header = "X-Forwarded-Port: 443"' + if [[ -n "$token_file" ]]; then + printf 'header = "Authorization: Bearer ' + tr -d '\r\n' <"$token_file" + printf '"\n' + fi + } >"$destination" + chmod 0600 -- "$destination" + private_file_ok "$destination" || fail 'curl configuration is unsafe' +} + +authenticate_keycloak() { + local token_response token_file public_config auth_config status + token_response="$temporary_dir/token-response.json" + token_file="$temporary_dir/access-token" + public_config="$temporary_dir/curl-public.conf" + auth_config="$temporary_dir/curl-auth.conf" + write_curl_config "$public_config" + status="$(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable \ + --config "$public_config" --request POST \ + --output "$token_response" --write-out '%{http_code}' \ + --data-urlencode 'grant_type=password' \ + --data-urlencode 'client_id=admin-cli' \ + --data-urlencode "username@$temporary_dir/admin-username" \ + --data-urlencode "password@$temporary_dir/admin-password" \ + "$local_base/realms/master/protocol/openid-connect/token" 2>/dev/null)" || + fail 'temporary Keycloak administrator authentication failed' + chmod 0600 -- "$token_response" + [[ "$status" == 200 ]] || fail 'temporary Keycloak administrator authentication returned an unexpected status' + jq -erj '.access_token | select(type == "string" and length >= 16)' "$token_response" >"$token_file" || + fail 'Keycloak token response is invalid' + chmod 0600 -- "$token_file" + private_file_ok "$token_file" || fail 'temporary Keycloak token file is unsafe' + write_curl_config "$auth_config" "$token_file" + rm -f -- "$token_response" "$temporary_dir/admin-username" "$temporary_dir/admin-password" +} + +admin_request() { + local method=$1 url=$2 body=${3:-} output status_file rc=0 + request_counter=$((request_counter + 1)) + output="$temporary_dir/admin-response-$request_counter.json" + status_file="$temporary_dir/admin-status-$request_counter" + : >"$output"; : >"$status_file"; chmod 0600 -- "$output" "$status_file" + local -a command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable + --config "$temporary_dir/curl-auth.conf" --request "$method" --output "$output" + --write-out '%{http_code}' "$url") + if [[ -n "$body" ]]; then + private_file_ok "$body" || fail 'request body is not a private snapshot' + command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable + --config "$temporary_dir/curl-auth.conf" --request "$method" --header 'Content-Type: application/json' + --data-binary "@$body" --output "$output" --write-out '%{http_code}' "$url") + fi + "${command[@]}" >"$status_file" 2>/dev/null || rc=$? + private_file_ok "$output" true && private_file_ok "$status_file" true || + fail 'Admin API response files are unsafe' + last_body=$output + last_status_file=$status_file + last_curl_rc=$rc + last_http_status='' + if (( rc == 0 )); then + IFS= read -r last_http_status <"$status_file" || true + [[ "$last_http_status" =~ ^[0-9]{3}$ ]] || { last_curl_rc=90; last_http_status=''; } + fi +} + +admin_get_query() { + local url=$1 query_name=$2 query_value=$3 max=$4 output status_file rc=0 + request_counter=$((request_counter + 1)) + output="$temporary_dir/admin-response-$request_counter.json" + status_file="$temporary_dir/admin-status-$request_counter" + : >"$output"; : >"$status_file"; chmod 0600 -- "$output" "$status_file" + /usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable \ + --config "$temporary_dir/curl-auth.conf" --get \ + --data-urlencode "$query_name=$query_value" --data-urlencode "max=$max" \ + --output "$output" --write-out '%{http_code}' "$url" >"$status_file" 2>/dev/null || rc=$? + private_file_ok "$output" true && private_file_ok "$status_file" true || + fail 'Admin API query response files are unsafe' + last_body=$output; last_status_file=$status_file; last_curl_rc=$rc; last_http_status='' + if (( rc == 0 )); then IFS= read -r last_http_status <"$status_file" || true; fi +} + +require_http() { + local expected=$1 operation=$2 + (( last_curl_rc == 0 )) || fail "$operation transport failed" + [[ "$last_http_status" == "$expected" ]] || fail "$operation returned HTTP ${last_http_status:-none}" +} + +build_desired_client() { + client_desired="$temporary_dir/client-desired.json" + jq -n --arg client_id "$CLIENT_ID" --arg root "$GRAFANA_URL" --arg redirect "$REDIRECT_URI" \ + --arg logout "$POST_LOGOUT_URI" '{ + clientId:$client_id, + name:"Grafana", + description:"Grafana confidential OIDC client managed by the platform bootstrap", + enabled:true, + protocol:"openid-connect", + clientAuthenticatorType:"client-secret", + publicClient:false, + standardFlowEnabled:true, + implicitFlowEnabled:false, + directAccessGrantsEnabled:false, + serviceAccountsEnabled:false, + authorizationServicesEnabled:false, + consentRequired:false, + fullScopeAllowed:false, + rootUrl:$root, + baseUrl:$root, + redirectUris:[$redirect], + webOrigins:[$root], + attributes:{ + "post.logout.redirect.uris":$logout, + "oauth2.device.authorization.grant.enabled":"false", + "oidc.ciba.grant.enabled":"false" + } + }' >"$client_desired" + chmod 0600 -- "$client_desired" +} + +build_desired_mapper() { + mapper_desired="$temporary_dir/mapper-desired.json" + jq -n --arg name "$MAPPER_NAME" '{ + name:$name, + protocol:"openid-connect", + protocolMapper:"oidc-group-membership-mapper", + consentRequired:false, + config:{ + "claim.name":"groups", + "full.path":"true", + "id.token.claim":"true", + "access.token.claim":"true", + "userinfo.token.claim":"true" + } + }' >"$mapper_desired" + chmod 0600 -- "$mapper_desired" +} + +lookup_client() { + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2 + require_http 200 'Grafana client lookup' + [[ "$(jq 'length' "$last_body")" == 0 || "$(jq 'length' "$last_body")" == 1 ]] || + fail 'Grafana client lookup is ambiguous' + if [[ "$(jq 'length' "$last_body")" == 0 ]]; then + client_state=absent; client_id=''; client_before='' + else + client_state=existing + client_id="$(jq -er '.[0].id | select(type == "string" and length > 0)' "$last_body")" || + fail 'existing Grafana client has no exact internal id' + client_before="$temporary_dir/client-before.json" + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" + require_http 200 'existing Grafana client snapshot' + jq -e --arg id "$client_id" --arg client_id "$CLIENT_ID" \ + '.id == $id and .clientId == $client_id' "$last_body" >/dev/null || fail 'existing client identity changed' + cp -- "$last_body" "$client_before"; chmod 0600 -- "$client_before" + fi +} + +lookup_group() { + local label=$1 name=$2 path=$3 state_var=$4 id_var=$5 before_var=$6 + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2 + require_http 200 "$label group lookup" + local count + count="$(jq 'length' "$last_body")" || + fail "$label group response is invalid" + [[ "$count" == 0 || "$count" == 1 ]] || fail "$label group lookup is ambiguous" + if [[ "$count" == 0 ]]; then + printf -v "$state_var" '%s' absent; printf -v "$id_var" '%s' ''; printf -v "$before_var" '%s' '' + else + local gid before + gid="$(jq -er --arg path "$path" '.[0] | select(.path == $path) | .id | select(type == "string" and length > 0)' "$last_body")" || + fail "$label group has no exact internal id" + before="$temporary_dir/${label}-group-before.json" + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid" + require_http 200 "$label group snapshot" + jq -e --arg id "$gid" --arg name "$name" --arg path "$path" \ + '.id == $id and .name == $name and .path == $path' "$last_body" >/dev/null || + fail "$label group identity changed" + cp -- "$last_body" "$before"; chmod 0600 -- "$before" + printf -v "$state_var" '%s' existing; printf -v "$id_var" '%s' "$gid"; printf -v "$before_var" '%s' "$before" + fi +} + +lookup_mapper() { + [[ "$client_state" == existing ]] || { mapper_state=absent; mapper_id=''; mapper_before=''; return; } + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" + require_http 200 'Grafana mapper lookup' + local count_by_name count_by_claim + count_by_name="$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)] | length' "$last_body")" || + fail 'mapper response is invalid' + count_by_claim="$(jq '[.[] | select(.config["claim.name"] == "groups")] | length' "$last_body")" || + fail 'mapper response is invalid' + [[ "$count_by_name" == 0 || "$count_by_name" == 1 ]] || fail 'Grafana mapper name is ambiguous' + [[ "$count_by_claim" == 0 || "$count_by_claim" == 1 ]] || fail 'Grafana groups claim mapper is ambiguous' + if [[ "$count_by_name" == 1 ]]; then + mapper_state=existing + mapper_id="$(jq -er --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)][0].id | select(type == "string" and length > 0)' "$last_body")" || + fail 'existing Grafana mapper has no exact id' + [[ "$count_by_claim" == 0 || + "$(jq -r --arg id "$mapper_id" '[.[] | select(.config["claim.name"] == "groups")][0].id // empty' "$last_body")" == "$mapper_id" ]] || + fail 'a different mapper already owns the groups claim' + mapper_before="$temporary_dir/mapper-before.json" + jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_before" || + fail 'cannot snapshot exact existing mapper' + chmod 0600 -- "$mapper_before" + else + [[ "$count_by_claim" == 0 ]] || fail 'groups claim is owned by a differently named mapper' + mapper_state=absent; mapper_id=''; mapper_before='' + fi +} + +lookup_user() { + local label=$1 username=$2 id_var=$3 + [[ -n "$username" ]] || return 0 + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/users?exact=true" username "$username" 2 + require_http 200 "$label user lookup" + [[ "$(jq 'length' "$last_body")" == 1 ]] || fail "$label username lookup must return exactly one user" + local uid + uid="$(jq -er --arg username "$username" '.[0] | select(.username == $username) | .id | select(type == "string" and length > 0)' "$last_body")" || + fail "$label user identity is invalid" + printf -v "$id_var" '%s' "$uid" +} + +snapshot_membership() { + local label=$1 user_id=$2 group_id=$3 state_var=$4 + [[ -n "$user_id" ]] || return 0 + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups" briefRepresentation false 101 + require_http 200 "$label membership lookup" + [[ "$(jq 'length' "$last_body")" -le 100 ]] || fail "$label membership list exceeds the bounded cardinality" + if jq -e --arg id "$group_id" 'any(.[]; .id == $id)' "$last_body" >/dev/null; then + printf -v "$state_var" '%s' true + else + printf -v "$state_var" '%s' false + fi +} + +retrieve_client_secret() { + client_secret_file="$temporary_dir/client-secret" + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret" + require_http 200 'Grafana client-secret retrieval' + jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$client_secret_file" || + fail 'Grafana client secret response is invalid' + chmod 0600 -- "$client_secret_file" + private_file_ok "$client_secret_file" || fail 'Grafana client secret file is unsafe' +} + +start_kubernetes_proxy() { + local log="$temporary_dir/kubernetes-proxy.log" attempt + kube_socket="$temporary_dir/kube-api.sock" + /usr/bin/timeout --signal=TERM --kill-after=1s "$PROXY_PROCESS_TIMEOUT" \ + "$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" proxy \ + --unix-socket="$kube_socket" --api-prefix=/ \ + --accept-paths="^/api/v1/namespaces/${OBSERVABILITY_NAMESPACE}/secrets(/${OIDC_SECRET})?$" \ + >"$log" 2>&1 & + proxy_pid=$! + proxy_start="$(process_start_time "$proxy_pid")" || fail 'cannot pin Kubernetes proxy identity' + for ((attempt=0; attempt<60; attempt++)); do + [[ -S "$kube_socket" ]] && break + kill -0 "$proxy_pid" 2>/dev/null || fail 'bounded Kubernetes proxy exited' + /usr/bin/sleep 0.05 + done + [[ -S "$kube_socket" && "$(process_start_time "$proxy_pid")" == "$proxy_start" ]] || + fail 'bounded Kubernetes proxy did not become ready safely' +} + +kube_request() { + local method=$1 path=$2 body=${3:-} output status_file rc=0 + request_counter=$((request_counter + 1)) + output="$temporary_dir/kube-response-$request_counter.json" + status_file="$temporary_dir/kube-status-$request_counter" + : >"$output"; : >"$status_file"; chmod 0600 -- "$output" "$status_file" + local -a command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable + --silent --show-error --noproxy '*' --connect-timeout "$CURL_CONNECT_TIMEOUT" --max-time "$CURL_MAX_TIME" + --unix-socket "$kube_socket" --request "$method" --output "$output" --write-out '%{http_code}' + "http://localhost$path") + if [[ -n "$body" ]]; then + private_file_ok "$body" || fail 'Kubernetes request body is unsafe' + command=(/usr/bin/timeout --signal=TERM --kill-after=1s "$CURL_PROCESS_TIMEOUT" /usr/bin/curl --disable + --silent --show-error --noproxy '*' --connect-timeout "$CURL_CONNECT_TIMEOUT" --max-time "$CURL_MAX_TIME" + --unix-socket "$kube_socket" --request "$method" --header 'Content-Type: application/json' + --data-binary "@$body" --output "$output" --write-out '%{http_code}' "http://localhost$path") + fi + "${command[@]}" >"$status_file" 2>/dev/null || rc=$? + private_file_ok "$output" true && private_file_ok "$status_file" true || + fail 'Kubernetes API response files are unsafe' + last_body=$output; last_status_file=$status_file; last_curl_rc=$rc; last_http_status='' + if (( rc == 0 )); then IFS= read -r last_http_status <"$status_file" || true; fi +} + +validate_secret_json() { + local source=$1 prefix=$2 + python3 -I -S - "$source" "$prefix" <<'PY' +import base64,binascii,json,os,pathlib,re,sys +source,prefix=sys.argv[1:] +try: item=json.loads(pathlib.Path(source).read_text()) +except Exception: raise SystemExit(1) +meta=item.get('metadata') or {}; data=item.get('data') +if item.get('apiVersion')!='v1' or item.get('kind')!='Secret' or item.get('type')!='Opaque' or meta.get('namespace')!='observability' or meta.get('name')!='grafana-keycloak-oidc' or sorted(data or {})!=['client-id','client-secret']: + raise SystemExit(1) +uid=meta.get('uid'); rv=meta.get('resourceVersion') +if not isinstance(uid,str) or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*',uid) or not isinstance(rv,str) or not re.fullmatch(r'[0-9]+',rv): raise SystemExit(1) +for key in ('client-id','client-secret'): + try: payload=base64.b64decode(data[key],validate=True) + except (binascii.Error,ValueError): raise SystemExit(1) + if (key=='client-id' and payload!=b'grafana') or (key=='client-secret' and not payload): raise SystemExit(1) + fd=os.open(prefix+'.'+key,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600) + try: os.write(fd,payload) + finally: os.close(fd) +print(uid+'\t'+rv,end='') +PY +} + +snapshot_oidc_secret() { + local prefix values + kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" + (( last_curl_rc == 0 )) || fail 'Grafana OIDC Secret lookup transport failed' + case "$last_http_status" in + 200) + secret_state=existing; secret_before="$temporary_dir/secret-before.json" + cp -- "$last_body" "$secret_before"; chmod 0600 -- "$secret_before" + prefix="$temporary_dir/secret-before" + values="$(validate_secret_json "$secret_before" "$prefix")" || fail 'existing Grafana OIDC Secret contract is invalid' + IFS=$'\t' read -r secret_uid secret_resource_version <<<"$values" + [[ "$client_state" == existing && -n "$client_secret_file" ]] || + fail 'Grafana OIDC Secret exists without one exact existing Keycloak client' + [[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] || fail 'existing Grafana OIDC Secret client-id differs' + cmp --silent -- "$prefix.client-secret" "$client_secret_file" || + fail 'existing Grafana OIDC Secret does not match the exact existing Keycloak credential; rotation refused' + ;; + 404) secret_state=absent; secret_before=''; secret_uid=''; secret_resource_version='' ;; + *) fail "Grafana OIDC Secret lookup returned HTTP ${last_http_status:-none}" ;; + esac +} + +sudo_refresh() { + "$SUDO_BIN" -v || fail 'sudo credential refresh failed' + "$SUDO_BIN" -n /usr/bin/true || fail 'narrow non-interactive sudo is unavailable' +} + +validate_evidence_stream() { + local file=$1 timestamp_var=$2 parsed + local -a lines=() + mapfile -t lines <"$file" + [[ "$(wc -l <"$file" | tr -d '[:space:]')" == 4 && ${#lines[@]} -eq 4 ]] || return 1 + [[ "${lines[0]}" == "schema=$EVIDENCE_SCHEMA" ]] || return 1 + [[ "${lines[1]}" == "context=$EXPECTED_CONTEXT" ]] || return 1 + [[ "${lines[2]}" == "resource=$EVIDENCE_RESOURCE" ]] || return 1 + [[ "${lines[3]}" =~ ^checked_at_utc=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)$ ]] || return 1 + parsed="${BASH_REMATCH[1]}" + timestamp_epoch "$parsed" >/dev/null || return 1 + printf -v "$timestamp_var" '%s' "$parsed" +} + +root_marker_helper() { + local output=$1 + shift + "$SUDO_BIN" -n /usr/bin/python3 -I -S - platform-grafana-oidc-marker-helper "$@" \ + >"$output" 2>/dev/null <<'PY' +import errno,os,pathlib,stat,sys +tag,action,directory,basename,uid_raw,gid_raw,dir_mode_raw,source,expected_kind,expected_source,source_uid_raw=sys.argv[1:] +if tag!='platform-grafana-oidc-marker-helper' or action not in {'check-parent','check-dir','ensure-dir','read','install','unlink-exact'} or basename!='keycloak.env': raise SystemExit(2) +if not directory.startswith('/') or '\x00' in directory: raise SystemExit(2) +uid=int(uid_raw); gid=int(gid_raw); directory_mode=int(dir_mode_raw,8); source_uid=int(source_uid_raw) +flags=os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW|os.O_CLOEXEC +def open_dir(path,mode): + fd=os.open('/',flags) + try: + for part in pathlib.PurePosixPath(path).parts[1:]: + if part in {'','.','..'}: raise OSError(errno.EINVAL,'component') + nxt=os.open(part,flags,dir_fd=fd); os.close(fd); fd=nxt + st=os.fstat(fd) + if not stat.S_ISDIR(st.st_mode) or stat.S_IMODE(st.st_mode)!=mode or st.st_uid!=uid or st.st_gid!=gid: raise OSError(errno.EPERM,'directory metadata') + return fd + except BaseException: + os.close(fd); raise +def read_file(directory_fd,name): + fd=os.open(name,os.O_RDONLY|os.O_NOFOLLOW|os.O_CLOEXEC,dir_fd=directory_fd) + try: + before=os.fstat(fd) + if not stat.S_ISREG(before.st_mode) or stat.S_IMODE(before.st_mode)!=0o600 or before.st_uid!=uid or before.st_gid!=gid or before.st_nlink!=1: raise OSError(errno.EPERM,'file metadata') + chunks=[] + while True: + chunk=os.read(fd,65536) + if not chunk: break + chunks.append(chunk) + after=os.fstat(fd) + a=(before.st_dev,before.st_ino,before.st_size,before.st_mtime_ns,before.st_ctime_ns,before.st_nlink) + b=(after.st_dev,after.st_ino,after.st_size,after.st_mtime_ns,after.st_ctime_ns,after.st_nlink) + if a!=b: raise OSError(errno.ESTALE,'changed') + payload=b''.join(chunks) + if len(payload)!=before.st_size: raise OSError(errno.EIO,'short read') + return payload + finally: os.close(fd) +def read_source(path): + fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW|os.O_CLOEXEC) + try: + before=os.fstat(fd) + if not stat.S_ISREG(before.st_mode) or stat.S_IMODE(before.st_mode)!=0o600 or before.st_uid!=source_uid or before.st_nlink!=1: raise OSError(errno.EPERM,'source metadata') + chunks=[] + while True: + chunk=os.read(fd,65536) + if not chunk: break + chunks.append(chunk) + after=os.fstat(fd) + if (before.st_dev,before.st_ino,before.st_size,before.st_mtime_ns,before.st_ctime_ns,before.st_nlink)!=(after.st_dev,after.st_ino,after.st_size,after.st_mtime_ns,after.st_ctime_ns,after.st_nlink): raise OSError(errno.ESTALE,'source changed') + payload=b''.join(chunks) + if len(payload)!=before.st_size: raise OSError(errno.EIO,'short source') + return payload + finally: os.close(fd) +if action in {'check-parent','check-dir','ensure-dir'}: + parent,leaf=os.path.split(directory.rstrip('/')) + if leaf!='recovery-evidence' or not parent: raise OSError(errno.EINVAL,'directory') + parent_fd=open_dir(parent,directory_mode) + if action=='check-parent': + os.close(parent_fd); raise SystemExit(0) + try: + try: directory_fd=os.open(leaf,flags,dir_fd=parent_fd) + except FileNotFoundError: + if action=='check-dir': raise SystemExit(4) + os.mkdir(leaf,directory_mode,dir_fd=parent_fd); os.fsync(parent_fd); directory_fd=os.open(leaf,flags,dir_fd=parent_fd) + try: + st=os.fstat(directory_fd) + if not stat.S_ISDIR(st.st_mode) or stat.S_IMODE(st.st_mode)!=directory_mode or st.st_uid!=uid or st.st_gid!=gid: raise OSError(errno.EPERM,'evidence directory') + finally: os.close(directory_fd) + finally: os.close(parent_fd) + raise SystemExit(0) +try: directory_fd=open_dir(directory,directory_mode) +except FileNotFoundError: + if action=='read': raise SystemExit(4) + raise +try: + if action=='read': + try: payload=read_file(directory_fd,basename) + except FileNotFoundError: raise SystemExit(4) + os.write(1,payload); raise SystemExit(0) + desired=read_source(source) + if expected_kind=='absent': + try: read_file(directory_fd,basename) + except FileNotFoundError: pass + else: raise OSError(errno.EEXIST,'expected absent') + elif expected_kind=='file': + if read_file(directory_fd,basename)!=read_source(expected_source): raise OSError(errno.ESTALE,'precondition') + else: raise OSError(errno.EINVAL,'kind') + if action=='unlink-exact': + if read_file(directory_fd,basename)!=desired: raise OSError(errno.ESTALE,'unlink precondition') + os.unlink(basename,dir_fd=directory_fd); os.fsync(directory_fd) + try: os.stat(basename,dir_fd=directory_fd,follow_symlinks=False) + except FileNotFoundError: raise SystemExit(0) + raise OSError(errno.EEXIST,'remained') + temporary=f'.{basename}.new.{os.getpid():x}' + fd=os.open(temporary,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW|os.O_CLOEXEC,0o600,dir_fd=directory_fd); exists=True + try: + os.fchown(fd,uid,gid); os.fchmod(fd,0o600); view=memoryview(desired) + while view: + n=os.write(fd,view) + if n<=0: raise OSError(errno.EIO,'short write') + view=view[n:] + os.fsync(fd) + if os.fstat(fd).st_nlink!=1: raise OSError(errno.EPERM,'links') + finally: os.close(fd) + try: + os.replace(temporary,basename,src_dir_fd=directory_fd,dst_dir_fd=directory_fd); exists=False; os.fsync(directory_fd) + if read_file(directory_fd,basename)!=desired: raise OSError(errno.EIO,'verify') + finally: + if exists: + try: os.unlink(temporary,dir_fd=directory_fd) + except FileNotFoundError: pass +finally: os.close(directory_fd) +PY +} + +root_marker_read() { + local destination=$1 rc + if root_marker_helper "$destination" read "$EVIDENCE_DIR" keycloak.env \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" '' '' '' "$EUID"; then + chmod 0600 -- "$destination"; return 0 + else rc=$?; rm -f -- "$destination"; return "$rc"; fi +} + +prepare_marker_snapshot() { + local rc timestamp + marker_prior="$temporary_dir/marker-prior.env" + marker_desired="$temporary_dir/marker-desired.env" + root_marker_helper /dev/null check-parent "$EVIDENCE_DIR" keycloak.env \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" '' '' '' "$EUID" || + fail 'recovery evidence parent boundary is missing or unsafe' + if root_marker_read "$marker_prior"; then + validate_evidence_stream "$marker_prior" timestamp || fail 'existing Keycloak recovery marker is malformed' + marker_had_prior=true + else + rc=$?; [[ "$rc" == 4 ]] || fail 'cannot safely inspect Keycloak recovery marker' + marker_had_prior=false + fi + printf '%s\n' "schema=$EVIDENCE_SCHEMA" "context=$EXPECTED_CONTEXT" \ + "resource=$EVIDENCE_RESOURCE" "checked_at_utc=$(now_utc)" >"$marker_desired" + chmod 0600 -- "$marker_desired" + validate_evidence_stream "$marker_desired" timestamp || fail 'generated recovery marker is invalid' +} + +ensure_evidence_directory() { + root_marker_helper /dev/null ensure-dir "$EVIDENCE_DIR" keycloak.env \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" '' '' '' "$EUID" || + fail 'cannot safely ensure recovery evidence directory' +} + +install_marker() { + pending_kind=marker + if [[ "$marker_had_prior" == true ]]; then + root_marker_helper /dev/null install "$EVIDENCE_DIR" keycloak.env \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_desired" file "$marker_prior" "$EUID" || + fail 'cannot atomically replace Keycloak recovery marker' + else + root_marker_helper /dev/null install "$EVIDENCE_DIR" keycloak.env \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_desired" absent '' "$EUID" || + fail 'cannot atomically install Keycloak recovery marker' + fi + local current="$temporary_dir/marker-installed.env" + root_marker_read "$current" && cmp --silent -- "$marker_desired" "$current" || + fail 'installed Keycloak recovery marker cannot be verified' + marker_written=true + pending_kind='' +} + +restore_marker() { + local current="$temporary_dir/marker-rollback-current.env" + [[ "$marker_written" == true ]] || return 0 + root_marker_read "$current" && cmp --silent -- "$marker_desired" "$current" || return 1 + if [[ "$marker_had_prior" == true ]]; then + root_marker_helper /dev/null install "$EVIDENCE_DIR" keycloak.env \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_prior" file "$marker_desired" "$EUID" + else + root_marker_helper /dev/null unlink-exact "$EVIDENCE_DIR" keycloak.env \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$EVIDENCE_DIR_MODE" "$marker_desired" file "$marker_desired" "$EUID" + fi +} + +check_recovery_evidence() { + local marker timestamp marker_epoch current_epoch age + validate_context_and_authority + sudo_refresh + marker="$temporary_dir/check-keycloak.env" + root_marker_read "$marker" || fail 'Keycloak recovery evidence is missing or unsafe' + validate_evidence_stream "$marker" timestamp || fail 'Keycloak recovery evidence is malformed' + marker_epoch="$(timestamp_epoch "$timestamp")" || fail 'recovery evidence timestamp is invalid' + current_epoch="$(now_epoch)" || fail 'current UTC clock is invalid' + age=$((current_epoch - marker_epoch)) + (( age >= 0 && age <= EVIDENCE_MAX_AGE_SECONDS )) || fail 'Keycloak recovery evidence is outside the 30-day window' + printf 'KEYCLOAK_RECOVERY_EVIDENCE=PASS\n' +} + +build_group_body() { + local name=$1 destination=$2 token=${3:-} + if [[ -n "$token" ]]; then + jq -n --arg name "$name" --arg token "$token" \ + '{name:$name,attributes:{"platform.observability.transaction":[$token]}}' >"$destination" + else + jq -n --arg name "$name" '{name:$name}' >"$destination" + fi + chmod 0600 -- "$destination" +} + +client_body_with_secret() { + local source=$1 destination=$2 + private_file_ok "$source" && private_file_ok "$client_secret_file" || return 1 + jq --rawfile secret "$client_secret_file" '. + {secret:$secret}' "$source" >"$destination" || return 1 + chmod 0600 -- "$destination" + private_file_ok "$destination" +} + +client_contract_matches() { + local source=$1 expected_id=$2 + jq -e --arg id "$expected_id" --arg root "$GRAFANA_URL" --arg redirect "$REDIRECT_URI" \ + --arg logout "$POST_LOGOUT_URI" ' + .id == $id and .clientId == "grafana" and .name == "Grafana" and + .description == "Grafana confidential OIDC client managed by the platform bootstrap" and + .enabled == true and .protocol == "openid-connect" and + .clientAuthenticatorType == "client-secret" and .publicClient == false and + .standardFlowEnabled == true and .implicitFlowEnabled == false and + .directAccessGrantsEnabled == false and .serviceAccountsEnabled == false and + .authorizationServicesEnabled == false and .consentRequired == false and + .fullScopeAllowed == false and .rootUrl == $root and .baseUrl == $root and + .redirectUris == [$redirect] and .webOrigins == [$root] and + .attributes["post.logout.redirect.uris"] == $logout and + .attributes["oauth2.device.authorization.grant.enabled"] == "false" and + .attributes["oidc.ciba.grant.enabled"] == "false" and + (.attributes["platform.observability.transaction"] // null) == null + ' "$source" >/dev/null +} + +mapper_contract_matches() { + local source=$1 expected_id=$2 + jq -e --arg id "$expected_id" --arg name "$MAPPER_NAME" ' + .id == $id and .name == $name and .protocol == "openid-connect" and + .protocolMapper == "oidc-group-membership-mapper" and .consentRequired == false and + .config == {"claim.name":"groups","full.path":"true","id.token.claim":"true", + "access.token.claim":"true","userinfo.token.claim":"true"} + ' "$source" >/dev/null +} + +capture_client_after() { + client_after="$temporary_dir/client-after.json" + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" + require_http 200 'post-mutation Grafana client snapshot' + client_contract_matches "$last_body" "$client_id" || fail 'post-mutation Grafana client contract differs' + cp -- "$last_body" "$client_after"; chmod 0600 -- "$client_after" + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret" + require_http 200 'post-mutation Grafana credential verification' + local verification="$temporary_dir/client-secret-verification" + jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$verification" || + fail 'post-mutation Grafana credential response is invalid' + chmod 0600 -- "$verification" + cmp --silent -- "$client_secret_file" "$verification" || + fail 'Grafana client PUT changed the existing credential unexpectedly' +} + +classify_client_create() { + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2 + (( last_curl_rc == 0 )) || return 2 + [[ "$last_http_status" == 200 ]] || return 2 + local count + count="$(jq 'length' "$last_body" 2>/dev/null)" || return 2 + if [[ "$count" == 0 ]]; then return 1; fi + [[ "$count" == 1 ]] || return 2 + local owned_id + owned_id="$(jq -er --arg token "$transaction_token" '.[0] | + select(.attributes["platform.observability.transaction"] == $token) | + .id | select(type == "string" and length > 0)' "$last_body" 2>/dev/null)" || return 2 + client_id=$owned_id + client_mutation=created + return 0 +} + +create_client() { + local create_body="$temporary_dir/client-create.json" status + jq --arg token "$transaction_token" \ + '.attributes["platform.observability.transaction"]=$token' "$client_desired" >"$create_body" + chmod 0600 -- "$create_body" + pending_kind=client-create; pending_desired=$create_body; pending_token=$transaction_token + admin_request POST "$local_base/admin/realms/$KEYCLOAK_REALM/clients" "$create_body" + status=$last_http_status + if (( last_curl_rc != 0 )) || [[ "$status" != 201 ]]; then + if classify_client_create; then + : + elif [[ "$?" != 1 ]]; then + emit_manual_recovery + fi + fail "Grafana client create was not cleanly confirmed" + fi + classify_client_create || { emit_manual_recovery; fail 'Grafana client create response could not be owned exactly'; } + client_after="$temporary_dir/client-created-tagged.json" + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" + require_http 200 'created Grafana client ownership snapshot' + cp -- "$last_body" "$client_after"; chmod 0600 -- "$client_after" + retrieve_client_secret + local update_body="$temporary_dir/client-created-final.json" + client_body_with_secret "$client_desired" "$update_body" || fail 'cannot build credential-preserving client finalization' + pending_kind=client-created-finalize; pending_before=$client_after; pending_desired=$client_desired; pending_id=$client_id + admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" "$update_body" + if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then + fail 'created Grafana client finalization was not cleanly confirmed' + fi + capture_client_after + pending_kind='' +} + +update_client() { + local update_body="$temporary_dir/client-update.json" + client_body_with_secret "$client_desired" "$update_body" || fail 'cannot build credential-preserving client update' + client_restore="$temporary_dir/client-restore.json" + client_body_with_secret "$client_before" "$client_restore" || fail 'cannot build exact credential-preserving client rollback snapshot' + pending_kind=client-update; pending_before=$client_before; pending_desired=$client_desired; pending_id=$client_id + admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" "$update_body" + if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then + fail 'Grafana client update was not cleanly confirmed' + fi + client_mutation=updated; pending_kind='' + capture_client_after +} + +classify_group_create() { + local name=$1 path=$2 id_var=$3 mutation_var=$4 + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2 + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 + local count gid + count="$(jq 'length' "$last_body" 2>/dev/null)" || return 2 + if [[ "$count" == 0 ]]; then return 1; fi + [[ "$count" == 1 ]] || return 2 + gid="$(jq -er --arg path "$path" --arg token "$transaction_token" ' + .[0] | select(.path == $path and .attributes["platform.observability.transaction"] == [$token]) | .id | + select(type == "string" and length > 0)' "$last_body" 2>/dev/null)" || return 2 + printf -v "$id_var" '%s' "$gid"; printf -v "$mutation_var" '%s' created + return 0 +} + +create_group() { + local label=$1 name=$2 path=$3 id_var=$4 mutation_var=$5 after_var=$6 + local create_body="$temporary_dir/${label}-group-create.json" final_body="$temporary_dir/${label}-group-final.json" + build_group_body "$name" "$create_body" "$transaction_token" + pending_kind="${label}-group-create"; pending_desired=$create_body; pending_token=$transaction_token + admin_request POST "$local_base/admin/realms/$KEYCLOAK_REALM/groups" "$create_body" + if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 201 ]]; then + if classify_group_create "$name" "$path" "$id_var" "$mutation_var"; then : + elif [[ "$?" != 1 ]]; then emit_manual_recovery; fi + fail "$label group create was not cleanly confirmed" + fi + classify_group_create "$name" "$path" "$id_var" "$mutation_var" || + { emit_manual_recovery; fail "$label group create response could not be owned exactly"; } + local gid=${!id_var} + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid" + require_http 200 "$label created group ownership snapshot" + local after="$temporary_dir/${label}-group-after.json" + cp -- "$last_body" "$after"; chmod 0600 -- "$after"; printf -v "$after_var" '%s' "$after" + pending_kind="${label}-group-finalize"; pending_before=$after; pending_id=$gid + build_group_body "$name" "$final_body" + pending_desired=$final_body + admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid" "$final_body" + if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then + if classify_pending; then + fail "$label group finalization response was lost after an exact state transition" + else + [[ "$?" != 2 ]] || emit_manual_recovery + fail "$label group finalization was not cleanly confirmed" + fi + fi + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$gid" + require_http 200 "$label group post-mutation snapshot" + jq -e --arg id "$gid" --arg name "$name" --arg path "$path" ' + .id == $id and .name == $name and .path == $path and + ((.attributes // {}) | has("platform.observability.transaction") | not) + ' "$last_body" >/dev/null || fail "$label group final contract differs" + cp -- "$last_body" "$after"; chmod 0600 -- "$after"; printf -v "$after_var" '%s' "$after" + pending_kind='' +} + +create_mapper() { + local create_body="$temporary_dir/mapper-create.json" + jq --arg token "$transaction_token" '.config["platform.observability.transaction"]=$token' \ + "$mapper_desired" >"$create_body"; chmod 0600 -- "$create_body" + pending_kind=mapper-create; pending_desired=$create_body; pending_token=$transaction_token + admin_request POST "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" "$create_body" + if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 201 ]]; then + fail 'Grafana groups mapper create was not cleanly confirmed' + fi + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" + require_http 200 'post-create Grafana mapper lookup' + [[ "$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)] | length' "$last_body")" == 1 ]] || + { emit_manual_recovery; fail 'created Grafana mapper is ambiguous'; } + mapper_id="$(jq -er --arg name "$MAPPER_NAME" --arg token "$transaction_token" ' + [.[] | select(.name == $name and .config["platform.observability.transaction"] == $token)][0].id | + select(type == "string" and length > 0)' "$last_body")" || + { emit_manual_recovery; fail 'created Grafana mapper ownership is not exact'; } + mapper_mutation=created + mapper_after="$temporary_dir/mapper-created-tagged.json" + jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_after" || + fail 'cannot snapshot created Grafana mapper ownership' + chmod 0600 -- "$mapper_after" + pending_kind=mapper-created-finalize; pending_before=$mapper_after; pending_desired=$mapper_desired; pending_id=$mapper_id + admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id" "$mapper_desired" + if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then + fail 'created Grafana mapper finalization was not cleanly confirmed' + fi + capture_mapper_after + pending_kind='' +} + +capture_mapper_after() { + mapper_after="$temporary_dir/mapper-after.json" + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" + require_http 200 'post-mutation Grafana mapper snapshot' + jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_after" || + fail 'cannot capture post-mutation Grafana mapper' + chmod 0600 -- "$mapper_after" + mapper_contract_matches "$mapper_after" "$mapper_id" || fail 'post-mutation Grafana mapper contract differs' +} + +update_mapper() { + pending_kind=mapper-update; pending_before=$mapper_before; pending_desired=$mapper_desired; pending_id=$mapper_id + admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id" "$mapper_desired" + if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 204 ]]; then fail 'Grafana mapper update was not cleanly confirmed'; fi + mapper_mutation=updated; pending_kind=''; capture_mapper_after +} + +build_secret_manifest() { + local destination=$1 + python3 -I -S - "$client_secret_file" "$transaction_token" "$destination" <<'PY' || return 1 +import base64,json,os,pathlib,re,sys +source,token,destination=sys.argv[1:] +if re.fullmatch(r'[0-9a-f]{32}',token) is None: raise SystemExit(1) +payload=pathlib.Path(source).read_bytes() +if not payload: raise SystemExit(1) +item={'apiVersion':'v1','kind':'Secret','type':'Opaque','metadata':{'namespace':'observability','name':'grafana-keycloak-oidc','annotations':{'observability.hyeonworks.com/create-transaction':token}},'data':{'client-id':base64.b64encode(b'grafana').decode(),'client-secret':base64.b64encode(payload).decode()}} +fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_CLOEXEC,0o600) +try: os.write(fd,(json.dumps(item,sort_keys=True,separators=(',',':'))+'\n').encode()) +finally: os.close(fd) +PY +} + +classify_secret_create() { + kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" + (( last_curl_rc == 0 )) || return 2 + if [[ "$last_http_status" == 404 ]]; then return 1; fi + [[ "$last_http_status" == 200 ]] || return 2 + local prefix="$temporary_dir/secret-classify" values + values="$(validate_secret_json "$last_body" "$prefix")" || return 2 + jq -e --arg token "$transaction_token" '.metadata.annotations["observability.hyeonworks.com/create-transaction"] == $token' "$last_body" >/dev/null || return 2 + [[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] || return 2 + cmp --silent -- "$prefix.client-secret" "$client_secret_file" || return 2 + IFS=$'\t' read -r secret_created_uid secret_created_resource_version <<<"$values" + secret_mutation=created + return 0 +} + +create_oidc_secret() { + local manifest="$temporary_dir/secret-create.json" + build_secret_manifest "$manifest" || fail 'cannot build private Grafana OIDC Secret manifest' + pending_kind=secret-create; pending_desired=$manifest; pending_token=$transaction_token + kube_request POST "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets" "$manifest" + if (( last_curl_rc != 0 )) || [[ "$last_http_status" != 201 ]]; then + if classify_secret_create; then :; elif [[ "$?" == 2 ]]; then emit_manual_recovery; fi + fail 'Grafana OIDC Secret create was not cleanly confirmed' + fi + classify_secret_create || { emit_manual_recovery; fail 'Grafana OIDC Secret create response could not be owned exactly'; } + pending_kind='' +} + +membership_present() { + local user_id=$1 group_id=$2 + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups" briefRepresentation false 101 + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 + [[ "$(jq 'length' "$last_body")" -le 100 ]] || return 2 + jq -e --arg id "$group_id" 'any(.[]; .id == $id)' "$last_body" >/dev/null +} + +add_membership() { + local label=$1 user_id=$2 group_id=$3 before=$4 added_var=$5 + [[ -n "$user_id" ]] || return 0 + [[ "$before" == false ]] || return 0 + pending_kind="${label}-membership"; pending_user_id=$user_id; pending_group_id=$group_id + admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups/$group_id" + if (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]]; then + membership_present "$user_id" "$group_id" || fail "$label membership was not observable after PUT" + printf -v "$added_var" '%s' true; pending_kind=''; return 0 + fi + if membership_present "$user_id" "$group_id"; then + printf -v "$added_var" '%s' true + else + [[ "$?" != 2 ]] || emit_manual_recovery + fi + fail "$label membership PUT was not cleanly confirmed" +} + +classify_mapper_create() { + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 + local count + count="$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name)] | length' "$last_body" 2>/dev/null)" || return 2 + if [[ "$count" == 0 ]]; then return 1; fi + [[ "$count" == 1 ]] || return 2 + mapper_id="$(jq -er --arg name "$MAPPER_NAME" --arg token "$transaction_token" ' + [.[] | select(.name == $name and .config["platform.observability.transaction"] == $token)][0].id | + select(type == "string" and length > 0)' "$last_body" 2>/dev/null)" || return 2 + mapper_after="$temporary_dir/mapper-after-pending.json" + jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$mapper_after" || return 2 + chmod 0600 -- "$mapper_after"; mapper_mutation=created; return 0 +} + +capture_pending_group_after() { + local label=$1 id=$2 after_var=$3 + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 + local after="$temporary_dir/${label}-group-after-pending.json" + cp -- "$last_body" "$after"; chmod 0600 -- "$after"; printf -v "$after_var" '%s' "$after" +} + +classify_pending() { + local result current target + case "$pending_kind" in + '') return 0 ;; + client-create) + if classify_client_create; then + client_after="$temporary_dir/client-after-pending.json" + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" + if (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]]; then + cp -- "$last_body" "$client_after"; chmod 0600 -- "$client_after"; pending_kind=''; return 0 + fi + return 2 + else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi + ;; + client-created-finalize) + current="$temporary_dir/client-created-finalize-current.json" + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 + cp -- "$last_body" "$current"; chmod 0600 -- "$current" + if client_contract_matches "$current" "$client_id" || + jq -e --arg id "$client_id" --arg token "$transaction_token" ' + .id == $id and .clientId == "grafana" and + .attributes["platform.observability.transaction"] == $token + ' "$current" >/dev/null; then + client_after=$current; client_mutation=created; pending_kind=''; return 0 + fi + return 2 + ;; + admin-group-create) + if classify_group_create "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" admin_group_id admin_group_mutation; then + capture_pending_group_after admin "$admin_group_id" admin_group_after || return 2 + pending_kind=''; return 0 + else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi + ;; + viewer-group-create) + if classify_group_create "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" viewer_group_id viewer_group_mutation; then + capture_pending_group_after viewer "$viewer_group_id" viewer_group_after || return 2 + pending_kind=''; return 0 + else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi + ;; + admin-group-finalize|viewer-group-finalize) + if [[ "$pending_kind" == admin-group-finalize ]]; then + target=admin; pending_id=$admin_group_id + else + target=viewer; pending_id=$viewer_group_id + fi + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$pending_id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 + current="$temporary_dir/${target}-group-finalize-current.json" + cp -- "$last_body" "$current"; chmod 0600 -- "$current" + if [[ "$target" == admin ]]; then + jq -e --arg id "$admin_group_id" --arg name "$ADMIN_GROUP_NAME" --arg path "$ADMIN_GROUP_PATH" --arg token "$transaction_token" ' + .id == $id and .name == $name and .path == $path and + ((.attributes // {}) == {} or .attributes["platform.observability.transaction"] == [$token]) + ' "$current" >/dev/null || return 2 + admin_group_after=$current + else + jq -e --arg id "$viewer_group_id" --arg name "$VIEWER_GROUP_NAME" --arg path "$VIEWER_GROUP_PATH" --arg token "$transaction_token" ' + .id == $id and .name == $name and .path == $path and + ((.attributes // {}) == {} or .attributes["platform.observability.transaction"] == [$token]) + ' "$current" >/dev/null || return 2 + viewer_group_after=$current + fi + pending_kind=''; return 0 + ;; + mapper-create) + if classify_mapper_create; then pending_kind=''; return 0; else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi + ;; + mapper-created-finalize) + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 + current="$temporary_dir/mapper-created-finalize-current.json" + jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 2 + chmod 0600 -- "$current" + if mapper_contract_matches "$current" "$mapper_id" || + jq -e --arg id "$mapper_id" --arg token "$transaction_token" ' + .id == $id and .name == "grafana-groups" and + .config["platform.observability.transaction"] == $token + ' "$current" >/dev/null; then + mapper_after=$current; mapper_mutation=created; pending_kind=''; return 0 + fi + return 2 + ;; + secret-create) + if classify_secret_create; then pending_kind=''; return 0; else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi + ;; + admin-membership) + if membership_present "$pending_user_id" "$pending_group_id"; then admin_membership_added=true; pending_kind=''; return 0 + else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi + ;; + viewer-membership) + if membership_present "$pending_user_id" "$pending_group_id"; then viewer_membership_added=true; pending_kind=''; return 0 + else result=$?; [[ "$result" == 1 ]] && { pending_kind=''; return 1; }; return 2; fi + ;; + marker) + current="$temporary_dir/marker-pending-current.env" + if root_marker_read "$current"; then + if cmp --silent -- "$marker_desired" "$current"; then marker_written=true; pending_kind=''; return 0; fi + if [[ "$marker_had_prior" == true ]] && cmp --silent -- "$marker_prior" "$current"; then pending_kind=''; return 1; fi + return 2 + else + result=$? + [[ "$result" == 4 && "$marker_had_prior" == false ]] && { pending_kind=''; return 1; } + return 2 + fi + ;; + client-update) + current="$temporary_dir/client-pending-current.json" + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 + cp -- "$last_body" "$current"; chmod 0600 -- "$current" + if client_contract_matches "$current" "$client_id"; then + client_after=$current; client_mutation=updated; pending_kind=''; return 0 + fi + if json_equal "$current" "$client_before"; then pending_kind=''; return 1; fi + return 2 + ;; + mapper-update) + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 2 + current="$temporary_dir/mapper-pending-current.json" + jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 2 + chmod 0600 -- "$current" + if mapper_contract_matches "$current" "$mapper_id"; then mapper_after=$current; mapper_mutation=updated; pending_kind=''; return 0; fi + if json_equal "$current" "$mapper_before"; then pending_kind=''; return 1; fi + return 2 + ;; + *) return 2 ;; + esac +} + +delete_created_secret() { + [[ "$secret_mutation" == created ]] || return 0 + local current="$temporary_dir/rollback-secret-current.json" prefix="$temporary_dir/rollback-secret" values options + kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 + cp -- "$last_body" "$current"; chmod 0600 -- "$current" + values="$(validate_secret_json "$current" "$prefix")" || return 1 + local uid rv + IFS=$'\t' read -r uid rv <<<"$values" + [[ "$uid" == "$secret_created_uid" && "$rv" == "$secret_created_resource_version" ]] || return 1 + jq -e --arg token "$transaction_token" '.metadata.annotations["observability.hyeonworks.com/create-transaction"] == $token' "$current" >/dev/null || return 1 + [[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] && cmp --silent -- "$prefix.client-secret" "$client_secret_file" || return 1 + options="$temporary_dir/secret-delete-options.json" + jq -n --arg uid "$uid" --arg rv "$rv" '{apiVersion:"meta.k8s.io/v1",kind:"DeleteOptions",propagationPolicy:"Background",preconditions:{uid:$uid,resourceVersion:$rv}}' >"$options" + chmod 0600 -- "$options" + kube_request DELETE "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" "$options" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 || "$last_http_status" == 202 ]] || return 1 + kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 404 ]] +} + +rollback_membership() { + local user_id=$1 group_id=$2 added=$3 + [[ "$added" == true ]] || return 0 + admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/users/$user_id/groups/$group_id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 + if membership_present "$user_id" "$group_id"; then return 1; else [[ "$?" == 1 ]]; fi +} + +rollback_mapper() { + [[ "$mapper_mutation" != none ]] || return 0 + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 + local current="$temporary_dir/rollback-mapper-current.json" + jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 1 + chmod 0600 -- "$current" + json_equal "$current" "$mapper_after" || return 1 + if [[ "$mapper_mutation" == created ]]; then + admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 + jq -e --arg id "$mapper_id" \ + 'type == "array" and ([.[] | select(.id == $id)] | length) == 0' "$last_body" >/dev/null + else + admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models/$mapper_id" "$mapper_before" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 + jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current" || return 1 + json_equal "$current" "$mapper_before" + fi +} + +rollback_client() { + [[ "$client_mutation" != none ]] || return 0 + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 + local current="$temporary_dir/rollback-client-current.json" + cp -- "$last_body" "$current"; chmod 0600 -- "$current" + json_equal "$current" "$client_after" || return 1 + if [[ "$client_mutation" == created ]]; then + if jq -e --arg token "$transaction_token" '.attributes["platform.observability.transaction"] == $token' "$current" >/dev/null; then : + elif client_contract_matches "$current" "$client_id"; then : + else return 1; fi + admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2 + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 && "$(jq 'length' "$last_body")" == 0 ]] + else + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 + local current_secret="$temporary_dir/rollback-client-current-secret" + jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$current_secret" || return 1 + chmod 0600 -- "$current_secret"; cmp --silent -- "$client_secret_file" "$current_secret" || return 1 + admin_request PUT "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" "$client_restore" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 + json_equal "$last_body" "$client_before" + fi +} + +rollback_group() { + local id=$1 mutation=$2 after=$3 + [[ "$mutation" == created ]] || return 0 + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || return 1 + json_equal "$last_body" "$after" || return 1 + admin_request DELETE "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 204 ]] || return 1 + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 404 ]] +} + +rollback_transaction() { + local rc=0 + [[ "$rollback_in_progress" == false ]] || return 1 + rollback_in_progress=true + restore_marker || rc=1 + rollback_membership "$viewer_user_id" "$viewer_group_id" "$viewer_membership_added" || rc=1 + rollback_membership "$admin_user_id" "$admin_group_id" "$admin_membership_added" || rc=1 + delete_created_secret || rc=1 + rollback_mapper || rc=1 + rollback_client || rc=1 + rollback_group "$viewer_group_id" "$viewer_group_mutation" "$viewer_group_after" || rc=1 + rollback_group "$admin_group_id" "$admin_group_mutation" "$admin_group_after" || rc=1 + if (( rc == 0 )); then printf 'GRAFANA_OIDC_ROLLBACK=PASS\n' >&2 + else printf 'GRAFANA_OIDC_ROLLBACK=FAIL\n' >&2; emit_manual_recovery; fi + rollback_in_progress=false + return "$rc" +} + +transaction_fail() { + local message=$1 status=${2:-1} classification + trap - ERR HUP INT TERM + set +e + if [[ -n "$pending_kind" ]]; then + classify_pending; classification=$? + [[ "$classification" != 2 ]] || emit_manual_recovery + fi + rollback_transaction || true + transaction_active=false + printf 'ERROR: %s\n' "$message" >&2 + exit "$status" +} + +on_signal() { transaction_fail "interrupted by $1" "$2"; } + +on_process_exit() { + local rc=$1 classification + [[ "$exit_handler_running" == false ]] || exit "$rc" + exit_handler_running=true + trap - EXIT ERR HUP INT TERM + set +e + if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then + if [[ -n "$pending_kind" ]]; then classify_pending; classification=$?; [[ "$classification" != 2 ]] || emit_manual_recovery; fi + rollback_transaction || true + transaction_active=false + (( rc != 0 )) || rc=1 + fi + cleanup + exit "$rc" +} + +verify_group_prestate() { + local label=$1 name=$2 path=$3 state=$4 id=$5 before=$6 count + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2 + require_http 200 "$label group prestate reinspection" + count="$(jq 'length' "$last_body")" || return 1 + if [[ "$state" == absent ]]; then [[ "$count" == 0 ]]; return; fi + [[ "$count" == 1 && "$(jq -r --arg path "$path" '.[0] | select(.path == $path) | .id' "$last_body")" == "$id" ]] || return 1 + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/groups/$id" + require_http 200 "$label group prestate snapshot reinspection" + json_equal "$last_body" "$before" +} + +verify_prestate() { + local count current_secret="$temporary_dir/prestate-current-secret" current_mapper="$temporary_dir/prestate-current-mapper.json" + validate_context_and_authority + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2 + require_http 200 'Grafana client prestate reinspection' + count="$(jq 'length' "$last_body")" + if [[ "$client_state" == absent ]]; then + [[ "$count" == 0 ]] || fail 'Grafana client appeared after snapshot' + else + [[ "$count" == 1 && "$(jq -r '.[0].id' "$last_body")" == "$client_id" ]] || fail 'Grafana client cardinality or identity changed' + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" + require_http 200 'Grafana client exact prestate reinspection' + json_equal "$last_body" "$client_before" || fail 'Grafana client changed after snapshot' + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/client-secret" + require_http 200 'Grafana credential prestate reinspection' + jq -erj '.value | select(type == "string" and length >= 16 and length <= 512)' "$last_body" >"$current_secret" || + fail 'Grafana credential reinspection response is invalid' + chmod 0600 -- "$current_secret" + cmp --silent -- "$client_secret_file" "$current_secret" || fail 'Grafana credential changed after snapshot' + fi + verify_group_prestate admin "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" "$admin_group_state" "$admin_group_id" "$admin_group_before" || + fail 'admin group changed after snapshot' + verify_group_prestate viewer "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" "$viewer_group_state" "$viewer_group_id" "$viewer_group_before" || + fail 'viewer group changed after snapshot' + if [[ "$client_state" == existing ]]; then + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" + require_http 200 'Grafana mapper prestate reinspection' + if [[ "$mapper_state" == absent ]]; then + [[ "$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name or .config["claim.name"] == "groups")] | length' "$last_body")" == 0 ]] || + fail 'Grafana mapper appeared after snapshot' + else + [[ "$(jq --arg id "$mapper_id" '[.[] | select(.id == $id)] | length' "$last_body")" == 1 ]] || + fail 'Grafana mapper identity changed' + jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current_mapper" + chmod 0600 -- "$current_mapper" + json_equal "$current_mapper" "$mapper_before" || fail 'Grafana mapper changed after snapshot' + fi + fi + if [[ -n "$admin_username" ]]; then + local prior_id=$admin_user_id; lookup_user admin "$admin_username" admin_user_id + [[ "$admin_user_id" == "$prior_id" ]] || fail 'admin user identity changed' + if [[ "$admin_group_state" == existing ]]; then + local membership_rc + if membership_present "$admin_user_id" "$admin_group_id"; then membership_rc=0; else membership_rc=$?; fi + case "$membership_rc" in + 0) [[ "$admin_membership_before" == true ]] || fail 'admin membership appeared after snapshot' ;; + 1) [[ "$admin_membership_before" == false ]] || fail 'admin membership disappeared after snapshot' ;; + *) fail 'admin membership reinspection failed' ;; + esac + fi + fi + if [[ -n "$viewer_username" ]]; then + local prior_viewer_id=$viewer_user_id; lookup_user viewer "$viewer_username" viewer_user_id + [[ "$viewer_user_id" == "$prior_viewer_id" ]] || fail 'viewer user identity changed' + if [[ "$viewer_group_state" == existing ]]; then + local viewer_membership_rc + if membership_present "$viewer_user_id" "$viewer_group_id"; then viewer_membership_rc=0; else viewer_membership_rc=$?; fi + case "$viewer_membership_rc" in + 0) [[ "$viewer_membership_before" == true ]] || fail 'viewer membership appeared after snapshot' ;; + 1) [[ "$viewer_membership_before" == false ]] || fail 'viewer membership disappeared after snapshot' ;; + *) fail 'viewer membership reinspection failed' ;; + esac + fi + fi + kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" + (( last_curl_rc == 0 )) || fail 'Grafana OIDC Secret prestate reinspection transport failed' + if [[ "$secret_state" == absent ]]; then + [[ "$last_http_status" == 404 ]] || fail 'Grafana OIDC Secret appeared after snapshot' + else + [[ "$last_http_status" == 200 ]] || fail 'Grafana OIDC Secret disappeared after snapshot' + json_equal "$last_body" "$secret_before" || fail 'Grafana OIDC Secret changed after snapshot' + fi +} + +snapshot_transaction_prestate() { + transaction_token="$(new_transaction_token)" || fail 'cannot create a transaction ownership token' + [[ "$transaction_token" =~ ^[0-9a-f]{32}$ ]] || fail 'transaction ownership token is invalid' + build_desired_client + build_desired_mapper + lookup_client + lookup_group admin "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" admin_group_state admin_group_id admin_group_before + lookup_group viewer "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" viewer_group_state viewer_group_id viewer_group_before + lookup_mapper + lookup_user admin "$admin_username" admin_user_id + lookup_user viewer "$viewer_username" viewer_user_id + if [[ -n "$admin_user_id" && "$admin_group_state" == existing ]]; then + snapshot_membership admin "$admin_user_id" "$admin_group_id" admin_membership_before + fi + if [[ -n "$viewer_user_id" && "$viewer_group_state" == existing ]]; then + snapshot_membership viewer "$viewer_user_id" "$viewer_group_id" viewer_membership_before + fi + if [[ "$client_state" == existing ]]; then retrieve_client_secret; fi + start_kubernetes_proxy + snapshot_oidc_secret + sudo_refresh + prepare_marker_snapshot +} + +verify_final_contract() { + local count current_mapper="$temporary_dir/final-mapper.json" prefix values + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/clients" clientId "$CLIENT_ID" 2 + require_http 200 'final Grafana client lookup' + [[ "$(jq 'length' "$last_body")" == 1 && "$(jq -r '.[0].id' "$last_body")" == "$client_id" ]] || + fail 'final Grafana client cardinality differs' + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id" + require_http 200 'final Grafana client read' + client_contract_matches "$last_body" "$client_id" || fail 'final Grafana client contract differs' + local label name path id + for label in admin viewer; do + if [[ "$label" == admin ]]; then name=$ADMIN_GROUP_NAME; path=$ADMIN_GROUP_PATH; id=$admin_group_id + else name=$VIEWER_GROUP_NAME; path=$VIEWER_GROUP_PATH; id=$viewer_group_id; fi + admin_get_query "$local_base/admin/realms/$KEYCLOAK_REALM/groups?exact=true&briefRepresentation=false" search "$name" 2 + require_http 200 "final $label group lookup" + [[ "$(jq 'length' "$last_body")" == 1 && + "$(jq -r --arg path "$path" '.[0] | select(.path == $path) | .id' "$last_body")" == "$id" ]] || + fail "final $label group contract differs" + done + admin_request GET "$local_base/admin/realms/$KEYCLOAK_REALM/clients/$client_id/protocol-mappers/models" + require_http 200 'final Grafana mapper lookup' + [[ "$(jq --arg name "$MAPPER_NAME" '[.[] | select(.name == $name or .config["claim.name"] == "groups")] | length' "$last_body")" == 1 ]] || + fail 'final Grafana mapper cardinality differs' + jq -e --arg id "$mapper_id" '.[] | select(.id == $id)' "$last_body" >"$current_mapper" || fail 'final Grafana mapper identity differs' + chmod 0600 -- "$current_mapper"; mapper_contract_matches "$current_mapper" "$mapper_id" || fail 'final Grafana mapper contract differs' + kube_request GET "/api/v1/namespaces/$OBSERVABILITY_NAMESPACE/secrets/$OIDC_SECRET" + (( last_curl_rc == 0 )) && [[ "$last_http_status" == 200 ]] || fail 'final Grafana OIDC Secret is absent' + prefix="$temporary_dir/final-secret"; values="$(validate_secret_json "$last_body" "$prefix")" || fail 'final Grafana OIDC Secret contract differs' + [[ "$(<"$prefix.client-id")" == "$CLIENT_ID" ]] && cmp --silent -- "$prefix.client-secret" "$client_secret_file" || + fail 'final Grafana OIDC Secret relationship differs' + if [[ -n "$admin_user_id" ]]; then membership_present "$admin_user_id" "$admin_group_id" || fail 'requested admin membership is absent'; fi + if [[ -n "$viewer_user_id" ]]; then membership_present "$viewer_user_id" "$viewer_group_id" || fail 'requested viewer membership is absent'; fi + admin_request GET "$local_base/realms/$KEYCLOAK_REALM/.well-known/openid-configuration" + require_http 200 'Keycloak OIDC discovery verification' + jq -e --arg issuer "https://$KEYCLOAK_HOST/realms/$KEYCLOAK_REALM" '.issuer == $issuer' "$last_body" >/dev/null || + fail 'Keycloak public issuer differs' +} + +reconcile_transaction() { + transaction_active=true + trap 'transaction_fail "unexpected command failure at line $LINENO"' ERR + trap 'on_signal HUP 129' HUP + trap 'on_signal INT 130' INT + trap 'on_signal TERM 143' TERM + if [[ "$admin_group_state" == absent ]]; then + create_group admin "$ADMIN_GROUP_NAME" "$ADMIN_GROUP_PATH" admin_group_id admin_group_mutation admin_group_after + fi + if [[ "$viewer_group_state" == absent ]]; then + create_group viewer "$VIEWER_GROUP_NAME" "$VIEWER_GROUP_PATH" viewer_group_id viewer_group_mutation viewer_group_after + fi + if [[ "$client_state" == absent ]]; then + create_client + elif client_contract_matches "$client_before" "$client_id"; then + client_mutation=none + else + update_client + fi + if [[ "$mapper_state" == absent ]]; then + create_mapper + elif mapper_contract_matches "$mapper_before" "$mapper_id"; then + mapper_mutation=none + else + update_mapper + fi + if [[ "$secret_state" == absent ]]; then create_oidc_secret; fi + add_membership admin "$admin_user_id" "$admin_group_id" "$admin_membership_before" admin_membership_added + add_membership viewer "$viewer_user_id" "$viewer_group_id" "$viewer_membership_before" viewer_membership_added + verify_final_contract + install_marker + trap - ERR HUP INT TERM + transaction_active=false + printf '%s\n' \ + 'GRAFANA_OIDC_CLIENT=RECONCILED' \ + 'GRAFANA_OIDC_GROUPS=RECONCILED' \ + 'GRAFANA_OIDC_MAPPER=RECONCILED' \ + "GRAFANA_OIDC_SECRET=$(if [[ "$secret_state" == absent ]]; then printf CREATE_CONFIRMED; else printf REUSED_UNCHANGED; fi)" \ + 'GRAFANA_OIDC_TRANSACTION=PASS' +} + +execute_transaction() { + run_common_gates + validate_context_and_authority + snapshot_kubernetes_objects + start_port_forward + authenticate_keycloak + snapshot_transaction_prestate + read_confirmation 'Type APPLY default: ' 'APPLY default' + read_confirmation 'Type RECOVERY KEYCLOAK default: ' 'RECOVERY KEYCLOAK default' + verify_prestate + run_common_gates + ensure_evidence_directory + reconcile_transaction +} + +grafana_oidc_main() { + parse_arguments "$@" + (( EUID != 0 )) || fail 'whole-script root execution is forbidden; use narrow sudo from the invoking user' + if [[ "$execute_requested" == false && "$check_requested" == false ]]; then print_dry_run; return 0; fi + require_commands + make_temporary_dir + trap 'on_process_exit "$?"' EXIT + if [[ "$check_requested" == true ]]; then check_recovery_evidence; else execute_transaction; fi +} + +platform_grafana_oidc_fixture_main() { + local fixture=${1:-} + [[ "${BASH_SOURCE[0]}" != "$0" ]] || fail 'fixture entrypoint must be sourced' + [[ -n "$fixture" ]] || fail 'fixture root is required' + shift + TEST_MODE=true + configure_test_boundaries "$fixture" + grafana_oidc_main "$@" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + reject_production_overrides + grafana_oidc_main "$@" +fi diff --git a/scripts/bootstrap/create-aistor-secrets.sh b/scripts/bootstrap/create-aistor-secrets.sh new file mode 100755 index 0000000..d29516f --- /dev/null +++ b/scripts/bootstrap/create-aistor-secrets.sh @@ -0,0 +1,324 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +# 비밀값이 명령 추적, argv 또는 표준 출력에 노출되지 않도록 한다. +set +x + +readonly REQUIRED_CONFIRMATION="APPLY AISTOR SECRETS" +readonly -a SECRET_CONTRACTS=( + "aistor/minio-license" + "object-storage/aistor-root-configuration" +) + +license_file="" +root_config_file="" +generate_root_config=false +execute_requested=false +license_created=false +root_config_created=false + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'USAGE' +Usage: + bash scripts/bootstrap/create-aistor-secrets.sh \ + --license-file /home/donghyeon/.secrets/aistor/minio.license \ + --root-config-file /home/donghyeon/.secrets/aistor/root.env \ + --generate-root-config \ + --execute + +Creates exactly these two Secrets only when both are absent: + aistor/minio-license + object-storage/aistor-root-configuration + +When both already exist, validates and reuses them unchanged. A partial state +is refused. --generate-root-config creates the local 0600 credential file only +when it is absent; it never overwrites or rotates an existing credential. +USAGE +} + +while (( $# > 0 )); do + case "$1" in + --license-file) + (( $# >= 2 )) || fail "--license-file requires a path" + license_file="$2" + shift 2 + ;; + --root-config-file) + (( $# >= 2 )) || fail "--root-config-file requires a path" + root_config_file="$2" + shift 2 + ;; + --generate-root-config) + generate_root_config=true + shift + ;; + --execute) + execute_requested=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage >&2 + fail "unsupported argument: $1" + ;; + esac +done + +[[ "$execute_requested" == true ]] || { + usage >&2 + exit 2 +} +[[ "$license_file" == /* ]] || fail "--license-file must be an absolute path" +[[ "$root_config_file" == /* ]] || \ + fail "--root-config-file must be an absolute path" + +for command_name in awk base64 chmod cmp find install jq kubectl mktemp \ + openssl sort stat wc; do + command -v "$command_name" >/dev/null 2>&1 || \ + fail "${command_name} is required" +done + +validate_private_file() { + local file="$1" + local description="$2" + + [[ -f "$file" && ! -L "$file" && -O "$file" && -r "$file" && -s "$file" ]] || \ + fail "${description} must be a readable, non-empty, current-user-owned regular file" + [[ "$(stat --format='%a' -- "$file")" == "600" ]] || \ + fail "${description} must have mode 0600: ${file}" +} + +validate_license_file() { + local payload + + validate_private_file "$license_file" "license file" + payload="$(<"$license_file")" + payload="${payload%$'\r'}" + [[ "$payload" =~ ^eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$ ]] || \ + fail "license file must contain one JWT value beginning with eyJ" + unset payload +} + +validate_root_config_file() { + local file="$1" + + validate_private_file "$file" "AIStor root configuration file" + awk ' + BEGIN { + user_prefix = "export MINIO_ROOT_USER=" + password_prefix = "export MINIO_ROOT_PASSWORD=" + } + NR == 1 && index($0, user_prefix) == 1 { + value = substr($0, length(user_prefix) + 1) + if (length(value) >= 3 && substr(value, 1, 1) == "\042" && substr(value, length(value), 1) == "\042") { + value = substr(value, 2, length(value) - 2) + users++ + user = value + if (length(value) < 8 || value == "minioadmin") bad = 1 + next + } + } + NR == 2 && index($0, password_prefix) == 1 { + value = substr($0, length(password_prefix) + 1) + if (length(value) >= 3 && substr(value, 1, 1) == "\042" && substr(value, length(value), 1) == "\042") { + value = substr(value, 2, length(value) - 2) + passwords++ + password = value + if (length(value) < 16 || value == "minioadmin") bad = 1 + next + } + } + { bad = 1 } + END { + if (NR != 2 || users != 1 || passwords != 1 || + user == password || bad) exit 1 + } + ' "$file" || \ + fail "root configuration must contain exactly valid MINIO_ROOT_USER and MINIO_ROOT_PASSWORD exports" +} + +generate_root_configuration() { + local parent_dir="${root_config_file%/*}" + local root_user + local root_password + + [[ "$parent_dir" != "$root_config_file" ]] || \ + fail "root configuration path has no parent directory" + [[ ! -e "$root_config_file" && ! -L "$root_config_file" ]] || \ + fail "refusing to overwrite existing root configuration: ${root_config_file}" + + install -d -m 0700 -- "$parent_dir" + [[ -d "$parent_dir" && ! -L "$parent_dir" && -O "$parent_dir" ]] || \ + fail "root configuration parent must be a current-user-owned directory" + [[ "$(stat --format='%a' -- "$parent_dir")" == "700" ]] || \ + fail "root configuration parent must have mode 0700: ${parent_dir}" + + root_user="hyeonworks-aistor-$(openssl rand -hex 4)" + root_password="$(openssl rand -hex 24)" + umask 077 + { + printf 'export MINIO_ROOT_USER="%s"\n' "$root_user" + printf 'export MINIO_ROOT_PASSWORD="%s"\n' "$root_password" + } >"$root_config_file" + chmod 0600 -- "$root_config_file" + unset root_user root_password + validate_root_config_file "$root_config_file" + printf 'Generated a local AIStor root configuration with mode 0600: %s\n' \ + "$root_config_file" +} + +validate_license_file + +for namespace in aistor object-storage; do + kubectl get namespace "$namespace" >/dev/null 2>&1 || \ + fail "namespace ${namespace} does not exist; apply AIStor namespaces first" +done + +existing_secret_count=0 +for contract in "${SECRET_CONTRACTS[@]}"; do + namespace="${contract%%/*}" + name="${contract#*/}" + if kubectl --namespace "$namespace" get secret "$name" >/dev/null 2>&1; then + (( existing_secret_count += 1 )) + fi +done + +if (( existing_secret_count > 0 && existing_secret_count < ${#SECRET_CONTRACTS[@]} )); then + fail "partial AIStor Secret state detected; no Secret was created or rotated" +fi + +umask 077 +secret_temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-aistor-secrets.XXXXXX")" +existing_license_file="${secret_temp_dir}/existing-minio-license" +existing_root_config_file="${secret_temp_dir}/existing-root-config" + +cleanup() { + case "$secret_temp_dir" in + /tmp/platform-aistor-secrets.*|"${TMPDIR:-/tmp}"/platform-aistor-secrets.*) + rm -rf -- "$secret_temp_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected temporary directory: %s\n' \ + "$secret_temp_dir" >&2 + ;; + esac +} +trap cleanup EXIT + +validate_secret_contract() { + local namespace="$1" + local name="$2" + local expected_key="$3" + local actual_type + local actual_keys + + actual_type="$( + kubectl --namespace "$namespace" get secret "$name" \ + --output=jsonpath='{.type}' + )" + [[ "$actual_type" == "Opaque" ]] || \ + fail "${namespace}/${name} must have type Opaque" + actual_keys="$( + kubectl --namespace "$namespace" get secret "$name" --output=json | + jq -r '.data | keys[]' | sort + )" + [[ "$actual_keys" == "$expected_key" ]] || \ + fail "${namespace}/${name} must contain only the ${expected_key} key" +} + +if (( existing_secret_count == ${#SECRET_CONTRACTS[@]} )); then + validate_secret_contract aistor minio-license minio.license + validate_secret_contract \ + object-storage aistor-root-configuration config.env + + kubectl --namespace aistor get secret minio-license \ + --output=jsonpath='{.data.minio\.license}' | + base64 --decode >"$existing_license_file" + cmp --silent -- "$license_file" "$existing_license_file" || \ + fail "the supplied license differs from the existing Secret; rotation was not performed" + + kubectl --namespace object-storage get secret aistor-root-configuration \ + --output=jsonpath='{.data.config\.env}' | + base64 --decode >"$existing_root_config_file" + chmod 0600 -- "$existing_root_config_file" + validate_root_config_file "$existing_root_config_file" + + if [[ -e "$root_config_file" || -L "$root_config_file" ]]; then + validate_root_config_file "$root_config_file" + cmp --silent -- "$root_config_file" "$existing_root_config_file" || \ + fail "local root configuration differs from the existing Secret; rotation was not performed" + else + fail "existing Secret is valid, but the local root configuration file is missing" + fi + + printf 'Existing AIStor Secret contracts are valid and were reused unchanged.\n' + printf 'No credential or license rotation was performed.\n' + exit 0 +fi + +if [[ ! -e "$root_config_file" && ! -L "$root_config_file" ]]; then + [[ "$generate_root_config" == true ]] || \ + fail "root configuration is absent; pass --generate-root-config to create it" + generate_root_configuration +else + validate_root_config_file "$root_config_file" +fi + +[[ -t 0 ]] || fail "an interactive terminal is required for initial Secret creation" +printf '\nThis will create exactly two AIStor Secrets. Type %s to continue: ' \ + "$REQUIRED_CONFIRMATION" +read -r confirmation +[[ "$confirmation" == "$REQUIRED_CONFIRMATION" ]] || fail "cancelled" + +rollback_new_secrets() { + set +e + if [[ "$root_config_created" == true ]]; then + kubectl --namespace object-storage delete secret \ + aistor-root-configuration --ignore-not-found >/dev/null + fi + if [[ "$license_created" == true ]]; then + kubectl --namespace aistor delete secret \ + minio-license --ignore-not-found >/dev/null + fi + if [[ "$root_config_created" == true || "$license_created" == true ]]; then + printf 'ROLLBACK: removed only Secrets created by this failed invocation.\n' >&2 + fi +} + +on_error() { + local status="$1" + local line="$2" + + trap - ERR + rollback_new_secrets + printf 'ERROR: Secret creation failed at line %s (exit %s).\n' \ + "$line" "$status" >&2 + exit "$status" +} +trap 'on_error "$?" "$LINENO"' ERR + +kubectl --namespace aistor create secret generic minio-license \ + --type=Opaque \ + --from-file="minio.license=${license_file}" +license_created=true + +kubectl --namespace object-storage create secret generic \ + aistor-root-configuration \ + --type=Opaque \ + --from-file="config.env=${root_config_file}" +root_config_created=true + +validate_secret_contract aistor minio-license minio.license +validate_secret_contract object-storage aistor-root-configuration config.env + +printf 'Created both AIStor Secret contracts without printing payloads.\n' +printf 'Local root credentials remain only in: %s\n' "$root_config_file" +printf 'This script does not perform credential or license rotation.\n' diff --git a/scripts/bootstrap/create-keycloak-secrets.sh b/scripts/bootstrap/create-keycloak-secrets.sh new file mode 100755 index 0000000..cc6b9ca --- /dev/null +++ b/scripts/bootstrap/create-keycloak-secrets.sh @@ -0,0 +1,298 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +# 호출자가 bash -x로 실행해도 비밀번호가 추적 출력에 노출되지 않도록 한다. +set +x + +readonly REQUIRED_CONFIRMATION="APPLY KEYCLOAK SECRETS" +readonly DB_USERNAME="keycloak" +readonly TOTAL_SECRET_CONTRACTS=2 +readonly -a SECRET_CONTRACTS=( + "platform-data/keycloak-db-credentials" + "keycloak/keycloak-db-credentials" +) + +mutation_started=false +secret_temp_dir="" + +report_retained_state() { + if [[ "$mutation_started" == true ]]; then + printf '%s\n' \ + 'SAFE STOP: no Secret or database data was deleted or rolled back.' \ + 'Any Secret created before the failure remains in the cluster.' \ + 'Inspect Secret names and events without printing Secret data, then rerun after resolving the cause.' >&2 + fi +} + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + report_retained_state + exit 1 +} + +on_error() { + local status="$1" + local line="$2" + + trap - ERR + set +e + printf 'ERROR: command failed at line %s (exit %s).\n' "$line" "$status" >&2 + report_retained_state + exit "$status" +} + +on_signal() { + local status="$1" + + trap - INT TERM + set +e + printf 'INTERRUPTED: stopping without deleting cluster state.\n' >&2 + report_retained_state + exit "$status" +} + +usage() { + cat <<'USAGE' +Usage: + bash scripts/bootstrap/create-keycloak-secrets.sh --execute + bash scripts/bootstrap/create-keycloak-secrets.sh --generate --execute + +Creates exactly these two Secrets only when both are absent: + platform-data/keycloak-db-credentials + keycloak/keycloak-db-credentials + +When both already exist, validates and reuses their data unchanged. A partial +state is refused. This script does not require or modify an AIStor license, +does not rotate credentials, and never prints Secret payloads. + +--execute prompts twice for a database password and requires the exact +confirmation text. --generate --execute is an explicit non-interactive mode: +it generates 32 random bytes with OpenSSL in a private 0600 temporary file. +USAGE +} + +execute_requested=false +generate_requested=false +while (( $# > 0 )); do + case "$1" in + --execute) + execute_requested=true + shift + ;; + --generate) + generate_requested=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage >&2 + fail "unsupported argument: $1" + ;; + esac +done +[[ "$execute_requested" == true ]] || { + usage >&2 + exit 2 +} + +for command_name in kubectl base64 cmp mktemp sort stat tr wc; do + command -v "$command_name" >/dev/null 2>&1 || \ + fail "${command_name} is required" +done +if [[ "$generate_requested" == true ]]; then + command -v openssl >/dev/null 2>&1 || fail "openssl is required for --generate" +fi + +for namespace in platform-data keycloak; do + kubectl get namespace "$namespace" >/dev/null 2>&1 || \ + fail "namespace ${namespace} does not exist" +done + +umask 077 +secret_temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-keycloak-secrets.XXXXXX")" +db_user_file="${secret_temp_dir}/expected-username" +db_password_file="${secret_temp_dir}/new-password" +platform_username_file="${secret_temp_dir}/platform-username" +keycloak_username_file="${secret_temp_dir}/keycloak-username" +platform_password_file="${secret_temp_dir}/platform-password" +keycloak_password_file="${secret_temp_dir}/keycloak-password" + +cleanup() { + unset db_password + if [[ -n "$secret_temp_dir" ]]; then + rm -f -- \ + "$db_user_file" \ + "$db_password_file" \ + "$platform_username_file" \ + "$keycloak_username_file" \ + "$platform_password_file" \ + "$keycloak_password_file" + rmdir -- "$secret_temp_dir" 2>/dev/null || true + fi +} +trap cleanup EXIT +trap 'on_error "$?" "$LINENO"' ERR +trap 'on_signal 130' INT +trap 'on_signal 143' TERM + +printf '%s' "$DB_USERNAME" >"$db_user_file" + +secret_data_b64() { + local namespace="$1" + local name="$2" + local key="$3" + + kubectl --namespace "$namespace" get secret "$name" \ + --output="go-template={{ index .data \"${key}\" }}" +} + +validate_secret_type_and_keys() { + local namespace="$1" + local name="$2" + local actual_type + local actual_keys + local actual_keys_sorted + + actual_type="$( + kubectl --namespace "$namespace" get secret "$name" \ + --output='jsonpath={.type}' + )" + [[ "$actual_type" == "kubernetes.io/basic-auth" ]] || \ + fail "${namespace}/${name} must have type kubernetes.io/basic-auth" + + actual_keys="$( + kubectl --namespace "$namespace" get secret "$name" \ + --output='go-template={{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}' + )" + actual_keys_sorted="$(printf '%s\n' "$actual_keys" | LC_ALL=C sort)" + [[ "$actual_keys_sorted" == $'password\nusername' ]] || \ + fail "${namespace}/${name} has an unexpected data key set" +} + +validate_existing_contracts() { + validate_secret_type_and_keys platform-data keycloak-db-credentials + validate_secret_type_and_keys keycloak keycloak-db-credentials + + secret_data_b64 platform-data keycloak-db-credentials username \ + | base64 --decode >"$platform_username_file" + secret_data_b64 keycloak keycloak-db-credentials username \ + | base64 --decode >"$keycloak_username_file" + cmp --silent -- "$db_user_file" "$platform_username_file" || \ + fail "platform-data/keycloak-db-credentials username must be keycloak" + cmp --silent -- "$db_user_file" "$keycloak_username_file" || \ + fail "keycloak/keycloak-db-credentials username must be keycloak" + + secret_data_b64 platform-data keycloak-db-credentials password \ + | base64 --decode >"$platform_password_file" + secret_data_b64 keycloak keycloak-db-credentials password \ + | base64 --decode >"$keycloak_password_file" + (( $(wc -c <"$platform_password_file") >= 16 )) || \ + fail "platform-data/keycloak-db-credentials password must contain at least 16 bytes" + (( $(wc -c <"$keycloak_password_file") >= 16 )) || \ + fail "keycloak/keycloak-db-credentials password must contain at least 16 bytes" + cmp --silent -- "$platform_password_file" "$keycloak_password_file" || \ + fail "the two Keycloak DB Secret passwords do not match" +} + +existing_secret_count=0 +for contract in "${SECRET_CONTRACTS[@]}"; do + namespace="${contract%%/*}" + name="${contract#*/}" + existing_resource="$( + kubectl --namespace "$namespace" get secret "$name" \ + --ignore-not-found --output=name + )" + if [[ -n "$existing_resource" ]]; then + ((existing_secret_count += 1)) + fi +done + +if (( existing_secret_count > 0 && existing_secret_count < TOTAL_SECRET_CONTRACTS )); then + fail "partial Keycloak Secret state detected (${existing_secret_count}/${TOTAL_SECRET_CONTRACTS}); refusing creation or rotation" +fi + +if (( existing_secret_count == TOTAL_SECRET_CONTRACTS )); then + validate_existing_contracts + + reload_label="$( + kubectl --namespace platform-data get secret keycloak-db-credentials \ + --output='jsonpath={.metadata.labels.cnpg\.io/reload}' + )" + if [[ "$reload_label" != "true" ]]; then + mutation_started=true + kubectl --namespace platform-data label secret keycloak-db-credentials \ + cnpg.io/reload=true --overwrite + mutation_started=false + printf 'Repaired cnpg.io/reload=true without changing Secret data.\n' + fi + + printf 'Existing Keycloak Secret contracts are valid and were reused unchanged.\n' + printf 'No credential rotation was performed.\n' + exit 0 +fi + +read_secret_twice() { + local prompt="$1" + local first + local second + + read -r -s -p "${prompt}: " first + printf '\n' >&2 + read -r -s -p "Confirm ${prompt}: " second + printf '\n' >&2 + + [[ "$first" == "$second" ]] || fail "the two values do not match" + (( ${#first} >= 16 )) || fail "${prompt} must contain at least 16 characters" + [[ "$first" != *$'\n'* && "$first" != *$'\r'* ]] || \ + fail "${prompt} contains an unsupported line break" + printf '%s' "$first" +} + +if [[ "$generate_requested" == true ]]; then + openssl rand -hex 32 | tr -d '\n' >"$db_password_file" + [[ "$(stat --format='%a' -- "$db_password_file")" == "600" ]] || \ + fail "generated password file must have mode 0600" + [[ "$(wc -c <"$db_password_file" | tr -d '[:space:]')" == "64" ]] || \ + fail "OpenSSL did not generate the expected 32-byte password" + printf '%s\n' \ + 'Authorized by explicit --generate --execute flags.' \ + 'A 32-byte random database password was generated without printing it.' +else + [[ -t 0 ]] || fail "an interactive terminal is required for initial Secret creation" + db_password="$(read_secret_twice 'Keycloak database password')" + printf '%s' "$db_password" >"$db_password_file" + unset db_password + + printf '\nThe script will create exactly two Keycloak DB Secrets.\n' + printf 'It will not read or modify any AIStor Secret.\n' + printf 'Type %s to continue: ' "$REQUIRED_CONFIRMATION" + read -r confirmation + [[ "$confirmation" == "$REQUIRED_CONFIRMATION" ]] || fail "cancelled" +fi + +create_basic_auth_secret() { + local namespace="$1" + + kubectl --namespace "$namespace" create secret generic keycloak-db-credentials \ + --type=kubernetes.io/basic-auth \ + --from-file="username=${db_user_file}" \ + --from-file="password=${db_password_file}" \ + --dry-run=client \ + --output=yaml \ + | kubectl create --filename=- +} + +mutation_started=true +create_basic_auth_secret platform-data +create_basic_auth_secret keycloak +kubectl --namespace platform-data label secret keycloak-db-credentials \ + cnpg.io/reload=true --overwrite + +validate_existing_contracts +mutation_started=false + +printf 'Initial Keycloak Secret contracts were created without printing payloads.\n' +printf 'This script does not perform credential rotation.\n' diff --git a/scripts/bootstrap/create-observability-secrets.sh b/scripts/bootstrap/create-observability-secrets.sh new file mode 100755 index 0000000..af6158a --- /dev/null +++ b/scripts/bootstrap/create-observability-secrets.sh @@ -0,0 +1,1826 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +# Secret payloads must never be exposed by a caller's xtrace setting. +set +x + +readonly SCRIPT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly EXPECTED_CONTEXT='default' +readonly NAMESPACE='observability' +readonly EVIDENCE_SCHEMA='platform-observability-recovery-evidence-v1' +readonly RISK_EVIDENCE_SCHEMA='platform-observability-slack-risk-acceptance-v1' +readonly RISK_APPROVAL_REF='2026-08-14-observability-slack-recovery-risk-acceptance-design' +readonly RISK_ACCEPTED_BY_UID='1000' +readonly EVIDENCE_MAX_AGE_SECONDS=2592000 +readonly PRODUCTION_EVIDENCE_DIR='/etc/hyeonworks/platform/recovery-evidence' +readonly PRODUCTION_EVIDENCE_ANCHOR='/etc' +readonly PRODUCTION_ENCRYPTION_SCRIPT="$SCRIPT_ROOT/scripts/validate/k3s-secret-encryption.sh" +readonly PRODUCTION_RESTORE_SCRIPT="$SCRIPT_ROOT/scripts/validate/k3s-secret-encryption-restore-evidence.sh" +readonly VALIDATOR_PATH='/usr/sbin:/usr/bin:/sbin:/bin' +readonly VALIDATOR_HOME='/home/donghyeon' +readonly KUBECTL_REQUEST_TIMEOUT='5s' +KUBECTL_PROCESS_TIMEOUT='10s' +KUBECTL_PROXY_PROCESS_TIMEOUT='15s' + +TEST_MODE=false +KUBECTL_BIN='kubectl' +SUDO_BIN='sudo' +ENCRYPTION_SCRIPT="$PRODUCTION_ENCRYPTION_SCRIPT" +RESTORE_SCRIPT="$PRODUCTION_RESTORE_SCRIPT" +EVIDENCE_DIR="$PRODUCTION_EVIDENCE_DIR" +EVIDENCE_HIERARCHY_PROFILE='production-hierarchy' +EVIDENCE_HIERARCHY_ANCHOR="$PRODUCTION_EVIDENCE_ANCHOR" +TEST_CONFIRMATIONS='' +TEST_NOW_UTC='' +MARKER_TEST_FAILURE_PHASE='' + +execute_requested=false +grafana_requested=false +slack_requested=false +check_grafana_requested=false +check_slack_requested=false +accept_slack_risk_requested=false +check_slack_deployment_requested=false +grafana_user_file='' +grafana_password_file='' +slack_webhook_file='' +grafana_user_snapshot='' +grafana_password_snapshot='' +slack_webhook_snapshot='' + +temporary_dir='' +transaction_active=false +manual_recovery_required=false +rollback_in_progress=false +exit_handler_running=false +declare -a OWNED_ASSETS=() +declare -a OWNED_NAMES=() +declare -a OWNED_UIDS=() +pending_asset='' +pending_name='' +pending_token='' +pending_manifest='' +pending_response='' +pending_marker_asset='' +declare -a WRITTEN_MARKERS=() +declare -A MARKER_HAD_PRIOR=() +declare -A MARKER_PRIOR_FILE=() +declare -A MARKER_DESIRED_FILE=() +declare -A MARKER_KIND=() +EVIDENCE_EXPECTED_UID=0 +EVIDENCE_EXPECTED_GID=0 +confirmation_index=0 +declare -a confirmation_answers=() + +usage() { + cat <<'USAGE' +Usage: + bash scripts/bootstrap/create-observability-secrets.sh + bash scripts/bootstrap/create-observability-secrets.sh \ + --execute --grafana-admin \ + --grafana-admin-user-file /absolute/0600/file \ + --grafana-admin-password-file /absolute/0600/file + bash scripts/bootstrap/create-observability-secrets.sh \ + --execute --slack-webhook \ + --slack-webhook-file /absolute/0600/file + bash scripts/bootstrap/create-observability-secrets.sh \ + --check-grafana-recovery-evidence + bash scripts/bootstrap/create-observability-secrets.sh \ + --check-slack-recovery-evidence + bash scripts/bootstrap/create-observability-secrets.sh \ + --check-slack-deployment-evidence + +With no arguments, prints a source-only plan and performs no system access. +Execute mode creates only absent selected Secrets, reuses exact existing +contracts, and never rotates or prints payloads. Both asset flags form one +create-only transaction. +USAGE +} + +usage_fail() { + printf 'ERROR: %s\n' "$*" >&2 + usage >&2 + exit 2 +} + +emit_manual_recovery() { + [[ "$manual_recovery_required" == false ]] || return 0 + manual_recovery_required=true + printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2 +} + +plain_fail() { + if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then + transaction_fail "$*" + fi + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +safe_temporary_path() { + local path=$1 + [[ -n "$temporary_dir" && "$temporary_dir" == /tmp/platform-observability-secrets.?????? ]] || return 1 + [[ "$path" == "$temporary_dir" || "$path" == "$temporary_dir"/* ]] +} + +cleanup() { + trap - EXIT + if [[ -n "$temporary_dir" ]] && safe_temporary_path "$temporary_dir"; then + rm -rf -- "$temporary_dir" + fi +} + +on_process_exit() { + local rc=$1 ambiguous=false classification + [[ "$exit_handler_running" == false ]] || exit "$rc" + exit_handler_running=true + trap - EXIT ERR HUP INT TERM + set +e + if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then + if [[ -n "$pending_marker_asset" ]]; then + classify_pending_marker + classification=$? + [[ "$classification" != 2 ]] || ambiguous=true + fi + if [[ -n "$pending_asset" ]]; then + classify_pending_create + classification=$? + [[ "$classification" == 0 || "$classification" == 1 ]] || ambiguous=true + fi + [[ "$ambiguous" != true ]] || emit_manual_recovery + rollback_transaction || true + transaction_active=false + printf 'ERROR: transaction exited before a confirmed terminal state\n' >&2 + (( rc != 0 )) || rc=1 + fi + cleanup + exit "$rc" +} +trap 'on_process_exit "$?"' EXIT + +reject_production_overrides() { + local variable + local -a forbidden=( + PLATFORM_OBSERVABILITY_SECRETS_TEST_MODE + PLATFORM_OBSERVABILITY_SECRETS_KUBECTL_BIN + PLATFORM_OBSERVABILITY_SECRETS_SUDO_BIN + PLATFORM_OBSERVABILITY_SECRETS_ENCRYPTION_SCRIPT + PLATFORM_OBSERVABILITY_SECRETS_RESTORE_SCRIPT + PLATFORM_OBSERVABILITY_SECRETS_EVIDENCE_DIR + PLATFORM_OBSERVABILITY_SECRETS_CONFIRMATIONS + PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC + PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE + ) + for variable in "${forbidden[@]}"; do + [[ ! -v "$variable" ]] || plain_fail "production boundary override is forbidden: $variable" + done +} + +attest_test_executable() { + local path=$1 metadata + [[ -f "$path" && ! -L "$path" && -O "$path" && -x "$path" ]] || return 1 + metadata="$(stat --format='%u:%a:%h:%F' -- "$path")" || return 1 + [[ "$metadata" == "${EUID}:755:1:regular file" ]] +} + +configure_test_boundaries() { + local fixture_root=$1 fixture_metadata physical_root + [[ "$fixture_root" =~ ^/tmp/platform-observability-secrets-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+$ ]] || + plain_fail 'test fixture root has an invalid shape' + [[ -d "$fixture_root" && ! -L "$fixture_root" && -O "$fixture_root" ]] || + plain_fail 'test fixture root is not an owned non-symlink directory' + physical_root="$(cd -- "$fixture_root" && pwd -P)" || plain_fail 'test fixture root cannot be resolved' + [[ "$physical_root" == "$fixture_root" ]] || plain_fail 'test fixture root contains a symlink boundary' + fixture_metadata="$(stat --format='%u:%a:%F' -- "$fixture_root")" || + plain_fail 'test fixture root cannot be attested' + [[ "$fixture_metadata" == "${EUID}:700:directory" ]] || + plain_fail 'test fixture root metadata is invalid' + [[ "$(cd -- "$fixture_root/bin" && pwd -P)" == "$fixture_root/bin" ]] || + plain_fail 'test binary directory contains a symlink boundary' + [[ "$(stat --format='%u:%a:%F' -- "$fixture_root/bin")" == "${EUID}:755:directory" ]] || + plain_fail 'test binary directory metadata is invalid' + + KUBECTL_BIN="$fixture_root/bin/kubectl" + SUDO_BIN="$fixture_root/bin/sudo" + ENCRYPTION_SCRIPT="$fixture_root/bin/encryption" + RESTORE_SCRIPT="$fixture_root/bin/restore" + EVIDENCE_DIR="$fixture_root/evidence-parent/platform/recovery-evidence" + EVIDENCE_HIERARCHY_PROFILE='fixture-hierarchy' + EVIDENCE_HIERARCHY_ANCHOR="$fixture_root" + TEST_CONFIRMATIONS="${PLATFORM_OBSERVABILITY_SECRETS_CONFIRMATIONS:-}" + TEST_NOW_UTC="${PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC:-}" + MARKER_TEST_FAILURE_PHASE="${PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE:-}" + KUBECTL_PROCESS_TIMEOUT='2s' + KUBECTL_PROXY_PROCESS_TIMEOUT='5s' + EVIDENCE_EXPECTED_UID=$EUID + EVIDENCE_EXPECTED_GID="$(/usr/bin/id -g)" + + [[ "$SUDO_BIN" != /usr/bin/sudo && "$SUDO_BIN" != /bin/sudo ]] || + plain_fail 'real sudo is forbidden in the isolated test entrypoint' + attest_test_executable "$KUBECTL_BIN" || plain_fail 'test kubectl boundary is not attested' + attest_test_executable "$SUDO_BIN" || plain_fail 'test sudo boundary is not attested' + attest_test_executable "$ENCRYPTION_SCRIPT" || plain_fail 'test encryption boundary is not attested' + attest_test_executable "$RESTORE_SCRIPT" || plain_fail 'test restore boundary is not attested' + [[ "$TEST_NOW_UTC" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || + plain_fail 'test clock is invalid' + [[ "$MARKER_TEST_FAILURE_PHASE" == '' || "$MARKER_TEST_FAILURE_PHASE" == first-write || + "$MARKER_TEST_FAILURE_PHASE" == pre-rename ]] || + plain_fail 'test marker failure phase is invalid' + mapfile -t confirmation_answers <<<"$TEST_CONFIRMATIONS" +} + +parse_arguments() { + local seen_execute=false seen_grafana=false seen_slack=false + local seen_user_file=false seen_password_file=false seen_slack_file=false + local seen_check_grafana=false seen_check_slack=false seen_accept_slack_risk=false + local seen_check_slack_deployment=false + + while (( $# > 0 )); do + case "$1" in + --execute) + [[ "$seen_execute" == false ]] || usage_fail 'duplicate --execute' + seen_execute=true; execute_requested=true; shift + ;; + --grafana-admin) + [[ "$seen_grafana" == false ]] || usage_fail 'duplicate --grafana-admin' + seen_grafana=true; grafana_requested=true; shift + ;; + --slack-webhook) + [[ "$seen_slack" == false ]] || usage_fail 'duplicate --slack-webhook' + seen_slack=true; slack_requested=true; shift + ;; + --grafana-admin-user-file) + [[ "$seen_user_file" == false && $# -ge 2 ]] || usage_fail 'invalid --grafana-admin-user-file' + seen_user_file=true; grafana_user_file=$2; shift 2 + ;; + --grafana-admin-password-file) + [[ "$seen_password_file" == false && $# -ge 2 ]] || usage_fail 'invalid --grafana-admin-password-file' + seen_password_file=true; grafana_password_file=$2; shift 2 + ;; + --slack-webhook-file) + [[ "$seen_slack_file" == false && $# -ge 2 ]] || usage_fail 'invalid --slack-webhook-file' + seen_slack_file=true; slack_webhook_file=$2; shift 2 + ;; + --check-grafana-recovery-evidence) + [[ "$seen_check_grafana" == false ]] || usage_fail 'duplicate Grafana recovery check' + seen_check_grafana=true; check_grafana_requested=true; shift + ;; + --check-slack-recovery-evidence) + [[ "$seen_check_slack" == false ]] || usage_fail 'duplicate Slack recovery check' + seen_check_slack=true; check_slack_requested=true; shift + ;; + --accept-no-off-host-slack-recovery) + [[ "$seen_accept_slack_risk" == false ]] || usage_fail 'duplicate Slack risk acceptance' + seen_accept_slack_risk=true; accept_slack_risk_requested=true; shift + ;; + --check-slack-deployment-evidence) + [[ "$seen_check_slack_deployment" == false ]] || usage_fail 'duplicate Slack deployment check' + seen_check_slack_deployment=true; check_slack_deployment_requested=true; shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage_fail "unsupported argument: $1" + ;; + esac + done + + if [[ "$check_grafana_requested" == true || "$check_slack_requested" == true || + "$check_slack_deployment_requested" == true ]]; then + if [[ "$check_grafana_requested" == true ]]; then + [[ "$check_slack_requested" == false && "$check_slack_deployment_requested" == false ]] || + usage_fail 'select exactly one recovery evidence check' + elif [[ "$check_slack_requested" == true ]]; then + [[ "$check_slack_deployment_requested" == false ]] || + usage_fail 'select exactly one recovery evidence check' + fi + [[ "$execute_requested" == false && "$grafana_requested" == false && "$slack_requested" == false ]] || + usage_fail 'recovery evidence checks cannot be mixed with execute mode' + [[ "$accept_slack_risk_requested" == false ]] || + usage_fail 'Slack risk acceptance cannot be mixed with evidence checks' + [[ "$seen_user_file" == false && "$seen_password_file" == false && "$seen_slack_file" == false ]] || + usage_fail 'recovery evidence checks do not accept payload files' + return + fi + + if [[ "$execute_requested" == false ]]; then + [[ "$grafana_requested" == false && "$slack_requested" == false ]] || + usage_fail 'asset selection requires --execute' + [[ "$seen_user_file" == false && "$seen_password_file" == false && "$seen_slack_file" == false ]] || + usage_fail 'payload files require --execute and an asset selection' + [[ "$accept_slack_risk_requested" == false ]] || + usage_fail 'Slack risk acceptance requires the Slack execute interface' + return + fi + + [[ "$grafana_requested" == true || "$slack_requested" == true ]] || + usage_fail '--execute requires at least one asset selection' + if [[ "$grafana_requested" == true ]]; then + [[ "$seen_user_file" == true && "$seen_password_file" == true ]] || + usage_fail '--grafana-admin requires both Grafana input files' + [[ "$grafana_user_file" == /* && "$grafana_password_file" == /* ]] || + usage_fail 'Grafana input files must be absolute paths' + else + [[ "$seen_user_file" == false && "$seen_password_file" == false ]] || + usage_fail 'Grafana input files require --grafana-admin' + fi + if [[ "$slack_requested" == true ]]; then + [[ "$seen_slack_file" == true ]] || usage_fail '--slack-webhook requires its input file' + [[ "$slack_webhook_file" == /* ]] || usage_fail 'Slack webhook file must be an absolute path' + else + [[ "$seen_slack_file" == false ]] || usage_fail 'Slack webhook file requires --slack-webhook' + fi + if [[ "$accept_slack_risk_requested" == true ]]; then + [[ "$slack_requested" == true && "$grafana_requested" == false ]] || + usage_fail 'Slack risk acceptance is only available for the Slack-only execute interface' + fi +} + +require_commands() { + local command_name + for command_name in bash base64 cmp curl date install kubectl mktemp python3 sort stat sudo timeout wc; do + if [[ "$command_name" == kubectl ]]; then + [[ -x "$KUBECTL_BIN" ]] || command -v "$KUBECTL_BIN" >/dev/null 2>&1 || + plain_fail 'kubectl is required' + elif [[ "$command_name" == sudo ]]; then + [[ -x "$SUDO_BIN" ]] || command -v "$SUDO_BIN" >/dev/null 2>&1 || + plain_fail 'sudo is required' + else + command -v "$command_name" >/dev/null 2>&1 || plain_fail "$command_name is required" + fi + done + [[ -f "$ENCRYPTION_SCRIPT" && ! -L "$ENCRYPTION_SCRIPT" && -r "$ENCRYPTION_SCRIPT" ]] || + plain_fail 'k3s encryption validator is unavailable' + [[ -f "$RESTORE_SCRIPT" && ! -L "$RESTORE_SCRIPT" && -r "$RESTORE_SCRIPT" ]] || + plain_fail 'k3s restore evidence validator is unavailable' +} + +make_temporary_dir() { + umask 077 + temporary_dir="$(mktemp -d /tmp/platform-observability-secrets.XXXXXX)" || + plain_fail 'could not create private temporary directory' + [[ "$temporary_dir" == /tmp/platform-observability-secrets.?????? && ! -L "$temporary_dir" ]] || + plain_fail 'private temporary directory boundary is invalid' + chmod 0700 -- "$temporary_dir" +} + +validate_private_input_metadata() { + local file=$1 description=$2 metadata + [[ "$file" == /* && -f "$file" && ! -L "$file" && -O "$file" && -r "$file" && -s "$file" ]] || + plain_fail "$description must be a readable non-empty current-user-owned non-symlink regular file" + metadata="$(stat --format='%u:%a:%h:%F' -- "$file")" || plain_fail "cannot stat $description" + [[ "$metadata" == "${EUID}:600:1:regular file" ]] || + plain_fail "$description must be current-user-owned mode 0600 with one hard link" +} + +validate_selected_input_metadata() { + if [[ "$grafana_requested" == true ]]; then + validate_private_input_metadata "$grafana_user_file" 'Grafana admin user file' + validate_private_input_metadata "$grafana_password_file" 'Grafana admin password file' + fi + if [[ "$slack_requested" == true ]]; then + validate_private_input_metadata "$slack_webhook_file" 'Slack webhook file' + fi +} + +snapshot_private_input() { + local source=$1 destination=$2 description=$3 + python3 -I -S - "$source" "$destination" <<'PY' || +import os, stat, sys +source, destination = sys.argv[1:] +required = ("O_NOFOLLOW", "O_CLOEXEC") +if any(not hasattr(os, name) for name in required): + raise SystemExit(1) +source_fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) +try: + before = os.fstat(source_fd) + if ( + not stat.S_ISREG(before.st_mode) + or stat.S_IMODE(before.st_mode) != 0o600 + or before.st_uid != os.geteuid() + or before.st_size <= 0 + or before.st_nlink != 1 + ): + raise SystemExit(1) + chunks = [] + while True: + chunk = os.read(source_fd, 65536) + if not chunk: + break + chunks.append(chunk) + after = os.fstat(source_fd) + identity_before = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) + identity_after = (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns) + if identity_before != identity_after: + raise SystemExit(1) + payload = b"".join(chunks) + if len(payload) != before.st_size: + raise SystemExit(1) +finally: + os.close(source_fd) +destination_fd = os.open( + destination, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, + 0o600, +) +try: + view = memoryview(payload) + while view: + written = os.write(destination_fd, view) + if written <= 0: + raise SystemExit(1) + view = view[written:] +finally: + os.close(destination_fd) +PY + plain_fail "could not take a stable private snapshot of $description" +} + +capture_selected_inputs() { + if [[ "$grafana_requested" == true ]]; then + grafana_user_snapshot="$temporary_dir/input-grafana-user" + grafana_password_snapshot="$temporary_dir/input-grafana-password" + snapshot_private_input "$grafana_user_file" "$grafana_user_snapshot" 'Grafana admin user file' + snapshot_private_input "$grafana_password_file" "$grafana_password_snapshot" 'Grafana admin password file' + fi + if [[ "$slack_requested" == true ]]; then + slack_webhook_snapshot="$temporary_dir/input-slack-webhook" + snapshot_private_input "$slack_webhook_file" "$slack_webhook_snapshot" 'Slack webhook file' + fi +} + +recheck_selected_inputs() { + local second + if [[ "$grafana_requested" == true ]]; then + second="$temporary_dir/recheck-grafana-user" + snapshot_private_input "$grafana_user_file" "$second" 'Grafana admin user file' + cmp --silent -- "$grafana_user_snapshot" "$second" || + plain_fail 'Grafana admin user input changed during confirmation' + second="$temporary_dir/recheck-grafana-password" + snapshot_private_input "$grafana_password_file" "$second" 'Grafana admin password file' + cmp --silent -- "$grafana_password_snapshot" "$second" || + plain_fail 'Grafana admin password input changed during confirmation' + fi + if [[ "$slack_requested" == true ]]; then + second="$temporary_dir/recheck-slack-webhook" + snapshot_private_input "$slack_webhook_file" "$second" 'Slack webhook file' + cmp --silent -- "$slack_webhook_snapshot" "$second" || + plain_fail 'Slack webhook input changed during confirmation' + fi +} + +validate_admin_user_content() { + python3 -I -S - "$grafana_user_snapshot" <<'PY' +import pathlib, sys +data = pathlib.Path(sys.argv[1]).read_bytes() +if not data or any(byte < 0x20 or byte > 0x7e for byte in data): + raise SystemExit(1) +PY +} + +validate_admin_password_content() { + python3 -I -S - "$grafana_password_snapshot" <<'PY' +import pathlib, sys +data = pathlib.Path(sys.argv[1]).read_bytes() +if len(data) < 20 or b"\x00" in data or b"\r" in data or b"\n" in data: + raise SystemExit(1) +PY +} + +validate_slack_webhook_content() { + python3 -I -S - "$slack_webhook_snapshot" <<'PY' +import pathlib, re, sys +data = pathlib.Path(sys.argv[1]).read_bytes() +try: + value = data.decode("ascii") +except UnicodeDecodeError: + raise SystemExit(1) +pattern = r"https://hooks[.]slack[.]com/services/[A-Za-z0-9_-]+/[A-Za-z0-9_-]+/[A-Za-z0-9_-]+" +if re.fullmatch(pattern, value) is None: + raise SystemExit(1) +PY +} + +validate_selected_input_content() { + if [[ "$grafana_requested" == true ]]; then + validate_admin_user_content || plain_fail 'Grafana admin user must be one non-empty printable ASCII line' + validate_admin_password_content || plain_fail 'Grafana admin password must be at least 20 bytes with no NUL or line break' + fi + if [[ "$slack_requested" == true ]]; then + validate_slack_webhook_content || + plain_fail 'Slack webhook must have the exact hooks.slack.com/services prefix and three path components' + fi +} + +run_common_gates() { + local -a safe_environment=( + /usr/bin/env -i + "PATH=$VALIDATOR_PATH" + "HOME=$VALIDATOR_HOME" + ) + if [[ "$TEST_MODE" == true ]]; then + safe_environment+=( + "PLATFORM_TEST_COMMAND_LOG=${PLATFORM_TEST_COMMAND_LOG:-}" + "PLATFORM_TEST_ENCRYPTION_COUNT=${PLATFORM_TEST_ENCRYPTION_COUNT:-}" + "PLATFORM_TEST_RESTORE_COUNT=${PLATFORM_TEST_RESTORE_COUNT:-}" + "PLATFORM_TEST_ENCRYPTION_FAIL_AT=${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" + "PLATFORM_TEST_RESTORE_FAIL_AT=${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" + "PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE=${PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE:-}" + "PLATFORM_TEST_VALIDATOR_ENV_LOG=${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" + "PLATFORM_TEST_STATE=${PLATFORM_TEST_STATE:-}" + ) + fi + "${safe_environment[@]}" /usr/bin/bash "$ENCRYPTION_SCRIPT" --expect-reencrypted >/dev/null || + plain_fail 'k3s Secret encryption is not fully re-encrypted' + "${safe_environment[@]}" /usr/bin/bash "$RESTORE_SCRIPT" --check >/dev/null || + plain_fail 'current k3s restore evidence is unavailable' +} + +require_context_and_namespace() { + local context + context="$(kubectl_bounded config current-context)" || plain_fail 'cannot read current Kubernetes context' + [[ "$context" == "$EXPECTED_CONTEXT" ]] || plain_fail 'current Kubernetes context is not default' + kubectl_bounded get namespace "$NAMESPACE" >/dev/null || + plain_fail 'observability namespace is absent' +} + +require_current_context() { + local context + context="$(kubectl_bounded config current-context)" || + plain_fail 'cannot read current Kubernetes context' + [[ "$context" == "$EXPECTED_CONTEXT" ]] || + plain_fail 'current Kubernetes context is not default' +} + +kubectl_bounded() { + local process_timeout=$KUBECTL_PROCESS_TIMEOUT + [[ "${1:-}" != proxy ]] || process_timeout=$KUBECTL_PROXY_PROCESS_TIMEOUT + /usr/bin/timeout --signal=TERM --kill-after=1s "$process_timeout" \ + "$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" "$@" +} + +secret_exists() { + local name=$1 result + result="$(kubectl_bounded --namespace "$NAMESPACE" get secret "$name" \ + --ignore-not-found --output=name)" || return 2 + if [[ -z "$result" ]]; then return 1; fi + [[ "$result" == "secret/$name" ]] || return 2 + return 0 +} + +validate_secret_json() { + local json_file=$1 name=$2 expected_keys=$3 decoded_prefix=$4 expected_token=${5:-} + python3 -I -S - "$json_file" "$name" "$expected_keys" "$decoded_prefix" "$expected_token" <<'PY' +import base64, binascii, json, os, pathlib, re, sys +json_file, expected_name, expected_keys_raw, decoded_prefix, expected_token = sys.argv[1:] +try: + item = json.loads(pathlib.Path(json_file).read_text(encoding="utf-8")) +except Exception: + raise SystemExit(1) +expected_keys = expected_keys_raw.split(",") +metadata = item.get("metadata") or {} +data = item.get("data") +uid = metadata.get("uid") +annotations = metadata.get("annotations") or {} +if ( + item.get("apiVersion") != "v1" + or item.get("kind") != "Secret" + or item.get("type") != "Opaque" + or metadata.get("namespace") != "observability" + or metadata.get("name") != expected_name + or not isinstance(data, dict) + or sorted(data) != sorted(expected_keys) + or not isinstance(uid, str) + or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", uid) is None +): + raise SystemExit(1) +if expected_token and annotations.get("observability.hyeonworks.com/create-transaction") != expected_token: + raise SystemExit(1) +for key in expected_keys: + encoded = data.get(key) + if not isinstance(encoded, str): + raise SystemExit(1) + try: + decoded = base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError): + raise SystemExit(1) + destination = pathlib.Path(decoded_prefix + "." + key) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + descriptor = os.open(destination, flags, 0o600) + try: + os.write(descriptor, decoded) + finally: + os.close(descriptor) +print(uid, end="") +PY +} + +new_transaction_token() { + python3 -I -S -c 'import secrets; print(secrets.token_hex(16))' +} + +build_secret_manifest() { + local asset=$1 token=$2 destination=$3 name + case "$asset" in + grafana) name='grafana-admin' ;; + slack) name='alertmanager-slack-webhook' ;; + *) return 1 ;; + esac + python3 -I -S - "$asset" "$name" "$token" "$destination" \ + "$grafana_user_snapshot" "$grafana_password_snapshot" "$slack_webhook_snapshot" <<'PY' +import base64, json, os, pathlib, re, sys +asset, name, token, destination, user_file, password_file, slack_file = sys.argv[1:] +if re.fullmatch(r"[0-9a-f]{32}", token) is None: + raise SystemExit(1) +if asset == "grafana": + sources = {"admin-user": user_file, "admin-password": password_file} +elif asset == "slack": + sources = {"url": slack_file} +else: + raise SystemExit(1) +data = { + key: base64.b64encode(pathlib.Path(path).read_bytes()).decode("ascii") + for key, path in sources.items() +} +item = { + "apiVersion": "v1", + "kind": "Secret", + "type": "Opaque", + "metadata": { + "namespace": "observability", + "name": name, + "annotations": { + "observability.hyeonworks.com/create-transaction": token, + }, + }, + "data": data, +} +descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, 0o600) +try: + encoded = json.dumps(item, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n" + view = memoryview(encoded) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise SystemExit(1) + view = view[written:] +finally: + os.close(descriptor) +PY +} + +inspect_contract() { + local asset=$1 phase=$2 state_var=$3 uid_var=$4 + local name expected_keys json_file decoded_prefix uid + case "$asset" in + grafana) + name='grafana-admin'; expected_keys='admin-user,admin-password' + ;; + slack) + name='alertmanager-slack-webhook'; expected_keys='url' + ;; + *) return 1 ;; + esac + + if secret_exists "$name"; then + json_file="$temporary_dir/${phase}-${asset}.json" + decoded_prefix="$temporary_dir/${phase}-${asset}" + kubectl_bounded --namespace "$NAMESPACE" get secret "$name" --output=json >"$json_file" || + plain_fail "cannot read existing $asset Secret contract" + chmod 0600 -- "$json_file" + uid="$(validate_secret_json "$json_file" "$name" "$expected_keys" "$decoded_prefix")" || + plain_fail "existing $asset Secret has wrong type, keys, metadata, or encoding" + case "$asset" in + grafana) + cmp --silent -- "$grafana_user_snapshot" "$decoded_prefix.admin-user" || + plain_fail 'existing Grafana admin user differs; rotation was refused' + cmp --silent -- "$grafana_password_snapshot" "$decoded_prefix.admin-password" || + plain_fail 'existing Grafana admin password differs; rotation was refused' + ;; + slack) + cmp --silent -- "$slack_webhook_snapshot" "$decoded_prefix.url" || + plain_fail 'existing Slack webhook differs; rotation was refused' + ;; + esac + printf -v "$state_var" '%s' existing + printf -v "$uid_var" '%s' "$uid" + else + case $? in + 1) + printf -v "$state_var" '%s' absent + printf -v "$uid_var" '%s' '' + ;; + *) plain_fail "cannot determine $asset Secret presence" ;; + esac + fi +} + +read_confirmation() { + local prompt=$1 expected=$2 answer='' + printf '%s' "$prompt" >&2 + if [[ "$TEST_MODE" == true ]]; then + (( confirmation_index < ${#confirmation_answers[@]} )) || plain_fail 'confirmation input is missing' + answer="${confirmation_answers[$confirmation_index]}" + confirmation_index=$((confirmation_index + 1)) + else + [[ -t 0 ]] || plain_fail 'an interactive terminal is required for execute confirmation' + IFS= read -r answer + fi + [[ "$answer" == "$expected" ]] || plain_fail 'confirmation was not exact' +} + +now_utc() { + if [[ "$TEST_MODE" == true ]]; then printf '%s\n' "$TEST_NOW_UTC"; else date -u +%Y-%m-%dT%H:%M:%SZ; fi +} + +timestamp_epoch() { + local timestamp=$1 normalized + normalized="$(date -u -d "$timestamp" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" || return 1 + [[ "$normalized" == "$timestamp" ]] || return 1 + date -u -d "$timestamp" +%s +} + +now_epoch() { + timestamp_epoch "$(now_utc)" +} + +evidence_asset_fields() { + local asset=$1 path_var=$2 resource_var=$3 + case "$asset" in + grafana) + printf -v "$path_var" '%s' "$EVIDENCE_DIR/grafana-local.env" + printf -v "$resource_var" '%s' 'observability/grafana-admin' + ;; + slack) + printf -v "$path_var" '%s' "$EVIDENCE_DIR/slack.env" + printf -v "$resource_var" '%s' 'observability/alertmanager-slack-webhook' + ;; + *) return 1 ;; + esac +} + +sudo_refresh() { + "$SUDO_BIN" -v || plain_fail 'sudo credential refresh failed' + "$SUDO_BIN" -n /usr/bin/true || plain_fail 'narrow non-interactive sudo is unavailable' +} + +validate_recovery_evidence_stream() { + local file=$1 expected_resource=$2 timestamp_var=$3 + local -a lines=() + local parsed_timestamp marker_epoch current_epoch + mapfile -t lines <"$file" + [[ "$(wc -l <"$file" | tr -d '[:space:]')" == 4 && ${#lines[@]} -eq 4 ]] || return 1 + [[ "${lines[0]}" == "schema=$EVIDENCE_SCHEMA" ]] || return 1 + [[ "${lines[1]}" == "context=$EXPECTED_CONTEXT" ]] || return 1 + [[ "${lines[2]}" == "resource=$expected_resource" ]] || return 1 + [[ "${lines[3]}" =~ ^checked_at_utc=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)$ ]] || return 1 + parsed_timestamp="${BASH_REMATCH[1]}" + timestamp_epoch "$parsed_timestamp" >/dev/null || return 1 + printf -v "$timestamp_var" '%s' "$parsed_timestamp" +} + +validate_slack_risk_evidence_stream() { + local file=$1 timestamp_var=$2 + local -a lines=() + local parsed_timestamp marker_epoch current_epoch + mapfile -t lines <"$file" + [[ "$(wc -l <"$file" | tr -d '[:space:]')" == 8 && ${#lines[@]} -eq 8 ]] || return 1 + [[ "${lines[0]}" == "schema=$RISK_EVIDENCE_SCHEMA" ]] || return 1 + [[ "${lines[1]}" == "context=$EXPECTED_CONTEXT" ]] || return 1 + [[ "${lines[2]}" == 'resource=observability/alertmanager-slack-webhook' ]] || return 1 + [[ "${lines[3]}" == 'reason=off-host-slack-recovery-deferred' ]] || return 1 + [[ "${lines[4]}" == "accepted_by_uid=$RISK_ACCEPTED_BY_UID" ]] || return 1 + [[ "${lines[5]}" == 'approval_method=interactive-exact-confirmation' ]] || return 1 + [[ "${lines[6]}" == "approval_ref=$RISK_APPROVAL_REF" ]] || return 1 + [[ "${lines[7]}" =~ ^checked_at_utc=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)$ ]] || return 1 + parsed_timestamp="${BASH_REMATCH[1]}" + timestamp_epoch "$parsed_timestamp" >/dev/null || return 1 + marker_epoch="$(timestamp_epoch "$parsed_timestamp")" || return 1 + current_epoch="$(now_epoch)" || return 1 + (( marker_epoch <= current_epoch )) || return 1 + printf -v "$timestamp_var" '%s' "$parsed_timestamp" +} + +classify_slack_evidence_stream() { + local file=$1 kind_var=$2 timestamp_var=$3 timestamp + if validate_recovery_evidence_stream "$file" 'observability/alertmanager-slack-webhook' timestamp; then + printf -v "$kind_var" '%s' RECOVERY + elif validate_slack_risk_evidence_stream "$file" timestamp; then + printf -v "$kind_var" '%s' RISK_ACCEPTED + else + return 1 + fi + printf -v "$timestamp_var" '%s' "$timestamp" +} + +validate_evidence_stream() { + validate_recovery_evidence_stream "$@" +} + +root_marker_helper() { + local output=$1 failure_phase='' + shift + if [[ "${1:-}" == install && "$TEST_MODE" == true ]]; then + failure_phase=$MARKER_TEST_FAILURE_PHASE + fi + "$SUDO_BIN" -n /usr/bin/python3 -I -S - platform-observability-marker-helper "$@" \ + "$failure_phase" >"$output" 2>/dev/null <<'PY' +import errno +import os +import pathlib +import re +import stat +import sys + +( + tag, + action, + directory, + basename, + expected_uid_raw, + expected_gid_raw, + source, + expected_kind, + expected_source, + source_uid_raw, + failure_phase, +) = sys.argv[1:] +if tag != "platform-observability-marker-helper": + raise SystemExit(2) +if action not in {"check-dir", "ensure-dir", "read", "install", "unlink-exact"}: + raise SystemExit(2) +if failure_phase not in {"", "first-write", "pre-rename"}: + raise SystemExit(2) +if failure_phase and action != "install": + raise SystemExit(2) +if not directory.startswith("/") or "\x00" in directory: + raise SystemExit(2) +expected_uid = int(expected_uid_raw) +expected_gid = int(expected_gid_raw) +source_uid = int(source_uid_raw) +if min(expected_uid, expected_gid, source_uid) < 0: + raise SystemExit(2) +if action in {"check-dir", "ensure-dir"}: + if basename not in {"production-hierarchy", "fixture-hierarchy"}: + raise SystemExit(2) + if source or expected_kind or expected_source or failure_phase: + raise SystemExit(2) +else: + if basename not in {"grafana-local.env", "slack.env"}: + raise SystemExit(2) + fixture_marker_directory = re.fullmatch( + r"/tmp/platform-observability-secrets-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+/evidence-parent/platform/recovery-evidence", + directory, + ) + if directory == "/etc/hyeonworks/platform/recovery-evidence": + if expected_uid != 0 or expected_gid != 0: + raise SystemExit(2) + elif fixture_marker_directory is not None: + if expected_uid != source_uid: + raise SystemExit(2) + else: + raise SystemExit(2) +directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC + + +def open_directory(path): + if path == "/etc/hyeonworks/platform/recovery-evidence": + profile = "production-hierarchy" + anchor = "/etc" + components = ("hyeonworks", "platform", "recovery-evidence") + else: + fixture_suffix = "/evidence-parent/platform/recovery-evidence" + if not path.endswith(fixture_suffix): + raise OSError(errno.EINVAL, "invalid marker directory") + profile = "fixture-hierarchy" + anchor = path[: -len(fixture_suffix)] + components = ("evidence-parent", "platform", "recovery-evidence") + + descriptors = [open_hierarchy_anchor(anchor, profile)] + identities = [] + try: + for component in components: + parent_descriptor = descriptors[-1] + following = os.open(component, directory_flags, dir_fd=parent_descriptor) + try: + identity = attest_managed_directory(following) + attest_managed_entry(parent_descriptor, component, identity) + except BaseException: + os.close(following) + raise + descriptors.append(following) + identities.append(identity) + for index, component in enumerate(components): + attest_managed_entry(descriptors[index], component, identities[index]) + result = descriptors.pop() + for descriptor in reversed(descriptors): + os.close(descriptor) + return result + except BaseException: + for descriptor in reversed(descriptors): + os.close(descriptor) + raise + + +def require_managed_metadata(metadata): + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o700 + or metadata.st_uid != expected_uid + or metadata.st_gid != expected_gid + ): + raise OSError(errno.EPERM, "managed directory metadata mismatch") + + +def attest_managed_directory(descriptor): + metadata = os.fstat(descriptor) + require_managed_metadata(metadata) + return metadata.st_dev, metadata.st_ino + + +def attest_managed_entry(parent_descriptor, component, expected_identity): + metadata = os.stat(component, dir_fd=parent_descriptor, follow_symlinks=False) + require_managed_metadata(metadata) + if (metadata.st_dev, metadata.st_ino) != expected_identity: + raise OSError(errno.ESTALE, "managed directory identity changed") + + +def open_hierarchy_anchor(path, profile): + descriptor = os.open(path, directory_flags) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISDIR(metadata.st_mode): + raise OSError(errno.ENOTDIR, "hierarchy anchor is not a directory") + if profile == "production-hierarchy": + if metadata.st_uid != 0 or metadata.st_gid != 0: + raise OSError(errno.EPERM, "production hierarchy anchor metadata mismatch") + elif ( + stat.S_IMODE(metadata.st_mode) != 0o700 + or metadata.st_uid != expected_uid + or metadata.st_gid != expected_gid + ): + raise OSError(errno.EPERM, "fixture hierarchy anchor metadata mismatch") + return descriptor + except BaseException: + os.close(descriptor) + raise + + +def manage_evidence_hierarchy(profile, anchor, create): + if profile == "production-hierarchy": + if anchor != "/etc" or expected_uid != 0 or expected_gid != 0: + raise OSError(errno.EINVAL, "invalid production hierarchy profile") + components = ("hyeonworks", "platform", "recovery-evidence") + elif profile == "fixture-hierarchy": + if ( + expected_uid != source_uid + or re.fullmatch( + r"/tmp/platform-observability-secrets-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+", + anchor, + ) + is None + ): + raise OSError(errno.EINVAL, "invalid fixture hierarchy profile") + components = ("evidence-parent", "platform", "recovery-evidence") + else: + raise OSError(errno.EINVAL, "unknown hierarchy profile") + + descriptors = [open_hierarchy_anchor(anchor, profile)] + identities = [] + + def revalidate_opened_prefix(): + for index, opened_component in enumerate(components[: len(identities)]): + attest_managed_entry(descriptors[index], opened_component, identities[index]) + + try: + for component in components: + parent_descriptor = descriptors[-1] + needs_parent_fsync = False + try: + following = os.open(component, directory_flags, dir_fd=parent_descriptor) + except FileNotFoundError: + if not create: + revalidate_opened_prefix() + return 4 + try: + previous_umask = os.umask(0) + try: + os.mkdir(component, 0o700, dir_fd=parent_descriptor) + finally: + os.umask(previous_umask) + except FileExistsError: + needs_parent_fsync = True + else: + needs_parent_fsync = True + following = os.open(component, directory_flags, dir_fd=parent_descriptor) + try: + identity = attest_managed_directory(following) + attest_managed_entry(parent_descriptor, component, identity) + if needs_parent_fsync: + os.fsync(parent_descriptor) + attest_managed_entry(parent_descriptor, component, identity) + except BaseException: + os.close(following) + raise + descriptors.append(following) + identities.append(identity) + revalidate_opened_prefix() + return 0 + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def read_open_file(directory_fd, name): + descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=directory_fd) + try: + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or stat.S_IMODE(before.st_mode) != 0o600 + or before.st_uid != expected_uid + or before.st_gid != expected_gid + or before.st_nlink != 1 + ): + raise OSError(errno.EPERM, "file metadata mismatch") + chunks = [] + while True: + chunk = os.read(descriptor, 65536) + if not chunk: + break + chunks.append(chunk) + after = os.fstat(descriptor) + identity_before = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + before.st_nlink, + ) + identity_after = ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + after.st_nlink, + ) + if identity_before != identity_after: + raise OSError(errno.ESTALE, "file changed while open") + payload = b"".join(chunks) + if len(payload) != before.st_size: + raise OSError(errno.EIO, "short read") + return payload + finally: + os.close(descriptor) + + +def read_source(path): + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or stat.S_IMODE(before.st_mode) != 0o600 + or before.st_uid != source_uid + or before.st_nlink != 1 + ): + raise OSError(errno.EPERM, "source metadata mismatch") + chunks = [] + while True: + chunk = os.read(descriptor, 65536) + if not chunk: + break + chunks.append(chunk) + after = os.fstat(descriptor) + before_identity = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns, before.st_nlink) + after_identity = (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns, after.st_nlink) + if before_identity != after_identity: + raise OSError(errno.ESTALE, "source changed") + payload = b"".join(chunks) + if len(payload) != before.st_size: + raise OSError(errno.EIO, "short source read") + return payload + finally: + os.close(descriptor) + + +if action in {"check-dir", "ensure-dir"}: + raise SystemExit( + manage_evidence_hierarchy( + basename, + directory, + action == "ensure-dir", + ) + ) + +try: + directory_fd = open_directory(directory) +except FileNotFoundError: + if action == "read": + raise SystemExit(4) + raise +try: + if action == "read": + try: + payload = read_open_file(directory_fd, basename) + except FileNotFoundError: + raise SystemExit(4) + os.write(1, payload) + raise SystemExit(0) + + desired = read_source(source) + if expected_kind == "absent": + try: + read_open_file(directory_fd, basename) + except FileNotFoundError: + pass + else: + raise OSError(errno.EEXIST, "expected absent marker") + elif expected_kind == "file": + if read_open_file(directory_fd, basename) != read_source(expected_source): + raise OSError(errno.ESTALE, "marker precondition mismatch") + else: + raise OSError(errno.EINVAL, "invalid marker precondition") + + if action == "unlink-exact": + if read_open_file(directory_fd, basename) != desired: + raise OSError(errno.ESTALE, "unlink precondition mismatch") + os.unlink(basename, dir_fd=directory_fd) + os.fsync(directory_fd) + try: + os.stat(basename, dir_fd=directory_fd, follow_symlinks=False) + except FileNotFoundError: + raise SystemExit(0) + raise OSError(errno.EEXIST, "marker remained after unlink") + + temporary_name = f".{basename}.new.{os.getpid():x}" + temporary_fd = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, + dir_fd=directory_fd, + ) + temporary_exists = True + try: + try: + os.fchown(temporary_fd, expected_uid, expected_gid) + os.fchmod(temporary_fd, 0o600) + view = memoryview(desired) + while view: + if failure_phase == "first-write": + written = 0 + failure_phase = "" + else: + written = os.write(temporary_fd, view) + if written <= 0: + raise OSError(errno.EIO, "short marker write") + view = view[written:] + os.fsync(temporary_fd) + metadata = os.fstat(temporary_fd) + if metadata.st_nlink != 1: + raise OSError(errno.EPERM, "temporary marker link count mismatch") + if failure_phase == "pre-rename": + raise OSError(errno.EIO, "injected marker pre-rename failure") + finally: + os.close(temporary_fd) + os.replace(temporary_name, basename, src_dir_fd=directory_fd, dst_dir_fd=directory_fd) + temporary_exists = False + os.fsync(directory_fd) + if read_open_file(directory_fd, basename) != desired: + raise OSError(errno.EIO, "installed marker verification failed") + finally: + if temporary_exists: + try: + os.unlink(temporary_name, dir_fd=directory_fd) + except FileNotFoundError: + pass +finally: + os.close(directory_fd) +PY +} + +root_marker_read() { + local asset=$1 destination=$2 path resource rc + evidence_asset_fields "$asset" path resource || return 1 + if root_marker_helper "$destination" read "$EVIDENCE_DIR" "${path##*/}" \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" '' '' '' "$EUID"; then + chmod 0600 -- "$destination" + return 0 + else + rc=$? + rm -f -- "$destination" + return "$rc" + fi +} + +read_root_evidence() { + local asset=$1 destination=$2 resource path timestamp + evidence_asset_fields "$asset" path resource || return 1 + root_marker_read "$asset" "$destination" || return 1 + validate_evidence_stream "$destination" "$resource" timestamp +} + +check_recovery_evidence() { + local asset=$1 marker timestamp marker_epoch current_epoch age + require_current_context + sudo_refresh + marker="$temporary_dir/check-${asset}.env" + read_root_evidence "$asset" "$marker" || plain_fail "$asset recovery evidence is missing or malformed" + case "$asset" in + grafana) validate_evidence_stream "$marker" 'observability/grafana-admin' timestamp ;; + slack) validate_evidence_stream "$marker" 'observability/alertmanager-slack-webhook' timestamp ;; + esac || plain_fail "$asset recovery evidence is malformed" + marker_epoch="$(timestamp_epoch "$timestamp")" || plain_fail 'recovery timestamp is invalid' + current_epoch="$(now_epoch)" || plain_fail 'current UTC clock is invalid' + age=$((current_epoch - marker_epoch)) + (( age >= 0 && age <= EVIDENCE_MAX_AGE_SECONDS )) || + plain_fail "$asset recovery evidence is outside the 30-day window" + case "$asset" in + grafana) printf 'GRAFANA_RECOVERY_EVIDENCE=PASS\n' ;; + slack) printf 'SLACK_RECOVERY_EVIDENCE=PASS\n' ;; + esac +} + +check_slack_deployment_evidence() { + local marker kind timestamp marker_epoch current_epoch age + require_current_context + sudo_refresh + marker="$temporary_dir/check-slack-deployment.env" + root_marker_read slack "$marker" || plain_fail 'Slack deployment evidence is missing or malformed' + classify_slack_evidence_stream "$marker" kind timestamp || + plain_fail 'Slack deployment evidence is malformed' + if [[ "$kind" == RECOVERY ]]; then + marker_epoch="$(timestamp_epoch "$timestamp")" || plain_fail 'recovery timestamp is invalid' + current_epoch="$(now_epoch)" || plain_fail 'current UTC clock is invalid' + age=$((current_epoch - marker_epoch)) + (( age >= 0 && age <= EVIDENCE_MAX_AGE_SECONDS )) || + plain_fail 'Slack recovery evidence is outside the 30-day window' + printf 'SLACK_DEPLOYMENT_GATE=RECOVERY\n' + else + printf 'SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n' + fi +} + +ensure_evidence_directory() { + root_marker_helper /dev/null ensure-dir "$EVIDENCE_HIERARCHY_ANCHOR" \ + "$EVIDENCE_HIERARCHY_PROFILE" \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" '' '' '' "$EUID" || + plain_fail 'cannot safely create or verify the recovery evidence directory' +} + +preflight_evidence_directory() { + local rc + if root_marker_helper /dev/null check-dir "$EVIDENCE_HIERARCHY_ANCHOR" \ + "$EVIDENCE_HIERARCHY_PROFILE" \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" '' '' '' "$EUID"; then + return 0 + else + rc=$? + [[ "$rc" == 4 ]] || plain_fail 'recovery evidence directory boundary is unsafe or drifted' + fi +} + +prepare_marker() { + local asset=$1 kind=${2:-RECOVERY} path resource snapshot timestamp desired read_rc prior_kind + evidence_asset_fields "$asset" path resource || plain_fail 'unknown recovery evidence asset' + snapshot="$temporary_dir/prior-${asset}.env" + desired="$temporary_dir/desired-${asset}.env" + MARKER_PRIOR_FILE["$asset"]="$snapshot" + MARKER_DESIRED_FILE["$asset"]="$desired" + MARKER_KIND["$asset"]="$kind" + MARKER_HAD_PRIOR["$asset"]=false + + if root_marker_read "$asset" "$snapshot"; then + if [[ "$asset" == slack ]]; then + classify_slack_evidence_stream "$snapshot" prior_kind timestamp || + plain_fail 'Slack recovery marker schema drifted' + if [[ "$kind" == RISK && "$prior_kind" == RECOVERY ]]; then + plain_fail 'Slack recovery evidence cannot be downgraded to risk acceptance' + fi + if [[ "$kind" == RISK && "$prior_kind" == RISK_ACCEPTED ]]; then + cp -- "$snapshot" "$desired" + chmod 0600 -- "$desired" + MARKER_HAD_PRIOR["$asset"]=true + return + fi + else + validate_recovery_evidence_stream "$snapshot" "$resource" timestamp || + plain_fail "$asset recovery marker schema drifted" + fi + MARKER_HAD_PRIOR["$asset"]=true + else + read_rc=$? + [[ "$read_rc" == 4 ]] || plain_fail "cannot safely read $asset recovery marker" + fi + + if [[ "$kind" == RISK ]]; then + printf '%s\n' \ + "schema=$RISK_EVIDENCE_SCHEMA" \ + "context=$EXPECTED_CONTEXT" \ + "resource=$resource" \ + 'reason=off-host-slack-recovery-deferred' \ + "accepted_by_uid=$RISK_ACCEPTED_BY_UID" \ + 'approval_method=interactive-exact-confirmation' \ + "approval_ref=$RISK_APPROVAL_REF" \ + "checked_at_utc=$(now_utc)" >"$desired" + else + printf '%s\n' \ + "schema=$EVIDENCE_SCHEMA" \ + "context=$EXPECTED_CONTEXT" \ + "resource=$resource" \ + "checked_at_utc=$(now_utc)" >"$desired" + fi + chmod 0600 -- "$desired" + if [[ "$kind" == RISK ]]; then + validate_slack_risk_evidence_stream "$desired" timestamp || + plain_fail 'generated Slack risk acceptance evidence is invalid' + else + validate_recovery_evidence_stream "$desired" "$resource" timestamp || + plain_fail 'generated recovery evidence is invalid' + fi +} + +root_marker_install() { + local asset=$1 source=$2 expected_kind=$3 expected_source=${4:-} path resource + evidence_asset_fields "$asset" path resource || return 1 + root_marker_helper /dev/null install "$EVIDENCE_DIR" "${path##*/}" \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$source" "$expected_kind" \ + "$expected_source" "$EUID" +} + +marker_prestate_unchanged() { + local asset=$1 current="$temporary_dir/prestate-${asset}-$RANDOM" read_rc + if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]]; then + root_marker_read "$asset" "$current" || return 1 + cmp --silent -- "${MARKER_PRIOR_FILE[$asset]}" "$current" + else + if root_marker_read "$asset" "$current"; then + return 1 + else + read_rc=$? + [[ "$read_rc" == 4 ]] + fi + fi +} + +marker_record_written() { + local asset=$1 existing + for existing in "${WRITTEN_MARKERS[@]}"; do + [[ "$existing" != "$asset" ]] || return 0 + done + WRITTEN_MARKERS+=("$asset") +} + +clear_pending_marker() { + pending_marker_asset='' +} + +classify_pending_marker() { + local asset=$pending_marker_asset current read_rc + [[ -n "$asset" ]] || return 0 + current="$temporary_dir/classify-marker-${asset}-$RANDOM" + if root_marker_read "$asset" "$current"; then + if cmp --silent -- "${MARKER_DESIRED_FILE[$asset]}" "$current"; then + marker_record_written "$asset" + clear_pending_marker + return 0 + fi + if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]] && + cmp --silent -- "${MARKER_PRIOR_FILE[$asset]}" "$current"; then + clear_pending_marker + return 1 + fi + return 2 + else + read_rc=$? + if [[ "$read_rc" == 4 && "${MARKER_HAD_PRIOR[$asset]}" == false ]]; then + clear_pending_marker + return 1 + fi + return 2 + fi +} + +install_marker() { + local asset=$1 install_rc classification + if [[ "$asset" == slack && "${MARKER_HAD_PRIOR[$asset]}" == true ]] && + cmp --silent -- "${MARKER_PRIOR_FILE[$asset]}" "${MARKER_DESIRED_FILE[$asset]}"; then + return 0 + fi + marker_prestate_unchanged "$asset" || return 1 + pending_marker_asset=$asset + if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]]; then + root_marker_install "$asset" "${MARKER_DESIRED_FILE[$asset]}" file \ + "${MARKER_PRIOR_FILE[$asset]}" || install_rc=$? + else + root_marker_install "$asset" "${MARKER_DESIRED_FILE[$asset]}" absent '' || install_rc=$? + fi + if [[ -n "${install_rc:-}" ]]; then + classify_pending_marker + classification=$? + if [[ "$asset" == slack && "${MARKER_KIND[$asset]}" == RISK && + "$classification" == 0 ]]; then + transaction_active=false + emit_manual_recovery + printf 'ERROR: Slack risk marker installation outcome is uncertain\n' >&2 + exit "$install_rc" + fi + [[ "$classification" != 2 ]] || emit_manual_recovery + return "$install_rc" + fi + classify_pending_marker || return 1 +} + +restore_written_markers() { + local index asset current desired prior read_rc rc=0 + for ((index=${#WRITTEN_MARKERS[@]} - 1; index >= 0; index--)); do + asset="${WRITTEN_MARKERS[$index]}" + desired="${MARKER_DESIRED_FILE[$asset]}" + current="$temporary_dir/rollback-marker-${asset}-$RANDOM" + if ! root_marker_read "$asset" "$current" || ! cmp --silent -- "$desired" "$current"; then + rc=1 + continue + fi + if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]]; then + prior="${MARKER_PRIOR_FILE[$asset]}" + root_marker_install "$asset" "$prior" file "$desired" || rc=1 + else + root_marker_helper /dev/null unlink-exact "$EVIDENCE_DIR" \ + "$(case "$asset" in grafana) printf grafana-local.env ;; slack) printf slack.env ;; esac)" \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$desired" file "$desired" "$EUID" || rc=1 + fi + done + return "$rc" +} + +stop_proxy() { + local pid=$1 attempt + /bin/kill -TERM "$pid" 2>/dev/null || true + for ((attempt=0; attempt<20; attempt++)); do + if ! /bin/kill -0 "$pid" 2>/dev/null; then wait "$pid" 2>/dev/null || true; return 0; fi + /usr/bin/sleep 0.05 + done + /bin/kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true +} + +delete_secret_with_uid_precondition() { + local name=$1 uid=$2 socket options response proxy_log proxy_pid attempt http_code curl_rc=0 + [[ "$name" == grafana-admin || "$name" == alertmanager-slack-webhook ]] || return 1 + [[ "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1 + socket="$temporary_dir/proxy-${name}.sock" + options="$temporary_dir/delete-${name}.json" + response="$temporary_dir/delete-${name}.response" + proxy_log="$temporary_dir/proxy-${name}.log" + printf '{"apiVersion":"meta.k8s.io/v1","kind":"DeleteOptions","propagationPolicy":"Background","preconditions":{"uid":"%s"}}\n' \ + "$uid" >"$options" + chmod 0600 -- "$options" "$response" 2>/dev/null || true + + /usr/bin/timeout --signal=TERM --kill-after=1s "$KUBECTL_PROXY_PROCESS_TIMEOUT" \ + "$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" proxy \ + --unix-socket="$socket" \ + --api-prefix=/ \ + --accept-paths="^/api/v1/namespaces/${NAMESPACE}/secrets/${name}$" \ + --reject-methods='^(GET|POST|PUT|PATCH)$' >"$proxy_log" 2>&1 & + proxy_pid=$! + for ((attempt=0; attempt<50; attempt++)); do + [[ -S "$socket" ]] && break + /bin/kill -0 "$proxy_pid" 2>/dev/null || { wait "$proxy_pid" 2>/dev/null || true; return 1; } + /usr/bin/sleep 0.05 + done + [[ -S "$socket" ]] || { stop_proxy "$proxy_pid"; return 1; } + + http_code="$(curl --silent --show-error --max-time 5 \ + --unix-socket "$socket" \ + --output "$response" \ + --write-out '%{http_code}' \ + --request DELETE \ + --header 'Content-Type: application/json' \ + --data-binary "@$options" \ + "http://localhost/api/v1/namespaces/${NAMESPACE}/secrets/${name}")" || curl_rc=$? + stop_proxy "$proxy_pid" + (( curl_rc == 0 )) || return 1 + [[ "$http_code" == 200 || "$http_code" == 202 ]] || return 1 + if secret_exists "$name"; then return 1; else [[ $? == 1 ]]; fi +} + +rollback_owned_secrets() { + local index rc=0 + for ((index=${#OWNED_NAMES[@]} - 1; index >= 0; index--)); do + if ! delete_secret_with_uid_precondition "${OWNED_NAMES[$index]}" "${OWNED_UIDS[$index]}"; then + rc=1 + fi + done + return "$rc" +} + +rollback_transaction() { + local rc=0 + [[ "$rollback_in_progress" == false ]] || return 1 + rollback_in_progress=true + restore_written_markers || rc=1 + rollback_owned_secrets || rc=1 + if (( rc == 0 )); then + printf 'OBSERVABILITY_SECRET_ROLLBACK=PASS\n' >&2 + else + printf 'OBSERVABILITY_SECRET_ROLLBACK=FAIL\n' >&2 + emit_manual_recovery + fi + rollback_in_progress=false + return "$rc" +} + +clear_pending_create() { + pending_asset='' + pending_name='' + pending_token='' + pending_manifest='' + pending_response='' +} + +classify_pending_create() { + local live_json live_prefix response_prefix response_uid live_uid presence_rc expected_keys + [[ -n "$pending_asset" && -n "$pending_name" && -n "$pending_token" && -n "$pending_manifest" ]] || return 0 + case "$pending_asset" in + grafana) expected_keys='admin-user,admin-password' ;; + slack) expected_keys='url' ;; + *) return 2 ;; + esac + + if [[ -n "$pending_response" && -f "$pending_response" && ! -L "$pending_response" && -s "$pending_response" ]]; then + response_prefix="$temporary_dir/pending-${pending_asset}-response" + response_uid="$(validate_secret_json "$pending_response" "$pending_name" "$expected_keys" \ + "$response_prefix" "$pending_token")" || response_uid='' + if [[ -n "$response_uid" ]]; then + case "$pending_asset" in + grafana) + cmp --silent -- "$grafana_user_snapshot" "$response_prefix.admin-user" || response_uid='' + cmp --silent -- "$grafana_password_snapshot" "$response_prefix.admin-password" || response_uid='' + ;; + slack) + cmp --silent -- "$slack_webhook_snapshot" "$response_prefix.url" || response_uid='' + ;; + esac + fi + else + response_uid='' + fi + if secret_exists "$pending_name"; then + live_json="$temporary_dir/pending-${pending_asset}-live.json" + live_prefix="$temporary_dir/pending-${pending_asset}-live" + kubectl_bounded --namespace "$NAMESPACE" get secret "$pending_name" --output=json >"$live_json" || return 2 + chmod 0600 -- "$live_json" + live_uid="$(validate_secret_json "$live_json" "$pending_name" "$expected_keys" \ + "$live_prefix" "$pending_token")" || return 2 + [[ -n "$response_uid" && "$live_uid" == "$response_uid" ]] || return 2 + case "$pending_asset" in + grafana) + cmp --silent -- "$grafana_user_snapshot" "$live_prefix.admin-user" || return 2 + cmp --silent -- "$grafana_password_snapshot" "$live_prefix.admin-password" || return 2 + ;; + slack) cmp --silent -- "$slack_webhook_snapshot" "$live_prefix.url" || return 2 ;; + esac + OWNED_ASSETS+=("$pending_asset") + OWNED_NAMES+=("$pending_name") + OWNED_UIDS+=("$response_uid") + clear_pending_create + return 0 + else + presence_rc=$? + [[ "$presence_rc" == 1 ]] || return 2 + clear_pending_create + return 1 + fi +} + +transaction_fail() { + local message=$1 ambiguous=${2:-false} exit_status=${3:-1} classification + trap - ERR HUP INT TERM + set +e + if [[ -n "$pending_marker_asset" ]]; then + classify_pending_marker + classification=$? + [[ "$classification" != 2 ]] || ambiguous=true + fi + if [[ -n "$pending_asset" ]]; then + classify_pending_create + case $? in + 0|1) ;; + *) ambiguous=true ;; + esac + fi + [[ "$ambiguous" != true ]] || emit_manual_recovery + if [[ "$transaction_active" == true ]]; then rollback_transaction || true; fi + transaction_active=false + printf 'ERROR: %s\n' "$message" >&2 + exit "$exit_status" +} + +on_error() { + local rc=$1 line=$2 + transaction_fail "unexpected failure at line $line (exit $rc)" +} + +on_signal() { + local rc=$1 + transaction_fail "interrupted (exit $rc)" false "$rc" +} + +create_asset_secret() { + local asset=$1 name uid ambiguous=false response decoded_prefix create_rc=0 + case "$asset" in + grafana) name='grafana-admin' ;; + slack) name='alertmanager-slack-webhook' ;; + *) transaction_fail 'unknown Secret asset' ;; + esac + + pending_asset="$asset" + pending_name="$name" + pending_token="$(new_transaction_token)" || transaction_fail 'could not create transaction ownership token' + [[ "$pending_token" =~ ^[0-9a-f]{32}$ ]] || transaction_fail 'transaction ownership token is invalid' + pending_manifest="$temporary_dir/create-${asset}.json" + response="$temporary_dir/create-${asset}-response.json" + pending_response="$response" + decoded_prefix="$temporary_dir/create-${asset}-response" + build_secret_manifest "$asset" "$pending_token" "$pending_manifest" || + transaction_fail "could not build $asset Secret create request" + + if kubectl_bounded create --filename="$pending_manifest" --output=json >"$response"; then + : + else + create_rc=$? + transaction_fail "create outcome for $NAMESPACE/$name was not confirmed (exit $create_rc)" + fi + chmod 0600 -- "$response" + case "$asset" in + grafana) + uid="$(validate_secret_json "$response" "$name" 'admin-user,admin-password' \ + "$decoded_prefix" "$pending_token")" || transaction_fail 'Grafana create response contract is invalid' + cmp --silent -- "$grafana_user_snapshot" "$decoded_prefix.admin-user" || + transaction_fail 'Grafana create response user differs' + cmp --silent -- "$grafana_password_snapshot" "$decoded_prefix.admin-password" || + transaction_fail 'Grafana create response password differs' + ;; + slack) + uid="$(validate_secret_json "$response" "$name" url "$decoded_prefix" "$pending_token")" || + transaction_fail 'Slack create response contract is invalid' + cmp --silent -- "$slack_webhook_snapshot" "$decoded_prefix.url" || + transaction_fail 'Slack create response webhook differs' + ;; + esac + OWNED_ASSETS+=("$asset") + OWNED_NAMES+=("$name") + OWNED_UIDS+=("$uid") + clear_pending_create +} + +verify_reinspection() { + local asset=$1 expected_state=$2 expected_uid=$3 actual_state actual_uid + inspect_contract "$asset" postprompt actual_state actual_uid + [[ "$actual_state" == "$expected_state" && "$actual_uid" == "$expected_uid" ]] || + plain_fail "$asset Secret state changed before the last gate" +} + +execute_transaction() { + local grafana_state='' grafana_uid='' slack_state='' slack_uid='' + local post_state post_uid index asset + + validate_selected_input_metadata + run_common_gates + validate_selected_input_metadata + capture_selected_inputs + validate_selected_input_content + require_context_and_namespace + + if [[ "$grafana_requested" == true ]]; then inspect_contract grafana preprompt grafana_state grafana_uid; fi + if [[ "$slack_requested" == true ]]; then inspect_contract slack preprompt slack_state slack_uid; fi + + sudo_refresh + read_confirmation 'Type APPLY default: ' 'APPLY default' + if [[ "$grafana_requested" == true ]]; then + read_confirmation 'Type RECOVERY GRAFANA-LOCAL default: ' 'RECOVERY GRAFANA-LOCAL default' + fi + if [[ "$slack_requested" == true ]]; then + if [[ "$accept_slack_risk_requested" == true ]]; then + read_confirmation 'Type ACCEPT NO OFF-HOST SLACK RECOVERY default: ' \ + 'ACCEPT NO OFF-HOST SLACK RECOVERY default' + else + read_confirmation 'Type RECOVERY SLACK default: ' 'RECOVERY SLACK default' + fi + fi + + preflight_evidence_directory + if [[ "$grafana_requested" == true ]]; then prepare_marker grafana; fi + if [[ "$slack_requested" == true ]]; then + if [[ "$accept_slack_risk_requested" == true ]]; then + prepare_marker slack RISK + else + prepare_marker slack RECOVERY + fi + fi + recheck_selected_inputs + require_context_and_namespace + if [[ "$grafana_requested" == true ]]; then verify_reinspection grafana "$grafana_state" "$grafana_uid"; fi + if [[ "$slack_requested" == true ]]; then verify_reinspection slack "$slack_state" "$slack_uid"; fi + + run_common_gates + transaction_active=true + trap 'on_error "$?" "$LINENO"' ERR + trap 'on_signal 129' HUP + trap 'on_signal 130' INT + trap 'on_signal 143' TERM + + if [[ "$grafana_requested" == true && "$grafana_state" == absent ]]; then create_asset_secret grafana; fi + if [[ "$slack_requested" == true && "$slack_state" == absent ]]; then create_asset_secret slack; fi + + if [[ "$grafana_requested" == true ]]; then + inspect_contract grafana postcreate post_state post_uid + if [[ "$grafana_state" == absent ]]; then + [[ "$post_uid" == "${OWNED_UIDS[0]}" ]] || transaction_fail 'Grafana Secret UID changed after creation' true + else + [[ "$post_uid" == "$grafana_uid" ]] || transaction_fail 'existing Grafana Secret UID changed' true + fi + fi + if [[ "$slack_requested" == true ]]; then + inspect_contract slack postcreate post_state post_uid + if [[ "$slack_state" == absent ]]; then + for ((index=0; index<${#OWNED_ASSETS[@]}; index++)); do + if [[ "${OWNED_ASSETS[$index]}" == slack ]]; then + [[ "$post_uid" == "${OWNED_UIDS[$index]}" ]] || transaction_fail 'Slack Secret UID changed after creation' true + fi + done + else + [[ "$post_uid" == "$slack_uid" ]] || transaction_fail 'existing Slack Secret UID changed' true + fi + fi + + ensure_evidence_directory + if [[ "$grafana_requested" == true ]]; then install_marker grafana; fi + if [[ "$slack_requested" == true ]]; then install_marker slack; fi + + trap - ERR HUP INT TERM + transaction_active=false + if [[ "$grafana_requested" == true ]]; then + if [[ "$grafana_state" == absent ]]; then + printf 'OBSERVABILITY_GRAFANA_ADMIN=CREATE_CONFIRMED\n' + else + printf 'OBSERVABILITY_GRAFANA_ADMIN=REUSED_UNCHANGED\n' + fi + fi + if [[ "$slack_requested" == true ]]; then + if [[ "$slack_state" == absent ]]; then + printf 'OBSERVABILITY_SLACK_WEBHOOK=CREATE_CONFIRMED\n' + else + printf 'OBSERVABILITY_SLACK_WEBHOOK=REUSED_UNCHANGED\n' + fi + fi + printf 'OBSERVABILITY_SECRETS_TRANSACTION=PASS\n' +} + +observability_secrets_main() { + parse_arguments "$@" + + if [[ "$execute_requested" == false && "$check_grafana_requested" == false && + "$check_slack_requested" == false && "$check_slack_deployment_requested" == false ]]; then + printf '%s\n' \ + 'OBSERVABILITY_SECRETS_DRY_RUN=PASS' \ + 'CONTEXT=default' \ + 'GRAFANA_ADMIN_CONTRACT=observability/grafana-admin:Opaque:admin-user,admin-password' \ + 'SLACK_WEBHOOK_CONTRACT=observability/alertmanager-slack-webhook:Opaque:url' \ + 'MUTATION=NOT_REQUESTED' + return 0 + fi + + (( EUID != 0 )) || plain_fail 'run this script as the current user, not through sudo' + require_commands + make_temporary_dir + + if [[ "$check_grafana_requested" == true ]]; then + check_recovery_evidence grafana + elif [[ "$check_slack_requested" == true ]]; then + check_recovery_evidence slack + elif [[ "$check_slack_deployment_requested" == true ]]; then + check_slack_deployment_evidence + else + execute_transaction + fi +} + +platform_observability_secrets_fixture_main() { + local fixture_root=${1:-} + [[ "${BASH_SOURCE[0]}" != "$0" ]] || plain_fail 'test fixture entrypoint must be sourced' + [[ -n "$fixture_root" ]] || plain_fail 'test fixture root is required' + shift + TEST_MODE=true + configure_test_boundaries "$fixture_root" + observability_secrets_main "$@" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + reject_production_overrides + observability_secrets_main "$@" +fi diff --git a/scripts/bootstrap/create-phase1-secrets.sh b/scripts/bootstrap/create-phase1-secrets.sh new file mode 100755 index 0000000..83c05ab --- /dev/null +++ b/scripts/bootstrap/create-phase1-secrets.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +# 비밀값이 명령 추적 출력에 노출되지 않도록 호출자가 활성화한 xtrace도 끈다. +set +x + +readonly REQUIRED_CONFIRMATION="APPLY" +readonly DB_USERNAME="gitea" +readonly TOTAL_SECRET_CONTRACTS=3 +readonly -a SECRET_CONTRACTS=( + "platform-data/gitea-db-credentials" + "gitea/gitea-db-credentials" + "gitea/gitea-admin" +) + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'USAGE' +Usage: bash scripts/bootstrap/create-phase1-secrets.sh --execute + +Creates these Secrets only when all three are absent: + platform-data/gitea-db-credentials + gitea/gitea-db-credentials + gitea/gitea-admin + +When all three already exist, validates and reuses their data unchanged. The +script refuses a partial state and does not perform credential rotation. It may +repair only the cnpg.io/reload=true label on the platform-data DB Secret. +USAGE +} + +[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || { + usage + exit 2 +} + +command -v kubectl >/dev/null 2>&1 || fail "kubectl is required" +command -v base64 >/dev/null 2>&1 || fail "base64 is required" + +for namespace in platform-data gitea; do + kubectl get namespace "$namespace" >/dev/null 2>&1 || \ + fail "namespace ${namespace} does not exist; apply namespaces first" +done + +existing_secret_count=0 +for contract in "${SECRET_CONTRACTS[@]}"; do + namespace="${contract%%/*}" + name="${contract#*/}" + existing_resource="$( + kubectl --namespace "$namespace" get secret "$name" \ + --ignore-not-found --output=name + )" + if [[ -n "$existing_resource" ]]; then + ((existing_secret_count += 1)) + fi +done + +if (( existing_secret_count > 0 && existing_secret_count < TOTAL_SECRET_CONTRACTS )); then + fail "partial Phase 1 Secret state detected (${existing_secret_count}/${TOTAL_SECRET_CONTRACTS}); refusing creation or rotation" +fi + +secret_data_b64() { + local namespace="$1" + local name="$2" + local key="$3" + + kubectl --namespace "$namespace" get secret "$name" \ + --output="jsonpath={.data.${key}}" +} + +validate_basic_auth_secret() { + local namespace="$1" + local name="$2" + local secret_type + local username_b64 + local password_b64 + + secret_type="$( + kubectl --namespace "$namespace" get secret "$name" \ + --output='jsonpath={.type}' + )" + [[ "$secret_type" == "kubernetes.io/basic-auth" ]] || \ + fail "${namespace}/${name} must have type kubernetes.io/basic-auth" + + username_b64="$(secret_data_b64 "$namespace" "$name" username)" + password_b64="$(secret_data_b64 "$namespace" "$name" password)" + [[ -n "$username_b64" ]] || fail "${namespace}/${name} is missing non-empty data.username" + [[ -n "$password_b64" ]] || fail "${namespace}/${name} is missing non-empty data.password" +} + +if (( existing_secret_count == TOTAL_SECRET_CONTRACTS )); then + validate_basic_auth_secret platform-data gitea-db-credentials + validate_basic_auth_secret gitea gitea-db-credentials + validate_basic_auth_secret gitea gitea-admin + + platform_db_username_b64="$( + secret_data_b64 platform-data gitea-db-credentials username + )" + platform_db_password_b64="$( + secret_data_b64 platform-data gitea-db-credentials password + )" + gitea_db_username_b64="$( + secret_data_b64 gitea gitea-db-credentials username + )" + gitea_db_password_b64="$( + secret_data_b64 gitea gitea-db-credentials password + )" + expected_db_username_b64="$(printf '%s' "$DB_USERNAME" | base64)" + + [[ "$platform_db_username_b64" == "$expected_db_username_b64" ]] || \ + fail "platform-data/gitea-db-credentials username must be gitea" + [[ "$gitea_db_username_b64" == "$expected_db_username_b64" ]] || \ + fail "gitea/gitea-db-credentials username must be gitea" + [[ "$platform_db_username_b64" == "$gitea_db_username_b64" ]] || \ + fail "the two DB Secret usernames do not match" + [[ "$platform_db_password_b64" == "$gitea_db_password_b64" ]] || \ + fail "the two DB Secret passwords do not match" + + reload_label="$( + kubectl --namespace platform-data get secret gitea-db-credentials \ + --output='jsonpath={.metadata.labels.cnpg\.io/reload}' + )" + if [[ "$reload_label" != "true" ]]; then + kubectl --namespace platform-data label secret gitea-db-credentials \ + cnpg.io/reload=true --overwrite + printf 'Repaired cnpg.io/reload=true without changing Secret data.\n' + fi + + unset platform_db_username_b64 platform_db_password_b64 + unset gitea_db_username_b64 gitea_db_password_b64 expected_db_username_b64 + printf 'Existing Phase 1 Secret contracts are valid and were reused unchanged.\n' + printf 'No credential rotation was performed.\n' + exit 0 +fi + +[[ -t 0 ]] || fail "an interactive terminal is required for initial Secret creation" + +read_secret_twice() { + local prompt="$1" + local first + local second + + read -r -s -p "${prompt}: " first + printf '\n' >&2 + read -r -s -p "Confirm ${prompt}: " second + printf '\n' >&2 + + [[ "$first" == "$second" ]] || fail "the two values do not match" + (( ${#first} >= 16 )) || fail "use at least 16 characters" + printf '%s' "$first" +} + +read -r -p 'Gitea administrator username: ' admin_username +[[ "$admin_username" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || \ + fail "administrator username contains unsupported characters" + +db_password="$(read_secret_twice 'Gitea database password')" +admin_password="$(read_secret_twice 'Gitea administrator password')" + +secret_temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-phase1-secrets.XXXXXX")" +db_user_file="${secret_temp_dir}/db-username" +db_password_file="${secret_temp_dir}/db-password" +admin_user_file="${secret_temp_dir}/admin-username" +admin_password_file="${secret_temp_dir}/admin-password" + +cleanup() { + unset db_password admin_password + rm -f -- "$db_user_file" "$db_password_file" "$admin_user_file" "$admin_password_file" + rmdir -- "$secret_temp_dir" 2>/dev/null || true +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +umask 077 +printf '%s' "$DB_USERNAME" >"$db_user_file" +printf '%s' "$db_password" >"$db_password_file" +printf '%s' "$admin_username" >"$admin_user_file" +printf '%s' "$admin_password" >"$admin_password_file" +unset db_password admin_password + +printf '\nThe script will create exactly three Secrets. Type %s to continue: ' "$REQUIRED_CONFIRMATION" +read -r confirmation +[[ "$confirmation" == "$REQUIRED_CONFIRMATION" ]] || fail "cancelled" + +apply_basic_auth_secret() { + local namespace="$1" + local name="$2" + local username_file="$3" + local password_file="$4" + + kubectl --namespace "$namespace" create secret generic "$name" \ + --type=kubernetes.io/basic-auth \ + --from-file="username=${username_file}" \ + --from-file="password=${password_file}" \ + --dry-run=client \ + --output=yaml \ + | kubectl apply --filename=- +} + +apply_basic_auth_secret \ + platform-data gitea-db-credentials "$db_user_file" "$db_password_file" +kubectl --namespace platform-data label secret gitea-db-credentials \ + cnpg.io/reload=true --overwrite + +apply_basic_auth_secret \ + gitea gitea-db-credentials "$db_user_file" "$db_password_file" +apply_basic_auth_secret \ + gitea gitea-admin "$admin_user_file" "$admin_password_file" + +printf 'Initial Phase 1 Secret contracts were created. No values were written to the repository.\n' +printf 'This script does not perform credential rotation.\n' diff --git a/scripts/bootstrap/create-phase2-secrets.sh b/scripts/bootstrap/create-phase2-secrets.sh new file mode 100755 index 0000000..2747035 --- /dev/null +++ b/scripts/bootstrap/create-phase2-secrets.sh @@ -0,0 +1,323 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +# 비밀값이 명령 추적, argv 또는 표준 출력에 노출되지 않도록 한다. +set +x + +readonly REQUIRED_CONFIRMATION="APPLY PHASE 2 SECRETS" +readonly DB_USERNAME="keycloak" +readonly TOTAL_SECRET_CONTRACTS=4 +readonly -a SECRET_CONTRACTS=( + "platform-data/keycloak-db-credentials" + "keycloak/keycloak-db-credentials" + "aistor/minio-license" + "object-storage/aistor-root-configuration" +) + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'USAGE' +Usage: + bash scripts/bootstrap/create-phase2-secrets.sh \ + --license-file /absolute/path/to/minio.license \ + --execute + +Creates all four Phase 2 Secrets only when all four are absent. When all four +already exist, validates and reuses them unchanged. A partial state is refused. +This script never rotates credentials or the AIStor license. +USAGE +} + +license_file="" +execute_requested=false +while (( $# > 0 )); do + case "$1" in + --license-file) + (( $# >= 2 )) || fail "--license-file requires a path" + license_file="$2" + shift 2 + ;; + --execute) + execute_requested=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage >&2 + fail "unsupported argument: $1" + ;; + esac +done + +[[ "$execute_requested" == true ]] || { + usage >&2 + exit 2 +} +[[ -n "$license_file" ]] || fail "--license-file is required" +[[ "$license_file" == /* ]] || fail "--license-file must be an absolute path" +[[ -f "$license_file" && -r "$license_file" && -s "$license_file" ]] || \ + fail "license file must be a readable, non-empty regular file" + +license_payload="$(<"$license_file")" +license_payload="${license_payload%$'\r'}" +[[ "$license_payload" =~ ^eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$ ]] || \ + fail "license file must contain one decoded JWT value beginning with eyJ" +unset license_payload + +command -v kubectl >/dev/null 2>&1 || fail "kubectl is required" +command -v base64 >/dev/null 2>&1 || fail "base64 is required" +command -v cmp >/dev/null 2>&1 || fail "cmp is required" +command -v wc >/dev/null 2>&1 || fail "wc is required" + +for namespace in platform-data keycloak aistor object-storage; do + kubectl get namespace "$namespace" >/dev/null 2>&1 || \ + fail "namespace ${namespace} does not exist; apply Phase 2 namespaces first" +done + +existing_secret_count=0 +for contract in "${SECRET_CONTRACTS[@]}"; do + namespace="${contract%%/*}" + name="${contract#*/}" + existing_resource="$( + kubectl --namespace "$namespace" get secret "$name" \ + --ignore-not-found --output=name + )" + if [[ -n "$existing_resource" ]]; then + ((existing_secret_count += 1)) + fi +done + +if (( existing_secret_count > 0 && existing_secret_count < TOTAL_SECRET_CONTRACTS )); then + fail "partial Phase 2 Secret state detected (${existing_secret_count}/${TOTAL_SECRET_CONTRACTS}); refusing creation or rotation" +fi + +umask 077 +secret_temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-phase2-secrets.XXXXXX")" +db_user_file="${secret_temp_dir}/db-username" +db_password_file="${secret_temp_dir}/db-password" +root_config_file="${secret_temp_dir}/config.env" +existing_license_file="${secret_temp_dir}/existing-minio-license" +existing_root_config_file="${secret_temp_dir}/existing-config.env" +existing_platform_db_password_file="${secret_temp_dir}/existing-platform-db-password" +existing_keycloak_db_password_file="${secret_temp_dir}/existing-keycloak-db-password" + +cleanup() { + unset db_password root_user root_password + rm -f -- \ + "$db_user_file" "$db_password_file" "$root_config_file" \ + "$existing_license_file" "$existing_root_config_file" \ + "$existing_platform_db_password_file" \ + "$existing_keycloak_db_password_file" + rmdir -- "$secret_temp_dir" 2>/dev/null || true +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +secret_data_b64() { + local namespace="$1" + local name="$2" + local key="$3" + + kubectl --namespace "$namespace" get secret "$name" \ + --output="go-template={{ index .data \"${key}\" }}" +} + +validate_secret_type_and_keys() { + local namespace="$1" + local name="$2" + local expected_type="$3" + local expected_keys="$4" + local actual_type + local actual_keys + local actual_keys_sorted + local expected_keys_sorted + + actual_type="$( + kubectl --namespace "$namespace" get secret "$name" \ + --output='jsonpath={.type}' + )" + [[ "$actual_type" == "$expected_type" ]] || \ + fail "${namespace}/${name} must have type ${expected_type}" + + actual_keys="$( + kubectl --namespace "$namespace" get secret "$name" \ + --output='go-template={{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}' + )" + actual_keys_sorted="$(printf '%s\n' "$actual_keys" | LC_ALL=C sort)" + expected_keys_sorted="$(printf '%s\n' "$expected_keys" | LC_ALL=C sort)" + [[ "$actual_keys_sorted" == "$expected_keys_sorted" ]] || \ + fail "${namespace}/${name} has an unexpected data key set" +} + +validate_root_config_file() { + local file="$1" + + awk ' + BEGIN { + user_prefix = "export MINIO_ROOT_USER=\042" + password_prefix = "export MINIO_ROOT_PASSWORD=\042" + } + NR == 1 && index($0, user_prefix) == 1 && + substr($0, length($0), 1) == "\042" { + value = substr($0, length(user_prefix) + 1, + length($0) - length(user_prefix) - 1) + users++ + user = value + if (length(value) < 8 || value == "minioadmin" || index(value, "\042") > 0) bad = 1 + next + } + NR == 2 && index($0, password_prefix) == 1 && + substr($0, length($0), 1) == "\042" { + value = substr($0, length(password_prefix) + 1, + length($0) - length(password_prefix) - 1) + passwords++ + password = value + if (length(value) < 16 || value == "minioadmin" || index(value, "\042") > 0) bad = 1 + next + } + { bad = 1 } + END { + if (NR != 2 || users != 1 || passwords != 1 || + user == password || bad) exit 1 + } + ' "$file" || fail "object-storage/aistor-root-configuration has an invalid config.env contract" +} + +if (( existing_secret_count == TOTAL_SECRET_CONTRACTS )); then + validate_secret_type_and_keys \ + platform-data keycloak-db-credentials kubernetes.io/basic-auth $'password\nusername' + validate_secret_type_and_keys \ + keycloak keycloak-db-credentials kubernetes.io/basic-auth $'password\nusername' + validate_secret_type_and_keys aistor minio-license Opaque 'minio.license' + validate_secret_type_and_keys \ + object-storage aistor-root-configuration Opaque 'config.env' + + platform_username_b64="$(secret_data_b64 platform-data keycloak-db-credentials username)" + keycloak_username_b64="$(secret_data_b64 keycloak keycloak-db-credentials username)" + expected_username_b64="$(printf '%s' "$DB_USERNAME" | base64)" + + [[ "$platform_username_b64" == "$expected_username_b64" ]] || \ + fail "platform-data/keycloak-db-credentials username must be keycloak" + [[ "$keycloak_username_b64" == "$expected_username_b64" ]] || \ + fail "keycloak/keycloak-db-credentials username must be keycloak" + [[ "$platform_username_b64" == "$keycloak_username_b64" ]] || \ + fail "the two Keycloak DB Secret usernames do not match" + secret_data_b64 platform-data keycloak-db-credentials password \ + | base64 --decode >"$existing_platform_db_password_file" + secret_data_b64 keycloak keycloak-db-credentials password \ + | base64 --decode >"$existing_keycloak_db_password_file" + (( $(wc -c <"$existing_platform_db_password_file") >= 16 )) || \ + fail "platform-data/keycloak-db-credentials password must contain at least 16 bytes" + (( $(wc -c <"$existing_keycloak_db_password_file") >= 16 )) || \ + fail "keycloak/keycloak-db-credentials password must contain at least 16 bytes" + cmp --silent -- "$existing_platform_db_password_file" \ + "$existing_keycloak_db_password_file" || \ + fail "the two Keycloak DB Secret passwords do not match" + + reload_label="$( + kubectl --namespace platform-data get secret keycloak-db-credentials \ + --output='jsonpath={.metadata.labels.cnpg\.io/reload}' + )" + [[ "$reload_label" == "true" ]] || \ + fail "platform-data/keycloak-db-credentials must have cnpg.io/reload=true" + + secret_data_b64 aistor minio-license minio.license \ + | base64 --decode >"$existing_license_file" + [[ -s "$existing_license_file" ]] || \ + fail "aistor/minio-license has an empty minio.license payload" + cmp --silent -- "$license_file" "$existing_license_file" || \ + fail "the supplied license file differs from the existing Secret; rotation was not performed" + + secret_data_b64 object-storage aistor-root-configuration config.env \ + | base64 --decode >"$existing_root_config_file" + [[ -s "$existing_root_config_file" ]] || \ + fail "object-storage/aistor-root-configuration has an empty config.env payload" + validate_root_config_file "$existing_root_config_file" + + unset platform_username_b64 keycloak_username_b64 expected_username_b64 + printf 'Existing Phase 2 Secret contracts are valid and were reused unchanged.\n' + printf 'No credential or license rotation was performed.\n' + exit 0 +fi + +[[ -t 0 ]] || fail "an interactive terminal is required for initial Secret creation" + +read_secret_twice() { + local prompt="$1" + local minimum_length="$2" + local first + local second + + read -r -s -p "${prompt}: " first + printf '\n' >&2 + read -r -s -p "Confirm ${prompt}: " second + printf '\n' >&2 + + [[ "$first" == "$second" ]] || fail "the two values do not match" + (( ${#first} >= minimum_length )) || \ + fail "${prompt} must contain at least ${minimum_length} characters" + [[ "$first" != *$'\n'* && "$first" != *$'\r'* && "$first" != *"'"* ]] || \ + fail "${prompt} contains a character unsupported by config.env" + printf '%s' "$first" +} + +db_password="$(read_secret_twice 'Keycloak database password' 16)" +root_user="$(read_secret_twice 'AIStor root username' 8)" +root_password="$(read_secret_twice 'AIStor root password' 16)" + +[[ "$root_user" != "minioadmin" ]] || fail "do not use the default root username" +[[ "$root_password" != "minioadmin" ]] || fail "do not use the default root password" +[[ "$root_user" != "$root_password" ]] || fail "root username and password must differ" + +printf '%s' "$DB_USERNAME" >"$db_user_file" +printf '%s' "$db_password" >"$db_password_file" +printf 'export MINIO_ROOT_USER="%s"\n' "$root_user" >"$root_config_file" +printf 'export MINIO_ROOT_PASSWORD="%s"\n' "$root_password" >>"$root_config_file" +validate_root_config_file "$root_config_file" +unset db_password root_user root_password + +printf '\nThis will create exactly four Phase 2 Secrets. Type %s to continue: ' \ + "$REQUIRED_CONFIRMATION" +read -r confirmation +[[ "$confirmation" == "$REQUIRED_CONFIRMATION" ]] || fail "cancelled" + +create_secret_from_files() { + local namespace="$1" + local name="$2" + local type="$3" + shift 3 + + kubectl --namespace "$namespace" create secret generic "$name" \ + --type="$type" "$@" \ + --dry-run=client --output=yaml \ + | kubectl create --filename=- +} + +create_secret_from_files \ + platform-data keycloak-db-credentials kubernetes.io/basic-auth \ + --from-file="username=${db_user_file}" \ + --from-file="password=${db_password_file}" +kubectl --namespace platform-data label secret keycloak-db-credentials \ + cnpg.io/reload=true --overwrite + +create_secret_from_files \ + keycloak keycloak-db-credentials kubernetes.io/basic-auth \ + --from-file="username=${db_user_file}" \ + --from-file="password=${db_password_file}" +create_secret_from_files \ + aistor minio-license Opaque \ + --from-file="minio.license=${license_file}" +create_secret_from_files \ + object-storage aistor-root-configuration Opaque \ + --from-file="config.env=${root_config_file}" + +printf 'Initial Phase 2 Secret contracts were created without printing payloads.\n' +printf 'This script does not perform credential or license rotation.\n' diff --git a/scripts/bootstrap/open-k3s-local-recovery.sh b/scripts/bootstrap/open-k3s-local-recovery.sh new file mode 100644 index 0000000..0da73cc --- /dev/null +++ b/scripts/bootstrap/open-k3s-local-recovery.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +PATH='/usr/sbin:/usr/bin:/sbin:/bin' +export PATH +LC_ALL=C +export LC_ALL +umask 077 + +_k3slr_wrapper_initial_guard() { + local effective_uid="${1-}" shell_options="${2-}" + [[ "$effective_uid" =~ ^[0-9]+$ && "$effective_uid" != 0 ]] || return 1 + [[ "$shell_options" != *x* ]] +} + +if ! _k3slr_wrapper_initial_guard "${EUID:-}" "$-"; then + printf 'ERROR: lifecycle wrapper refuses root or xtrace execution\n' >&2 + return 1 2>/dev/null || exit 1 +fi + +if [[ "${BASH_SOURCE[0]}" == */* ]]; then + K3SLR_WRAPPER_DIRECTORY="${BASH_SOURCE[0]%/*}" +else + K3SLR_WRAPPER_DIRECTORY='.' +fi +K3SLR_WRAPPER_ROOT="$(cd -- "${K3SLR_WRAPPER_DIRECTORY}/../.." && pwd -P)" || { + return 1 2>/dev/null || exit 1 +} +K3SLR_WRAPPER_CONTRACT="${K3SLR_WRAPPER_ROOT}/infrastructure/security/k3s/local-recovery.env" +# shellcheck source=/dev/null +source "${K3SLR_WRAPPER_ROOT}/scripts/lib/k3s-local-recovery.sh" || { + return 1 2>/dev/null || exit 1 +} + +_k3slr_open_usage() { + printf 'Usage: bash scripts/bootstrap/open-k3s-local-recovery.sh [--execute]\n' >&2 +} + +_k3slr_open_main() { + local execution_mode + if ! _k3slr_parse_lifecycle_cli execution_mode "$@"; then + _k3slr_open_usage + return 2 + fi + _k3slr_lifecycle_main open "$execution_mode" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + _k3slr_open_main "$@" +fi diff --git a/scripts/bootstrap/prepare-k3s-local-recovery.sh b/scripts/bootstrap/prepare-k3s-local-recovery.sh new file mode 100644 index 0000000..cc42633 --- /dev/null +++ b/scripts/bootstrap/prepare-k3s-local-recovery.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +PATH='/usr/sbin:/usr/bin:/sbin:/bin' +export PATH +LC_ALL=C +export LC_ALL +umask 077 + +_k3slr_wrapper_initial_guard() { + local effective_uid="${1-}" shell_options="${2-}" + [[ "$effective_uid" =~ ^[0-9]+$ && "$effective_uid" != 0 ]] || return 1 + [[ "$shell_options" != *x* ]] +} + +if ! _k3slr_wrapper_initial_guard "${EUID:-}" "$-"; then + printf 'ERROR: lifecycle wrapper refuses root or xtrace execution\n' >&2 + return 1 2>/dev/null || exit 1 +fi + +unset -f _k3slr_prove_header_restore 2>/dev/null || : + +if [[ "${BASH_SOURCE[0]}" == */* ]]; then + K3SLR_WRAPPER_DIRECTORY="${BASH_SOURCE[0]%/*}" +else + K3SLR_WRAPPER_DIRECTORY='.' +fi +K3SLR_WRAPPER_ROOT="$(cd -- "${K3SLR_WRAPPER_DIRECTORY}/../.." && pwd -P)" || { + return 1 2>/dev/null || exit 1 +} +K3SLR_WRAPPER_CONTRACT="${K3SLR_WRAPPER_ROOT}/infrastructure/security/k3s/local-recovery.env" +# shellcheck source=/dev/null +source "${K3SLR_WRAPPER_ROOT}/scripts/lib/k3s-local-recovery.sh" || { + return 1 2>/dev/null || exit 1 +} + +_k3slr_prepare_usage() { + printf 'Usage: bash scripts/bootstrap/prepare-k3s-local-recovery.sh [--execute]\n' >&2 +} + +_k3slr_prepare_main() { + local execution_mode + if ! _k3slr_parse_lifecycle_cli execution_mode "$@"; then + _k3slr_prepare_usage + return 2 + fi + _k3slr_lifecycle_main prepare "$execution_mode" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + _k3slr_prepare_main "$@" +fi diff --git a/scripts/bootstrap/prepare-observability-local-paths.sh b/scripts/bootstrap/prepare-observability-local-paths.sh new file mode 100644 index 0000000..5e0f63e --- /dev/null +++ b/scripts/bootstrap/prepare-observability-local-paths.sh @@ -0,0 +1,564 @@ +#!/usr/bin/env bash + +# Intentionally sourceable. Privileged filesystem work crosses exactly one +# helper seam and runs descriptor-relative inside one process. + +_olp_error() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +_olp_target_rows() { + local base="$1" + printf '%s\t%s\t%s\n' \ + "$base/observability/prometheus" 1000 2000 \ + "$base/observability/grafana" 472 472 \ + "$base/observability/alertmanager" 1000 2000 \ + "$base/observability/alloy" 473 473 \ + "$base/observability/loki" 10001 10001 \ + "$base/observability/tempo" 10001 10001 +} + +_olp_helper_program() { + /usr/bin/cat <<'PY' +import errno +import os +import signal +import stat +import sys + +TARGETS = ( + ("prometheus", 1000, 2000), + ("grafana", 472, 472), + ("alertmanager", 1000, 2000), + ("alloy", 473, 473), + ("loki", 10001, 10001), + ("tempo", 10001, 10001), +) +MIN_AVAILABLE = 50 * 1024 * 1024 * 1024 +MAX_USE = 1024 * 1024 * 1024 +MAX_INTEGER = (1 << 63) - 1 +OPEN_DIR = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW +SIGNALS = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM) + + +class Rejected(Exception): + pass + + +class Interrupted(Exception): + def __init__(self, signum): + self.signum = signum + + +def reject(message): + raise Rejected(message) + + +if len(sys.argv) != 5 or sys.argv[1] not in {"preflight", "apply"}: + print("REJECT: helper argv differs", file=sys.stderr) + raise SystemExit(97) + +action, root_path, ssd_path, aistor_path = sys.argv[1:] + + +def normalized_absolute(path): + return os.path.isabs(path) and os.path.normpath(path) == path and "//" not in path + + +for candidate in (root_path, ssd_path, aistor_path): + if not normalized_absolute(candidate): + reject("path is not normalized absolute") + +fixture_root = os.environ.get("OLP_HELPER_FIXTURE_ROOT", "") +fixture = bool(fixture_root) and os.geteuid() != 0 +if fixture: + if not normalized_absolute(fixture_root): + reject("fixture root is invalid") + for candidate in (root_path, ssd_path, aistor_path): + if os.path.commonpath((fixture_root, candidate)) != fixture_root: + reject("fixture path escaped fixture root") +elif (root_path, ssd_path, aistor_path) != ("/", "/srv/k3s/ssd", "/srv/k3s/aistor"): + reject("production helper paths differ from exact contract") + + +def fixture_value(name, default=""): + return os.environ.get(name, default) if fixture else default + + +audit_path = fixture_value("OLP_HELPER_FIXTURE_AUDIT") +if audit_path and os.path.commonpath((fixture_root, audit_path)) != fixture_root: + reject("fixture audit escaped fixture root") + + +def audit(message): + if audit_path: + with open(audit_path, "a", encoding="utf-8") as stream: + stream.write(message + "\n") + + +def open_physical_absolute(path): + descriptor = os.open("/", OPEN_DIR) + try: + for component in path.split("/")[1:]: + if not component: + continue + child = os.open(component, OPEN_DIR, dir_fd=descriptor) + os.close(descriptor) + descriptor = child + return descriptor + except BaseException: + os.close(descriptor) + raise + + +def open_child(parent, name, missing_ok=False): + try: + return os.open(name, OPEN_DIR, dir_fd=parent) + except FileNotFoundError: + if missing_ok: + return None + raise + + +def identity(descriptor): + metadata = os.fstat(descriptor) + return metadata.st_dev, metadata.st_ino + + +def entry_identity(parent, name): + metadata = os.stat(name, dir_fd=parent, follow_symlinks=False) + if not stat.S_ISDIR(metadata.st_mode): + reject("entry is not a physical directory") + return metadata.st_dev, metadata.st_ino + + +def directory_use(descriptor): + total = os.fstat(descriptor).st_blocks * 512 + for name in os.listdir(descriptor): + metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if stat.S_ISLNK(metadata.st_mode): + reject("symbolic link below target is forbidden") + if stat.S_ISDIR(metadata.st_mode): + child = os.open(name, OPEN_DIR, dir_fd=descriptor) + try: + total += directory_use(child) + finally: + os.close(child) + else: + total += metadata.st_blocks * 512 + if total > MAX_USE: + return total + return total + + +def filesystem_snapshot(root_fd, ssd_fd, aistor_fd, obs_fd): + root_dev = os.fstat(root_fd).st_dev + if os.fstat(ssd_fd).st_dev != root_dev: + reject("SSD base is not on root filesystem") + aistor_dev = os.fstat(aistor_fd).st_dev + if fixture_value("OLP_HELPER_FIXTURE_VIRTUAL_AISTOR") == "1": + aistor_dev = root_dev + 1 + if aistor_dev == root_dev: + reject("AIStor must use another filesystem") + fs = os.fstatvfs(root_fd) + available = fs.f_bavail * fs.f_frsize + if available < 0 or available > MAX_INTEGER or available < MIN_AVAILABLE: + reject("root available bytes fail boundary") + uses = [] + if obs_fd is None: + uses = [0] * len(TARGETS) + else: + for name, _, _ in TARGETS: + target_fd = open_child(obs_fd, name, True) + if target_fd is None: + uses.append(0) + continue + try: + value = directory_use(target_fd) + if value < 0 or value > MAX_USE: + reject("individual target use exceeds 1GiB") + uses.append(value) + if os.listdir(target_fd): + reject("existing target is non-empty") + finally: + os.close(target_fd) + total = 0 + for value in uses: + if value < 0 or value > MAX_USE: + reject("individual target use exceeds 1GiB") + total += value + if total > MAX_USE: + reject("aggregate target use exceeds 1GiB") + return available, uses + + +root_fd = ssd_fd = aistor_fd = obs_fd = None +created = [] +restored = [] +active_created = None + + +def close_all(): + seen = set() + values = [record[2] for record in created] + values.extend(record[1] for record in restored) + values.extend((obs_fd, aistor_fd, ssd_fd, root_fd)) + for descriptor in values: + if descriptor is None or descriptor in seen: + continue + seen.add(descriptor) + try: + os.close(descriptor) + except OSError: + pass + + +def rollback(): + global active_created + failed = False + if active_created is not None: + parent_fd, name, expected = active_created + try: + if entry_identity(parent_fd, name) == expected: + descriptor = open_child(parent_fd, name) + try: + if os.listdir(descriptor): + failed = True + else: + os.rmdir(name, dir_fd=parent_fd) + audit(f"remove-active-created {name}") + finally: + os.close(descriptor) + else: + failed = True + except OSError: + failed = True + active_created = None + for parent_fd, name, target_fd, expected in reversed(created): + try: + if entry_identity(parent_fd, name) != expected: + failed = True + continue + if os.listdir(target_fd): + failed = True + continue + os.rmdir(name, dir_fd=parent_fd) + audit(f"remove-created {name}") + except OSError: + failed = True + for name, target_fd, uid, gid, mode, _expected in reversed(restored): + try: + os.fchown(target_fd, uid, gid) + audit(f"restore-chown {name} {uid} {gid}") + os.fchmod(target_fd, mode) + audit(f"restore-chmod {name} {mode:04o}") + except OSError: + failed = True + return not failed + + +def signal_handler(signum, _frame): + raise Interrupted(signum) + + +def block_signals(): + return signal.pthread_sigmask(signal.SIG_BLOCK, SIGNALS) + + +def restore_signal_mask(previous): + signal.pthread_sigmask(signal.SIG_SETMASK, previous) + + +def requested_owner(uid, gid): + if fixture_value("OLP_HELPER_FIXTURE_OWNERS") == "1": + return os.getuid(), os.getgid() + return uid, gid + + +def maybe_fixture_race(kind, name=""): + race = fixture_value("OLP_HELPER_FIXTURE_RACE") + external = fixture_value("OLP_HELPER_FIXTURE_EXTERNAL") + if not race: + return + if not external or os.path.commonpath((fixture_root, external)) != fixture_root: + reject("fixture race external path is invalid") + if kind == "ancestor" and race == "ancestor": + os.rename("observability", "observability.pinned", src_dir_fd=ssd_fd, dst_dir_fd=ssd_fd) + os.symlink(external, "observability", dir_fd=ssd_fd) + audit("fixture-race ancestor") + elif kind == "target" and race == f"target:{name}": + os.rename(name, f"{name}.pinned", src_dir_fd=obs_fd, dst_dir_fd=obs_fd) + os.symlink(external, name, dir_fd=obs_fd) + audit(f"fixture-race target {name}") + + +def maybe_fixture_signal(name): + configured = fixture_value("OLP_HELPER_FIXTURE_SIGNAL") + target = fixture_value("OLP_HELPER_FIXTURE_SIGNAL_TARGET") + if configured and target == name: + signum = {"HUP": signal.SIGHUP, "INT": signal.SIGINT, "TERM": signal.SIGTERM}.get(configured) + if signum is None: + reject("fixture signal is invalid") + os.kill(os.getpid(), signum) + + +exit_status = 0 +try: + root_fd = open_physical_absolute(root_path) + ssd_fd = open_physical_absolute(ssd_path) + aistor_fd = open_physical_absolute(aistor_path) + obs_fd = open_child(ssd_fd, "observability", True) + available, uses = filesystem_snapshot(root_fd, ssd_fd, aistor_fd, obs_fd) + if action == "preflight": + print("\t".join(["PREFLIGHT", str(available), *(str(value) for value in uses)])) + else: + for signum in SIGNALS: + signal.signal(signum, signal_handler) + if obs_fd is None: + previous = block_signals() + try: + os.mkdir("observability", 0o770, dir_fd=ssd_fd) + active_created = (ssd_fd, "observability", entry_identity(ssd_fd, "observability")) + obs_fd = open_child(ssd_fd, "observability") + if identity(obs_fd) != active_created[2]: + reject("new observability ancestor identity changed before open") + created.append((ssd_fd, "observability", obs_fd, identity(obs_fd))) + active_created = None + finally: + restore_signal_mask(previous) + obs_expected = identity(obs_fd) + maybe_fixture_race("ancestor") + for name, requested_uid, requested_gid in TARGETS: + target_fd = open_child(obs_fd, name, True) + if target_fd is None: + previous = block_signals() + try: + os.mkdir(name, 0o770, dir_fd=obs_fd) + active_created = (obs_fd, name, entry_identity(obs_fd, name)) + target_fd = open_child(obs_fd, name) + if identity(target_fd) != active_created[2]: + reject("new target identity changed before open") + created.append((obs_fd, name, target_fd, identity(target_fd))) + active_created = None + actual_uid, actual_gid = requested_owner(requested_uid, requested_gid) + os.fchown(target_fd, actual_uid, actual_gid) + os.fchmod(target_fd, 0o770) + audit(f"set-owner-mode {name} {requested_uid} {requested_gid} 0770") + finally: + restore_signal_mask(previous) + maybe_fixture_signal(name) + else: + metadata = os.fstat(target_fd) + if os.listdir(target_fd): + reject("existing target is non-empty") + restored.append(( + name, + target_fd, + metadata.st_uid, + metadata.st_gid, + stat.S_IMODE(metadata.st_mode), + identity(target_fd), + )) + maybe_fixture_race("target", name) + actual_uid, actual_gid = requested_owner(requested_uid, requested_gid) + os.fchown(target_fd, actual_uid, actual_gid) + os.fchmod(target_fd, 0o770) + audit(f"set-owner-mode {name} {requested_uid} {requested_gid} 0770") + metadata = os.fstat(target_fd) + actual_uid, actual_gid = requested_owner(requested_uid, requested_gid) + if (metadata.st_uid, metadata.st_gid, stat.S_IMODE(metadata.st_mode)) != (actual_uid, actual_gid, 0o770): + reject("post-create owner or mode differs") + for parent_fd, name, target_fd, expected in created: + if entry_identity(parent_fd, name) != expected: + reject("created entry identity changed") + for name, _target_fd, _uid, _gid, _mode, expected in restored: + if entry_identity(obs_fd, name) != expected: + reject("pre-existing target entry identity changed") + if entry_identity(ssd_fd, "observability") != obs_expected: + reject("observability ancestor identity changed") + print("APPLIED") +except Interrupted as error: + for signum in SIGNALS: + signal.signal(signum, signal.SIG_IGN) + rollback() + print(f"REJECT: interrupted by signal {error.signum}", file=sys.stderr) + exit_status = 128 + error.signum +except (Rejected, OSError) as error: + for signum in SIGNALS: + signal.signal(signum, signal.SIG_IGN) + rollback() + message = str(error) if str(error) else "privileged helper failed" + print(f"REJECT: {message}", file=sys.stderr) + exit_status = 23 +finally: + close_all() + +raise SystemExit(exit_status) +PY +} + +_olp_privileged_helper() { + if (( $# != 4 )) || [[ "$1" != preflight && "$1" != apply ]]; then + return 97 + fi + [[ "$2" == / && "$3" == /srv/k3s/ssd && "$4" == /srv/k3s/aistor ]] || \ + return 97 + local program + program="$(_olp_helper_program)" || return + /usr/bin/printf '%s\n' "$program" | \ + /usr/bin/sudo -- /usr/bin/python3 - "$@" +} + +_olp_root_facts() { + /usr/bin/findmnt --kernel --first-only --noheadings --output SOURCE,FSTYPE \ + --target "$1" | /usr/bin/awk 'NF == 2 { print $1 "\t" $2 }' +} + +_olp_available_bytes() { + /usr/bin/df --block-size=1 --output=avail "$1" | /usr/bin/tail -n 1 | \ + /usr/bin/tr -d '[:space:]' +} + +_olp_current_context() { + local kubectl_bin + kubectl_bin="$(command -v kubectl)" || return 1 + "$kubectl_bin" config current-context +} + +_olp_node_names() { + local kubectl_bin + kubectl_bin="$(command -v kubectl)" || return 1 + "$kubectl_bin" get nodes \ + -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}' +} + +_olp_read_confirmation() { + local value + printf 'Type APPLY default to prepare the six Local PV paths: ' >&2 + IFS= read -r value /dev/null)" || _olp_error "privileged filesystem preflight failed" || return + _olp_validate_preflight_reply "$reply" || \ + _olp_error "privileged preflight protocol or capacity boundary failed" || return + context="$(_olp_current_context)" || \ + _olp_error "could not read Kubernetes context" || return + [[ "$context" == default ]] || \ + _olp_error "Kubernetes context must be exactly default" || return + nodes="$(_olp_node_names)" || \ + _olp_error "could not read Kubernetes node names" || return + [[ "$nodes" == donghyeon-system-product-name ]] || \ + _olp_error "Kubernetes node must be exactly donghyeon-system-product-name" || return +} + +observability_local_paths_execute() { + local root_path="$1" ssd_base="$2" aistor_base="$3" confirmation reply + _olp_execute_preflight "$root_path" "$ssd_base" "$aistor_base" || return + confirmation="$(_olp_read_confirmation)" || \ + _olp_error "TTY confirmation was not read" || return + [[ "$confirmation" == 'APPLY default' ]] || \ + _olp_error "confirmation did not match APPLY default" || return + _olp_execute_preflight "$root_path" "$ssd_base" "$aistor_base" || return + reply="$(_olp_privileged_helper apply "$root_path" "$ssd_base" "$aistor_base")" || \ + _olp_error "descriptor-relative path preparation failed" || return + [[ "$reply" == APPLIED ]] || \ + _olp_error "privileged apply protocol differed" || return + printf 'Prepared six observability Local PV paths on the root filesystem.\n' + printf 'No Kubernetes resources were applied.\n' +} + +_olp_usage() { + printf 'Usage: bash scripts/bootstrap/prepare-observability-local-paths.sh [--execute]\n' >&2 +} + +_olp_main() { + if (( $# == 0 )); then + observability_local_paths_dry_run / /srv/k3s/ssd /srv/k3s/aistor + elif (( $# == 1 )) && [[ "$1" == --execute ]]; then + observability_local_paths_execute / /srv/k3s/ssd /srv/k3s/aistor + else + _olp_usage + return 2 + fi +} + +_olp_entry() ( + set -Eeuo pipefail + _olp_main "$@" +) + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + _olp_entry "$@" +fi diff --git a/scripts/bootstrap/prepare-ssd-local-paths.sh b/scripts/bootstrap/prepare-ssd-local-paths.sh new file mode 100755 index 0000000..94daf4f --- /dev/null +++ b/scripts/bootstrap/prepare-ssd-local-paths.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly SSD_BASE_PATH="/srv/k3s/ssd" +readonly POSTGRES_PATH="${SSD_BASE_PATH}/platform-postgres" +readonly GITEA_PATH="${SSD_BASE_PATH}/gitea" +readonly DECLARED_CAPACITY_BYTES=$((70 * 1024 * 1024 * 1024)) +readonly SUDO_BIN="/usr/bin/sudo" +readonly FINDMNT_BIN="/usr/bin/findmnt" +readonly INSTALL_BIN="/usr/bin/install" +readonly TEST_BIN="/usr/bin/test" + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +mount_source_for() { + # The Local PV directories are intentionally root:root 0750. An unprivileged + # findmnt cannot canonicalize a child below /srv/k3s/ssd after that directory + # has been created, so perform the mount lookup with the same privilege that + # creates and owns the paths. + "$SUDO_BIN" -- "$FINDMNT_BIN" --kernel --first-only \ + --noheadings --output SOURCE --target "$1" +} + +mount_target_for() { + "$SUDO_BIN" -- "$FINDMNT_BIN" --kernel --first-only \ + --noheadings --output TARGET --target "$1" +} + +sudo_test() { + "$SUDO_BIN" -- "$TEST_BIN" "$@" +} + +validate_existing_path_kind() { + local target="$1" + + if sudo_test -L "$target"; then + fail "refusing symbolic-link target: ${target}" + fi + + if sudo_test -e "$target"; then + sudo_test -d "$target" || \ + fail "target exists but is not a directory: ${target}" + return 0 + fi + + return 1 +} + +ensure_plain_directory() { + local target="$1" + + if validate_existing_path_kind "$target"; then + printf 'Already present; left ownership and mode unchanged: %s\n' "$target" + return 0 + fi + + "$SUDO_BIN" -- "$INSTALL_BIN" -d -o root -g root -m 0750 -- "$target" + validate_existing_path_kind "$target" || \ + fail "post-create target is missing: ${target}" + printf 'Created %s\n' "$target" +} + +for required_binary in "$SUDO_BIN" "$FINDMNT_BIN" "$INSTALL_BIN" "$TEST_BIN"; do + [[ -x "$required_binary" ]] || \ + fail "required executable is missing: ${required_binary}" +done +for command_name in df tail tr; do + command -v "$command_name" >/dev/null 2>&1 || fail "${command_name} is required" +done +[[ -d /srv ]] || fail "/srv does not exist" + +# Authenticate before command substitutions call sudo so a failed or cancelled +# prompt cannot be confused with an empty mount-source result. +printf 'Validating sudo access for SSD path checks...\n' +"$SUDO_BIN" -v || fail "sudo authentication failed" + +readonly ROOT_SOURCE="$(mount_source_for /)" +readonly SRV_SOURCE="$(mount_source_for /srv)" +readonly SRV_MOUNT_TARGET="$(mount_target_for /srv)" + +[[ -n "$ROOT_SOURCE" ]] || fail "could not identify the root filesystem source" +[[ "$SRV_SOURCE" == "$ROOT_SOURCE" ]] || fail "/srv is not on the root filesystem (${SRV_SOURCE} != ${ROOT_SOURCE})" +[[ "$SRV_MOUNT_TARGET" == "/" ]] || fail "/srv is covered by a separate mount (${SRV_MOUNT_TARGET})" + +for target in "$SSD_BASE_PATH" "$POSTGRES_PATH" "$GITEA_PATH"; do + if validate_existing_path_kind "$target"; then + [[ "$(mount_source_for "$target")" == "$ROOT_SOURCE" ]] || fail "${target} is not on the root filesystem" + [[ "$(mount_target_for "$target")" == "/" ]] || fail "${target} is covered by a separate mount" + fi +done + +readonly ROOT_AVAILABLE_BYTES="$(df --block-size=1 --output=avail / | tail -n 1 | tr -d '[:space:]')" +if [[ "$ROOT_AVAILABLE_BYTES" =~ ^[0-9]+$ ]] && (( ROOT_AVAILABLE_BYTES < DECLARED_CAPACITY_BYTES )); then + printf 'WARNING: root filesystem has less than 70 GiB free; Local PV capacity is not a quota.\n' >&2 +fi + +ensure_plain_directory "$SSD_BASE_PATH" +ensure_plain_directory "$POSTGRES_PATH" +ensure_plain_directory "$GITEA_PATH" + +for target in "$SSD_BASE_PATH" "$POSTGRES_PATH" "$GITEA_PATH"; do + validate_existing_path_kind "$target" || fail "post-create target is missing: ${target}" + [[ "$(mount_source_for "$target")" == "$ROOT_SOURCE" ]] || fail "post-create filesystem check failed for ${target}" + [[ "$(mount_target_for "$target")" == "/" ]] || fail "post-create mount check failed for ${target}" +done + +printf 'SSD Local PV paths are ready on root source %s.\n' "$ROOT_SOURCE" +printf 'No Kubernetes resources were applied.\n' diff --git a/scripts/bootstrap/prove-k3s-local-recovery-header.sh b/scripts/bootstrap/prove-k3s-local-recovery-header.sh new file mode 100644 index 0000000..3909a3f --- /dev/null +++ b/scripts/bootstrap/prove-k3s-local-recovery-header.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +PATH='/usr/sbin:/usr/bin:/sbin:/bin' +export PATH +LC_ALL=C +export LC_ALL +umask 077 + +_k3slrhp_inherited_namespace_is_clean() { + if compgen -A function _k3slrh_ >/dev/null; then return 1; fi + if compgen -A variable K3SLRH_ >/dev/null; then return 1; fi + if compgen -A variable _K3SLRH_ >/dev/null; then return 1; fi + if compgen -A variable K3SLR_ >/dev/null; then return 1; fi +} + +if ! _k3slrhp_inherited_namespace_is_clean; then + return 1 2>/dev/null || exit 1 +fi + +_k3slrh_wrapper_initial_guard() { + local effective_uid="${1-}" shell_options="${2-}" + (( $# == 2 )) || return 1 + [[ "$effective_uid" =~ ^[0-9]+$ && "$effective_uid" != 0 ]] || return 1 + [[ "$shell_options" != *x* ]] +} + +if ! _k3slrh_wrapper_initial_guard "${EUID:-}" "$-"; then + return 1 2>/dev/null || exit 1 +fi + +if [[ "${BASH_SOURCE[0]}" == */* ]]; then + K3SLRH_WRAPPER_DIRECTORY="${BASH_SOURCE[0]%/*}" +else + K3SLRH_WRAPPER_DIRECTORY='.' +fi +K3SLRH_REPOSITORY_ROOT="$(cd -- "${K3SLRH_WRAPPER_DIRECTORY}/../.." && pwd -P)" || { + return 1 2>/dev/null || exit 1 +} +K3SLRH_CONTRACT_PATH="${K3SLRH_REPOSITORY_ROOT}/infrastructure/security/k3s/local-recovery.env" +readonly K3SLRH_WRAPPER_DIRECTORY K3SLRH_REPOSITORY_ROOT K3SLRH_CONTRACT_PATH + +# shellcheck source=/dev/null +builtin source "${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery.sh" || { + return 1 2>/dev/null || exit 1 +} +# shellcheck source=/dev/null +builtin source "${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery-a1.sh" || { + return 1 2>/dev/null || exit 1 +} +# shellcheck source=/dev/null +builtin source "${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery-header-proof.sh" || { + return 1 2>/dev/null || exit 1 +} + +_k3slr_load_contract "$K3SLRH_CONTRACT_PATH" || { + return 1 2>/dev/null || exit 1 +} +readonly K3SLR_SCHEMA_VERSION K3SLR_RECOVERY_DISK_BY_ID K3SLR_RECOVERY_PARTITION_BY_ID +readonly K3SLR_RECOVERY_FS_UUID K3SLR_RECOVERY_PARTUUID K3SLR_RECOVERY_MODEL +readonly K3SLR_RECOVERY_SERIAL K3SLR_RECOVERY_WWN K3SLR_K3S_DISK_BY_ID +readonly K3SLR_K3S_PARTITION_BY_ID K3SLR_K3S_FS_UUID K3SLR_K3S_PARTUUID +readonly K3SLR_K3S_MODEL K3SLR_K3S_SERIAL K3SLR_K3S_WWN K3SLR_OWNER_UID +readonly K3SLR_OWNER_GID K3SLR_OUTER_MOUNT K3SLR_INNER_MOUNT K3SLR_ROOT_RELATIVE +readonly K3SLR_DATABASE_RELATIVE K3SLR_CONTAINER_RELATIVE K3SLR_RUNTIME_METADATA_RELATIVE +readonly K3SLR_MAPPING_NAME K3SLR_PROOF_MAPPING_NAME K3SLR_INNER_LABEL +readonly K3SLR_CONTAINER_SIZE_BYTES K3SLR_MINIMUM_FREE_BYTES K3SLR_OUTER_MIN_REMAINING_PERCENT + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + _k3slrh_header_proof_main "$@" +fi diff --git a/scripts/lib/k3s-local-recovery-a1.sh b/scripts/lib/k3s-local-recovery-a1.sh new file mode 100644 index 0000000..c8d6ed3 --- /dev/null +++ b/scripts/lib/k3s-local-recovery-a1.sh @@ -0,0 +1,1142 @@ +#!/usr/bin/env bash + +# Source-only strict A1 recovery payload verifier. + +readonly K3SLRA1_MANIFEST_MAX_BYTES=65536 +readonly K3SLRA1_PATH_MAX_BYTES=512 +readonly K3SLRA1_MOUNT_RECORD_MAX_BYTES=4096 +readonly K3SLRA1_TARGET_COUNT=80 +readonly K3SLRA1_TARGET_BYTES=22857829 +readonly K3SLRA1_METADATA_FORMAT='%d:%i|%h|%s|%y|%F|%u:%g|%a' +readonly K3SLRA1_CANONICAL_DATABASE='./datastore/state.db' +readonly K3SLRA1_LEGACY_DATABASE='./rootfs/var/lib/rancher/k3s/server/db/state.db' +readonly K3SLRA1_MAIN_MAPPER='/dev/mapper/k3s-recovery' +readonly K3SLRA1_PROOF_MAPPER='/dev/mapper/k3s-recovery-proof' +readonly K3SLRA1_MAIN_MAPPING_NAME=k3s-recovery +readonly K3SLRA1_PROOF_MAPPING_NAME=k3s-recovery-proof +readonly K3SLRA1_SQLITE_QUERY='PRAGMA query_only=ON; PRAGMA quick_check;' + +_k3slra1_command() { + "$@" +} + +_k3slra1_output_name_is_safe() { + local output_name="${1-}" + (( $# == 1 )) || return 1 + [[ "$output_name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || return 1 + [[ "$output_name" != _k3slra1_* ]] +} + +_k3slra1_metadata() { + local path="${1-}" output_name="${2-}" line='' + (( $# == 2 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + [[ "$output_name" != path ]] || return 1 + [[ "$output_name" != output_name ]] || return 1 + [[ "$output_name" != line ]] || return 1 + _k3slra1_packet_line metadata "$path" line || return 1 + printf -v "$output_name" '%s' "$line" +} + +_k3slra1_canonical_path() { + local path="${1-}" output_name="${2-}" line='' + (( $# == 2 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + [[ "$output_name" != path ]] || return 1 + [[ "$output_name" != output_name ]] || return 1 + [[ "$output_name" != line ]] || return 1 + _k3slra1_packet_line readlink "$path" line || return 1 + printf -v "$output_name" '%s' "$line" +} + +_k3slra1_metadata_packet() { + local path="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/stat --format="$K3SLRA1_METADATA_FORMAT" -- "$path" 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1_fd_metadata_packet() { + local path="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/stat --dereference --format="$K3SLRA1_METADATA_FORMAT" -- "$path" 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1_mapper_stat_packet() { + local mapper="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/stat --format='%F|%Hr:%Lr' -- "$mapper" 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1_size_stat_packet() { + local path="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/stat --format='%s' -- "$path" 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1_wc_packet() { + local path="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/wc -c -- "$path" 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1_od_packet() { + local path="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/od -An -v -tx1 -- "$path" 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1_sha256_packet() { + local path="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/sha256sum -- "$path" 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1_findmnt_packet() { + local mount_root="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/findmnt --noheadings --raw --mountpoint "$mount_root" --output SOURCE,TARGET,FSTYPE,VFS-OPTIONS,FS-OPTIONS,MAJ:MIN 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1_readlink_packet() { + local mapper="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/readlink --canonicalize-existing -- "$mapper" 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1_lsblk_packet() { + local mapper="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/lsblk --noheadings --nodeps --raw --output MAJ:MIN -- "$mapper" 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1_blockdev_packet() { + local mapper="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/sbin/blockdev --getro "$mapper" 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1_stderr_marker() { + local chunk='' read_rc=0 + if IFS= read -r -d '' -n 1 chunk; then + read_rc=0 + else + read_rc=$? + fi + if (( read_rc == 0 )); then + printf '\0STDERR\0' + while IFS= read -r -d '' -n 4096 chunk; do + : + done + else + if [[ -n "$chunk" ]]; then + printf '\0STDERR\0' + fi + fi +} + +_k3slra1_packet_value() { + local producer="${1-}" argument="${2-}" output_name="${3-}" + local -a records=() + (( $# == 3 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + [[ "$output_name" != producer ]] || return 1 + [[ "$output_name" != argument ]] || return 1 + [[ "$output_name" != output_name ]] || return 1 + [[ "$output_name" != records ]] || return 1 + case "$producer" in + findmnt) mapfile -d '' -t records < <(_k3slra1_findmnt_packet "$argument") ;; + metadata) mapfile -d '' -t records < <(_k3slra1_metadata_packet "$argument") ;; + fd-metadata) mapfile -d '' -t records < <(_k3slra1_fd_metadata_packet "$argument") ;; + mapper-stat) mapfile -d '' -t records < <(_k3slra1_mapper_stat_packet "$argument") ;; + size-stat) mapfile -d '' -t records < <(_k3slra1_size_stat_packet "$argument") ;; + wc) mapfile -d '' -t records < <(_k3slra1_wc_packet "$argument") ;; + od) mapfile -d '' -t records < <(_k3slra1_od_packet "$argument") ;; + sha256) mapfile -d '' -t records < <(_k3slra1_sha256_packet "$argument") ;; + readlink) mapfile -d '' -t records < <(_k3slra1_readlink_packet "$argument") ;; + lsblk) mapfile -d '' -t records < <(_k3slra1_lsblk_packet "$argument") ;; + blockdev) mapfile -d '' -t records < <(_k3slra1_blockdev_packet "$argument") ;; + *) return 1 ;; + esac + (( ${#records[@]} == 2 )) || return 1 + [[ "${records[1]}" == RC=0 ]] || return 1 + printf -v "$output_name" '%s' "${records[0]}" +} + +_k3slra1_packet_line() { + local producer="${1-}" argument="${2-}" output_name="${3-}" value='' + (( $# == 3 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + case "$output_name" in producer|argument|output_name|value) return 1 ;; esac + _k3slra1_packet_value "$producer" "$argument" value || return 1 + [[ "$value" == *$'\n' ]] || return 1 + [[ "$value" != *$'\r'* ]] || return 1 + value="${value%$'\n'}" + [[ -n "$value" ]] || return 1 + [[ "$value" != *$'\n'* ]] || return 1 + printf -v "$output_name" '%s' "$value" +} + +_k3slra1_option_has() { + local options="${1-}" wanted="${2-}" option + local -a option_list=() + (( $# == 2 )) || return 1 + IFS=, read -r -a option_list <<<"$options" + for option in "${option_list[@]}"; do + [[ "$option" == "$wanted" ]] && return 0 + done + return 1 +} + +_k3slra1_expected_mapper() { + local role="${1-}" output_name="${2-}" value='' + (( $# == 2 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + [[ "$output_name" != role ]] || return 1 + [[ "$output_name" != output_name ]] || return 1 + [[ "$output_name" != value ]] || return 1 + case "$role" in + proof) value="$K3SLRA1_PROOF_MAPPER" ;; + original-baseline|original-final) value="$K3SLRA1_MAIN_MAPPER" ;; + *) return 1 ;; + esac + printf -v "$output_name" '%s' "$value" +} + +_k3slra1_capture_mount_mapping_evidence() { + local mount_root="${1-}" role="${2-}" output_name="${3-}" + local expected_mapper='' mount_record='' mount_line='' source='' target='' fstype='' + local vfs_options='' fs_options='' mount_major_minor='' extra='' + local source_canonical='' canonical_mapper='' mapper_major_minor='' mapper_readonly='' mapper_stat='' + (( $# == 3 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + case "$output_name" in + mount_root|role|output_name|expected_mapper|mount_record|mount_line|source|target|fstype|vfs_options|fs_options|mount_major_minor|extra|source_canonical|canonical_mapper|mapper_major_minor|mapper_readonly|mapper_stat) return 1 ;; + esac + _k3slra1_expected_mapper "$role" expected_mapper || return 1 + _k3slra1_packet_value findmnt "$mount_root" mount_record || return 1 + (( ${#mount_record} >= 1 )) || return 1 + (( ${#mount_record} <= K3SLRA1_MOUNT_RECORD_MAX_BYTES )) || return 1 + [[ "$mount_record" == *$'\n' ]] || return 1 + [[ "$mount_record" != *$'\r'* ]] || return 1 + mount_line="${mount_record%$'\n'}" + [[ -n "$mount_line" ]] || return 1 + [[ "$mount_line" != *$'\n'* ]] || return 1 + [[ "$mount_line" =~ ^([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)$ ]] || return 1 + source="${BASH_REMATCH[1]}" + target="${BASH_REMATCH[2]}" + fstype="${BASH_REMATCH[3]}" + vfs_options="${BASH_REMATCH[4]}" + fs_options="${BASH_REMATCH[5]}" + mount_major_minor="${BASH_REMATCH[6]}" + [[ -n "$source" ]] || return 1 + [[ -n "$target" ]] || return 1 + [[ -n "$fstype" ]] || return 1 + [[ -n "$vfs_options" ]] || return 1 + [[ -n "$fs_options" ]] || return 1 + [[ -n "$mount_major_minor" ]] || return 1 + [[ -z "$extra" ]] || return 1 + [[ "$target" == "$mount_root" ]] || return 1 + [[ "$fstype" == ext4 ]] || return 1 + _k3slra1_option_has "$vfs_options" ro || return 1 + _k3slra1_option_has "$vfs_options" nodev || return 1 + _k3slra1_option_has "$vfs_options" nosuid || return 1 + _k3slra1_option_has "$vfs_options" noexec || return 1 + case ",${vfs_options}," in *,rw,*|*,dev,*|*,suid,*|*,exec,*) return 1 ;; esac + _k3slra1_option_has "$fs_options" noload || return 1 + _k3slra1_packet_line readlink "$source" source_canonical || return 1 + _k3slra1_packet_line readlink "$expected_mapper" canonical_mapper || return 1 + [[ "$canonical_mapper" == "$source_canonical" ]] || return 1 + [[ "$canonical_mapper" =~ ^/dev/dm-[0-9]+$ ]] || return 1 + _k3slra1_packet_line mapper-stat "$canonical_mapper" mapper_stat || return 1 + [[ "$mapper_stat" == "block special file|${mount_major_minor}" ]] || return 1 + _k3slra1_packet_line lsblk "$canonical_mapper" mapper_major_minor || return 1 + [[ "$mapper_major_minor" == "$mount_major_minor" ]] || return 1 + _k3slra1_packet_value blockdev "$canonical_mapper" mapper_readonly || return 1 + [[ "$mapper_readonly" == $'1\n' ]] || return 1 + printf -v "$output_name" '%s' "${mount_record}|${canonical_mapper}|${mapper_major_minor}|${mapper_readonly}" +} + +_k3slra1_output_names_are_distinct() { + local name seen_name + local -A seen=() + (( $# >= 1 )) || return 1 + for name in "$@"; do + _k3slra1_output_name_is_safe "$name" || return 1 + [[ "${seen[$name]+set}" != set ]] || return 1 + printf -v 'seen[$name]' '%s' 1 + done +} + +_k3slra1_output_names_avoid() { + local output_count="${1-}" output_index=0 output_name forbidden_name + local -a output_names=() + (( $# >= 2 )) || return 1 + [[ "$output_count" =~ ^[1-9][0-9]*$ ]] || return 1 + shift + (( $# >= 10#$output_count + 1 )) || return 1 + for (( output_index = 0; output_index < 10#$output_count; output_index += 1 )); do + output_names+=("${1-}") + shift + done + for forbidden_name in "$@"; do + for output_name in "${output_names[@]}"; do + [[ "$output_name" != "$forbidden_name" ]] || return 1 + done + done +} + +_k3slra1_metadata_fields() { + local path="${1-}" out_identity="${2-}" out_nlink="${3-}" out_size="${4-}" + local out_mtime="${5-}" out_type="${6-}" out_owner="${7-}" out_mode="${8-}" + local metadata='' _k3slra1_type_value='' + (( $# == 8 )) || return 1 + _k3slra1_output_names_are_distinct "$out_identity" "$out_nlink" "$out_size" "$out_mtime" "$out_type" "$out_owner" "$out_mode" || return 1 + case "$out_identity|$out_nlink|$out_size|$out_mtime|$out_type|$out_owner|$out_mode" in + *'|path|'*|path\|*|*\|path|*'|out_identity|'*|out_identity\|*|*\|out_identity|*'|out_nlink|'*|out_nlink\|*|*\|out_nlink|*'|out_size|'*|out_size\|*|*\|out_size|*'|out_mtime|'*|out_mtime\|*|*\|out_mtime|*'|out_type|'*|out_type\|*|*\|out_type|*'|out_owner|'*|out_owner\|*|*\|out_owner|*'|out_mode|'*|out_mode\|*|*\|out_mode|*'|metadata|'*|metadata\|*|*\|metadata) return 1 ;; + esac + _k3slra1_metadata "$path" metadata || return 1 + [[ "$metadata" =~ ^([0-9]+:[0-9]+)\|([0-9]+)\|([0-9]+)\|([^|]+)\|(directory|regular[[:space:]]file|regular[[:space:]]empty[[:space:]]file|symbolic[[:space:]]link)\|([0-9]+:[0-9]+)\|([0-9]+)$ ]] || return 1 + _k3slra1_type_value="${BASH_REMATCH[5]}" + if [[ "$_k3slra1_type_value" == 'regular empty file' ]]; then + printf -v _k3slra1_type_value '%s' 'regular file' + fi + printf -v "$out_identity" '%s' "${BASH_REMATCH[1]}" + printf -v "$out_nlink" '%s' "${BASH_REMATCH[2]}" + printf -v "$out_size" '%s' "${BASH_REMATCH[3]}" + printf -v "$out_mtime" '%s' "${BASH_REMATCH[4]}" + printf -v "$out_type" '%s' "$_k3slra1_type_value" + printf -v "$out_owner" '%s' "${BASH_REMATCH[6]}" + printf -v "$out_mode" '%s' "${BASH_REMATCH[7]}" +} + +_k3slra1_fd_metadata_fields() { + local path="${1-}" out_identity="${2-}" out_nlink="${3-}" out_size="${4-}" + local out_mtime="${5-}" out_type="${6-}" out_owner="${7-}" out_mode="${8-}" + local metadata='' _k3slra1_type_value='' + (( $# == 8 )) || return 1 + _k3slra1_output_names_are_distinct "$out_identity" "$out_nlink" "$out_size" "$out_mtime" "$out_type" "$out_owner" "$out_mode" || return 1 + case "$out_identity|$out_nlink|$out_size|$out_mtime|$out_type|$out_owner|$out_mode" in + *'|path|'*|path\|*|*\|path|*'|out_identity|'*|out_identity\|*|*\|out_identity|*'|out_nlink|'*|out_nlink\|*|*\|out_nlink|*'|out_size|'*|out_size\|*|*\|out_size|*'|out_mtime|'*|out_mtime\|*|*\|out_mtime|*'|out_type|'*|out_type\|*|*\|out_type|*'|out_owner|'*|out_owner\|*|*\|out_owner|*'|out_mode|'*|out_mode\|*|*\|out_mode|*'|metadata|'*|metadata\|*|*\|metadata) return 1 ;; + esac + _k3slra1_packet_line fd-metadata "$path" metadata || return 1 + [[ "$metadata" =~ ^([0-9]+:[0-9]+)\|([0-9]+)\|([0-9]+)\|([^|]+)\|(directory|regular[[:space:]]file|regular[[:space:]]empty[[:space:]]file|symbolic[[:space:]]link)\|([0-9]+:[0-9]+)\|([0-9]+)$ ]] || return 1 + _k3slra1_type_value="${BASH_REMATCH[5]}" + if [[ "$_k3slra1_type_value" == 'regular empty file' ]]; then + printf -v _k3slra1_type_value '%s' 'regular file' + fi + printf -v "$out_identity" '%s' "${BASH_REMATCH[1]}" + printf -v "$out_nlink" '%s' "${BASH_REMATCH[2]}" + printf -v "$out_size" '%s' "${BASH_REMATCH[3]}" + printf -v "$out_mtime" '%s' "${BASH_REMATCH[4]}" + printf -v "$out_type" '%s' "$_k3slra1_type_value" + printf -v "$out_owner" '%s' "${BASH_REMATCH[6]}" + printf -v "$out_mode" '%s' "${BASH_REMATCH[7]}" +} + +_k3slra1_find_mount_children_packet() { + local mount_root="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/find "$mount_root" -mindepth 1 -maxdepth 1 -print0 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf 'RC=%s\0' "$rc" +} + +_k3slra1_find_regular_packet() { + local pre_root="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1_command /usr/bin/find "$pre_root" -xdev -type f -print0 2> >(_k3slra1_stderr_marker); then + rc=0 + else + rc=$? + fi + printf 'RC=%s\0' "$rc" +} + +_k3slra1_select_pre_bundle() { + local mount_root="${1-}" output_name="${2-}" entry base _k3slra1_selected_value='' + local canonical_mount='' canonical_bundle='' canonical_pre='' canonical_manifest='' + local mount_identity='' mount_nlink='' mount_size='' mount_mtime='' mount_type='' mount_owner='' mount_mode='' + local bundle_identity='' bundle_nlink='' bundle_size='' bundle_mtime='' bundle_type='' bundle_owner='' bundle_mode='' + local pre_identity='' pre_nlink='' pre_size='' pre_mtime='' pre_type='' pre_owner='' pre_mode='' + local manifest_identity='' manifest_nlink='' manifest_size='' manifest_mtime='' manifest_type='' manifest_owner='' manifest_mode='' + local pre_root='' manifest='' match_count=0 last_index=0 + local -a entries=() + (( $# == 2 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + _k3slra1_output_names_avoid 1 "$output_name" mount_root output_name entry base canonical_mount canonical_bundle canonical_pre canonical_manifest mount_identity mount_nlink mount_size mount_mtime mount_type mount_owner mount_mode bundle_identity bundle_nlink bundle_size bundle_mtime bundle_type bundle_owner bundle_mode pre_identity pre_nlink pre_size pre_mtime pre_type pre_owner pre_mode manifest_identity manifest_nlink manifest_size manifest_mtime manifest_type manifest_owner manifest_mode pre_root manifest match_count last_index entries || return 1 + mapfile -d '' -t entries < <(_k3slra1_find_mount_children_packet "$mount_root") + (( ${#entries[@]} >= 1 )) || return 1 + (( last_index = ${#entries[@]} - 1 )) + [[ "${entries[$last_index]}" == RC=0 ]] || return 1 + unset 'entries[last_index]' + for entry in "${entries[@]}"; do + [[ -n "$entry" ]] || return 1 + [[ "$entry" != STDERR ]] || return 1 + base="${entry##*/}" + if [[ "$base" =~ ^k3s-secrets-encryption-[0-9]{8}T[0-9]{6}Z$ ]]; then + (( match_count += 1 )) + _k3slra1_selected_value="$entry" + fi + done + (( match_count == 1 )) || return 1 + pre_root="${_k3slra1_selected_value}/pre" + manifest="${pre_root}/verification.manifest" + _k3slra1_canonical_path "$mount_root" canonical_mount || return 1 + _k3slra1_canonical_path "$_k3slra1_selected_value" canonical_bundle || return 1 + _k3slra1_canonical_path "$pre_root" canonical_pre || return 1 + _k3slra1_canonical_path "$manifest" canonical_manifest || return 1 + [[ "$canonical_mount" == "$mount_root" ]] || return 1 + [[ "$canonical_bundle" == "$_k3slra1_selected_value" ]] || return 1 + [[ "$canonical_pre" == "$pre_root" ]] || return 1 + [[ "$canonical_manifest" == "$manifest" ]] || return 1 + _k3slra1_metadata_fields "$mount_root" mount_identity mount_nlink mount_size mount_mtime mount_type mount_owner mount_mode || return 1 + _k3slra1_metadata_fields "$_k3slra1_selected_value" bundle_identity bundle_nlink bundle_size bundle_mtime bundle_type bundle_owner bundle_mode || return 1 + _k3slra1_metadata_fields "$pre_root" pre_identity pre_nlink pre_size pre_mtime pre_type pre_owner pre_mode || return 1 + _k3slra1_metadata_fields "$manifest" manifest_identity manifest_nlink manifest_size manifest_mtime manifest_type manifest_owner manifest_mode || return 1 + [[ "$mount_type" == directory ]] || return 1 + [[ "$bundle_type" == directory ]] || return 1 + [[ "$pre_type" == directory ]] || return 1 + [[ "$manifest_type" == 'regular file' ]] || return 1 + [[ "$bundle_owner" == 0:0 ]] || return 1 + [[ "$pre_owner" == 0:0 ]] || return 1 + [[ "$manifest_owner" == 0:0 ]] || return 1 + [[ "$bundle_mode" == 700 ]] || return 1 + [[ "$pre_mode" == 700 ]] || return 1 + [[ "$manifest_mode" == 600 ]] || return 1 + [[ "$manifest_nlink" == 1 ]] || return 1 + [[ "${bundle_identity%%:*}" == "${mount_identity%%:*}" ]] || return 1 + [[ "${pre_identity%%:*}" == "${mount_identity%%:*}" ]] || return 1 + [[ "${manifest_identity%%:*}" == "${mount_identity%%:*}" ]] || return 1 + printf -v "$output_name" '%s' "$_k3slra1_selected_value" +} + +_k3slra1_capture_manifest_hex() { + local manifest="${1-}" output_name="${2-}" stat_size='' byte_count='' od_output='' hex='' + (( $# == 2 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + [[ "$output_name" != manifest ]] || return 1 + [[ "$output_name" != output_name ]] || return 1 + [[ "$output_name" != size ]] || return 1 + [[ "$output_name" != stat_size ]] || return 1 + [[ "$output_name" != byte_count ]] || return 1 + [[ "$output_name" != od_output ]] || return 1 + [[ "$output_name" != hex ]] || return 1 + _k3slra1_packet_line size-stat "$manifest" stat_size || return 1 + [[ "$stat_size" =~ ^[0-9]+$ ]] || return 1 + (( 10#$stat_size >= 1 )) || return 1 + (( 10#$stat_size <= K3SLRA1_MANIFEST_MAX_BYTES )) || return 1 + _k3slra1_packet_line wc "$manifest" byte_count || return 1 + [[ "$byte_count" == *"$manifest" ]] || return 1 + byte_count="${byte_count%"$manifest"}" + [[ "$byte_count" == *[[:space:]] ]] || return 1 + byte_count="${byte_count//[[:space:]]/}" + [[ "$byte_count" == "$stat_size" ]] || return 1 + _k3slra1_packet_value od "$manifest" od_output || return 1 + [[ "$od_output" == *$'\n' ]] || return 1 + [[ "$od_output" != *$'\r'* ]] || return 1 + od_output="${od_output%$'\n'}" + [[ -n "$od_output" ]] || return 1 + [[ "$od_output" != *$'\n' ]] || return 1 + [[ "$od_output" != *$'\n\n'* ]] || return 1 + hex="${od_output//[[:space:]]/}" + [[ "$hex" =~ ^[0-9a-f]+$ ]] || return 1 + (( ${#hex} == 10#$stat_size * 2 )) || return 1 + [[ " $od_output " != *[[:space:]]00[[:space:]]* ]] || return 1 + [[ " $od_output " != *[[:space:]]0d[[:space:]]* ]] || return 1 + [[ "${hex: -2}" == 0a ]] || return 1 + printf -v "$output_name" '%s' "$hex" +} + +_k3slra1_verify_payload() { + # Internal-only: the proof orchestrator supplies a reviewed helper staged in + # its root-owned runtime tmpfs before this privileged dispatch boundary. + local mount_root="${1-}" role="${2-}" capture_root="${3-}" + local helper='' + (( $# == 3 )) || return 1 + case "$role" in + original-baseline|proof|original-final) ;; + *) return 1 ;; + esac + helper="${capture_root}/libexec/k3s-local-recovery-a1-check.sh" + _k3slra1_command /usr/bin/sudo --non-interactive -- /usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin LC_ALL=C /usr/bin/bash --noprofile --norc -- "$helper" "$mount_root" "$role" "$capture_root" >/dev/null 2>&1 +} + +_k3slra1_verify_payload_root() { + local mount_root="${1-}" role="${2-}" capture_root="${3-}" + local _bundle='' _layout='' _count_before='' _bytes_before='' _snapshot_before='' + local _count_after='' _bytes_after='' _snapshot_after='' + local _mount_before='' _mount_sqlite='' _mount_after='' _sqlite_rc=0 + (( $# == 3 )) || return 1 + case "$role" in + original-baseline|proof|original-final) ;; + *) return 1 ;; + esac + _k3slra1_capture_mount_mapping_evidence "$mount_root" "$role" _mount_before || return 1 + _k3slra1_select_pre_bundle "$mount_root" _bundle || return 1 + _k3slra1_analyze_manifest "$_bundle" _count_before _bytes_before _snapshot_before || return 1 + [[ "$_count_before" == "$K3SLRA1_TARGET_COUNT" ]] || return 1 + [[ "$_bytes_before" == "$K3SLRA1_TARGET_BYTES" ]] || return 1 + _k3slra1_select_sqlite_layout_from_snapshot "$_bundle" "$_snapshot_before" _layout || return 1 + _k3slra1_capture_mount_mapping_evidence "$mount_root" "$role" _mount_sqlite || return 1 + [[ "$_mount_sqlite" == "$_mount_before" ]] || return 1 + if _k3slra1_sqlite_quick_check "$_bundle" "$_layout" "$capture_root"; then + _sqlite_rc=0 + else + _sqlite_rc=$? + fi + case "$_sqlite_rc" in + 0) ;; + 130|143) return "$_sqlite_rc" ;; + *) return 1 ;; + esac + _k3slra1_analyze_manifest "$_bundle" _count_after _bytes_after _snapshot_after || return 1 + [[ "$_count_after" == "$_count_before" ]] || return 1 + [[ "$_bytes_after" == "$_bytes_before" ]] || return 1 + [[ "$_snapshot_after" == "$_snapshot_before" ]] || return 1 + _k3slra1_capture_mount_mapping_evidence "$mount_root" "$role" _mount_after || return 1 + [[ "$_mount_after" == "$_mount_before" ]] +} + +_k3slra1_sha256_record() { + local path="${1-}" output_name="${2-}" line='' + (( $# == 2 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + [[ "$output_name" != path ]] || return 1 + [[ "$output_name" != output_name ]] || return 1 + [[ "$output_name" != value ]] || return 1 + [[ "$output_name" != line ]] || return 1 + _k3slra1_packet_line sha256 "$path" line || return 1 + printf -v "$output_name" '%s' "$line" +} + +_k3slra1_analyze_manifest() { + local bundle="${1-}" out_count="${2-}" out_bytes="${3-}" out_snapshot="${4-}" + local mount_root='' _selected_value='' pre_root='' manifest='' manifest_hex_before='' manifest_hex_after='' + local bundle_identity='' bundle_nlink='' bundle_size='' bundle_mtime='' bundle_type='' bundle_owner='' bundle_mode='' + local pre_identity='' pre_nlink='' pre_size='' pre_mtime='' pre_type='' pre_owner='' pre_mode='' + local manifest_identity_before='' manifest_nlink_before='' manifest_size_before='' manifest_mtime_before='' + local manifest_type_before='' manifest_owner_before='' manifest_mode_before='' manifest_record_before='' + local manifest_identity_after='' manifest_nlink_after='' manifest_size_after='' manifest_mtime_after='' + local manifest_type_after='' manifest_owner_after='' manifest_mode_after='' manifest_record_after='' + local line='' hash='' relative='' target='' canonical_target='' target_record_before='' target_record_after='' + local target_identity='' target_nlink='' target_size='' target_mtime='' target_type='' target_owner='' target_mode='' + local inventory_path='' _snapshot_value='' _count_value=0 _bytes_value=0 + local last_index=0 inventory_count=0 + local -a inventory=() manifest_lines=() + local -A seen_paths=() seen_inodes=() expected_inventory=() seen_inventory=() + (( $# == 4 )) || return 1 + _k3slra1_output_names_are_distinct "$out_count" "$out_bytes" "$out_snapshot" || return 1 + _k3slra1_output_names_avoid 3 "$out_count" "$out_bytes" "$out_snapshot" bundle out_count out_bytes out_snapshot mount_root _selected_value pre_root manifest manifest_hex_before manifest_hex_after bundle_identity bundle_nlink bundle_size bundle_mtime bundle_type bundle_owner bundle_mode pre_identity pre_nlink pre_size pre_mtime pre_type pre_owner pre_mode manifest_identity_before manifest_nlink_before manifest_size_before manifest_mtime_before manifest_type_before manifest_owner_before manifest_mode_before manifest_record_before manifest_identity_after manifest_nlink_after manifest_size_after manifest_mtime_after manifest_type_after manifest_owner_after manifest_mode_after manifest_record_after line hash relative target canonical_target target_record_before target_record_after target_identity target_nlink target_size target_mtime target_type target_owner target_mode inventory_path _snapshot_value _count_value _bytes_value last_index inventory_count inventory manifest_lines seen_paths seen_inodes expected_inventory seen_inventory || return 1 + mount_root="${bundle%/*}" + [[ -n "$mount_root" ]] || return 1 + _k3slra1_select_pre_bundle "$mount_root" _selected_value || return 1 + [[ "$_selected_value" == "$bundle" ]] || return 1 + pre_root="${bundle}/pre" + manifest="${pre_root}/verification.manifest" + _k3slra1_metadata_fields "$bundle" bundle_identity bundle_nlink bundle_size bundle_mtime bundle_type bundle_owner bundle_mode || return 1 + _k3slra1_metadata_fields "$pre_root" pre_identity pre_nlink pre_size pre_mtime pre_type pre_owner pre_mode || return 1 + _k3slra1_metadata_fields "$manifest" manifest_identity_before manifest_nlink_before manifest_size_before manifest_mtime_before manifest_type_before manifest_owner_before manifest_mode_before || return 1 + _k3slra1_capture_manifest_hex "$manifest" manifest_hex_before || return 1 + _k3slra1_sha256_record "$manifest" manifest_record_before || return 1 + printf -v "expected_inventory[$manifest]" '%s' 1 + mapfile -t manifest_lines <"$manifest" + for line in "${manifest_lines[@]}"; do + (( _count_value += 1 )) + (( _count_value <= K3SLRA1_TARGET_COUNT )) || return 1 + [[ "$line" =~ ^[0-9a-f]{64}\ \ \./[A-Za-z0-9._+/-]+$ ]] || return 1 + hash="${line:0:64}" + relative="${line:66}" + (( ${#relative} <= K3SLRA1_PATH_MAX_BYTES )) || return 1 + [[ "$relative" != './verification.manifest' ]] || return 1 + [[ "$relative" != *//* ]] || return 1 + [[ "$relative" != *'/./'* ]] || return 1 + [[ "$relative" != *'/../'* ]] || return 1 + [[ "$relative" != */. ]] || return 1 + [[ "$relative" != */.. ]] || return 1 + [[ "$relative" != */ ]] || return 1 + [[ "${seen_paths[$relative]+set}" != set ]] || return 1 + printf -v "seen_paths[$relative]" '%s' 1 + target="${pre_root}/${relative#./}" + _k3slra1_canonical_path "$target" canonical_target || return 1 + [[ "$canonical_target" == "$target" ]] || return 1 + _k3slra1_metadata_fields "$target" target_identity target_nlink target_size target_mtime target_type target_owner target_mode || return 1 + [[ "$target_type" == 'regular file' ]] || return 1 + [[ "$target_nlink" == 1 ]] || return 1 + [[ "${target_identity%%:*}" == "${pre_identity%%:*}" ]] || return 1 + [[ "${seen_inodes[$target_identity]+set}" != set ]] || return 1 + printf -v "seen_inodes[$target_identity]" '%s' 1 + target_record_before="${target_identity}|${target_nlink}|${target_size}|${target_mtime}|${target_type}|${target_owner}|${target_mode}" + _k3slra1_sha256_record "$target" target_record_after || return 1 + [[ "$target_record_after" == "${hash} ${target}" ]] || return 1 + _k3slra1_metadata_fields "$target" target_identity target_nlink target_size target_mtime target_type target_owner target_mode || return 1 + [[ "${target_identity}|${target_nlink}|${target_size}|${target_mtime}|${target_type}|${target_owner}|${target_mode}" == "$target_record_before" ]] || return 1 + (( _bytes_value += 10#$target_size )) + printf -v _snapshot_value '%s%s|%s|%s\n' "$_snapshot_value" "$relative" "$target_record_before" "$hash" + printf -v "expected_inventory[$target]" '%s' 1 + done + (( _count_value == K3SLRA1_TARGET_COUNT )) || return 1 + (( _bytes_value == K3SLRA1_TARGET_BYTES )) || return 1 + mapfile -d '' -t inventory < <(_k3slra1_find_regular_packet "$pre_root") + (( ${#inventory[@]} >= 1 )) || return 1 + (( last_index = ${#inventory[@]} - 1 )) + [[ "${inventory[$last_index]}" == RC=0 ]] || return 1 + unset 'inventory[last_index]' + for inventory_path in "${inventory[@]}"; do + [[ -n "$inventory_path" ]] || return 1 + [[ "$inventory_path" != STDERR ]] || return 1 + [[ "${expected_inventory[$inventory_path]+set}" == set ]] || return 1 + [[ "${seen_inventory[$inventory_path]+set}" != set ]] || return 1 + printf -v "seen_inventory[$inventory_path]" '%s' 1 + (( inventory_count += 1 )) + done + (( inventory_count == K3SLRA1_TARGET_COUNT + 1 )) || return 1 + _k3slra1_metadata_fields "$manifest" manifest_identity_after manifest_nlink_after manifest_size_after manifest_mtime_after manifest_type_after manifest_owner_after manifest_mode_after || return 1 + _k3slra1_capture_manifest_hex "$manifest" manifest_hex_after || return 1 + _k3slra1_sha256_record "$manifest" manifest_record_after || return 1 + [[ "$manifest_identity_after" == "$manifest_identity_before" ]] || return 1 + [[ "$manifest_nlink_after" == "$manifest_nlink_before" ]] || return 1 + [[ "$manifest_size_after" == "$manifest_size_before" ]] || return 1 + [[ "$manifest_mtime_after" == "$manifest_mtime_before" ]] || return 1 + [[ "$manifest_type_after" == "$manifest_type_before" ]] || return 1 + [[ "$manifest_owner_after" == "$manifest_owner_before" ]] || return 1 + [[ "$manifest_mode_after" == "$manifest_mode_before" ]] || return 1 + [[ "$manifest_hex_after" == "$manifest_hex_before" ]] || return 1 + [[ "$manifest_record_after" == "$manifest_record_before" ]] || return 1 + printf -v _snapshot_value '%s%s\n%s\n%s\n' "$_snapshot_value" "$bundle_identity|$bundle_nlink|$bundle_size|$bundle_mtime|$bundle_type|$bundle_owner|$bundle_mode" "$pre_identity|$pre_nlink|$pre_size|$pre_mtime|$pre_type|$pre_owner|$pre_mode" "$manifest_identity_before|$manifest_nlink_before|$manifest_size_before|$manifest_mtime_before|$manifest_type_before|$manifest_owner_before|$manifest_mode_before|$manifest_record_before" + printf -v "$out_count" '%s' "$_count_value" + printf -v "$out_bytes" '%s' "$_bytes_value" + printf -v "$out_snapshot" '%s' "$_snapshot_value" +} + +_k3slra1_verify_manifest() { + local bundle="${1-}" out_count="${2-}" out_bytes="${3-}" + local _count_result='' _bytes_result='' _snapshot_result='' + (( $# == 3 )) || return 1 + _k3slra1_output_names_are_distinct "$out_count" "$out_bytes" || return 1 + _k3slra1_output_names_avoid 2 "$out_count" "$out_bytes" bundle out_count out_bytes _count_result _bytes_result _snapshot_result || return 1 + _k3slra1_analyze_manifest "$bundle" _count_result _bytes_result _snapshot_result || return 1 + printf -v "$out_count" '%s' "$_count_result" + printf -v "$out_bytes" '%s' "$_bytes_result" +} + +_k3slra1_snapshot_metadata_for_relative() { + local snapshot="${1-}" relative="${2-}" output_name="${3-}" + local line='' value='' match_count=0 + local -a snapshot_lines=() + (( $# == 3 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + _k3slra1_output_names_avoid 1 "$output_name" snapshot relative output_name line value match_count snapshot_lines || return 1 + [[ "$relative" == ./* ]] || return 1 + mapfile -t snapshot_lines <<<"$snapshot" + for line in "${snapshot_lines[@]}"; do + if [[ "$line" == "${relative}|"* ]]; then + (( match_count += 1 )) + value="${line#"${relative}|"}" + [[ "$value" =~ \|([0-9a-f]{64})$ ]] || return 1 + value="${value%|${BASH_REMATCH[1]}}" + fi + done + (( match_count == 1 )) || return 1 + [[ "$value" =~ ^[0-9]+:[0-9]+\|1\|[0-9]+\|.+\|regular[[:space:]]file\|[0-9]+:[0-9]+\|[0-9]+$ ]] || return 1 + printf -v "$output_name" '%s' "$value" +} + +_k3slra1_sqlite_selected_path_matches() { + local path="${1-}" expected_device="${2-}" expected_record="${3-}" canonical='' + local identity='' nlink='' size='' mtime='' type='' owner='' mode='' current_record='' + (( $# == 3 )) || return 1 + _k3slra1_canonical_path "$path" canonical || return 1 + [[ "$canonical" == "$path" ]] || return 1 + _k3slra1_metadata_fields "$path" identity nlink size mtime type owner mode || return 1 + [[ "${identity%%:*}" == "$expected_device" ]] || return 1 + [[ "$nlink" == 1 ]] || return 1 + [[ "$type" == 'regular file' ]] || return 1 + current_record="${identity}|${nlink}|${size}|${mtime}|${type}|${owner}|${mode}" + [[ "$current_record" == "$expected_record" ]] +} + +_k3slra1_select_sqlite_layout_from_snapshot() { + local bundle="${1-}" snapshot="${2-}" output_name="${3-}" path relative='' + local expected_record='' canonical_count=0 legacy_count=0 _k3slra1_layout_value='' + local pre_identity='' pre_nlink='' pre_size='' pre_mtime='' pre_type='' pre_owner='' pre_mode='' pre_device='' + local -a selected_paths=() canonical_paths=("${bundle}/pre/${K3SLRA1_CANONICAL_DATABASE#./}" "${bundle}/pre/${K3SLRA1_CANONICAL_DATABASE#./}-wal" "${bundle}/pre/${K3SLRA1_CANONICAL_DATABASE#./}-shm") legacy_paths=("${bundle}/pre/${K3SLRA1_LEGACY_DATABASE#./}" "${bundle}/pre/${K3SLRA1_LEGACY_DATABASE#./}-wal" "${bundle}/pre/${K3SLRA1_LEGACY_DATABASE#./}-shm") + (( $# == 3 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + _k3slra1_output_names_avoid 1 "$output_name" bundle snapshot output_name path relative expected_record canonical_count legacy_count pre_identity pre_nlink pre_size pre_mtime pre_type pre_owner pre_mode pre_device selected_paths canonical_paths legacy_paths || return 1 + _k3slra1_metadata_fields "${bundle}/pre" pre_identity pre_nlink pre_size pre_mtime pre_type pre_owner pre_mode || return 1 + pre_device="${pre_identity%%:*}" + for path in "${canonical_paths[@]}"; do + if [[ -e "$path" ]]; then + (( canonical_count += 1 )) + else + if [[ -L "$path" ]]; then + (( canonical_count += 1 )) + fi + fi + done + for path in "${legacy_paths[@]}"; do + if [[ -e "$path" ]]; then + (( legacy_count += 1 )) + else + if [[ -L "$path" ]]; then + (( legacy_count += 1 )) + fi + fi + done + if (( canonical_count == 3 )); then + (( legacy_count == 0 )) || return 1 + _k3slra1_layout_value=canonical + selected_paths=("${canonical_paths[@]}") + else + if (( legacy_count == 3 )); then + (( canonical_count == 0 )) || return 1 + _k3slra1_layout_value=legacy + selected_paths=("${legacy_paths[@]}") + else + return 1 + fi + fi + for path in "${selected_paths[@]}"; do + relative="./${path#"${bundle}/pre/"}" + _k3slra1_snapshot_metadata_for_relative "$snapshot" "$relative" expected_record || return 1 + _k3slra1_sqlite_selected_path_matches "$path" "$pre_device" "$expected_record" || return 1 + done + printf -v "$output_name" '%s' "$_k3slra1_layout_value" +} + +_k3slra1_select_sqlite_layout() { + local bundle="${1-}" output_name="${2-}" count='' bytes='' snapshot='' + (( $# == 2 )) || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + _k3slra1_output_names_avoid 1 "$output_name" bundle output_name count bytes snapshot path || return 1 + _k3slra1_analyze_manifest "$bundle" count bytes snapshot || return 1 + _k3slra1_select_sqlite_layout_from_snapshot "$bundle" "$snapshot" "$output_name" +} + +_k3slra1_capture_file_hex() { + local file="${1-}" maximum="${2-}" output_name="${3-}" size='' count='' od_output='' hex='' + (( $# == 3 )) || return 1 + [[ "$maximum" =~ ^[1-9][0-9]*$ ]] || return 1 + _k3slra1_output_name_is_safe "$output_name" || return 1 + [[ "$output_name" != file ]] || return 1 + [[ "$output_name" != maximum ]] || return 1 + [[ "$output_name" != output_name ]] || return 1 + [[ "$output_name" != size ]] || return 1 + [[ "$output_name" != count ]] || return 1 + [[ "$output_name" != od_output ]] || return 1 + [[ "$output_name" != hex ]] || return 1 + _k3slra1_packet_line size-stat "$file" size || return 1 + [[ "$size" =~ ^[0-9]+$ ]] || return 1 + (( 10#$size <= 10#$maximum )) || return 1 + _k3slra1_packet_line wc "$file" count || return 1 + [[ "$count" == *"$file" ]] || return 1 + count="${count%"$file"}" + [[ "$count" == *[[:space:]] ]] || return 1 + count="${count//[[:space:]]/}" + [[ "$count" == "$size" ]] || return 1 + _k3slra1_packet_value od "$file" od_output || return 1 + [[ "$od_output" != *$'\r'* ]] || return 1 + if [[ -n "$od_output" ]]; then + [[ "$od_output" == *$'\n' ]] || return 1 + od_output="${od_output%$'\n'}" + [[ -n "$od_output" ]] || return 1 + [[ "$od_output" != *$'\n' ]] || return 1 + [[ "$od_output" != *$'\n\n'* ]] || return 1 + fi + hex="${od_output//[[:space:]]/}" + if (( 10#$size == 0 )); then + [[ -z "$hex" ]] || return 1 + else + [[ "$hex" =~ ^[0-9a-f]+$ ]] || return 1 + (( ${#hex} == 10#$size * 2 )) || return 1 + fi + printf -v "$output_name" '%s' "$hex" +} + +_k3slra1_sqlite_capture_transaction() { + local database="${1-}" capture_root="${2-}" stdout_path="${3-}" stderr_path="${4-}" + local parent_identity_before='' parent_nlink='' parent_size='' parent_mtime='' parent_type='' parent_owner='' parent_mode='' + local parent_fd_identity_after='' parent_fd_nlink_after='' parent_fd_size_after='' + local parent_fd_mtime_after='' parent_fd_type_after='' parent_fd_owner_after='' parent_fd_mode_after='' + local stdout_identity_before='' stdout_nlink='' stdout_size='' stdout_mtime='' stdout_type='' stdout_owner='' stdout_mode='' + local stdout_fd_identity_after='' stdout_fd_nlink_after='' stdout_fd_size_after='' + local stdout_fd_mtime_after='' stdout_fd_type_after='' stdout_fd_owner_after='' stdout_fd_mode_after='' + local stderr_identity_before='' stderr_nlink='' stderr_size='' stderr_mtime='' stderr_type='' stderr_owner='' stderr_mode='' + local stderr_fd_identity='' stderr_fd_nlink='' stderr_fd_size='' stderr_fd_mtime='' stderr_fd_type='' stderr_fd_owner='' stderr_fd_mode='' + local stderr_fd_identity_after='' stderr_fd_nlink_after='' stderr_fd_size_after='' + local stderr_fd_mtime_after='' stderr_fd_type_after='' stderr_fd_owner_after='' stderr_fd_mode_after='' + local parent_identity_after='' parent_nlink_after='' parent_size_after='' parent_mtime_after='' parent_type_after='' parent_owner_after='' parent_mode_after='' + local stdout_identity_after='' stdout_nlink_after='' stdout_size_after='' stdout_mtime_after='' stdout_type_after='' stdout_owner_after='' stdout_mode_after='' + local stderr_identity_after='' stderr_nlink_after='' stderr_size_after='' stderr_mtime_after='' stderr_type_after='' stderr_owner_after='' stderr_mode_after='' + local stdout_hex='' stderr_hex='' validation_rc=0 child_rc=1 cleanup_rc=0 + (( $# == 4 )) || return 1 + _capture_entered=1 + if _k3slra1_metadata_fields "$capture_root" parent_identity_before parent_nlink parent_size parent_mtime parent_type parent_owner parent_mode; then + : + else + validation_rc=1 + fi + if _k3slra1_metadata_fields "$stdout_path" stdout_identity_before stdout_nlink stdout_size stdout_mtime stdout_type stdout_owner stdout_mode; then + : + else + validation_rc=1 + fi + if _k3slra1_metadata_fields "$stderr_path" stderr_identity_before stderr_nlink stderr_size stderr_mtime stderr_type stderr_owner stderr_mode; then + : + else + validation_rc=1 + fi + if _k3slra1_fd_metadata_fields "/proc/${BASHPID}/fd/8" stderr_fd_identity stderr_fd_nlink stderr_fd_size stderr_fd_mtime stderr_fd_type stderr_fd_owner stderr_fd_mode; then + : + else + validation_rc=1 + fi + [[ "$parent_type" == directory ]] || validation_rc=1 + [[ "$parent_owner" == 0:0 ]] || validation_rc=1 + [[ "$parent_mode" == 700 ]] || validation_rc=1 + [[ "$_stage_parent_fd_type" == directory ]] || validation_rc=1 + [[ "$_stage_parent_fd_owner" == 0:0 ]] || validation_rc=1 + [[ "$_stage_parent_fd_mode" == 700 ]] || validation_rc=1 + [[ "$_stage_parent_identity" == "$parent_identity_before" ]] || validation_rc=1 + [[ "$_stage_parent_fd_identity" == "$parent_identity_before" ]] || validation_rc=1 + [[ "$stdout_type" == 'regular file' ]] || validation_rc=1 + [[ "$stderr_type" == 'regular file' ]] || validation_rc=1 + [[ "$stdout_owner" == 0:0 ]] || validation_rc=1 + [[ "$stderr_owner" == 0:0 ]] || validation_rc=1 + [[ "$stdout_mode" == 600 ]] || validation_rc=1 + [[ "$stderr_mode" == 600 ]] || validation_rc=1 + [[ "$stdout_nlink" == 1 ]] || validation_rc=1 + [[ "$stderr_nlink" == 1 ]] || validation_rc=1 + [[ "$_stage_stdout_fd_type" == 'regular file' ]] || validation_rc=1 + [[ "$stderr_fd_type" == 'regular file' ]] || validation_rc=1 + [[ "$_stage_stdout_fd_owner" == 0:0 ]] || validation_rc=1 + [[ "$stderr_fd_owner" == 0:0 ]] || validation_rc=1 + [[ "$_stage_stdout_fd_mode" == 600 ]] || validation_rc=1 + [[ "$stderr_fd_mode" == 600 ]] || validation_rc=1 + [[ "$_stage_stdout_fd_nlink" == 1 ]] || validation_rc=1 + [[ "$stderr_fd_nlink" == 1 ]] || validation_rc=1 + [[ "$stdout_identity_before" == "$_stage_stdout_identity" ]] || validation_rc=1 + [[ "$stdout_identity_before" == "$_stage_stdout_fd_identity" ]] || validation_rc=1 + [[ "$stderr_identity_before" == "$stderr_fd_identity" ]] || validation_rc=1 + [[ "$stdout_size" == "$_stage_stdout_fd_size" ]] || validation_rc=1 + [[ "$stderr_size" == "$stderr_fd_size" ]] || validation_rc=1 + [[ "$stdout_mtime" == "$_stage_stdout_fd_mtime" ]] || validation_rc=1 + [[ "$stderr_mtime" == "$stderr_fd_mtime" ]] || validation_rc=1 + [[ "$stdout_type" == "$_stage_stdout_fd_type" ]] || validation_rc=1 + [[ "$stderr_type" == "$stderr_fd_type" ]] || validation_rc=1 + [[ "$stdout_owner" == "$_stage_stdout_fd_owner" ]] || validation_rc=1 + [[ "$stderr_owner" == "$stderr_fd_owner" ]] || validation_rc=1 + [[ "$stdout_mode" == "$_stage_stdout_fd_mode" ]] || validation_rc=1 + [[ "$stderr_mode" == "$stderr_fd_mode" ]] || validation_rc=1 + [[ "$stderr_size" == 0 ]] || validation_rc=1 + [[ "$stdout_identity_before" != "$stderr_identity_before" ]] || validation_rc=1 + if (( validation_rc == 0 )); then + if _k3slra1_command /usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin LC_ALL=C "HOME=${capture_root}/home" /usr/bin/sqlite3 -safe -nofollow -readonly -batch -bail -noheader -init /dev/null "$database" "$K3SLRA1_SQLITE_QUERY" >&7 2>&8; then + child_rc=0 + else + child_rc=$? + fi + fi + if _k3slra1_metadata_fields "$stdout_path" stdout_identity_after stdout_nlink_after stdout_size_after stdout_mtime_after stdout_type_after stdout_owner_after stdout_mode_after; then + : + else + validation_rc=1 + fi + if _k3slra1_metadata_fields "$stderr_path" stderr_identity_after stderr_nlink_after stderr_size_after stderr_mtime_after stderr_type_after stderr_owner_after stderr_mode_after; then + : + else + validation_rc=1 + fi + if _k3slra1_metadata_fields "$capture_root" parent_identity_after parent_nlink_after parent_size_after parent_mtime_after parent_type_after parent_owner_after parent_mode_after; then + : + else + validation_rc=1 + fi + if _k3slra1_fd_metadata_fields "/proc/${BASHPID}/fd/6" parent_fd_identity_after parent_fd_nlink_after parent_fd_size_after parent_fd_mtime_after parent_fd_type_after parent_fd_owner_after parent_fd_mode_after; then + : + else + validation_rc=1 + fi + if _k3slra1_fd_metadata_fields "/proc/${BASHPID}/fd/7" stdout_fd_identity_after stdout_fd_nlink_after stdout_fd_size_after stdout_fd_mtime_after stdout_fd_type_after stdout_fd_owner_after stdout_fd_mode_after; then + : + else + validation_rc=1 + fi + if _k3slra1_fd_metadata_fields "/proc/${BASHPID}/fd/8" stderr_fd_identity_after stderr_fd_nlink_after stderr_fd_size_after stderr_fd_mtime_after stderr_fd_type_after stderr_fd_owner_after stderr_fd_mode_after; then + : + else + validation_rc=1 + fi + [[ "$parent_identity_after" == "$parent_identity_before" ]] || validation_rc=1 + [[ "$parent_identity_after" == "$parent_fd_identity_after" ]] || validation_rc=1 + [[ "$parent_fd_identity_after" == "$_stage_parent_fd_identity" ]] || validation_rc=1 + [[ "$parent_type_after" == directory ]] || validation_rc=1 + [[ "$parent_fd_type_after" == directory ]] || validation_rc=1 + [[ "$parent_owner_after" == 0:0 ]] || validation_rc=1 + [[ "$parent_fd_owner_after" == 0:0 ]] || validation_rc=1 + [[ "$parent_mode_after" == 700 ]] || validation_rc=1 + [[ "$parent_fd_mode_after" == 700 ]] || validation_rc=1 + [[ "$stdout_identity_after" == "$stdout_identity_before" ]] || validation_rc=1 + [[ "$stdout_identity_after" == "$stdout_fd_identity_after" ]] || validation_rc=1 + [[ "$stdout_nlink_after" == 1 ]] || validation_rc=1 + [[ "$stdout_fd_nlink_after" == 1 ]] || validation_rc=1 + [[ "$stdout_type_after" == 'regular file' ]] || validation_rc=1 + [[ "$stdout_fd_type_after" == 'regular file' ]] || validation_rc=1 + [[ "$stdout_owner_after" == 0:0 ]] || validation_rc=1 + [[ "$stdout_fd_owner_after" == 0:0 ]] || validation_rc=1 + [[ "$stdout_mode_after" == 600 ]] || validation_rc=1 + [[ "$stdout_fd_mode_after" == 600 ]] || validation_rc=1 + [[ "$stdout_size_after" == "$stdout_fd_size_after" ]] || validation_rc=1 + [[ "$stdout_mtime_after" == "$stdout_fd_mtime_after" ]] || validation_rc=1 + [[ "$stderr_identity_after" == "$stderr_identity_before" ]] || validation_rc=1 + [[ "$stderr_identity_after" == "$stderr_fd_identity_after" ]] || validation_rc=1 + [[ "$stderr_nlink_after" == 1 ]] || validation_rc=1 + [[ "$stderr_fd_nlink_after" == 1 ]] || validation_rc=1 + [[ "$stderr_type_after" == 'regular file' ]] || validation_rc=1 + [[ "$stderr_fd_type_after" == 'regular file' ]] || validation_rc=1 + [[ "$stderr_owner_after" == 0:0 ]] || validation_rc=1 + [[ "$stderr_fd_owner_after" == 0:0 ]] || validation_rc=1 + [[ "$stderr_mode_after" == 600 ]] || validation_rc=1 + [[ "$stderr_fd_mode_after" == 600 ]] || validation_rc=1 + [[ "$stderr_size_after" == "$stderr_fd_size_after" ]] || validation_rc=1 + [[ "$stderr_mtime_after" == "$stderr_fd_mtime_after" ]] || validation_rc=1 + if (( validation_rc == 0 )); then + _k3slra1_capture_file_hex "$stdout_path" 16 stdout_hex || validation_rc=1 + _k3slra1_capture_file_hex "$stderr_path" 16 stderr_hex || validation_rc=1 + [[ "$stdout_hex" == 6f6b0a ]] || validation_rc=1 + [[ -z "$stderr_hex" ]] || validation_rc=1 + (( child_rc == 0 )) || validation_rc=1 + fi + _k3slra1_remove_owned_capture "$stdout_path" "$stdout_identity_before" || cleanup_rc=1 + _k3slra1_remove_owned_capture "$stderr_path" "$stderr_identity_before" || cleanup_rc=1 + if _k3slra1_metadata_fields "$capture_root" parent_identity_after parent_nlink_after parent_size_after parent_mtime_after parent_type_after parent_owner_after parent_mode_after; then + : + else + cleanup_rc=1 + fi + [[ "$parent_identity_after" == "$parent_identity_before" ]] || cleanup_rc=1 + (( cleanup_rc == 0 )) || return 1 + case "${_capture_signal-0}" in + 130|143) return "$_capture_signal" ;; + esac + (( validation_rc == 0 )) || return 1 +} + +_k3slra1_remove_owned_capture() { + local path="${1-}" expected_identity="${2-}" canonical='' + local identity='' nlink='' size='' mtime='' type='' owner='' mode='' + (( $# == 2 )) || return 1 + [[ -n "$expected_identity" ]] || return 1 + _k3slra1_canonical_path "$path" canonical || return 1 + [[ "$canonical" == "$path" ]] || return 1 + _k3slra1_metadata_fields "$path" identity nlink size mtime type owner mode || return 1 + [[ "$identity" == "$expected_identity" ]] || return 1 + [[ "$nlink" =~ ^[1-9][0-9]*$ ]] || return 1 + [[ "$type" == 'regular file' ]] || return 1 + _k3slra1_command /usr/bin/unlink "$path" >/dev/null 2>&1 || return 1 + [[ ! -e "$path" ]] || return 1 + [[ ! -L "$path" ]] +} + +_k3slra1_remove_partial_capture() { + local path="${1-}" expected_device="${2-}" expected_identity="${3-}" canonical='' + local identity='' nlink='' size='' mtime='' type='' owner='' mode='' + (( $# == 3 )) || return 1 + _k3slra1_canonical_path "$path" canonical || return 1 + [[ "$canonical" == "$path" ]] || return 1 + _k3slra1_metadata_fields "$path" identity nlink size mtime type owner mode || return 1 + [[ "$identity" == "$expected_identity" ]] || return 1 + [[ "$type" == 'regular file' ]] || return 1 + [[ "$nlink" == 1 ]] || return 1 + [[ "$owner" == 0:0 ]] || return 1 + [[ "$mode" == 600 ]] || return 1 + [[ "${identity%%:*}" == "$expected_device" ]] || return 1 + _k3slra1_command /usr/bin/unlink "$path" >/dev/null 2>&1 || return 1 + [[ ! -e "$path" ]] || return 1 + [[ ! -L "$path" ]] +} + +_k3slra1_stderr_open_boundary() { + : +} + +_k3slra1_sqlite_stdout_stage() { + local database="${1-}" capture_root="${2-}" stdout_path="${3-}" stderr_path="${4-}" + local _stage_parent_identity='' _stage_parent_nlink='' _stage_parent_size='' + local _stage_parent_mtime='' _stage_parent_type='' _stage_parent_owner='' _stage_parent_mode='' + local _stage_parent_fd_identity='' _stage_parent_fd_nlink='' _stage_parent_fd_size='' + local _stage_parent_fd_mtime='' _stage_parent_fd_type='' _stage_parent_fd_owner='' _stage_parent_fd_mode='' + local _stage_stdout_identity='' _stage_stdout_nlink='' _stage_stdout_size='' + local _stage_stdout_mtime='' _stage_stdout_type='' _stage_stdout_owner='' _stage_stdout_mode='' + local _stage_stdout_fd_identity='' _stage_stdout_fd_nlink='' _stage_stdout_fd_size='' + local _stage_stdout_fd_mtime='' _stage_stdout_fd_type='' _stage_stdout_fd_owner='' _stage_stdout_fd_mode='' + local stage_rc=1 cleanup_rc=0 validation_rc=0 + (( $# == 4 )) || return 1 + if _k3slra1_metadata_fields "$capture_root" _stage_parent_identity _stage_parent_nlink _stage_parent_size _stage_parent_mtime _stage_parent_type _stage_parent_owner _stage_parent_mode; then + : + else + validation_rc=1 + fi + if _k3slra1_fd_metadata_fields "/proc/${BASHPID}/fd/6" _stage_parent_fd_identity _stage_parent_fd_nlink _stage_parent_fd_size _stage_parent_fd_mtime _stage_parent_fd_type _stage_parent_fd_owner _stage_parent_fd_mode; then + : + else + validation_rc=1 + fi + if _k3slra1_metadata_fields "$stdout_path" _stage_stdout_identity _stage_stdout_nlink _stage_stdout_size _stage_stdout_mtime _stage_stdout_type _stage_stdout_owner _stage_stdout_mode; then + : + else + validation_rc=1 + fi + if _k3slra1_fd_metadata_fields "/proc/${BASHPID}/fd/7" _stage_stdout_fd_identity _stage_stdout_fd_nlink _stage_stdout_fd_size _stage_stdout_fd_mtime _stage_stdout_fd_type _stage_stdout_fd_owner _stage_stdout_fd_mode; then + : + else + validation_rc=1 + fi + [[ "$_stage_parent_identity" == "$_stage_parent_fd_identity" ]] || validation_rc=1 + [[ "$_stage_parent_type" == directory ]] || validation_rc=1 + [[ "$_stage_parent_fd_type" == directory ]] || validation_rc=1 + [[ "$_stage_parent_owner" == 0:0 ]] || validation_rc=1 + [[ "$_stage_parent_fd_owner" == 0:0 ]] || validation_rc=1 + [[ "$_stage_parent_mode" == 700 ]] || validation_rc=1 + [[ "$_stage_parent_fd_mode" == 700 ]] || validation_rc=1 + [[ "$_stage_stdout_identity" == "$_stage_stdout_fd_identity" ]] || validation_rc=1 + [[ "$_stage_stdout_type" == 'regular file' ]] || validation_rc=1 + [[ "$_stage_stdout_fd_type" == 'regular file' ]] || validation_rc=1 + [[ "$_stage_stdout_nlink" == 1 ]] || validation_rc=1 + [[ "$_stage_stdout_fd_nlink" == 1 ]] || validation_rc=1 + [[ "$_stage_stdout_size" == 0 ]] || validation_rc=1 + [[ "$_stage_stdout_fd_size" == 0 ]] || validation_rc=1 + [[ "$_stage_stdout_identity" != "$_stage_parent_identity" ]] || validation_rc=1 + if (( validation_rc == 0 )); then + if _k3slra1_stderr_open_boundary; then + if _k3slra1_sqlite_capture_transaction "$database" "$capture_root" "$stdout_path" "$stderr_path" 8>"$stderr_path"; then + stage_rc=0 + else + stage_rc=$? + fi + fi + fi + if (( _capture_entered == 0 )); then + _k3slra1_remove_partial_capture "$stdout_path" "${_stage_parent_identity%%:*}" "$_stage_stdout_identity" || cleanup_rc=1 + fi + (( cleanup_rc == 0 )) || return 1 + case "$stage_rc" in + 0) + case "${_capture_signal-0}" in + 130|143) return "$_capture_signal" ;; + esac + (( validation_rc == 0 )) || return 1 + return 0 + ;; + 130|143) return "$stage_rc" ;; + *) + if (( _capture_entered == 0 )); then + case "${_capture_signal-0}" in + 130|143) return "$_capture_signal" ;; + esac + fi + return 1 + ;; + esac +} + +_k3slra1_sqlite_quick_check() ( + local bundle="${1-}" layout="${2-}" capture_root="${3-}" database_relative='' + local stdout_path='' stderr_path='' transaction_rc=1 + local capture_canonical='' capture_identity='' capture_nlink='' capture_size='' capture_mtime='' + local capture_type='' capture_owner='' capture_mode='' + local _capture_entered=0 _capture_signal=0 + (( $# == 3 )) || return 1 + case "$layout" in + canonical) database_relative="$K3SLRA1_CANONICAL_DATABASE" ;; + legacy) database_relative="$K3SLRA1_LEGACY_DATABASE" ;; + *) return 1 ;; + esac + stdout_path="${capture_root}/sqlite.stdout" + stderr_path="${capture_root}/sqlite.stderr" + _k3slra1_canonical_path "$capture_root" capture_canonical || return 1 + [[ "$capture_canonical" == "$capture_root" ]] || return 1 + _k3slra1_metadata_fields "$capture_root" capture_identity capture_nlink capture_size capture_mtime capture_type capture_owner capture_mode || return 1 + [[ "$capture_type" == directory ]] || return 1 + [[ "$capture_owner" == 0:0 ]] || return 1 + [[ "$capture_mode" == 700 ]] || return 1 + [[ ! -e "$stdout_path" ]] || return 1 + [[ ! -L "$stdout_path" ]] || return 1 + [[ ! -e "$stderr_path" ]] || return 1 + [[ ! -L "$stderr_path" ]] || return 1 + umask 077 + set -o noclobber + trap '_capture_signal=130; if (( transaction_rc == 0 )); then transaction_rc=130; fi' INT + trap '_capture_signal=143; if (( transaction_rc == 0 )); then transaction_rc=143; fi' TERM + if _k3slra1_sqlite_stdout_stage "${bundle}/pre/${database_relative#./}" "$capture_root" "$stdout_path" "$stderr_path" 2>/dev/null 6<"$capture_root" 7>"$stdout_path"; then + transaction_rc=0 + else + transaction_rc=$? + fi + if (( _capture_signal != 0 && transaction_rc == 0 )); then + transaction_rc=$_capture_signal + fi + case "$transaction_rc" in + 0) return 0 ;; + 130|143) return "$transaction_rc" ;; + *) return 1 ;; + esac +) diff --git a/scripts/lib/k3s-local-recovery-header-proof.sh b/scripts/lib/k3s-local-recovery-header-proof.sh new file mode 100644 index 0000000..e2deeb3 --- /dev/null +++ b/scripts/lib/k3s-local-recovery-header-proof.sh @@ -0,0 +1,344 @@ +#!/usr/bin/env bash + +# Source-only foundation for the standalone K3s LUKS header proof. + +readonly K3SLRH_PROOF_RELATIVE='HyeonworksRecovery/containers/k3s-recovery.header-proof.luks' +readonly K3SLRH_PRE_KDBX_PREFIX='hyeonworks-recovery.header-proof-pre.' +readonly K3SLRH_POST_KDBX_PREFIX='hyeonworks-recovery.header-proof-post.' +readonly K3SLRH_HEADER_BACKUP_NAME='header-original.bin' +readonly K3SLRH_HEADER_EXPORT_NAME='header-exported.bin' +readonly K3SLRH_SQLITE_PACKAGE_VERSION='3.45.1-1ubuntu2.7' +readonly K3SLRH_SHARED_LIBRARY_SHA256='a206d81539e6c37fa085457ba01cff9de57de50f7235e3a4085989780e99585d' +readonly K3SLRH_A1_LIBRARY_SHA256='8b39fc46229ebd4c77ff20040f942c4ef7b756d564d344a8ecdd11a7428e2968' +readonly K3SLRH_A1_HELPER_SHA256='06831fa35943a328ecd183d3c6f721b24c4607b2f349abf6201a12e690d08ebe' +readonly K3SLRH_CONTRACT_SHA256='c045f42c2dc3eda09c37a123bcc05d88d94f275e61504a2d5816948237e516df' +readonly K3SLRH_VALIDATOR_SHA256='6b869032fc325759ee25b219fad4ea6190a3389a561c496f0664a51b802f73e6' + +_k3slrh_command() { + "$@" +} + +_k3slrh_output_name_is_safe() { + local _k3slrh_output_candidate="${1-}" + (( $# == 1 )) || return 1 + [[ "$_k3slrh_output_candidate" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || return 1 + [[ "$_k3slrh_output_candidate" != _k3slrh_* ]] +} + +_k3slrh_stderr_marker() { + local _k3slrh_stderr_chunk='' _k3slrh_stderr_rc=0 + if IFS= read -r -d '' -n 1 _k3slrh_stderr_chunk; then + _k3slrh_stderr_rc=0 + else + _k3slrh_stderr_rc=$? + fi + if (( _k3slrh_stderr_rc == 0 )); then + printf '\0STDERR\0' + while IFS= read -r -d '' -n 4096 _k3slrh_stderr_chunk; do + : + done + elif [[ -n "$_k3slrh_stderr_chunk" ]]; then + printf '\0STDERR\0' + fi +} + +_k3slrh_command_packet() { + local _k3slrh_packet_rc=0 + (( $# > 0 )) || return 1 + if _k3slrh_command "$@" 2> >(_k3slrh_stderr_marker); then + _k3slrh_packet_rc=0 + else + _k3slrh_packet_rc=$? + fi + printf '\0RC=%s\0' "$_k3slrh_packet_rc" +} + +_k3slrh_capture_command() { + local _k3slrh_capture_output="${1-}" _k3slrh_capture_expected_rc="${2-}" + local -a _k3slrh_capture_records=() + shift 2 || return 1 + (( $# > 0 )) || return 1 + _k3slrh_output_name_is_safe "$_k3slrh_capture_output" || return 1 + [[ "$_k3slrh_capture_expected_rc" =~ ^(0|[1-9][0-9]{0,2})$ ]] || return 1 + (( 10#$_k3slrh_capture_expected_rc <= 255 )) || return 1 + mapfile -d '' -t _k3slrh_capture_records < <(_k3slrh_command_packet "$@") + (( ${#_k3slrh_capture_records[@]} == 2 )) || return 1 + [[ "${_k3slrh_capture_records[1]}" == "RC=${_k3slrh_capture_expected_rc}" ]] || return 1 + printf -v "$_k3slrh_capture_output" '%s' "${_k3slrh_capture_records[0]}" +} + +_k3slrh_capture_one_line() { + local _k3slrh_line_output="${1-}" _k3slrh_line_value='' + shift || return 1 + _k3slrh_output_name_is_safe "$_k3slrh_line_output" || return 1 + _k3slrh_capture_command _k3slrh_line_value 0 "$@" || return 1 + [[ "$_k3slrh_line_value" == *$'\n' ]] || return 1 + [[ "$_k3slrh_line_value" != *$'\r'* ]] || return 1 + _k3slrh_line_value="${_k3slrh_line_value%$'\n'}" + [[ -n "$_k3slrh_line_value" && "$_k3slrh_line_value" != *$'\n'* ]] || return 1 + printf -v "$_k3slrh_line_output" '%s' "$_k3slrh_line_value" +} + +_k3slrh_parse_cli() { + local _k3slrh_cli_output="${1-}" _k3slrh_cli_mode='' + shift || return 1 + _k3slrh_output_name_is_safe "$_k3slrh_cli_output" || return 1 + case "$#" in + 0) _k3slrh_cli_mode=dry-run ;; + 1) + [[ "$1" == --execute ]] || return 1 + _k3slrh_cli_mode=execute + ;; + *) return 1 ;; + esac + printf -v "$_k3slrh_cli_output" '%s' "$_k3slrh_cli_mode" +} + +_k3slrh_target_state_is_consistent() { + local _k3slrh_target_utc="${1-}" _k3slrh_target_pre="${2-}" + local _k3slrh_target_post="${3-}" _k3slrh_target_expected_pre='' + local _k3slrh_target_expected_post='' + (( $# == 3 )) || return 1 + [[ "$_k3slrh_target_utc" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || return 1 + _k3slrh_target_expected_pre="HyeonworksRecovery/vault/backups/${K3SLRH_PRE_KDBX_PREFIX}${_k3slrh_target_utc}.kdbx" + _k3slrh_target_expected_post="HyeonworksRecovery/vault/backups/${K3SLRH_POST_KDBX_PREFIX}${_k3slrh_target_utc}.kdbx" + [[ "$_k3slrh_target_pre" == "$_k3slrh_target_expected_pre" ]] || return 1 + [[ "$_k3slrh_target_post" == "$_k3slrh_target_expected_post" ]] || return 1 + [[ "$_k3slrh_target_pre" != "$_k3slrh_target_post" ]] +} + +_k3slrh_derive_targets() { + local _k3slrh_derive_utc="${1-}" _k3slrh_derive_pre='' _k3slrh_derive_post='' + (( $# == 1 )) || return 1 + [[ "$_k3slrh_derive_utc" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || return 1 + [[ "${K3SLRH_RUN_UTC+x}" != x && "${K3SLRH_PRE_KDBX_RELATIVE+x}" != x && + "${K3SLRH_POST_KDBX_RELATIVE+x}" != x ]] || return 1 + _k3slrh_derive_pre="HyeonworksRecovery/vault/backups/${K3SLRH_PRE_KDBX_PREFIX}${_k3slrh_derive_utc}.kdbx" + _k3slrh_derive_post="HyeonworksRecovery/vault/backups/${K3SLRH_POST_KDBX_PREFIX}${_k3slrh_derive_utc}.kdbx" + _k3slrh_target_state_is_consistent "$_k3slrh_derive_utc" "$_k3slrh_derive_pre" "$_k3slrh_derive_post" || return 1 + declare -gr K3SLRH_RUN_UTC="$_k3slrh_derive_utc" + declare -gr K3SLRH_PRE_KDBX_RELATIVE="$_k3slrh_derive_pre" + declare -gr K3SLRH_POST_KDBX_RELATIVE="$_k3slrh_derive_post" +} + +_k3slrh_source_file_is_exact() { + local _k3slrh_source_path="${1-}" _k3slrh_source_hash="${2-}" + local _k3slrh_source_canonical='' _k3slrh_source_metadata='' _k3slrh_source_digest='' + (( $# == 2 )) || return 1 + [[ "$_k3slrh_source_path" == /* && "$_k3slrh_source_hash" =~ ^[0-9a-f]{64}$ ]] || return 1 + _k3slrh_command /usr/bin/test -f "$_k3slrh_source_path" >/dev/null 2>&1 || return 1 + _k3slrh_command /usr/bin/test ! -L "$_k3slrh_source_path" >/dev/null 2>&1 || return 1 + _k3slrh_capture_one_line _k3slrh_source_canonical /usr/bin/readlink -e -- "$_k3slrh_source_path" || return 1 + [[ "$_k3slrh_source_canonical" == "$_k3slrh_source_path" ]] || return 1 + _k3slrh_capture_one_line _k3slrh_source_metadata /usr/bin/stat '--format=%F|%u|%g|%a' -- "$_k3slrh_source_path" || return 1 + [[ "$_k3slrh_source_metadata" == 'regular file|'* || "$_k3slrh_source_metadata" == 'regular empty file|'* ]] || return 1 + _k3slrh_capture_one_line _k3slrh_source_digest /usr/bin/sha256sum -- "$_k3slrh_source_path" || return 1 + [[ "$_k3slrh_source_digest" == "${_k3slrh_source_hash} ${_k3slrh_source_path}" ]] +} + +_k3slrh_required_sources_are_exact() { + local _k3slrh_source_root="${K3SLRH_REPOSITORY_ROOT:-}" + (( $# == 0 )) || return 1 + [[ "$_k3slrh_source_root" == /* ]] || return 1 + _k3slrh_source_file_is_exact "${_k3slrh_source_root}/scripts/lib/k3s-local-recovery.sh" "$K3SLRH_SHARED_LIBRARY_SHA256" || return 1 + _k3slrh_source_file_is_exact "${_k3slrh_source_root}/scripts/lib/k3s-local-recovery-a1.sh" "$K3SLRH_A1_LIBRARY_SHA256" || return 1 + _k3slrh_source_file_is_exact "${_k3slrh_source_root}/scripts/libexec/k3s-local-recovery-a1-check.sh" "$K3SLRH_A1_HELPER_SHA256" || return 1 + _k3slrh_source_file_is_exact "${_k3slrh_source_root}/infrastructure/security/k3s/local-recovery.env" "$K3SLRH_CONTRACT_SHA256" +} + +_k3slrh_required_binaries_are_trusted() { + local _k3slrh_binary='' + local -a _k3slrh_binaries=( + /usr/bin/bash + /usr/bin/date + /usr/bin/dpkg-query + /usr/bin/env + /usr/bin/find + /usr/bin/findmnt + /usr/bin/keepassxc-cli + /usr/bin/readlink + /usr/bin/sha256sum + /usr/bin/sqlite3 + /usr/bin/stat + /usr/bin/sudo + /usr/bin/systemctl + /usr/bin/test + /usr/bin/timeout + /usr/local/bin/k3s + /usr/sbin/cryptsetup + /usr/sbin/losetup + ) + (( $# == 0 )) || return 1 + for _k3slrh_binary in "${_k3slrh_binaries[@]}"; do + _k3slr_trusted_root_executable "$_k3slrh_binary" || return 1 + done +} + +_k3slrh_package_is_exact() { + local _k3slrh_package_name="${1-}" _k3slrh_package_expected="${2-}" + local _k3slrh_package_record='' + (( $# == 2 )) || return 1 + case "$_k3slrh_package_name|$_k3slrh_package_expected" in + 'keepassxc|2.7.6+dfsg.1-1build3'|'cryptsetup-bin|2:2.7.0-1ubuntu4.2'|"sqlite3|${K3SLRH_SQLITE_PACKAGE_VERSION}") ;; + *) return 1 ;; + esac + _k3slrh_capture_command _k3slrh_package_record 0 /usr/bin/dpkg-query --show '--showformat=${Status}|${Version}\n' "$_k3slrh_package_name" || return 1 + [[ "$_k3slrh_package_record" == "install ok installed|${_k3slrh_package_expected}"$'\n' ]] +} + +_k3slrh_packages_are_exact() { + (( $# == 0 )) || return 1 + _k3slrh_package_is_exact keepassxc '2.7.6+dfsg.1-1build3' || return 1 + _k3slrh_package_is_exact cryptsetup-bin '2:2.7.0-1ubuntu4.2' || return 1 + _k3slrh_package_is_exact sqlite3 "$K3SLRH_SQLITE_PACKAGE_VERSION" +} + +_k3slrh_cached_sudo_is_available() { + (( $# == 0 )) || return 1 + _k3slrh_command /usr/bin/sudo --non-interactive --validate >/dev/null 2>&1 +} + +_k3slrh_k3s_health() { + local _k3slrh_health_systemctl='' _k3slrh_health_readyz='' + (( $# == 0 )) || return 1 + _k3slrh_capture_command _k3slrh_health_systemctl 0 /usr/bin/systemctl is-active --quiet k3s || return 1 + [[ -z "$_k3slrh_health_systemctl" ]] || return 1 + _k3slrh_capture_command _k3slrh_health_readyz 0 \ + /usr/bin/sudo --non-interactive -- \ + /usr/bin/env -i 'PATH=/usr/sbin:/usr/bin:/sbin:/bin' LC_ALL=C \ + /usr/bin/timeout --signal=TERM --kill-after=1s 30s \ + /usr/local/bin/k3s kubectl get --raw=/readyz || return 1 + [[ "$_k3slrh_health_readyz" == $'ok\n' ]] +} + +_k3slrh_run_closed_validator() { + local _k3slrh_validator_path='' _k3slrh_validator_record='' + local _k3slrh_validator_command_rc=0 _k3slrh_validator_post_rc=0 + (( $# == 0 )) || return 1 + [[ "${K3SLRH_REPOSITORY_ROOT:-}" == /* ]] || return 1 + _k3slrh_validator_path="${K3SLRH_REPOSITORY_ROOT}/scripts/validate/k3s-local-recovery.sh" + _k3slrh_source_file_is_exact "$_k3slrh_validator_path" "$K3SLRH_VALIDATOR_SHA256" || return 1 + _k3slrh_capture_command _k3slrh_validator_record 0 \ + /usr/bin/env -i 'PATH=/usr/sbin:/usr/bin:/sbin:/bin' LC_ALL=C \ + /usr/bin/bash --noprofile --norc -- "$_k3slrh_validator_path" --expect-closed || + _k3slrh_validator_command_rc=1 + _k3slrh_source_file_is_exact "$_k3slrh_validator_path" "$K3SLRH_VALIDATOR_SHA256" || + _k3slrh_validator_post_rc=1 + (( _k3slrh_validator_command_rc == 0 && _k3slrh_validator_post_rc == 0 )) || return 1 + [[ "$_k3slrh_validator_record" == $'Recovery device: match\nRecovery state: closed\nLineage: match\nLatest bundle: not_checked\n' ]] +} + +_k3slrh_mount_is_absent() { + local _k3slrh_mount_path="${1-}" _k3slrh_mount_record='' + (( $# == 1 )) || return 1 + [[ "$_k3slrh_mount_path" == /* ]] || return 1 + _k3slrh_capture_command _k3slrh_mount_record 1 /usr/bin/findmnt -rn -M "$_k3slrh_mount_path" -o TARGET || return 1 + [[ -z "$_k3slrh_mount_record" ]] +} + +_k3slrh_mapping_is_absent() { + local _k3slrh_mapping_path="${1-}" + (( $# == 1 )) || return 1 + case "$_k3slrh_mapping_path" in + "/dev/mapper/${K3SLR_MAPPING_NAME}"|"/dev/mapper/${K3SLR_PROOF_MAPPING_NAME}") ;; + *) return 1 ;; + esac + _k3slrh_command /usr/bin/test ! -e "$_k3slrh_mapping_path" >/dev/null 2>&1 || return 1 + _k3slrh_command /usr/bin/test ! -L "$_k3slrh_mapping_path" >/dev/null 2>&1 +} + +_k3slrh_related_loops_are_absent() { + local _k3slrh_loop_record='' _k3slrh_loop_line='' + local _k3slrh_loop_original="${K3SLR_OUTER_MOUNT}/${K3SLR_CONTAINER_RELATIVE}" + local _k3slrh_loop_proof="${K3SLR_OUTER_MOUNT}/${K3SLRH_PROOF_RELATIVE}" + (( $# == 0 )) || return 1 + _k3slrh_capture_command _k3slrh_loop_record 0 /usr/sbin/losetup --list --noheadings --output NAME,BACK-FILE || return 1 + while IFS= read -r _k3slrh_loop_line; do + [[ -n "$_k3slrh_loop_line" ]] || continue + case "$_k3slrh_loop_line" in + *" ${_k3slrh_loop_original}"|*" ${_k3slrh_loop_proof}") return 1 ;; + esac + done <<<"$_k3slrh_loop_record" +} + +_k3slrh_outer_mountpoint_is_clean() { + local _k3slrh_outer_canonical='' _k3slrh_outer_metadata='' _k3slrh_outer_children='' + (( $# == 0 )) || return 1 + _k3slrh_command /usr/bin/test -d "$K3SLR_OUTER_MOUNT" >/dev/null 2>&1 || return 1 + _k3slrh_command /usr/bin/test ! -L "$K3SLR_OUTER_MOUNT" >/dev/null 2>&1 || return 1 + _k3slrh_capture_one_line _k3slrh_outer_canonical /usr/bin/readlink -e -- "$K3SLR_OUTER_MOUNT" || return 1 + [[ "$_k3slrh_outer_canonical" == "$K3SLR_OUTER_MOUNT" ]] || return 1 + _k3slrh_capture_one_line _k3slrh_outer_metadata /usr/bin/stat '--format=%F|%u|%g|%a' -- "$K3SLR_OUTER_MOUNT" || return 1 + [[ "$_k3slrh_outer_metadata" == 'directory|0|0|755' ]] || return 1 + _k3slrh_capture_command _k3slrh_outer_children 0 /usr/bin/find "$K3SLR_OUTER_MOUNT" -mindepth 1 -maxdepth 1 -print -quit || return 1 + [[ -z "$_k3slrh_outer_children" ]] +} + +_k3slrh_runtime_residue_is_absent() { + local _k3slrh_runtime_record='' + (( $# == 0 )) || return 1 + _k3slrh_capture_command _k3slrh_runtime_record 0 /usr/bin/find /run -mindepth 1 -maxdepth 1 -name 'hyeonworks-k3s-recovery.*' -print -quit || return 1 + [[ -z "$_k3slrh_runtime_record" ]] +} + +_k3slrh_closed_resources_are_absent() { + (( $# == 0 )) || return 1 + _k3slrh_mount_is_absent "$K3SLR_OUTER_MOUNT" || return 1 + _k3slrh_mount_is_absent "$K3SLR_INNER_MOUNT" || return 1 + _k3slrh_mapping_is_absent "/dev/mapper/${K3SLR_MAPPING_NAME}" || return 1 + _k3slrh_mapping_is_absent "/dev/mapper/${K3SLR_PROOF_MAPPING_NAME}" || return 1 + _k3slrh_related_loops_are_absent || return 1 + _k3slrh_outer_mountpoint_is_clean || return 1 + _k3slrh_runtime_residue_is_absent +} + +_k3slrh_pre_mount_checks() { + (( $# == 0 )) || return 1 + _k3slrh_required_sources_are_exact || return 1 + _k3slrh_required_binaries_are_trusted || return 1 + _k3slrh_packages_are_exact || return 1 + _k3slrh_cached_sudo_is_available || return 1 + _k3slrh_k3s_health || return 1 + _k3slrh_run_closed_validator || return 1 + _k3slrh_closed_resources_are_absent +} + +_k3slrh_pre_mount_capability() { + (( $# == 0 )) || return 1 + _k3slrh_pre_mount_checks || return 1 + printf 'Header proof capability: pass\n' +} + +_k3slrh_current_utc() { + local _k3slrh_utc_output="${1-}" _k3slrh_utc_value='' + (( $# == 1 )) || return 1 + _k3slrh_output_name_is_safe "$_k3slrh_utc_output" || return 1 + _k3slrh_capture_one_line _k3slrh_utc_value /usr/bin/date --utc +%Y%m%dT%H%M%SZ || return 1 + [[ "$_k3slrh_utc_value" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || return 1 + printf -v "$_k3slrh_utc_output" '%s' "$_k3slrh_utc_value" +} + +_k3slrh_header_proof_main() { + local _k3slrh_main_mode='' _k3slrh_main_utc='' _k3slrh_main_parse_rc=0 + if _k3slrh_parse_cli _k3slrh_main_mode "$@"; then + _k3slrh_main_parse_rc=0 + else + _k3slrh_main_parse_rc=$? + fi + (( _k3slrh_main_parse_rc == 0 )) || return 2 + [[ "${EUID:-}" =~ ^[0-9]+$ && "${EUID:-}" != 0 ]] || return 1 + _k3slr_xtrace_is_disabled || return 1 + if [[ "$_k3slrh_main_mode" == execute ]]; then + _k3slr_require_execute_tty || return 1 + declare -F _k3slrh_prove_header_restore_a1 >/dev/null || return 1 + return 1 + fi + _k3slrh_current_utc _k3slrh_main_utc || return 1 + _k3slrh_derive_targets "$_k3slrh_main_utc" || return 1 + _k3slrh_pre_mount_checks || return 1 + printf '%s\n' \ + 'Header proof state: closed' \ + 'Header proof capability: pass' \ + 'Execute-time checks: pending' \ + 'DRY RUN PASS: no changes' +} diff --git a/scripts/lib/k3s-local-recovery.sh b/scripts/lib/k3s-local-recovery.sh new file mode 100644 index 0000000..281916a --- /dev/null +++ b/scripts/lib/k3s-local-recovery.sh @@ -0,0 +1,2038 @@ +#!/usr/bin/env bash + +# Source-only helpers for the non-secret local recovery hardware contract. +# The contract is parsed as data; it must never be sourced as shell code. + +if [[ "${K3SLR_KEEPASS_ENTRY+x}" == x ]]; then + [[ "$K3SLR_KEEPASS_ENTRY" == 'K3s Recovery LUKS' ]] || return 1 + readonly K3SLR_KEEPASS_ENTRY +else + readonly K3SLR_KEEPASS_ENTRY='K3s Recovery LUKS' +fi +if [[ "${K3SLR_KEEPASS_HEADER_ATTACHMENT+x}" == x ]]; then + [[ "$K3SLR_KEEPASS_HEADER_ATTACHMENT" == 'luks-header-backup.bin' ]] || return 1 + readonly K3SLR_KEEPASS_HEADER_ATTACHMENT +else + readonly K3SLR_KEEPASS_HEADER_ATTACHMENT='luks-header-backup.bin' +fi +if [[ "${_K3SLR_INT64_MAX+x}" == x ]]; then + [[ "$_K3SLR_INT64_MAX" == 9223372036854775807 ]] || return 1 + readonly _K3SLR_INT64_MAX +else + readonly _K3SLR_INT64_MAX=9223372036854775807 +fi +if [[ "${_K3SLR_REQUIRED_MINIMUM_FREE_BYTES+x}" == x ]]; then + [[ "$_K3SLR_REQUIRED_MINIMUM_FREE_BYTES" == 10737418240 ]] || return 1 + readonly _K3SLR_REQUIRED_MINIMUM_FREE_BYTES +else + readonly _K3SLR_REQUIRED_MINIMUM_FREE_BYTES=10737418240 +fi +if [[ "${_K3SLR_KEEPASSXC_VERSION+x}" == x ]]; then + [[ "$_K3SLR_KEEPASSXC_VERSION" == '2.7.6+dfsg.1-1build3' ]] || return 1 + readonly _K3SLR_KEEPASSXC_VERSION +else + readonly _K3SLR_KEEPASSXC_VERSION='2.7.6+dfsg.1-1build3' +fi +if [[ "${_K3SLR_CRYPTSETUP_VERSION+x}" == x ]]; then + [[ "$_K3SLR_CRYPTSETUP_VERSION" == '2:2.7.0-1ubuntu4.2' ]] || return 1 + readonly _K3SLR_CRYPTSETUP_VERSION +else + readonly _K3SLR_CRYPTSETUP_VERSION='2:2.7.0-1ubuntu4.2' +fi + +_k3slr_uint() { + local value="${1-}" + [[ "$value" =~ ^(0|[1-9][0-9]*)$ ]] || return 1 + (( ${#value} < 19 )) && return 0 + (( ${#value} == 19 )) && + [[ "$value" < '9223372036854775807' || "$value" == '9223372036854775807' ]] && return 0 + return 1 +} + +_k3slr_contract_value_is_valid() { + local key="$1" value="$2" + + case "$key" in + K3SLR_SCHEMA_VERSION) + [[ "$value" == 1 ]] + ;; + K3SLR_RECOVERY_DISK_BY_ID|K3SLR_K3S_DISK_BY_ID) + [[ "$value" =~ ^/dev/disk/by-id/wwn-0x[0-9a-f]{16}$ ]] + ;; + K3SLR_RECOVERY_PARTITION_BY_ID|K3SLR_K3S_PARTITION_BY_ID) + [[ "$value" =~ ^/dev/disk/by-id/wwn-0x[0-9a-f]{16}-part[1-9][0-9]*$ ]] + ;; + K3SLR_RECOVERY_FS_UUID) + [[ "$value" =~ ^[0-9A-F]{16}$ ]] + ;; + K3SLR_K3S_FS_UUID|K3SLR_RECOVERY_PARTUUID|K3SLR_K3S_PARTUUID) + [[ "$value" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] + ;; + K3SLR_RECOVERY_MODEL|K3SLR_K3S_MODEL) + [[ "$value" =~ ^[A-Za-z0-9._-]+$ ]] + ;; + K3SLR_RECOVERY_SERIAL|K3SLR_K3S_SERIAL) + [[ "$value" =~ ^[A-Za-z0-9]+$ ]] + ;; + K3SLR_RECOVERY_WWN|K3SLR_K3S_WWN) + [[ "$value" =~ ^0x[0-9a-f]{16}$ ]] + ;; + K3SLR_OWNER_UID|K3SLR_OWNER_GID) + _k3slr_uint "$value" + ;; + K3SLR_OUTER_MOUNT|K3SLR_INNER_MOUNT) + [[ "$value" =~ ^/[A-Za-z0-9._/-]+$ && "$value" != *'//'* && "$value" != *'/./'* && "$value" != *'/../'* && "$value" != */. && "$value" != */.. && "$value" != */ ]] + ;; + K3SLR_ROOT_RELATIVE|K3SLR_DATABASE_RELATIVE|K3SLR_CONTAINER_RELATIVE|K3SLR_RUNTIME_METADATA_RELATIVE) + [[ "$value" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ && "$value" != *'//'* && "$value" != *'/./'* && "$value" != *'/../'* && "$value" != */. && "$value" != */.. && "$value" != */ ]] + ;; + K3SLR_MAPPING_NAME|K3SLR_PROOF_MAPPING_NAME) + [[ "$value" =~ ^[a-z][a-z0-9-]*$ ]] + ;; + K3SLR_INNER_LABEL) + [[ "$value" =~ ^[A-Z][A-Z0-9_]*$ ]] + ;; + K3SLR_CONTAINER_SIZE_BYTES|K3SLR_MINIMUM_FREE_BYTES) + _k3slr_uint "$value" && [[ "$value" != 0 ]] + ;; + K3SLR_OUTER_MIN_REMAINING_PERCENT) + _k3slr_uint "$value" && [[ "$value" != 0 ]] && (( 10#$value <= 99 )) + ;; + *) + return 1 + ;; + esac +} + +_k3slr_parse_contract() { + local file="$1" result_name="$2" line key value + local -n result="$result_name" + local -A allowed=( + [K3SLR_SCHEMA_VERSION]=1 + [K3SLR_RECOVERY_DISK_BY_ID]=1 [K3SLR_RECOVERY_PARTITION_BY_ID]=1 + [K3SLR_RECOVERY_FS_UUID]=1 [K3SLR_RECOVERY_PARTUUID]=1 + [K3SLR_RECOVERY_MODEL]=1 [K3SLR_RECOVERY_SERIAL]=1 [K3SLR_RECOVERY_WWN]=1 + [K3SLR_K3S_DISK_BY_ID]=1 [K3SLR_K3S_PARTITION_BY_ID]=1 + [K3SLR_K3S_FS_UUID]=1 [K3SLR_K3S_PARTUUID]=1 + [K3SLR_K3S_MODEL]=1 [K3SLR_K3S_SERIAL]=1 [K3SLR_K3S_WWN]=1 + [K3SLR_OWNER_UID]=1 [K3SLR_OWNER_GID]=1 + [K3SLR_OUTER_MOUNT]=1 [K3SLR_INNER_MOUNT]=1 + [K3SLR_ROOT_RELATIVE]=1 [K3SLR_DATABASE_RELATIVE]=1 + [K3SLR_CONTAINER_RELATIVE]=1 [K3SLR_RUNTIME_METADATA_RELATIVE]=1 + [K3SLR_MAPPING_NAME]=1 [K3SLR_PROOF_MAPPING_NAME]=1 [K3SLR_INNER_LABEL]=1 + [K3SLR_CONTAINER_SIZE_BYTES]=1 [K3SLR_MINIMUM_FREE_BYTES]=1 + [K3SLR_OUTER_MIN_REMAINING_PERCENT]=1 + ) + local -A seen=() + + [[ -f "$file" && ! -L "$file" && -r "$file" ]] || return 1 + result=() + while IFS= read -r line || [[ -n "$line" ]]; do + [[ "$line" != *$'\r'* && "$line" =~ ^([A-Z][A-Z0-9_]*)=(.*)$ ]] || return 1 + key="${BASH_REMATCH[1]}" + value="${BASH_REMATCH[2]}" + [[ "${allowed[$key]+present}" == present && "${seen[$key]+present}" != present ]] || return 1 + _k3slr_contract_value_is_valid "$key" "$value" || return 1 + seen["$key"]=1 + result["$key"]="$value" + done <"$file" + + for key in "${!allowed[@]}"; do + [[ "${seen[$key]+present}" == present ]] || return 1 + done +} + +_k3slr_config_value() { + local file="$1" key="$2" + local -A values=() + + _k3slr_parse_contract "$file" values || return 1 + [[ "${values[$key]+present}" == present ]] || return 1 + printf '%s\n' "${values[$key]}" +} + +_k3slr_load_contract() { + local file="$1" key + local -A values=() + + _k3slr_parse_contract "$file" values || return 1 + for key in "${!values[@]}"; do + printf -v "$key" '%s' "${values[$key]}" + done +} + +_k3slr_required_bytes() { + local phase_bytes="$1" used_bytes="$2" padded maximum_used + + _k3slr_uint "$phase_bytes" || return 1 + _k3slr_uint "$used_bytes" || return 1 + (( phase_bytes <= 1844674407370955160 )) || return 1 + padded=$(((phase_bytes * 5 + 3) / 4)) + (( padded <= (_K3SLR_INT64_MAX - _K3SLR_REQUIRED_MINIMUM_FREE_BYTES) / 2 )) || return 1 + maximum_used=$((_K3SLR_INT64_MAX - 2 * padded - _K3SLR_REQUIRED_MINIMUM_FREE_BYTES)) + (( used_bytes <= maximum_used )) || return 1 + printf '%s\n' "$((used_bytes + 2 * padded + _K3SLR_REQUIRED_MINIMUM_FREE_BYTES))" +} + +_k3slr_inner_capacity_fits() { + local total_bytes="$1" used_bytes="$2" phase_bytes="$3" required + + _k3slr_uint "$total_bytes" || return 1 + required="$(_k3slr_required_bytes "$phase_bytes" "$used_bytes")" || return 1 + (( total_bytes >= required )) +} + +_k3slr_outer_capacity_fits() { + local total_bytes="$1" free_bytes="$2" container_bytes="$3" + local minimum_free_bytes="${K3SLR_MINIMUM_FREE_BYTES:-}" + local remaining_percent="${K3SLR_OUTER_MIN_REMAINING_PERCENT:-}" + local percent_reserve required + + _k3slr_uint "$total_bytes" || return 1 + _k3slr_uint "$free_bytes" || return 1 + _k3slr_uint "$container_bytes" || return 1 + _k3slr_uint "$minimum_free_bytes" || return 1 + _k3slr_uint "$remaining_percent" || return 1 + [[ "$container_bytes" != 0 && "$minimum_free_bytes" != 0 && "$remaining_percent" != 0 ]] || return 1 + (( remaining_percent <= 99 && free_bytes <= total_bytes )) || return 1 + + (( total_bytes <= (_K3SLR_INT64_MAX - 99) / remaining_percent )) || return 1 + percent_reserve=$(((total_bytes * remaining_percent + 99) / 100)) + (( container_bytes <= (_K3SLR_INT64_MAX - minimum_free_bytes) / 2 )) || return 1 + required=$((minimum_free_bytes + 2 * container_bytes)) + (( required <= _K3SLR_INT64_MAX - percent_reserve )) || return 1 + required=$((required + percent_reserve)) + (( free_bytes >= required )) +} + +_k3slr_xtrace_is_disabled() { + [[ "$-" != *x* ]] +} + +_k3slr_parse_lifecycle_cli() { + local destination_name="${1-}" mode + shift || return 1 + _k3slr_output_name_is_safe "$destination_name" _k3slr_cli_ || return 1 + case "$#" in + 0) mode=dry-run ;; + 1) + [[ "$1" == --execute ]] || return 1 + mode=execute + ;; + *) return 1 ;; + esac + printf -v "$destination_name" '%s' "$mode" +} + +_k3slr_tty_capability() { + local tty_fd + (( $# == 0 )) || return 1 + [[ -t 0 ]] || return 1 + exec {tty_fd}<>/dev/tty || return 1 + if [[ ! -t "$tty_fd" ]]; then + exec {tty_fd}>&- + return 1 + fi + exec {tty_fd}>&- +} + +_k3slr_require_execute_tty() { + (( $# == 0 )) || return 1 + _k3slr_tty_capability +} + +_k3slr_open_close_capability_gate() { + local keepass_state='' cryptsetup_state='' binary + local -a required_binaries=( + /usr/bin/keepassxc-cli + /usr/sbin/cryptsetup + /usr/sbin/losetup + /usr/sbin/blkid + /usr/sbin/smartctl + /usr/bin/mount + /usr/bin/umount + /usr/bin/findmnt + /usr/bin/lsblk + /usr/bin/readlink + /usr/bin/stat + /usr/bin/ntfsinfo + /usr/bin/ntfs-3g.probe + /usr/bin/fuser + /usr/bin/sync + /usr/bin/cmp + /usr/bin/cat + /usr/bin/sha256sum + /usr/bin/mawk + /usr/bin/od + /usr/bin/bash + /usr/bin/du + /usr/bin/test + /usr/bin/dd + /usr/bin/id + /usr/bin/ps + /usr/bin/sleep + /usr/bin/mkdir + /usr/bin/dpkg-query + /usr/bin/mktemp + /usr/bin/rm + /usr/bin/rmdir + /usr/bin/sudo + ) + local keepass_rc=0 cryptsetup_rc=0 + + _k3slr_query_package_state keepass_state keepassxc "$_K3SLR_KEEPASSXC_VERSION" || keepass_rc=$? + _k3slr_query_package_state cryptsetup_state cryptsetup-bin "$_K3SLR_CRYPTSETUP_VERSION" || cryptsetup_rc=$? + (( keepass_rc == 0 && cryptsetup_rc == 0 )) || return 1 + [[ "$keepass_state" == installed && "$cryptsetup_state" == installed ]] || return 1 + for binary in "${required_binaries[@]}"; do + _k3slr_trusted_root_executable "$binary" || return 1 + done +} + +_k3slr_lifecycle_step() { + (( $# == 1 )) +} + +_k3slr_lifecycle_action() { + local lifecycle="${1-}" action="${2-}" + case "$lifecycle" in + prepare) + (( $# == 3 )) || return 1 + ;; + open|close) + (( $# == 2 )) || return 1 + ;; + *) return 1 ;; + esac + case "$lifecycle/$action" in + prepare/sudo) + [[ "${3-}" == _k3slr_prepare_invocation ]] || return 1 + _k3slr_command /usr/bin/sudo --validate + ;; + prepare/*|open/*|close/*) + return 1 + ;; + *) + return 1 + ;; + esac +} + +_k3slr_lifecycle_dispatch() { + local lifecycle="${1-}" header_provider="${2-}" step + local -a steps=() action_arguments=() + (( $# == 2 )) || return 1 + case "$lifecycle" in + prepare) + local -A _k3slr_prepare_invocation=() + steps=( + sudo + context + device-validator + estimate-phase + capacity-preflight + confirm-prepare + install-packages + feasibility + revalidate-device + mount-outer + revalidate-capacity + create-layout + create-kdbx + create-container + validate-allocation + create-loop + luks-format + luks-open + mkfs-ext4 + mount-inner + write-marker-metadata + header-proof + close-inner + close-mapping + detach-loop + unmount-outer + closed-validator + ) + ;; + open) + steps=( + sudo + context + confirm-open + 'device-validator(--expect-device-ready)' + revalidate-context-device + mount-outer + validate-layout-kdbx-container + validate-allocation-and-ntfs-attributes + validate-outer-metadata + collision-check + create-loop + luks-open + mount-inner + validate-inner-metadata-marker-capacity + 'open-validator(--expect-open)' + ) + ;; + close) + steps=( + sudo + context + confirm-close + 'open-validator(--expect-open)' + validate-metadata-and-full-lineage + busy-process-query-pass-1 + busy-process-query-pass-2 + sync-inner-filesystem + unmount-inner + close-main-mapping + detach-original-loop + unmount-outer + 'closed-validator(--expect-closed)' + ) + ;; + *) return 1 ;; + esac + + for step in "${steps[@]}"; do + _k3slr_lifecycle_step "$step" || return 1 + action_arguments=("$lifecycle" "$step") + if [[ "$lifecycle" == prepare ]]; then + action_arguments+=(_k3slr_prepare_invocation) + fi + _k3slr_lifecycle_action "${action_arguments[@]}" || return 1 + done +} + +_k3slr_lifecycle_main() { + local lifecycle="${1-}" execution_mode="${2-}" + (( $# == 2 )) || return 1 + [[ "$lifecycle" == prepare || "$lifecycle" == open || "$lifecycle" == close ]] || return 1 + [[ -n "${K3SLR_WRAPPER_CONTRACT:-}" ]] || return 1 + _k3slr_load_contract "$K3SLR_WRAPPER_CONTRACT" || return 1 + case "$execution_mode" in + dry-run) + _k3slr_prepackage_device_preflight || return 1 + printf 'DRY RUN: %s lifecycle; no changes made.\n' "$lifecycle" + ;; + execute) + if [[ "$lifecycle" == prepare ]]; then + declare -F _k3slr_prove_header_restore >/dev/null || return 1 + else + _k3slr_open_close_capability_gate || return 1 + fi + _k3slr_require_execute_tty || return 1 + _k3slr_lifecycle_dispatch "$lifecycle" _k3slr_prove_header_restore + ;; + *) return 1 ;; + esac +} + +_k3slr_command() { + "$@" +} + +_k3slr_output_name_is_safe() { + [[ "$1" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ && "$1" != "$2"* ]] +} + +_k3slr_capture_directory_is_safe() { + [[ "${1-}" =~ ^/tmp/k3slr-capture\.[A-Za-z0-9]{8,}$ ]] +} + +_k3slr_remove_capture_directory() { + local _k3slr_process_capture_directory="${1-}" _k3slr_process_capture_rc=0 + _k3slr_capture_directory_is_safe "$_k3slr_process_capture_directory" || return 1 + _k3slr_command /usr/bin/rm -- \ + "${_k3slr_process_capture_directory}/stdout" \ + "${_k3slr_process_capture_directory}/stderr" >/dev/null 2>&1 || _k3slr_process_capture_rc=1 + _k3slr_command /usr/bin/rmdir -- "$_k3slr_process_capture_directory" \ + >/dev/null 2>&1 || _k3slr_process_capture_rc=1 + return "$_k3slr_process_capture_rc" +} + +_k3slr_capture_process() { + local _k3slr_process_capture_stdout_name="${1-}" + local _k3slr_process_capture_stderr_name="${2-}" + local _k3slr_process_capture_rc_name="${3-}" + local _k3slr_process_capture_directory='' _k3slr_process_capture_stdout='' + local _k3slr_process_capture_stderr='' _k3slr_process_capture_status=0 + shift 3 || return 1 + (( $# > 0 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_process_capture_stdout_name" _k3slr_process_capture_ || return 1 + _k3slr_output_name_is_safe "$_k3slr_process_capture_stderr_name" _k3slr_process_capture_ || return 1 + _k3slr_output_name_is_safe "$_k3slr_process_capture_rc_name" _k3slr_process_capture_ || return 1 + [[ "$_k3slr_process_capture_stdout_name" != "$_k3slr_process_capture_stderr_name" && + "$_k3slr_process_capture_stdout_name" != "$_k3slr_process_capture_rc_name" && + "$_k3slr_process_capture_stderr_name" != "$_k3slr_process_capture_rc_name" ]] || return 1 + + _k3slr_capture_one_line _k3slr_process_capture_directory \ + /usr/bin/mktemp --directory --tmpdir=/tmp 'k3slr-capture.XXXXXXXX' || return 1 + _k3slr_capture_directory_is_safe "$_k3slr_process_capture_directory" || return 1 + if _k3slr_command "$@" \ + >"${_k3slr_process_capture_directory}/stdout" \ + 2>"${_k3slr_process_capture_directory}/stderr"; then + _k3slr_process_capture_status=0 + else + _k3slr_process_capture_status=$? + fi + + if IFS= read -r -d '' _k3slr_process_capture_stdout \ + <"${_k3slr_process_capture_directory}/stdout"; then + _k3slr_remove_capture_directory "$_k3slr_process_capture_directory" || : + return 1 + fi + if IFS= read -r -d '' _k3slr_process_capture_stderr \ + <"${_k3slr_process_capture_directory}/stderr"; then + _k3slr_remove_capture_directory "$_k3slr_process_capture_directory" || : + return 1 + fi + _k3slr_remove_capture_directory "$_k3slr_process_capture_directory" || return 1 + printf -v "$_k3slr_process_capture_stdout_name" '%s' "$_k3slr_process_capture_stdout" + printf -v "$_k3slr_process_capture_stderr_name" '%s' "$_k3slr_process_capture_stderr" + printf -v "$_k3slr_process_capture_rc_name" '%s' "$_k3slr_process_capture_status" +} + +_k3slr_parse_current_context_record() { + local _k3slr_context_record="${1-}" _k3slr_context_destination="${2-}" + (( $# == 2 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_context_destination" _k3slr_context_ || return 1 + [[ "$_k3slr_context_record" == $'default\n' ]] || return 1 + printf -v "$_k3slr_context_destination" '%s' default +} + +_k3slr_current_context() { + local _k3slr_context_destination="${1-}" _k3slr_context_stdout='' + local _k3slr_context_stderr='' _k3slr_context_rc='' + (( $# == 1 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_context_destination" _k3slr_context_ || return 1 + _k3slr_capture_process _k3slr_context_stdout _k3slr_context_stderr _k3slr_context_rc \ + /usr/local/bin/k3s kubectl config current-context || return 1 + [[ "$_k3slr_context_rc" == 0 && -z "$_k3slr_context_stderr" ]] || return 1 + _k3slr_parse_current_context_record "$_k3slr_context_stdout" "$_k3slr_context_destination" +} + +_k3slr_parse_package_query_record() { + local _k3slr_package_destination="${1-}" _k3slr_package_name="${2-}" + local _k3slr_package_expected="${3-}" _k3slr_package_rc="${4-}" + local _k3slr_package_stdout="${5-}" _k3slr_package_stderr="${6-}" + local _k3slr_package_state=malformed _k3slr_package_prefix _k3slr_package_version + (( $# == 6 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_package_destination" _k3slr_package_ || return 1 + case "${_k3slr_package_name}|${_k3slr_package_expected}" in + "keepassxc|${_K3SLR_KEEPASSXC_VERSION}"|"cryptsetup-bin|${_K3SLR_CRYPTSETUP_VERSION}") ;; + *) return 1 ;; + esac + _k3slr_uint "$_k3slr_package_rc" || return 1 + (( _k3slr_package_rc <= 255 )) || return 1 + + if [[ "$_k3slr_package_rc" == 0 && -z "$_k3slr_package_stderr" ]]; then + if [[ "$_k3slr_package_stdout" == "install ok installed|${_k3slr_package_expected}"$'\n' ]]; then + _k3slr_package_state=installed + else + _k3slr_package_prefix='install ok installed|' + if [[ "$_k3slr_package_stdout" == "$_k3slr_package_prefix"*$'\n' ]]; then + _k3slr_package_version="${_k3slr_package_stdout#"$_k3slr_package_prefix"}" + _k3slr_package_version="${_k3slr_package_version%$'\n'}" + if [[ -n "$_k3slr_package_version" && "$_k3slr_package_version" != *$'\n'* && + "$_k3slr_package_version" != *$'\r'* && + "$_k3slr_package_stdout" == "${_k3slr_package_prefix}${_k3slr_package_version}"$'\n' ]]; then + _k3slr_package_state=wrong-version + fi + fi + fi + elif [[ "$_k3slr_package_rc" == 1 && -z "$_k3slr_package_stdout" && + "$_k3slr_package_stderr" == "dpkg-query: no packages found matching ${_k3slr_package_name}"$'\n' ]]; then + _k3slr_package_state=absent + fi + printf -v "$_k3slr_package_destination" '%s' "$_k3slr_package_state" +} + +_k3slr_query_package_state() { + local _k3slr_package_destination="${1-}" _k3slr_package_name="${2-}" + local _k3slr_package_expected="${3-}" _k3slr_package_stdout='' + local _k3slr_package_stderr='' _k3slr_package_rc='' _k3slr_query_result='' + (( $# == 3 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_package_destination" _k3slr_package_ || return 1 + _k3slr_capture_process _k3slr_package_stdout _k3slr_package_stderr _k3slr_package_rc \ + /usr/bin/dpkg-query --show '--showformat=${Status}|${Version}\n' "$_k3slr_package_name" || return 1 + _k3slr_parse_package_query_record _k3slr_query_result "$_k3slr_package_name" \ + "$_k3slr_package_expected" "$_k3slr_package_rc" \ + "$_k3slr_package_stdout" "$_k3slr_package_stderr" || return 1 + printf -v "$_k3slr_package_destination" '%s' "$_k3slr_query_result" +} + +_k3slr_parse_phase_bytes_record() { + local _k3slr_phase_record="${1-}" _k3slr_phase_destination="${2-}" + local _k3slr_phase_value + (( $# == 2 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_phase_destination" _k3slr_phase_ || return 1 + [[ "$_k3slr_phase_record" == phase_bytes=*$'\n' ]] || return 1 + _k3slr_phase_value="${_k3slr_phase_record#phase_bytes=}" + _k3slr_phase_value="${_k3slr_phase_value%$'\n'}" + [[ "$_k3slr_phase_record" == "phase_bytes=${_k3slr_phase_value}"$'\n' ]] || return 1 + _k3slr_uint "$_k3slr_phase_value" || return 1 + printf -v "$_k3slr_phase_destination" '%s' "$_k3slr_phase_value" +} + +_k3slr_max_phase_bytes() { + local _k3slr_phase_destination="${1-}" _k3slr_phase_value _k3slr_phase_max=0 + (( $# == 4 )) || return 1 + shift + _k3slr_output_name_is_safe "$_k3slr_phase_destination" _k3slr_phase_ || return 1 + for _k3slr_phase_value in "$@"; do + _k3slr_uint "$_k3slr_phase_value" || return 1 + if (( _k3slr_phase_value > _k3slr_phase_max )); then + _k3slr_phase_max="$_k3slr_phase_value" + fi + done + printf -v "$_k3slr_phase_destination" '%s' "$_k3slr_phase_max" +} + +_k3slr_parse_statfs_record() { + local _k3slr_statfs_record="${1-}" _k3slr_statfs_total_name="${2-}" + local _k3slr_statfs_free_name="${3-}" _k3slr_statfs_used_name="${4-}" + local _k3slr_statfs_block_size _k3slr_statfs_blocks _k3slr_statfs_available + local _k3slr_statfs_total _k3slr_statfs_free _k3slr_statfs_used + (( $# == 4 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_statfs_total_name" _k3slr_statfs_ || return 1 + _k3slr_output_name_is_safe "$_k3slr_statfs_free_name" _k3slr_statfs_ || return 1 + _k3slr_output_name_is_safe "$_k3slr_statfs_used_name" _k3slr_statfs_ || return 1 + [[ "$_k3slr_statfs_total_name" != "$_k3slr_statfs_free_name" && + "$_k3slr_statfs_total_name" != "$_k3slr_statfs_used_name" && + "$_k3slr_statfs_free_name" != "$_k3slr_statfs_used_name" ]] || return 1 + [[ "$_k3slr_statfs_record" =~ ^(0|[1-9][0-9]*)\|(0|[1-9][0-9]*)\|(0|[1-9][0-9]*)$'\n'$ ]] || return 1 + _k3slr_statfs_block_size="${BASH_REMATCH[1]}" + _k3slr_statfs_blocks="${BASH_REMATCH[2]}" + _k3slr_statfs_available="${BASH_REMATCH[3]}" + _k3slr_uint "$_k3slr_statfs_block_size" || return 1 + _k3slr_uint "$_k3slr_statfs_blocks" || return 1 + _k3slr_uint "$_k3slr_statfs_available" || return 1 + (( _k3slr_statfs_block_size > 0 && _k3slr_statfs_blocks > 0 && + _k3slr_statfs_available <= _k3slr_statfs_blocks )) || return 1 + (( _k3slr_statfs_blocks <= _K3SLR_INT64_MAX / _k3slr_statfs_block_size )) || return 1 + _k3slr_statfs_total=$((_k3slr_statfs_block_size * _k3slr_statfs_blocks)) + _k3slr_statfs_free=$((_k3slr_statfs_block_size * _k3slr_statfs_available)) + _k3slr_statfs_used=$((_k3slr_statfs_total - _k3slr_statfs_free)) + printf -v "$_k3slr_statfs_total_name" '%s' "$_k3slr_statfs_total" + printf -v "$_k3slr_statfs_free_name" '%s' "$_k3slr_statfs_free" + printf -v "$_k3slr_statfs_used_name" '%s' "$_k3slr_statfs_used" +} + +_k3slr_conservative_inner_capacity_fits() { + local _k3slr_inner_container="${1-}" _k3slr_inner_phase="${2-}" + local _k3slr_inner_used + (( $# == 2 )) || return 1 + _k3slr_uint "$_k3slr_inner_container" || return 1 + _k3slr_uint "$_k3slr_inner_phase" || return 1 + (( _k3slr_inner_container > 0 )) || return 1 + _k3slr_inner_used=$((_k3slr_inner_container / 2 + _k3slr_inner_container % 2)) + _k3slr_inner_capacity_fits "$_k3slr_inner_container" "$_k3slr_inner_used" "$_k3slr_inner_phase" +} + +_k3slr_marker_sha256_is_valid() { + [[ "${1-}" =~ ^[0-9a-f]{64}$ ]] +} + +_k3slr_encoded_uuid_is_valid() { + local _k3slr_uuid_hex="${1-}" _k3slr_uuid_kind="${2-}" + local _k3slr_uuid_index _k3slr_uuid_byte + (( $# == 2 && ${#_k3slr_uuid_hex} == 72 )) || return 1 + [[ "$_k3slr_uuid_kind" == canonical || "$_k3slr_uuid_kind" == v4 ]] || return 1 + for (( _k3slr_uuid_index=0; _k3slr_uuid_index<36; _k3slr_uuid_index++ )); do + _k3slr_uuid_byte="${_k3slr_uuid_hex:_k3slr_uuid_index * 2:2}" + case "$_k3slr_uuid_index" in + 8|13|18|23) + [[ "$_k3slr_uuid_byte" == 2d ]] || return 1 + ;; + 14) + if [[ "$_k3slr_uuid_kind" == v4 ]]; then + [[ "$_k3slr_uuid_byte" == 34 ]] || return 1 + else + [[ "$_k3slr_uuid_byte" =~ ^(3[0-9]|6[1-6])$ ]] || return 1 + fi + ;; + 19) + if [[ "$_k3slr_uuid_kind" == v4 ]]; then + [[ "$_k3slr_uuid_byte" =~ ^(38|39|61|62)$ ]] || return 1 + else + [[ "$_k3slr_uuid_byte" =~ ^(3[0-9]|6[1-6])$ ]] || return 1 + fi + ;; + *) + [[ "$_k3slr_uuid_byte" =~ ^(3[0-9]|6[1-6])$ ]] || return 1 + ;; + esac + done +} + +_k3slr_marker_hex_is_valid() { + local _k3slr_marker_hex_record="${1-}" _k3slr_marker_hex + (( $# == 1 )) || return 1 + [[ "$_k3slr_marker_hex_record" =~ ^[0-9a-f]+$'\n'$ ]] || return 1 + _k3slr_marker_hex="${_k3slr_marker_hex_record%$'\n'}" + [[ ${#_k3slr_marker_hex} == 74 && "${_k3slr_marker_hex:72:2}" == 0a ]] || return 1 + _k3slr_encoded_uuid_is_valid "${_k3slr_marker_hex:0:72}" v4 +} + +_k3slr_parse_runtime_metadata_hex() { + local _k3slr_metadata_hex_record="${1-}" _k3slr_metadata_hash_name="${2-}" + local _k3slr_metadata_hex _k3slr_metadata_prefix _k3slr_metadata_remainder + local _k3slr_metadata_uuid_hex _k3slr_metadata_tail_prefix _k3slr_metadata_hash_hex + local _k3slr_metadata_contract_hex='' _k3slr_metadata_label_hex='' + local _k3slr_metadata_hash='' _k3slr_metadata_character _k3slr_metadata_byte + local _k3slr_metadata_index + (( $# == 2 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_metadata_hash_name" _k3slr_metadata_ || return 1 + [[ -n "${K3SLR_CONTAINER_SIZE_BYTES:-}" && -n "${K3SLR_INNER_LABEL:-}" ]] || return 1 + _k3slr_uint "$K3SLR_CONTAINER_SIZE_BYTES" || return 1 + [[ "$K3SLR_INNER_LABEL" =~ ^[A-Z][A-Z0-9_]*$ ]] || return 1 + [[ "$_k3slr_metadata_hex_record" =~ ^[0-9a-f]+$'\n'$ ]] || return 1 + _k3slr_metadata_hex="${_k3slr_metadata_hex_record%$'\n'}" + (( ${#_k3slr_metadata_hex} <= 4096 && ${#_k3slr_metadata_hex} % 2 == 0 )) || return 1 + + for (( _k3slr_metadata_index=0; + _k3slr_metadata_index<${#K3SLR_CONTAINER_SIZE_BYTES}; + _k3slr_metadata_index++ )); do + _k3slr_metadata_character="${K3SLR_CONTAINER_SIZE_BYTES:_k3slr_metadata_index:1}" + printf -v _k3slr_metadata_byte '%02x' "'$_k3slr_metadata_character" + _k3slr_metadata_contract_hex+="$_k3slr_metadata_byte" + done + for (( _k3slr_metadata_index=0; + _k3slr_metadata_index<${#K3SLR_INNER_LABEL}; + _k3slr_metadata_index++ )); do + _k3slr_metadata_character="${K3SLR_INNER_LABEL:_k3slr_metadata_index:1}" + printf -v _k3slr_metadata_byte '%02x' "'$_k3slr_metadata_character" + _k3slr_metadata_label_hex+="$_k3slr_metadata_byte" + done + + _k3slr_metadata_prefix='736368656d613d6b33736c722d72756e74696d652d76310a6c756b735f757569643d' + [[ "$_k3slr_metadata_hex" == "$_k3slr_metadata_prefix"* ]] || return 1 + _k3slr_metadata_remainder="${_k3slr_metadata_hex#"$_k3slr_metadata_prefix"}" + (( ${#_k3slr_metadata_remainder} >= 72 )) || return 1 + _k3slr_metadata_uuid_hex="${_k3slr_metadata_remainder:0:72}" + _k3slr_encoded_uuid_is_valid "$_k3slr_metadata_uuid_hex" canonical || return 1 + _k3slr_metadata_remainder="${_k3slr_metadata_remainder:72}" + _k3slr_metadata_tail_prefix="0a636f6e7461696e65725f62797465733d${_k3slr_metadata_contract_hex}0a696e6e65725f6c6162656c3d${_k3slr_metadata_label_hex}0a6d61726b65725f7368613235363d" + [[ "$_k3slr_metadata_remainder" == "$_k3slr_metadata_tail_prefix"* ]] || return 1 + _k3slr_metadata_hash_hex="${_k3slr_metadata_remainder#"$_k3slr_metadata_tail_prefix"}" + [[ ${#_k3slr_metadata_hash_hex} == 130 && + "${_k3slr_metadata_hash_hex:128:2}" == 0a ]] || return 1 + _k3slr_metadata_hash_hex="${_k3slr_metadata_hash_hex:0:128}" + for (( _k3slr_metadata_index=0; _k3slr_metadata_index<128; _k3slr_metadata_index+=2 )); do + _k3slr_metadata_byte="${_k3slr_metadata_hash_hex:_k3slr_metadata_index:2}" + case "$_k3slr_metadata_byte" in + 30) _k3slr_metadata_hash+=0 ;; 31) _k3slr_metadata_hash+=1 ;; + 32) _k3slr_metadata_hash+=2 ;; 33) _k3slr_metadata_hash+=3 ;; + 34) _k3slr_metadata_hash+=4 ;; 35) _k3slr_metadata_hash+=5 ;; + 36) _k3slr_metadata_hash+=6 ;; 37) _k3slr_metadata_hash+=7 ;; + 38) _k3slr_metadata_hash+=8 ;; 39) _k3slr_metadata_hash+=9 ;; + 61) _k3slr_metadata_hash+=a ;; 62) _k3slr_metadata_hash+=b ;; + 63) _k3slr_metadata_hash+=c ;; 64) _k3slr_metadata_hash+=d ;; + 65) _k3slr_metadata_hash+=e ;; 66) _k3slr_metadata_hash+=f ;; + *) return 1 ;; + esac + done + _k3slr_marker_sha256_is_valid "$_k3slr_metadata_hash" || return 1 + printf -v "$_k3slr_metadata_hash_name" '%s' "$_k3slr_metadata_hash" +} + +_k3slr_normalize_od_hex_record() { + local _k3slr_od_record="${1-}" _k3slr_od_destination="${2-}" + local _k3slr_od_remaining _k3slr_od_line _k3slr_od_token + local _k3slr_od_hex='' _k3slr_od_bytes=0 + local -a _k3slr_od_tokens=() + (( $# == 2 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_od_destination" _k3slr_od_ || return 1 + [[ -n "$_k3slr_od_record" && "$_k3slr_od_record" == *$'\n' && + "$_k3slr_od_record" != *$'\r'* ]] || return 1 + _k3slr_od_remaining="$_k3slr_od_record" + while [[ -n "$_k3slr_od_remaining" ]]; do + [[ "$_k3slr_od_remaining" == *$'\n'* ]] || return 1 + _k3slr_od_line="${_k3slr_od_remaining%%$'\n'*}" + _k3slr_od_remaining="${_k3slr_od_remaining#*$'\n'}" + read -r -a _k3slr_od_tokens <<<"$_k3slr_od_line" + for _k3slr_od_token in "${_k3slr_od_tokens[@]}"; do + [[ "$_k3slr_od_token" =~ ^[0-9a-f]{2}$ ]] || return 1 + _k3slr_od_bytes=$((_k3slr_od_bytes + 1)) + (( _k3slr_od_bytes <= 65536 )) || return 1 + _k3slr_od_hex+="$_k3slr_od_token" + done + done + (( _k3slr_od_bytes > 0 )) || return 1 + printf -v "$_k3slr_od_destination" '%s\n' "$_k3slr_od_hex" +} + +_k3slr_parse_unmounted_ntfs_capacity_hex() { + local _k3slr_ntfs_record="${1-}" _k3slr_ntfs_total_name="${2-}" + local _k3slr_ntfs_free_name="${3-}" _k3slr_ntfs_hex _k3slr_ntfs_text='' + local _k3slr_ntfs_byte _k3slr_ntfs_character _k3slr_ntfs_remaining _k3slr_ntfs_line + local _k3slr_ntfs_trimmed _k3slr_ntfs_cluster_size='' _k3slr_ntfs_volume='' + local _k3slr_ntfs_allocated='' _k3slr_ntfs_free_clusters='' + local _k3slr_ntfs_allocated_percent='' _k3slr_ntfs_free_percent='' + local _k3slr_ntfs_integer _k3slr_ntfs_fraction _k3slr_ntfs_allocated_tenths + local _k3slr_ntfs_free_tenths _k3slr_ntfs_total _k3slr_ntfs_free + local _k3slr_ntfs_left _k3slr_ntfs_right _k3slr_ntfs_difference + local _k3slr_ntfs_index + (( $# == 3 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_ntfs_total_name" _k3slr_ntfs_ || return 1 + _k3slr_output_name_is_safe "$_k3slr_ntfs_free_name" _k3slr_ntfs_ || return 1 + [[ "$_k3slr_ntfs_total_name" != "$_k3slr_ntfs_free_name" ]] || return 1 + [[ "$_k3slr_ntfs_record" =~ ^[0-9a-f]+$'\n'$ ]] || return 1 + _k3slr_ntfs_hex="${_k3slr_ntfs_record%$'\n'}" + (( ${#_k3slr_ntfs_hex} > 0 && ${#_k3slr_ntfs_hex} <= 131072 && + ${#_k3slr_ntfs_hex} % 2 == 0 )) || return 1 + [[ "${_k3slr_ntfs_hex: -2}" == 0a ]] || return 1 + + for (( _k3slr_ntfs_index=0; + _k3slr_ntfs_index<${#_k3slr_ntfs_hex}; + _k3slr_ntfs_index+=2 )); do + _k3slr_ntfs_byte="${_k3slr_ntfs_hex:_k3slr_ntfs_index:2}" + [[ "$_k3slr_ntfs_byte" != 00 && "$_k3slr_ntfs_byte" != 0d ]] || return 1 + if [[ "$_k3slr_ntfs_byte" == 09 ]]; then + _k3slr_ntfs_character=$'\t' + elif [[ "$_k3slr_ntfs_byte" == 0a ]]; then + _k3slr_ntfs_character=$'\n' + elif (( 16#$_k3slr_ntfs_byte >= 16#20 && 16#$_k3slr_ntfs_byte <= 16#7e )); then + printf -v _k3slr_ntfs_character '%b' "\\x${_k3slr_ntfs_byte}" + else + return 1 + fi + _k3slr_ntfs_text+="$_k3slr_ntfs_character" + done + [[ "$_k3slr_ntfs_text" == *$'\n' ]] || return 1 + + _k3slr_ntfs_remaining="$_k3slr_ntfs_text" + while [[ -n "$_k3slr_ntfs_remaining" ]]; do + [[ "$_k3slr_ntfs_remaining" == *$'\n'* ]] || return 1 + _k3slr_ntfs_line="${_k3slr_ntfs_remaining%%$'\n'*}" + _k3slr_ntfs_remaining="${_k3slr_ntfs_remaining#*$'\n'}" + _k3slr_ntfs_trimmed="$_k3slr_ntfs_line" + while [[ "$_k3slr_ntfs_trimmed" == ' '* || "$_k3slr_ntfs_trimmed" == $'\t'* ]]; do + _k3slr_ntfs_trimmed="${_k3slr_ntfs_trimmed:1}" + done + if [[ "$_k3slr_ntfs_trimmed" =~ ^Cluster\ Size:\ (0|[1-9][0-9]*)$ ]]; then + [[ -z "$_k3slr_ntfs_cluster_size" ]] || return 1 + _k3slr_ntfs_cluster_size="${BASH_REMATCH[1]}" + elif [[ "$_k3slr_ntfs_trimmed" =~ ^Volume\ Size\ in\ Clusters:\ (0|[1-9][0-9]*)$ ]]; then + [[ -z "$_k3slr_ntfs_volume" ]] || return 1 + _k3slr_ntfs_volume="${BASH_REMATCH[1]}" + elif [[ "$_k3slr_ntfs_trimmed" =~ ^Allocated\ clusters\ (0|[1-9][0-9]*)\ \(((0|[1-9][0-9]{0,2})\.([0-9]))%\)$ ]]; then + [[ -z "$_k3slr_ntfs_allocated" ]] || return 1 + _k3slr_ntfs_allocated="${BASH_REMATCH[1]}" + _k3slr_ntfs_allocated_percent="${BASH_REMATCH[2]}" + elif [[ "$_k3slr_ntfs_trimmed" =~ ^Free\ Clusters:\ (0|[1-9][0-9]*)\ \(((0|[1-9][0-9]{0,2})\.([0-9]))%\)$ ]]; then + [[ -z "$_k3slr_ntfs_free_clusters" ]] || return 1 + _k3slr_ntfs_free_clusters="${BASH_REMATCH[1]}" + _k3slr_ntfs_free_percent="${BASH_REMATCH[2]}" + elif [[ "$_k3slr_ntfs_trimmed" == 'Cluster Size:'* || + "$_k3slr_ntfs_trimmed" == 'Volume Size in Clusters:'* || + "$_k3slr_ntfs_trimmed" == 'Allocated clusters'* || + "$_k3slr_ntfs_trimmed" == 'Free Clusters:'* ]]; then + return 1 + fi + done + [[ -n "$_k3slr_ntfs_cluster_size" && -n "$_k3slr_ntfs_volume" && + -n "$_k3slr_ntfs_allocated" && -n "$_k3slr_ntfs_free_clusters" ]] || return 1 + _k3slr_uint "$_k3slr_ntfs_cluster_size" || return 1 + _k3slr_uint "$_k3slr_ntfs_volume" || return 1 + _k3slr_uint "$_k3slr_ntfs_allocated" || return 1 + _k3slr_uint "$_k3slr_ntfs_free_clusters" || return 1 + (( _k3slr_ntfs_cluster_size > 0 && _k3slr_ntfs_volume > 0 )) || return 1 + (( _k3slr_ntfs_allocated <= _K3SLR_INT64_MAX - _k3slr_ntfs_free_clusters && + _k3slr_ntfs_allocated + _k3slr_ntfs_free_clusters == _k3slr_ntfs_volume )) || return 1 + (( _k3slr_ntfs_volume <= _K3SLR_INT64_MAX / _k3slr_ntfs_cluster_size )) || return 1 + _k3slr_ntfs_total=$((_k3slr_ntfs_cluster_size * _k3slr_ntfs_volume)) + _k3slr_ntfs_free=$((_k3slr_ntfs_cluster_size * _k3slr_ntfs_free_clusters)) + + _k3slr_ntfs_integer="${_k3slr_ntfs_allocated_percent%.*}" + _k3slr_ntfs_fraction="${_k3slr_ntfs_allocated_percent#*.}" + _k3slr_ntfs_allocated_tenths=$((10#$_k3slr_ntfs_integer * 10 + 10#$_k3slr_ntfs_fraction)) + _k3slr_ntfs_integer="${_k3slr_ntfs_free_percent%.*}" + _k3slr_ntfs_fraction="${_k3slr_ntfs_free_percent#*.}" + _k3slr_ntfs_free_tenths=$((10#$_k3slr_ntfs_integer * 10 + 10#$_k3slr_ntfs_fraction)) + (( _k3slr_ntfs_allocated_tenths <= 1000 && _k3slr_ntfs_free_tenths <= 1000 )) || return 1 + (( _k3slr_ntfs_volume <= _K3SLR_INT64_MAX / 1000 && + _k3slr_ntfs_allocated <= _K3SLR_INT64_MAX / 1000 && + _k3slr_ntfs_free_clusters <= _K3SLR_INT64_MAX / 1000 )) || return 1 + + _k3slr_ntfs_left=$((_k3slr_ntfs_allocated_tenths * _k3slr_ntfs_volume)) + _k3slr_ntfs_right=$((_k3slr_ntfs_allocated * 1000)) + if (( _k3slr_ntfs_left >= _k3slr_ntfs_right )); then + _k3slr_ntfs_difference=$((_k3slr_ntfs_left - _k3slr_ntfs_right)) + else + _k3slr_ntfs_difference=$((_k3slr_ntfs_right - _k3slr_ntfs_left)) + fi + (( _k3slr_ntfs_difference <= _K3SLR_INT64_MAX / 2 && + 2 * _k3slr_ntfs_difference <= _k3slr_ntfs_volume )) || return 1 + _k3slr_ntfs_left=$((_k3slr_ntfs_free_tenths * _k3slr_ntfs_volume)) + _k3slr_ntfs_right=$((_k3slr_ntfs_free_clusters * 1000)) + if (( _k3slr_ntfs_left >= _k3slr_ntfs_right )); then + _k3slr_ntfs_difference=$((_k3slr_ntfs_left - _k3slr_ntfs_right)) + else + _k3slr_ntfs_difference=$((_k3slr_ntfs_right - _k3slr_ntfs_left)) + fi + (( _k3slr_ntfs_difference <= _K3SLR_INT64_MAX / 2 && + 2 * _k3slr_ntfs_difference <= _k3slr_ntfs_volume )) || return 1 + printf -v "$_k3slr_ntfs_total_name" '%s' "$_k3slr_ntfs_total" + printf -v "$_k3slr_ntfs_free_name" '%s' "$_k3slr_ntfs_free" +} + +_k3slr_parse_prepackage_snapshot_fields() { + local _k3slr_snapshot_record="${1-}" _k3slr_snapshot_result_name="${2-}" + local _k3slr_snapshot_remaining _k3slr_snapshot_line _k3slr_snapshot_key _k3slr_snapshot_value + local -A _k3slr_snapshot_allowed=( + [canonical_partition]=1 [partition_major_minor]=1 + [canonical_parent_disk]=1 [disk_major_minor]=1 + [filesystem_uuid]=1 [partuuid]=1 [filesystem_type]=1 + [model]=1 [serial]=1 [wwn]=1 + [canonical_k3s_partition]=1 [k3s_partition_major_minor]=1 + [canonical_k3s_disk]=1 [k3s_disk_major_minor]=1 + [k3s_filesystem_uuid]=1 [k3s_partuuid]=1 [k3s_filesystem_type]=1 + [k3s_model]=1 [k3s_serial]=1 [k3s_wwn]=1 + [smart_health]=1 [smart_reallocated]=1 [smart_pending]=1 [smart_uncorrectable]=1 + [outer_mount]=1 [inner_mount]=1 [source_mounts]=1 [loops]=1 + [mapper_directory]=1 [mapping_path]=1 [proof_mapping_path]=1 [mapping_names]=1 + [ntfs_probe]=1 + ) + (( $# == 2 )) || return 1 + [[ "$_k3slr_snapshot_result_name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ && + "$_k3slr_snapshot_result_name" != _k3slr_snapshot_* ]] || return 1 + local -n _k3slr_snapshot_result="$_k3slr_snapshot_result_name" + [[ -n "$_k3slr_snapshot_record" && "$_k3slr_snapshot_record" == *$'\n' && + "$_k3slr_snapshot_record" != *$'\r'* ]] || return 1 + _k3slr_snapshot_result=() + _k3slr_snapshot_remaining="$_k3slr_snapshot_record" + while [[ -n "$_k3slr_snapshot_remaining" ]]; do + [[ "$_k3slr_snapshot_remaining" == *$'\n'* ]] || return 1 + _k3slr_snapshot_line="${_k3slr_snapshot_remaining%%$'\n'*}" + _k3slr_snapshot_remaining="${_k3slr_snapshot_remaining#*$'\n'}" + [[ "$_k3slr_snapshot_line" =~ ^([a-z][a-z0-9_]*)=([^[:space:]]+)$ ]] || return 1 + _k3slr_snapshot_key="${BASH_REMATCH[1]}" + _k3slr_snapshot_value="${BASH_REMATCH[2]}" + [[ "${_k3slr_snapshot_allowed[$_k3slr_snapshot_key]+present}" == present && + "${_k3slr_snapshot_result[$_k3slr_snapshot_key]+present}" != present ]] || return 1 + _k3slr_snapshot_result["$_k3slr_snapshot_key"]="$_k3slr_snapshot_value" + done + for _k3slr_snapshot_key in "${!_k3slr_snapshot_allowed[@]}"; do + [[ "${_k3slr_snapshot_result[$_k3slr_snapshot_key]+present}" == present ]] || return 1 + done +} + +_k3slr_validate_prepackage_snapshot() { + local _k3slr_snapshot_record="${1-}" + local -A snapshot_fields=() + (( $# == 1 )) || return 1 + _k3slr_parse_prepackage_snapshot_fields "$_k3slr_snapshot_record" snapshot_fields || return 1 + [[ "${snapshot_fields[canonical_partition]}" =~ ^/dev/[A-Za-z0-9._/-]+$ && + "${snapshot_fields[canonical_parent_disk]}" =~ ^/dev/[A-Za-z0-9._/-]+$ && + "${snapshot_fields[canonical_k3s_partition]}" =~ ^/dev/[A-Za-z0-9._/-]+$ && + "${snapshot_fields[canonical_k3s_disk]}" =~ ^/dev/[A-Za-z0-9._/-]+$ ]] || return 1 + [[ "${snapshot_fields[partition_major_minor]}" =~ ^[0-9]+:[0-9]+$ && + "${snapshot_fields[disk_major_minor]}" =~ ^[0-9]+:[0-9]+$ && + "${snapshot_fields[k3s_partition_major_minor]}" =~ ^[0-9]+:[0-9]+$ && + "${snapshot_fields[k3s_disk_major_minor]}" =~ ^[0-9]+:[0-9]+$ ]] || return 1 + [[ "${snapshot_fields[canonical_parent_disk]}" != "${snapshot_fields[canonical_k3s_disk]}" && + "${snapshot_fields[disk_major_minor]}" != "${snapshot_fields[k3s_disk_major_minor]}" ]] || return 1 + [[ "${snapshot_fields[filesystem_uuid]}" == "$K3SLR_RECOVERY_FS_UUID" && + "${snapshot_fields[partuuid]}" == "$K3SLR_RECOVERY_PARTUUID" && + "${snapshot_fields[filesystem_type]}" == ntfs && + "${snapshot_fields[model]}" == "$K3SLR_RECOVERY_MODEL" && + "${snapshot_fields[serial]}" == "$K3SLR_RECOVERY_SERIAL" && + "${snapshot_fields[wwn]}" == "$K3SLR_RECOVERY_WWN" ]] || return 1 + [[ "${snapshot_fields[k3s_filesystem_uuid]}" == "$K3SLR_K3S_FS_UUID" && + "${snapshot_fields[k3s_partuuid]}" == "$K3SLR_K3S_PARTUUID" && + "${snapshot_fields[k3s_filesystem_type]}" == ext4 && + "${snapshot_fields[k3s_model]}" == "$K3SLR_K3S_MODEL" && + "${snapshot_fields[k3s_serial]}" == "$K3SLR_K3S_SERIAL" && + "${snapshot_fields[k3s_wwn]}" == "$K3SLR_K3S_WWN" ]] || return 1 + [[ "${snapshot_fields[smart_health]}" == PASSED && + "${snapshot_fields[smart_reallocated]}" == 0 && + "${snapshot_fields[smart_pending]}" == 0 && + "${snapshot_fields[smart_uncorrectable]}" == 0 && + "${snapshot_fields[outer_mount]}" == absent && + "${snapshot_fields[inner_mount]}" == absent && + "${snapshot_fields[source_mounts]}" == absent && + "${snapshot_fields[loops]}" == absent && + "${snapshot_fields[mapper_directory]}" == physical-root-safe && + "${snapshot_fields[mapping_path]}" == absent && + "${snapshot_fields[proof_mapping_path]}" == absent && + "${snapshot_fields[mapping_names]}" == absent && + "${snapshot_fields[ntfs_probe]}" == pass ]] || return 1 +} + +_k3slr_prepackage_snapshot_identity() { + local _k3slr_snapshot_record="${1-}" _k3slr_snapshot_identity_name="${2-}" + local _k3slr_snapshot_identity + local -A snapshot_fields=() + (( $# == 2 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_snapshot_identity_name" _k3slr_snapshot_ || return 1 + _k3slr_validate_prepackage_snapshot "$_k3slr_snapshot_record" || return 1 + _k3slr_parse_prepackage_snapshot_fields "$_k3slr_snapshot_record" snapshot_fields || return 1 + printf -v _k3slr_snapshot_identity '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \ + "${snapshot_fields[canonical_partition]}" "${snapshot_fields[partition_major_minor]}" \ + "${snapshot_fields[canonical_parent_disk]}" "${snapshot_fields[disk_major_minor]}" \ + "${snapshot_fields[filesystem_uuid]}" "${snapshot_fields[partuuid]}" \ + "${snapshot_fields[filesystem_type]}" "${snapshot_fields[model]}" \ + "${snapshot_fields[serial]}" "${snapshot_fields[wwn]}" \ + "${snapshot_fields[k3s_disk_major_minor]}" + printf -v "$_k3slr_snapshot_identity_name" '%s' "$_k3slr_snapshot_identity" +} + +_k3slr_prepackage_device_preflight() { + local _k3slr_preflight_before='' _k3slr_preflight_after='' + (( $# == 0 )) || return 1 + _k3slr_collect_prepackage_snapshot _k3slr_preflight_before || return 1 + _k3slr_validate_prepackage_snapshot "$_k3slr_preflight_before" || return 1 + _k3slr_collect_prepackage_snapshot _k3slr_preflight_after || return 1 + _k3slr_validate_prepackage_snapshot "$_k3slr_preflight_after" || return 1 + [[ "$_k3slr_preflight_before" == "$_k3slr_preflight_after" ]] +} + +_k3slr_collect_unmounted_ntfs_capacity() { + local _k3slr_capacity_hex_name="${1-}" _k3slr_capacity_identity_name="${2-}" + local _k3slr_capacity_before='' _k3slr_capacity_after='' + local _k3slr_capacity_hex='' _k3slr_capacity_identity='' + local -A snapshot_fields=() + (( $# == 2 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_capacity_hex_name" _k3slr_capacity_ || return 1 + _k3slr_output_name_is_safe "$_k3slr_capacity_identity_name" _k3slr_capacity_ || return 1 + [[ "$_k3slr_capacity_hex_name" != "$_k3slr_capacity_identity_name" ]] || return 1 + _k3slr_collect_prepackage_snapshot _k3slr_capacity_before || return 1 + _k3slr_validate_prepackage_snapshot "$_k3slr_capacity_before" || return 1 + _k3slr_parse_prepackage_snapshot_fields "$_k3slr_capacity_before" snapshot_fields || return 1 + _k3slr_capture_ntfsinfo_hex _k3slr_capacity_hex "${snapshot_fields[canonical_partition]}" || return 1 + _k3slr_collect_prepackage_snapshot _k3slr_capacity_after || return 1 + _k3slr_validate_prepackage_snapshot "$_k3slr_capacity_after" || return 1 + [[ "$_k3slr_capacity_before" == "$_k3slr_capacity_after" ]] || return 1 + _k3slr_prepackage_snapshot_identity "$_k3slr_capacity_before" _k3slr_capacity_identity || return 1 + printf -v "$_k3slr_capacity_hex_name" '%s' "$_k3slr_capacity_hex" + printf -v "$_k3slr_capacity_identity_name" '%s' "$_k3slr_capacity_identity" +} + +_k3slr_capture_exact_one_line_record() { + local _k3slr_record_destination="${1-}" _k3slr_record_stdout='' + local _k3slr_record_stderr='' _k3slr_record_rc='' _k3slr_record_value + shift || return 1 + (( $# > 0 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_record_destination" _k3slr_record_ || return 1 + _k3slr_capture_process _k3slr_record_stdout _k3slr_record_stderr _k3slr_record_rc "$@" || return 1 + [[ "$_k3slr_record_rc" == 0 && -z "$_k3slr_record_stderr" && + "$_k3slr_record_stdout" == *$'\n' ]] || return 1 + _k3slr_record_value="${_k3slr_record_stdout%$'\n'}" + [[ -n "$_k3slr_record_value" && "$_k3slr_record_value" != *$'\n'* && + "$_k3slr_record_value" != *$'\r'* && + "$_k3slr_record_stdout" == "${_k3slr_record_value}"$'\n' ]] || return 1 + printf -v "$_k3slr_record_destination" '%s' "$_k3slr_record_value" +} + +_k3slr_capture_absent_record() { + local _k3slr_absent_stdout='' _k3slr_absent_stderr='' _k3slr_absent_rc='' + (( $# > 0 )) || return 1 + _k3slr_capture_process _k3slr_absent_stdout _k3slr_absent_stderr _k3slr_absent_rc "$@" || return 1 + [[ "$_k3slr_absent_rc" == 1 && -z "$_k3slr_absent_stdout" && -z "$_k3slr_absent_stderr" ]] +} + +_k3slr_capture_empty_success() { + local _k3slr_empty_stdout='' _k3slr_empty_stderr='' _k3slr_empty_rc='' + (( $# > 0 )) || return 1 + _k3slr_capture_process _k3slr_empty_stdout _k3slr_empty_stderr _k3slr_empty_rc "$@" || return 1 + [[ "$_k3slr_empty_rc" == 0 && -z "$_k3slr_empty_stdout" && -z "$_k3slr_empty_stderr" ]] +} + +_k3slr_capture_raw_success() { + local _k3slr_raw_destination="${1-}" _k3slr_raw_stdout='' + local _k3slr_raw_stderr='' _k3slr_raw_rc='' + shift || return 1 + (( $# > 0 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_raw_destination" _k3slr_raw_ || return 1 + _k3slr_capture_process _k3slr_raw_stdout _k3slr_raw_stderr _k3slr_raw_rc "$@" || return 1 + [[ "$_k3slr_raw_rc" == 0 && -z "$_k3slr_raw_stderr" ]] || return 1 + printf -v "$_k3slr_raw_destination" '%s' "$_k3slr_raw_stdout" +} + +_k3slr_parse_smart_evidence() { + local _k3slr_smart_record="${1-}" _k3slr_smart_destination="${2-}" + local _k3slr_smart_remaining _k3slr_smart_line _k3slr_smart_id _k3slr_smart_raw + local _k3slr_smart_health='' _k3slr_smart_reallocated='' + local _k3slr_smart_pending='' _k3slr_smart_uncorrectable='' + local -a _k3slr_smart_fields=() + (( $# == 2 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_smart_destination" _k3slr_smart_ || return 1 + [[ -n "$_k3slr_smart_record" && "$_k3slr_smart_record" == *$'\n' && + "$_k3slr_smart_record" != *$'\r'* ]] || return 1 + _k3slr_smart_remaining="$_k3slr_smart_record" + while [[ -n "$_k3slr_smart_remaining" ]]; do + _k3slr_smart_line="${_k3slr_smart_remaining%%$'\n'*}" + _k3slr_smart_remaining="${_k3slr_smart_remaining#*$'\n'}" + if [[ "$_k3slr_smart_line" =~ ^SMART[[:space:]]overall-health[[:space:]]self-assessment[[:space:]]test[[:space:]]result:[[:space:]]*PASSED[[:space:]]*$ ]]; then + [[ -z "$_k3slr_smart_health" ]] || return 1 + _k3slr_smart_health=PASSED + continue + fi + [[ "$_k3slr_smart_line" =~ ^[[:space:]]*(5|197|198)[[:space:]] ]] || continue + _k3slr_smart_id="${BASH_REMATCH[1]}" + read -r -a _k3slr_smart_fields <<<"$_k3slr_smart_line" + (( ${#_k3slr_smart_fields[@]} >= 2 )) || return 1 + _k3slr_smart_raw="${_k3slr_smart_fields[${#_k3slr_smart_fields[@]} - 1]}" + _k3slr_uint "$_k3slr_smart_raw" || return 1 + case "$_k3slr_smart_id" in + 5) + [[ -z "$_k3slr_smart_reallocated" ]] || return 1 + _k3slr_smart_reallocated="$_k3slr_smart_raw" + ;; + 197) + [[ -z "$_k3slr_smart_pending" ]] || return 1 + _k3slr_smart_pending="$_k3slr_smart_raw" + ;; + 198) + [[ -z "$_k3slr_smart_uncorrectable" ]] || return 1 + _k3slr_smart_uncorrectable="$_k3slr_smart_raw" + ;; + esac + done + [[ "$_k3slr_smart_health" == PASSED && "$_k3slr_smart_reallocated" == 0 && + "$_k3slr_smart_pending" == 0 && "$_k3slr_smart_uncorrectable" == 0 ]] || return 1 + printf -v "$_k3slr_smart_destination" '%s' 'PASSED|0|0|0' +} + +_k3slr_all_loops_exclude_backing_device() { + local _k3slr_loops_record="${1-}" _k3slr_loops_backing_major_minor="${2-}" + local _k3slr_loops_remaining _k3slr_loops_line + local -a _k3slr_loops_fields=() + (( $# == 2 )) || return 1 + [[ "$_k3slr_loops_backing_major_minor" =~ ^[0-9]+:[0-9]+$ ]] || return 1 + [[ "$_k3slr_loops_record" != *$'\r'* ]] || return 1 + [[ -z "$_k3slr_loops_record" || "$_k3slr_loops_record" == *$'\n' ]] || return 1 + _k3slr_loops_remaining="$_k3slr_loops_record" + while [[ -n "$_k3slr_loops_remaining" ]]; do + _k3slr_loops_line="${_k3slr_loops_remaining%%$'\n'*}" + _k3slr_loops_remaining="${_k3slr_loops_remaining#*$'\n'}" + read -r -a _k3slr_loops_fields <<<"$_k3slr_loops_line" + (( ${#_k3slr_loops_fields[@]} == 7 )) || return 1 + [[ "${_k3slr_loops_fields[0]}" =~ ^/dev/loop[0-9]+$ && + "${_k3slr_loops_fields[1]}" =~ ^/[^[:space:]]+$ && + "${_k3slr_loops_fields[2]}" =~ ^(0|[1-9][0-9]*)$ && + "${_k3slr_loops_fields[3]}" =~ ^[0-9]+:[0-9]+$ && + "${_k3slr_loops_fields[4]}" =~ ^[0-9]+:[0-9]+$ && + "${_k3slr_loops_fields[5]}" =~ ^(0|[1-9][0-9]*)$ && + "${_k3slr_loops_fields[6]}" =~ ^(0|[1-9][0-9]*)$ ]] || return 1 + [[ "${_k3slr_loops_fields[3]}" != "$_k3slr_loops_backing_major_minor" ]] || return 1 + done +} + +_k3slr_mapper_name_scan_is_clear() { + local _k3slr_names_record="${1-}" _k3slr_names_remaining _k3slr_names_line + local -a _k3slr_names_fields=() + (( $# == 1 )) || return 1 + [[ -n "$_k3slr_names_record" && "$_k3slr_names_record" == *$'\n' && + "$_k3slr_names_record" != *$'\r'* ]] || return 1 + _k3slr_names_remaining="$_k3slr_names_record" + while [[ -n "$_k3slr_names_remaining" ]]; do + _k3slr_names_line="${_k3slr_names_remaining%%$'\n'*}" + _k3slr_names_remaining="${_k3slr_names_remaining#*$'\n'}" + read -r -a _k3slr_names_fields <<<"$_k3slr_names_line" + (( ${#_k3slr_names_fields[@]} == 2 )) || return 1 + [[ "${_k3slr_names_fields[0]}" =~ ^[A-Za-z0-9._/-]+$ && + "${_k3slr_names_fields[1]}" =~ ^[a-z][a-z0-9_-]*$ ]] || return 1 + [[ "${_k3slr_names_fields[0]}" != "$K3SLR_MAPPING_NAME" && + "${_k3slr_names_fields[0]}" != "$K3SLR_PROOF_MAPPING_NAME" ]] || return 1 + done +} + +_k3slr_mapper_directory_is_safe() { + local _k3slr_mapper_record="${1-}" _k3slr_mapper_mode _k3slr_mapper_group _k3slr_mapper_other + (( $# == 1 )) || return 1 + [[ "$_k3slr_mapper_record" =~ ^directory\|0\|0\|([0-7]{3,4})$'\n'$ ]] || return 1 + _k3slr_mapper_mode="${BASH_REMATCH[1]}" + _k3slr_mapper_group="${_k3slr_mapper_mode: -2:1}" + _k3slr_mapper_other="${_k3slr_mapper_mode: -1}" + (( (10#$_k3slr_mapper_group & 2) == 0 && (10#$_k3slr_mapper_other & 2) == 0 )) +} + +_k3slr_collect_prepackage_snapshot() { + local _k3slr_system_destination="${1-}" + local _k3slr_system_recovery_partition _k3slr_system_recovery_disk + local _k3slr_system_k3s_partition _k3slr_system_k3s_disk + local _k3slr_system_recovery_parent _k3slr_system_partition_mm + local _k3slr_system_recovery_model _k3slr_system_recovery_serial _k3slr_system_recovery_wwn + local _k3slr_system_recovery_mm _k3slr_system_k3s_parent _k3slr_system_k3s_partition_mm + local _k3slr_system_k3s_model _k3slr_system_k3s_serial _k3slr_system_k3s_wwn _k3slr_system_k3s_mm + local _k3slr_system_recovery_uuid _k3slr_system_recovery_partuuid _k3slr_system_recovery_type + local _k3slr_system_k3s_uuid _k3slr_system_k3s_partuuid _k3slr_system_k3s_type + local _k3slr_system_smart_raw _k3slr_system_smart _k3slr_system_loops + local _k3slr_system_mapper_metadata _k3slr_system_names _k3slr_system_snapshot + (( $# == 1 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_system_destination" _k3slr_system_ || return 1 + + _k3slr_capture_exact_one_line_record _k3slr_system_recovery_partition \ + /usr/bin/readlink -f -- "$K3SLR_RECOVERY_PARTITION_BY_ID" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_recovery_disk \ + /usr/bin/readlink -f -- "$K3SLR_RECOVERY_DISK_BY_ID" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_k3s_partition \ + /usr/bin/readlink -f -- "$K3SLR_K3S_PARTITION_BY_ID" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_k3s_disk \ + /usr/bin/readlink -f -- "$K3SLR_K3S_DISK_BY_ID" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_recovery_parent \ + /usr/bin/lsblk --noheadings --paths --output PKNAME -- "$_k3slr_system_recovery_partition" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_partition_mm \ + /usr/bin/lsblk --noheadings --paths --output MAJ:MIN -- "$_k3slr_system_recovery_partition" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_recovery_model \ + /usr/bin/lsblk --noheadings --paths --output MODEL -- "$_k3slr_system_recovery_disk" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_recovery_serial \ + /usr/bin/lsblk --noheadings --paths --output SERIAL -- "$_k3slr_system_recovery_disk" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_recovery_wwn \ + /usr/bin/lsblk --noheadings --paths --output WWN -- "$_k3slr_system_recovery_disk" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_recovery_mm \ + /usr/bin/lsblk --noheadings --paths --output MAJ:MIN -- "$_k3slr_system_recovery_disk" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_k3s_parent \ + /usr/bin/lsblk --noheadings --paths --output PKNAME -- "$_k3slr_system_k3s_partition" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_k3s_partition_mm \ + /usr/bin/lsblk --noheadings --paths --output MAJ:MIN -- "$_k3slr_system_k3s_partition" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_k3s_model \ + /usr/bin/lsblk --noheadings --paths --output MODEL -- "$_k3slr_system_k3s_disk" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_k3s_serial \ + /usr/bin/lsblk --noheadings --paths --output SERIAL -- "$_k3slr_system_k3s_disk" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_k3s_wwn \ + /usr/bin/lsblk --noheadings --paths --output WWN -- "$_k3slr_system_k3s_disk" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_k3s_mm \ + /usr/bin/lsblk --noheadings --paths --output MAJ:MIN -- "$_k3slr_system_k3s_disk" || return 1 + [[ "$_k3slr_system_recovery_parent" == "$_k3slr_system_recovery_disk" && + "$_k3slr_system_k3s_parent" == "$_k3slr_system_k3s_disk" ]] || return 1 + + _k3slr_capture_exact_one_line_record _k3slr_system_recovery_uuid \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value --match-tag UUID -- "$_k3slr_system_recovery_partition" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_recovery_partuuid \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value --match-tag PARTUUID -- "$_k3slr_system_recovery_partition" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_recovery_type \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value --match-tag TYPE -- "$_k3slr_system_recovery_partition" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_k3s_uuid \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value --match-tag UUID -- "$_k3slr_system_k3s_partition" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_k3s_partuuid \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value --match-tag PARTUUID -- "$_k3slr_system_k3s_partition" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_k3s_type \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value --match-tag TYPE -- "$_k3slr_system_k3s_partition" || return 1 + _k3slr_capture_raw_success _k3slr_system_smart_raw \ + /usr/bin/sudo --non-interactive -- /usr/sbin/smartctl -H -A "$_k3slr_system_recovery_disk" || return 1 + _k3slr_parse_smart_evidence "$_k3slr_system_smart_raw" _k3slr_system_smart || return 1 + + _k3slr_capture_absent_record /usr/bin/sudo --non-interactive -- /usr/bin/findmnt \ + --noheadings --raw --output SOURCE,FSTYPE,OPTIONS,ID,MAJ:MIN --mountpoint "$K3SLR_OUTER_MOUNT" || return 1 + _k3slr_capture_absent_record /usr/bin/sudo --non-interactive -- /usr/bin/findmnt \ + --noheadings --raw --output SOURCE,FSTYPE,OPTIONS,ID,MAJ:MIN --mountpoint "$K3SLR_INNER_MOUNT" || return 1 + _k3slr_capture_absent_record /usr/bin/sudo --non-interactive -- /usr/bin/findmnt \ + --noheadings --raw --output TARGET --source "$_k3slr_system_recovery_partition" || return 1 + _k3slr_capture_raw_success _k3slr_system_loops /usr/bin/sudo --non-interactive -- \ + /usr/sbin/losetup --list --noheadings --raw \ + --output NAME,BACK-FILE,BACK-INO,BACK-MAJ:MIN,MAJ:MIN,OFFSET,SIZELIMIT || return 1 + _k3slr_all_loops_exclude_backing_device "$_k3slr_system_loops" "$_k3slr_system_partition_mm" || return 1 + _k3slr_capture_exact_one_line_record _k3slr_system_mapper_metadata \ + /usr/bin/stat '--format=%F|%u|%g|%a' -- /dev/mapper || return 1 + _k3slr_mapper_directory_is_safe "${_k3slr_system_mapper_metadata}"$'\n' || return 1 + _k3slr_capture_empty_success /usr/bin/test ! -L /dev/mapper || return 1 + _k3slr_capture_empty_success /usr/bin/test ! -e "/dev/mapper/${K3SLR_MAPPING_NAME}" || return 1 + _k3slr_capture_empty_success /usr/bin/test ! -L "/dev/mapper/${K3SLR_MAPPING_NAME}" || return 1 + _k3slr_capture_empty_success /usr/bin/test ! -e "/dev/mapper/${K3SLR_PROOF_MAPPING_NAME}" || return 1 + _k3slr_capture_empty_success /usr/bin/test ! -L "/dev/mapper/${K3SLR_PROOF_MAPPING_NAME}" || return 1 + _k3slr_capture_raw_success _k3slr_system_names \ + /usr/bin/lsblk --noheadings --raw --output NAME,TYPE || return 1 + _k3slr_mapper_name_scan_is_clear "$_k3slr_system_names" || return 1 + _k3slr_capture_empty_success /usr/bin/sudo --non-interactive -- \ + /usr/bin/ntfs-3g.probe --readwrite "$_k3slr_system_recovery_partition" || return 1 + + printf -v _k3slr_system_snapshot \ + 'canonical_partition=%s\npartition_major_minor=%s\ncanonical_parent_disk=%s\ndisk_major_minor=%s\nfilesystem_uuid=%s\npartuuid=%s\nfilesystem_type=%s\nmodel=%s\nserial=%s\nwwn=%s\ncanonical_k3s_partition=%s\nk3s_partition_major_minor=%s\ncanonical_k3s_disk=%s\nk3s_disk_major_minor=%s\nk3s_filesystem_uuid=%s\nk3s_partuuid=%s\nk3s_filesystem_type=%s\nk3s_model=%s\nk3s_serial=%s\nk3s_wwn=%s\nsmart_health=PASSED\nsmart_reallocated=0\nsmart_pending=0\nsmart_uncorrectable=0\nouter_mount=absent\ninner_mount=absent\nsource_mounts=absent\nloops=absent\nmapper_directory=physical-root-safe\nmapping_path=absent\nproof_mapping_path=absent\nmapping_names=absent\nntfs_probe=pass\n' \ + "$_k3slr_system_recovery_partition" "$_k3slr_system_partition_mm" \ + "$_k3slr_system_recovery_disk" "$_k3slr_system_recovery_mm" \ + "$_k3slr_system_recovery_uuid" "$_k3slr_system_recovery_partuuid" \ + "$_k3slr_system_recovery_type" "$_k3slr_system_recovery_model" \ + "$_k3slr_system_recovery_serial" "$_k3slr_system_recovery_wwn" \ + "$_k3slr_system_k3s_partition" "$_k3slr_system_k3s_partition_mm" \ + "$_k3slr_system_k3s_disk" "$_k3slr_system_k3s_mm" \ + "$_k3slr_system_k3s_uuid" "$_k3slr_system_k3s_partuuid" \ + "$_k3slr_system_k3s_type" "$_k3slr_system_k3s_model" \ + "$_k3slr_system_k3s_serial" "$_k3slr_system_k3s_wwn" + _k3slr_validate_prepackage_snapshot "$_k3slr_system_snapshot" || return 1 + printf -v "$_k3slr_system_destination" '%s' "$_k3slr_system_snapshot" +} + +_k3slr_capture_shell_function() { + local _k3slr_function_capture_stdout_name="${1-}" + local _k3slr_function_capture_stderr_name="${2-}" + local _k3slr_function_capture_rc_name="${3-}" + local _k3slr_function_capture_directory='' _k3slr_function_capture_stdout='' + local _k3slr_function_capture_stderr='' _k3slr_function_capture_status=0 + shift 3 || return 1 + (( $# > 0 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_function_capture_stdout_name" _k3slr_function_capture_ || return 1 + _k3slr_output_name_is_safe "$_k3slr_function_capture_stderr_name" _k3slr_function_capture_ || return 1 + _k3slr_output_name_is_safe "$_k3slr_function_capture_rc_name" _k3slr_function_capture_ || return 1 + [[ "$_k3slr_function_capture_stdout_name" != "$_k3slr_function_capture_stderr_name" && + "$_k3slr_function_capture_stdout_name" != "$_k3slr_function_capture_rc_name" && + "$_k3slr_function_capture_stderr_name" != "$_k3slr_function_capture_rc_name" ]] || return 1 + declare -F "$1" >/dev/null || return 1 + _k3slr_capture_one_line _k3slr_function_capture_directory \ + /usr/bin/mktemp --directory --tmpdir=/tmp 'k3slr-capture.XXXXXXXX' || return 1 + _k3slr_capture_directory_is_safe "$_k3slr_function_capture_directory" || return 1 + if "$@" >"${_k3slr_function_capture_directory}/stdout" \ + 2>"${_k3slr_function_capture_directory}/stderr"; then + _k3slr_function_capture_status=0 + else + _k3slr_function_capture_status=$? + fi + if IFS= read -r -d '' _k3slr_function_capture_stdout \ + <"${_k3slr_function_capture_directory}/stdout"; then + _k3slr_remove_capture_directory "$_k3slr_function_capture_directory" || : + return 1 + fi + if IFS= read -r -d '' _k3slr_function_capture_stderr \ + <"${_k3slr_function_capture_directory}/stderr"; then + _k3slr_remove_capture_directory "$_k3slr_function_capture_directory" || : + return 1 + fi + _k3slr_remove_capture_directory "$_k3slr_function_capture_directory" || return 1 + printf -v "$_k3slr_function_capture_stdout_name" '%s' "$_k3slr_function_capture_stdout" + printf -v "$_k3slr_function_capture_stderr_name" '%s' "$_k3slr_function_capture_stderr" + printf -v "$_k3slr_function_capture_rc_name" '%s' "$_k3slr_function_capture_status" +} + +_k3slr_run_ntfsinfo_hex_pipeline() { + local _k3slr_pipeline_partition="${1-}" _k3slr_pipeline_program + local -a _k3slr_pipeline_statuses=() + (( $# == 1 )) || return 1 + [[ "$_k3slr_pipeline_partition" =~ ^/dev/[A-Za-z0-9._/-]+$ ]] || return 1 + _k3slr_pipeline_program=$'{\n for (i = 1; i <= NF; i++) {\n if ($i !~ /^[0-9a-f]{2}$/ || ++bytes > 65536) {\n bad = 1\n exit\n }\n hex = hex $i\n }\n}\nEND {\n if (bad || bytes == 0) exit 1\n print hex\n}' + _k3slr_command /usr/bin/sudo --non-interactive -- \ + /usr/bin/ntfsinfo --mft --notime "$_k3slr_pipeline_partition" | + _k3slr_command /usr/bin/od -An -v -tx1 | + _k3slr_command /usr/bin/mawk "$_k3slr_pipeline_program" + _k3slr_pipeline_statuses=("${PIPESTATUS[@]}") + (( ${#_k3slr_pipeline_statuses[@]} == 3 && + _k3slr_pipeline_statuses[0] == 0 && + _k3slr_pipeline_statuses[1] == 0 && + _k3slr_pipeline_statuses[2] == 0 )) +} + +_k3slr_capture_ntfsinfo_hex() { + local _k3slr_ntfs_capture_destination="${1-}" _k3slr_ntfs_capture_partition="${2-}" + local _k3slr_ntfs_capture_stdout='' _k3slr_ntfs_capture_stderr='' _k3slr_ntfs_capture_rc='' + (( $# == 2 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_ntfs_capture_destination" _k3slr_ntfs_capture_ || return 1 + _k3slr_capture_shell_function _k3slr_ntfs_capture_stdout _k3slr_ntfs_capture_stderr \ + _k3slr_ntfs_capture_rc _k3slr_run_ntfsinfo_hex_pipeline "$_k3slr_ntfs_capture_partition" || return 1 + [[ "$_k3slr_ntfs_capture_rc" == 0 && -z "$_k3slr_ntfs_capture_stderr" && + "$_k3slr_ntfs_capture_stdout" =~ ^[0-9a-f]+$'\n'$ ]] || return 1 + (( ${#_k3slr_ntfs_capture_stdout} > 1 && ${#_k3slr_ntfs_capture_stdout} <= 131073 && + (${#_k3slr_ntfs_capture_stdout} - 1) % 2 == 0 )) || return 1 + printf -v "$_k3slr_ntfs_capture_destination" '%s' "$_k3slr_ntfs_capture_stdout" +} + +_k3slr_capture_one_line() { + local _k3slr_capture_framed _k3slr_capture_record _k3slr_capture_value + _k3slr_output_name_is_safe "$1" _k3slr_capture_ || return 1 + _k3slr_capture_framed="$( + _k3slr_command "${@:2}" + _k3slr_capture_command_rc=$? + printf '\034%s' "$_k3slr_capture_command_rc" + )" || return 1 + [[ "$_k3slr_capture_framed" == *$'\0340' ]] || return 1 + _k3slr_capture_record="${_k3slr_capture_framed%$'\0340'}" + [[ "$_k3slr_capture_record" == *$'\n' ]] || return 1 + _k3slr_capture_value="${_k3slr_capture_record%$'\n'}" + [[ -n "$_k3slr_capture_value" && "$_k3slr_capture_value" != *$'\n'* && + "$_k3slr_capture_value" != *$'\r'* && "$_k3slr_capture_value" != *$'\034'* ]] || return 1 + printf -v "$1" '%s' "$_k3slr_capture_value" +} + +_k3slr_trusted_root_executable() { + local binary="$1" metadata uid mode kind group_digit other_digit + [[ "$binary" == /* ]] || return 1 + _k3slr_command /usr/bin/test -f "$binary" >/dev/null 2>&1 || return 1 + _k3slr_command /usr/bin/test ! -L "$binary" >/dev/null 2>&1 || return 1 + _k3slr_command /usr/bin/test -x "$binary" >/dev/null 2>&1 || return 1 + _k3slr_capture_one_line metadata /usr/bin/stat --format='%u|%a|%F' -- "$binary" 2>/dev/null || return 1 + IFS='|' read -r uid mode kind <<<"$metadata" + [[ "$uid" == 0 && "$mode" =~ ^[0-7]{3,4}$ && "$kind" == 'regular file' ]] || return 1 + group_digit="${mode: -2:1}" + other_digit="${mode: -1}" + (( (10#$group_digit & 2) == 0 && (10#$other_digit & 2) == 0 )) +} + +_k3slr_pin_keepass_database() { + local database="$1" destination_name="$2" expected current canonical metadata uid + local component index=0 lineage='' + local -a components=() + [[ -n "${K3SLR_OUTER_MOUNT:-}" && -n "${K3SLR_DATABASE_RELATIVE:-}" ]] || return 1 + expected="${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" + [[ "$database" == "$expected" ]] || return 1 + _k3slr_capture_one_line uid /usr/bin/id -u || return 1 + [[ "$uid" =~ ^[0-9]+$ ]] || return 1 + current="$K3SLR_OUTER_MOUNT" + IFS='/' read -r -a components <<<"$K3SLR_DATABASE_RELATIVE" + for component in '' "${components[@]}"; do + if (( index > 0 )); then current="${current}/${component}"; fi + _k3slr_command /usr/bin/test ! -L "$current" >/dev/null 2>&1 || return 1 + _k3slr_capture_one_line canonical /usr/bin/readlink -e -- "$current" 2>/dev/null || return 1 + [[ "$canonical" == "$current" ]] || return 1 + _k3slr_capture_one_line metadata /usr/bin/stat --format='%d:%i|%F|%u|%a' -- "$current" 2>/dev/null || return 1 + if (( index < ${#components[@]} )); then + [[ "$metadata" == *'|directory|'* ]] || return 1 + else + [[ "$current" == "$database" && "$metadata" == *'|regular file|'* && + "$metadata" == *"|${uid}|600" ]] || return 1 + fi + lineage+="${metadata%|*|*}," + index=$((index + 1)) + done + printf -v "$destination_name" '%s' "${lineage%,}" +} + +_k3slr_keepass_database_lineage_matches() { + local database="$1" expected_snapshot="$2" current_snapshot='' + _k3slr_pin_keepass_database "$database" current_snapshot || return 1 + [[ "$current_snapshot" == "$expected_snapshot" ]] +} + +_k3slr_parse_keepass_attachment_listing() { + local _k3slr_attachment_record="${1-}" _k3slr_attachment_name="${2-}" + local _k3slr_attachment_destination="${3-}" _k3slr_attachment_prefix='' + local _k3slr_attachment_size='' + (( $# == 3 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_attachment_destination" _k3slr_attachment_ || return 1 + [[ "$_k3slr_attachment_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,254}$ ]] || return 1 + (( ${#_k3slr_attachment_record} >= 1 && ${#_k3slr_attachment_record} <= 8192 )) || return 1 + [[ "$_k3slr_attachment_record" != *$'\r'* && + "$_k3slr_attachment_record" == *$'\n' ]] || return 1 + + if [[ "$_k3slr_attachment_record" == "${K3SLR_KEEPASS_ENTRY}"$'\n\nNo attachments present.\n' ]]; then + printf -v "$_k3slr_attachment_destination" '%s' absent + return 0 + fi + + _k3slr_attachment_prefix="${K3SLR_KEEPASS_ENTRY}"$'\n\nAttachments:\n '"${_k3slr_attachment_name} (" + [[ "$_k3slr_attachment_record" == "$_k3slr_attachment_prefix"* ]] || return 1 + _k3slr_attachment_size="${_k3slr_attachment_record#"$_k3slr_attachment_prefix"}" + [[ "$_k3slr_attachment_size" == *$'\n' ]] || return 1 + _k3slr_attachment_size="${_k3slr_attachment_size%$'\n'}" + [[ "$_k3slr_attachment_size" =~ ^[0-9]+\.[0-9]\ (B|KiB|MiB|GiB)\)$ ]] || return 1 + printf -v "$_k3slr_attachment_destination" '%s' present +} + +_k3slr_capture_keepass_attachment_listing() { + local _k3slr_keepass_capture_destination="${1-}" + local _k3slr_keepass_capture_stdout_path="${2-}" + local _k3slr_keepass_capture_stderr_path="${3-}" + local _k3slr_keepass_capture_stdout_fd='' _k3slr_keepass_capture_stderr_fd='' + local _k3slr_keepass_capture_size='' _k3slr_keepass_capture_record='' + local _k3slr_keepass_capture_command_rc=0 _k3slr_keepass_capture_cleanup_rc=0 + local _k3slr_keepass_capture_had_noclobber=false + shift 3 || return 1 + (( $# > 0 )) || return 1 + _k3slr_output_name_is_safe "$_k3slr_keepass_capture_destination" \ + _k3slr_keepass_capture_ || return 1 + declare -F "$1" >/dev/null || return 1 + [[ "$_k3slr_keepass_capture_stdout_path" =~ ^/[A-Za-z0-9._/-]+$ && + "$_k3slr_keepass_capture_stderr_path" =~ ^/[A-Za-z0-9._/-]+$ && + "$_k3slr_keepass_capture_stdout_path" != *'//'* && + "$_k3slr_keepass_capture_stderr_path" != *'//'* && + "$_k3slr_keepass_capture_stdout_path" != *'/../'* && + "$_k3slr_keepass_capture_stderr_path" != *'/../'* && + "$_k3slr_keepass_capture_stdout_path" != "$_k3slr_keepass_capture_stderr_path" && + ! -e "$_k3slr_keepass_capture_stdout_path" && ! -L "$_k3slr_keepass_capture_stdout_path" && + ! -e "$_k3slr_keepass_capture_stderr_path" && ! -L "$_k3slr_keepass_capture_stderr_path" ]] || return 1 + + [[ -o noclobber ]] && _k3slr_keepass_capture_had_noclobber=true + set -o noclobber + if ! exec {_k3slr_keepass_capture_stdout_fd}>"$_k3slr_keepass_capture_stdout_path"; then + "$_k3slr_keepass_capture_had_noclobber" || set +o noclobber + return 1 + fi + if ! exec {_k3slr_keepass_capture_stderr_fd}>"$_k3slr_keepass_capture_stderr_path"; then + exec {_k3slr_keepass_capture_stdout_fd}>&- + "$_k3slr_keepass_capture_had_noclobber" || set +o noclobber + _k3slr_command /usr/bin/rm -- "$_k3slr_keepass_capture_stdout_path" >/dev/null 2>&1 || : + return 1 + fi + "$_k3slr_keepass_capture_had_noclobber" || set +o noclobber + + if "$@" >&"$_k3slr_keepass_capture_stdout_fd" 2>&"$_k3slr_keepass_capture_stderr_fd"; then + _k3slr_keepass_capture_command_rc=0 + else + _k3slr_keepass_capture_command_rc=$? + fi + exec {_k3slr_keepass_capture_stdout_fd}>&- + exec {_k3slr_keepass_capture_stderr_fd}>&- + + if (( _k3slr_keepass_capture_command_rc == 0 )) && + [[ -f "$_k3slr_keepass_capture_stdout_path" && ! -L "$_k3slr_keepass_capture_stdout_path" && + -f "$_k3slr_keepass_capture_stderr_path" && ! -L "$_k3slr_keepass_capture_stderr_path" && + ! -s "$_k3slr_keepass_capture_stderr_path" ]] && + _k3slr_capture_one_line _k3slr_keepass_capture_size /usr/bin/stat \ + --format='%s' -- "$_k3slr_keepass_capture_stdout_path" && + _k3slr_uint "$_k3slr_keepass_capture_size" && + (( 10#$_k3slr_keepass_capture_size >= 1 && 10#$_k3slr_keepass_capture_size <= 8192 )); then + if IFS= read -r -d '' _k3slr_keepass_capture_record \ + <"$_k3slr_keepass_capture_stdout_path"; then + _k3slr_keepass_capture_command_rc=1 + fi + else + _k3slr_keepass_capture_command_rc=1 + fi + + _k3slr_command /usr/bin/rm -- \ + "$_k3slr_keepass_capture_stdout_path" \ + "$_k3slr_keepass_capture_stderr_path" >/dev/null 2>&1 || + _k3slr_keepass_capture_cleanup_rc=1 + (( _k3slr_keepass_capture_command_rc == 0 && _k3slr_keepass_capture_cleanup_rc == 0 )) || return 1 + printf -v "$_k3slr_keepass_capture_destination" '%s' "$_k3slr_keepass_capture_record" +} + +_k3slr_keepass_password_stdout() { + local database="${1-}" entry="${2-}" + (( $# == 2 )) || return 1 + _k3slr_xtrace_is_disabled || return 1 + [[ "$entry" == "$K3SLR_KEEPASS_ENTRY" ]] || return 1 + _k3slr_command /usr/bin/test -t 0 >/dev/null 2>&1 || return 1 + _k3slr_command /usr/bin/keepassxc-cli show --show-protected \ + --attributes Password "$database" "$entry" +} + +_k3slr_validated_keepass_stdout() { + local database="$1" entry="$2" stage_fd='' stage_pid='' stage_rc + exec {stage_fd}< <( + set -o pipefail + _k3slr_keepass_password_stdout "$database" "$entry" | + _k3slr_command /usr/bin/od -An -v -tx1 | + _k3slr_command /usr/bin/mawk ' + function digit(c) { return index("0123456789abcdef", c) - 1 } + function byte_value(h) { return 16 * digit(substr(h, 1, 1)) + digit(substr(h, 2, 1)) } + { + for (i=1; i<=NF; i++) { + n++; byte=$i + if (n <= 40) { + if (byte ~ /^3[0-9]$/) numeric=1 + else if (byte ~ /^(4[1-9a-f]|5[0-9a])$/) uppercase=1 + else if (byte ~ /^(6[1-9a-f]|7[0-9a])$/) lowercase=1 + else invalid=1 + if (!invalid) value=value sprintf("%c", byte_value(byte)) + } else if (n == 41) { + if (byte != "0a") invalid=1 + } else invalid=1 + } + } + END { + if (n == 41 && !invalid && lowercase && uppercase && numeric) printf "%s\n", value + else exit 1 + } + ' + ) + stage_pid=$! + _k3slr_lifecycle_boundary validator-stage-published "$stage_pid" + if wait "$stage_pid"; then stage_rc=0; else stage_rc=$?; fi + if (( stage_rc == 0 )); then + _k3slr_command /usr/bin/dd iflag=fullblock bs=41 count=1 status=none <&"$stage_fd" || stage_rc=$? + fi + exec {stage_fd}<&- + return "$stage_rc" +} + +_k3slr_cached_sudo_is_available() { + _k3slr_trusted_root_executable /usr/bin/sudo || return 1 + _k3slr_command /usr/bin/sudo --non-interactive --validate >/dev/null 2>&1 +} + +_k3slr_loop_device_is_valid() { + local loop_device="$1" canonical kind + [[ "$loop_device" =~ ^/dev/loop(0|[1-9][0-9]*)$ ]] || return 1 + _k3slr_command /usr/bin/test -b "$loop_device" >/dev/null 2>&1 || return 1 + _k3slr_command /usr/bin/test ! -L "$loop_device" >/dev/null 2>&1 || return 1 + _k3slr_capture_one_line canonical /usr/bin/readlink -e -- "$loop_device" 2>/dev/null || return 1 + [[ "$canonical" == "$loop_device" ]] || return 1 + _k3slr_capture_one_line kind /usr/bin/stat --format='%F' -- "$loop_device" 2>/dev/null || return 1 + [[ "$kind" == 'block special file' ]] +} + +_k3slr_mapping_name_is_allowed() { + local mapping_name="$1" + [[ -n "${K3SLR_MAPPING_NAME:-}" && -n "${K3SLR_PROOF_MAPPING_NAME:-}" ]] || return 1 + [[ "$mapping_name" == "$K3SLR_MAPPING_NAME" || "$mapping_name" == "$K3SLR_PROOF_MAPPING_NAME" ]] +} + +_k3slr_cryptsetup_format_stdin() { + local loop_device="${1-}" + (( $# == 1 )) || return 1 + _k3slr_xtrace_is_disabled || return 1 + _k3slr_command /usr/bin/sudo --non-interactive -- /usr/sbin/cryptsetup luksFormat \ + --batch-mode --type luks2 --key-file=- "$loop_device" +} + +_k3slr_cryptsetup_open_stdin() { + local loop_device="${1-}" mapping_name="${2-}" + (( $# == 2 )) || return 1 + _k3slr_xtrace_is_disabled || return 1 + _k3slr_command /usr/bin/sudo --non-interactive -- /usr/sbin/cryptsetup open \ + --type luks2 --key-file=- "$loop_device" "$mapping_name" +} + +_k3slr_cryptsetup_open_readonly_stdin() { + local loop_device="${1-}" mapping_name="${2-}" + (( $# == 2 )) || return 1 + _k3slr_xtrace_is_disabled || return 1 + _k3slr_command /usr/bin/sudo --non-interactive -- /usr/sbin/cryptsetup open \ + --readonly --type luks2 --key-file=- "$loop_device" "$mapping_name" +} + +_k3slr_password_pipe_preflight() { + local destination_name="$1" database="$2" entry="$3" loop_device="$4" mapping_name="${5-}" + local snapshot='' binary + _k3slr_xtrace_is_disabled || return 1 + _k3slr_command /usr/bin/test -t 0 >/dev/null 2>&1 || return 1 + [[ "$entry" == "$K3SLR_KEEPASS_ENTRY" ]] || return 1 + for binary in /usr/bin/test /usr/bin/stat /usr/bin/readlink /usr/bin/id \ + /usr/bin/od /usr/bin/mawk /usr/bin/dd /usr/bin/ps /usr/bin/sleep /usr/bin/sudo \ + /usr/bin/keepassxc-cli /usr/sbin/cryptsetup; do + _k3slr_trusted_root_executable "$binary" || return 1 + done + _k3slr_pin_keepass_database "$database" snapshot || return 1 + _k3slr_loop_device_is_valid "$loop_device" || return 1 + if [[ -n "$mapping_name" ]]; then _k3slr_mapping_name_is_allowed "$mapping_name" || return 1; fi + _k3slr_cached_sudo_is_available || return 1 + printf -v "$destination_name" '%s' "$snapshot" +} + +_k3slr_process_group_for_pid() { + local _k3slr_group_capture='' _k3slr_group_resolved='' + _k3slr_output_name_is_safe "$1" _k3slr_group_ || return 1 + [[ "$2" =~ ^[1-9][0-9]*$ ]] || return 1 + _k3slr_capture_one_line _k3slr_group_capture /usr/bin/ps -o pgid= -p "$2" || return 1 + _k3slr_group_resolved="${_k3slr_group_capture//[[:space:]]/}" + [[ "$_k3slr_group_resolved" =~ ^[1-9][0-9]*$ ]] || return 1 + printf -v "$1" '%s' "$_k3slr_group_resolved" +} + +_k3slr_read_exact_record() { + local _k3slr_exact_value='' _k3slr_exact_trailing='' _k3slr_exact_fd='' + _k3slr_output_name_is_safe "$1" _k3slr_exact_ || return 1 + exec {_k3slr_exact_fd}<"$2" || return 1 + if ! IFS= read -r _k3slr_exact_value <&"$_k3slr_exact_fd"; then + exec {_k3slr_exact_fd}<&- + return 1 + fi + if IFS= read -r _k3slr_exact_trailing <&"$_k3slr_exact_fd" || [[ -n "$_k3slr_exact_trailing" ]]; then + exec {_k3slr_exact_fd}<&- + return 1 + fi + exec {_k3slr_exact_fd}<&- + [[ -n "$_k3slr_exact_value" && "$_k3slr_exact_value" != *$'\r'* && + "$_k3slr_exact_value" != *$'\n'* ]] || return 1 + printf -v "$1" '%s' "$_k3slr_exact_value" +} + +_k3slr_proc_stat_record() { + local _k3slr_stat_value='' + _k3slr_output_name_is_safe "$1" _k3slr_stat_ || return 1 + [[ "$2" =~ ^[1-9][0-9]*$ ]] || return 1 + _k3slr_read_exact_record _k3slr_stat_value "/proc/${2}/stat" || return 1 + printf -v "$1" '%s' "$_k3slr_stat_value" +} + +_k3slr_parse_proc_stat_record() { + local _k3slr_parse_suffix='' _k3slr_parse_state='' _k3slr_parse_parent='' + local _k3slr_parse_group='' _k3slr_parse_start='' _k3slr_parse_prefix='' + local _k3slr_parse_after_comm='' + local -a _k3slr_parse_fields=() + [[ "$1" != "$2" ]] && + _k3slr_output_name_is_safe "$1" _k3slr_parse_ && + _k3slr_output_name_is_safe "$2" _k3slr_parse_ || return 1 + [[ "$3" =~ ^[1-9][0-9]*$ ]] || return 1 + [[ -n "$4" && "$4" != *$'\r'* && "$4" != *$'\n'* ]] || return 1 + _k3slr_parse_prefix="${4%)*}" + _k3slr_parse_after_comm="${4##*)}" + [[ "$_k3slr_parse_prefix" != "$4" && + "$_k3slr_parse_prefix" == "${3} ("* && + "$_k3slr_parse_after_comm" == ' '* ]] || return 1 + _k3slr_parse_suffix="${_k3slr_parse_after_comm:1}" + [[ -n "$_k3slr_parse_suffix" && "$_k3slr_parse_suffix" != [[:space:]]* ]] || return 1 + read -r -a _k3slr_parse_fields <<<"$_k3slr_parse_suffix" + (( ${#_k3slr_parse_fields[@]} >= 20 )) || return 1 + _k3slr_parse_state="${_k3slr_parse_fields[0]}" + _k3slr_parse_parent="${_k3slr_parse_fields[1]}" + _k3slr_parse_group="${_k3slr_parse_fields[2]}" + _k3slr_parse_start="${_k3slr_parse_fields[19]}" + [[ "$_k3slr_parse_state" =~ ^[RSDZTWtXxKPI]$ && + "$_k3slr_parse_parent" =~ ^(0|[1-9][0-9]*)$ && + "$_k3slr_parse_group" =~ ^[1-9][0-9]*$ && + "$_k3slr_parse_start" =~ ^[1-9][0-9]*$ ]] || return 1 + printf -v "$1" '%s' "${3}|${_k3slr_parse_start}|${_k3slr_parse_parent}|${_k3slr_parse_group}" + printf -v "$2" '%s' "$_k3slr_parse_state" +} + +_k3slr_process_record() { + local _k3slr_record_value='' _k3slr_record_identity='' _k3slr_record_state='' + [[ "$1" != "$2" ]] && + _k3slr_output_name_is_safe "$1" _k3slr_record_ && + _k3slr_output_name_is_safe "$2" _k3slr_record_ || return 1 + _k3slr_proc_stat_record _k3slr_record_value "$3" || return 1 + _k3slr_parse_proc_stat_record _k3slr_record_identity _k3slr_record_state \ + "$3" "$_k3slr_record_value" || return 1 + printf -v "$1" '%s' "$_k3slr_record_identity" + printf -v "$2" '%s' "$_k3slr_record_state" +} + +_k3slr_process_identity() { + local _k3slr_identity_value='' _k3slr_identity_state='' + _k3slr_output_name_is_safe "$1" _k3slr_identity_ || return 1 + _k3slr_process_record _k3slr_identity_value _k3slr_identity_state "$2" || return 1 + printf -v "$1" '%s' "$_k3slr_identity_value" +} + +_k3slr_wait_for_stopped_group() { + local _k3slr_stopped_current_identity='' _k3slr_stopped_state='' _k3slr_stopped_attempt + local _k3slr_stopped_group_id='' _k3slr_stopped_identity_pid='' _k3slr_stopped_start_time='' + local _k3slr_stopped_parent_pid='' _k3slr_stopped_identity_group='' + [[ "$1" != "$2" ]] && + _k3slr_output_name_is_safe "$1" _k3slr_stopped_ && + _k3slr_output_name_is_safe "$2" _k3slr_stopped_ || return 1 + for ((_k3slr_stopped_attempt=0; _k3slr_stopped_attempt<200; _k3slr_stopped_attempt++)); do + if _k3slr_process_record _k3slr_stopped_current_identity _k3slr_stopped_state "$3"; then + IFS='|' read -r _k3slr_stopped_identity_pid _k3slr_stopped_start_time \ + _k3slr_stopped_parent_pid _k3slr_stopped_identity_group <<<"$_k3slr_stopped_current_identity" + if [[ "$_k3slr_stopped_state" =~ ^[Tt]$ && "$_k3slr_stopped_parent_pid" == "$BASHPID" ]]; then + printf -v "$2" '%s' "$_k3slr_stopped_current_identity" + _k3slr_process_group_for_pid _k3slr_stopped_group_id "$3" || return 1 + [[ "$_k3slr_stopped_group_id" == "$_k3slr_stopped_identity_group" ]] || return 1 + printf -v "$1" '%s' "$_k3slr_stopped_group_id" + return 0 + fi + fi + _k3slr_command /usr/bin/sleep 0.01 || return 1 + done + return 1 +} + +_k3slr_group_exists() { + local group_id="$1" + [[ "$group_id" =~ ^[1-9][0-9]*$ ]] || return 2 + kill -0 -- "-${group_id}" >/dev/null 2>&1 +} + +_k3slr_wait_for_group_exit() { + local group_id="$1" maximum_attempts="${2:-100}" attempt + [[ "$maximum_attempts" =~ ^[1-9][0-9]*$ ]] || return 1 + for ((attempt=0; attempt/dev/null || return 1 + _k3slr_process_group_for_pid observed_group "$child_pid" 2>/dev/null || return 1 + [[ "$observed_group" == "$group_id" && "$group_id" != "$caller_group" ]] +} + +_k3slr_signal_pid() { + local signal_name="$1" child_pid="$2" expected_identity="$3" expected_parent="$4" expected_group="${5-}" + local current_identity='' state='' identity_pid='' start_time='' parent_pid='' identity_group='' + local caller_group='' observed_group='' + [[ "$signal_name" =~ ^(TERM|CONT|KILL)$ ]] || return 1 + _k3slr_process_record current_identity state "$child_pid" || return 1 + [[ "$current_identity" == "$expected_identity" ]] || return 1 + IFS='|' read -r identity_pid start_time parent_pid identity_group <<<"$current_identity" + [[ "$identity_pid" == "$child_pid" && "$parent_pid" == "$expected_parent" ]] || return 1 + if [[ -n "$expected_group" ]]; then + [[ "$identity_group" == "$expected_group" ]] || return 1 + _k3slr_process_group_for_pid caller_group "$BASHPID" 2>/dev/null || return 1 + _k3slr_process_group_for_pid observed_group "$child_pid" 2>/dev/null || return 1 + [[ "$observed_group" == "$expected_group" && "$expected_group" != "$caller_group" ]] || return 1 + fi + kill -"$signal_name" "$child_pid" >/dev/null 2>&1 +} + +_k3slr_signal_group() { + local signal_name="$1" child_pid="$2" group_id="$3" expected_identity="$4" + [[ "$signal_name" =~ ^(TERM|CONT|KILL)$ ]] || return 1 + _k3slr_group_authority_is_current "$child_pid" "$group_id" "$expected_identity" || return 1 + kill -"$signal_name" -- "-${group_id}" >/dev/null 2>&1 +} + +_k3slr_signal_published_pid() { + local signal_name="$1" child_pid="$2" + [[ "$signal_name" == KILL && "$child_pid" =~ ^[1-9][0-9]*$ ]] || return 1 + kill -KILL "$child_pid" >/dev/null 2>&1 +} + +_k3slr_wait_child() { + wait "$1" >/dev/null 2>&1 +} + +_k3slr_terminate_published_child() { + local child_pid="$1" wait_rc + [[ "$child_pid" =~ ^[1-9][0-9]*$ ]] || return 1 + _k3slr_signal_published_pid KILL "$child_pid" || { + kill -0 "$child_pid" >/dev/null 2>&1 && return 1 + } + if _k3slr_wait_child "$child_pid"; then wait_rc=0; else wait_rc=$?; fi + (( wait_rc != 127 )) || return 1 + ! kill -0 "$child_pid" >/dev/null 2>&1 +} + +_k3slr_continue_owned_child() { + local child_pid="$1" group_id="$2" expected_identity="$3" + _k3slr_signal_pid CONT "$child_pid" "$expected_identity" "$BASHPID" "$group_id" +} + +_k3slr_terminate_direct_stopped_child() { + local child_pid="$1" expected_identity="$2" current_identity='' state='' wait_rc + _k3slr_process_record current_identity state "$child_pid" || return 1 + local identity_pid='' start_time='' parent_pid='' identity_group='' + IFS='|' read -r identity_pid start_time parent_pid identity_group <<<"$current_identity" + [[ "$current_identity" == "$expected_identity" && "$state" =~ ^[Tt]$ && + "$identity_pid" == "$child_pid" && "$parent_pid" == "$BASHPID" ]] || return 1 + _k3slr_signal_pid TERM "$child_pid" "$expected_identity" "$BASHPID" || return 1 + if kill -0 "$child_pid" >/dev/null 2>&1; then + _k3slr_process_record current_identity state "$child_pid" || return 1 + IFS='|' read -r identity_pid start_time parent_pid identity_group <<<"$current_identity" + [[ "$current_identity" == "$expected_identity" && "$identity_pid" == "$child_pid" && + "$parent_pid" == "$BASHPID" ]] || return 1 + _k3slr_signal_pid CONT "$child_pid" "$expected_identity" "$BASHPID" || return 1 + fi + for ((wait_rc=0; wait_rc<100; wait_rc++)); do + if ! kill -0 "$child_pid" >/dev/null 2>&1; then break; fi + _k3slr_command /usr/bin/sleep 0.01 || return 1 + done + if kill -0 "$child_pid" >/dev/null 2>&1; then + _k3slr_process_record current_identity state "$child_pid" || return 1 + IFS='|' read -r identity_pid start_time parent_pid identity_group <<<"$current_identity" + [[ "$current_identity" == "$expected_identity" && "$identity_pid" == "$child_pid" && + "$parent_pid" == "$BASHPID" ]] || return 1 + _k3slr_signal_pid KILL "$child_pid" "$expected_identity" "$BASHPID" || return 1 + fi + if _k3slr_wait_child "$child_pid"; then wait_rc=0; else wait_rc=$?; fi + (( wait_rc != 127 )) || return 1 + ! _k3slr_process_identity current_identity "$child_pid" 2>/dev/null +} + +_k3slr_terminate_and_reap() { + local child_pid="$1" group_id="${2-}" expected_identity="${3-}" + local caller_group='' observed_group='' current_identity='' state='' wait_rc + local identity_pid='' start_time='' parent_pid='' identity_group='' + [[ "$child_pid" =~ ^[1-9][0-9]*$ && + ( -z "$group_id" || "$group_id" =~ ^[1-9][0-9]*$ ) && + -n "$expected_identity" ]] || return 1 + _k3slr_process_record current_identity state "$child_pid" || return 1 + [[ "$current_identity" == "$expected_identity" ]] || return 1 + IFS='|' read -r identity_pid start_time parent_pid identity_group <<<"$current_identity" + [[ "$identity_pid" == "$child_pid" && "$parent_pid" == "$BASHPID" ]] || return 1 + + if [[ -z "$group_id" ]] || + ! _k3slr_process_group_for_pid caller_group "$BASHPID" 2>/dev/null || + ! _k3slr_process_group_for_pid observed_group "$child_pid" 2>/dev/null || + [[ "$observed_group" != "$identity_group" || "$observed_group" != "$group_id" || + "$group_id" == "$caller_group" ]]; then + _k3slr_terminate_direct_stopped_child "$child_pid" "$expected_identity" + return + fi + + _k3slr_signal_group TERM "$child_pid" "$group_id" "$expected_identity" || { + _k3slr_group_exists "$group_id" && return 1 + } + if _k3slr_group_exists "$group_id"; then + if _k3slr_group_authority_is_current "$child_pid" "$group_id" "$expected_identity"; then + _k3slr_signal_group CONT "$child_pid" "$group_id" "$expected_identity" || { + _k3slr_group_exists "$group_id" && return 1 + } + else + _k3slr_wait_for_group_exit "$group_id" || return 1 + fi + fi + if ! _k3slr_wait_for_group_exit "$group_id" 25; then + if _k3slr_group_authority_is_current "$child_pid" "$group_id" "$expected_identity"; then + _k3slr_signal_group KILL "$child_pid" "$group_id" "$expected_identity" || { + _k3slr_group_exists "$group_id" && return 1 + } + else + _k3slr_wait_for_group_exit "$group_id" || return 1 + fi + fi + if _k3slr_wait_child "$child_pid"; then wait_rc=0; else wait_rc=$?; fi + (( wait_rc != 127 )) || return 1 + _k3slr_wait_for_group_exit "$group_id" +} + +_k3slr_lifecycle_boundary() { + : +} + +_k3slr_run_password_pipe() ( + local operation="$1" database="$2" entry="$3" loop_device="$4" mapping_name="${5-}" + local database_snapshot='' producer_pid='' producer_pgid='' producer_source_fd='' pipe_fd='' tty_fd='' + local producer_identity='' stopped_identity='' current_identity='' caller_pgid='' monitor_enabled=false producer_rc consumer_rc + local lifecycle_state='empty' pending_signal='' pending_status='' + _k3slr_password_pipe_preflight database_snapshot "$database" "$entry" "$loop_device" "$mapping_name" || return 1 + _k3slr_process_group_for_pid caller_pgid "$BASHPID" || return 1 + + _k3slr_password_pipe_cleanup() { + local cleanup_rc=0 + if "$monitor_enabled"; then set +m; monitor_enabled=false; fi + if [[ -n "$tty_fd" ]]; then exec {tty_fd}<&- 2>/dev/null || true; tty_fd=''; fi + if [[ -n "$pipe_fd" ]]; then exec {pipe_fd}<&- 2>/dev/null || true; pipe_fd=''; fi + if [[ -n "$producer_source_fd" ]]; then exec {producer_source_fd}<&- 2>/dev/null || true; producer_source_fd=''; fi + if [[ -n "$producer_pid" ]]; then + if [[ "$lifecycle_state" == published ]]; then + _k3slr_terminate_published_child "$producer_pid" || cleanup_rc=$? + elif [[ -n "$producer_identity" ]]; then + _k3slr_terminate_and_reap "$producer_pid" "$producer_pgid" "$producer_identity" || cleanup_rc=$? + else + cleanup_rc=1 + fi + if (( cleanup_rc == 0 )); then + producer_pid='' + producer_pgid='' + producer_identity='' + lifecycle_state='reaped' + fi + fi + return "$cleanup_rc" + } + _k3slr_password_pipe_signal() { + if [[ "$1" == TERM || "$pending_signal" != TERM ]]; then + pending_signal="$1" + pending_status="$2" + fi + } + _k3slr_password_pipe_pending_checkpoint() { + local signal_status cleanup_rc=0 + [[ -n "$pending_signal" ]] || return 0 + signal_status="$pending_status" + if [[ -n "$producer_pid" ]]; then + [[ -n "$producer_identity" ]] || return 1 + _k3slr_password_pipe_cleanup || cleanup_rc=$? + fi + (( cleanup_rc == 0 )) || return 1 + return "$signal_status" + } + _k3slr_password_pipe_exit_cleanup() { + local original_status="$1" cleanup_rc=0 + trap - EXIT + _k3slr_password_pipe_cleanup || cleanup_rc=$? + (( cleanup_rc == 0 )) || exit 1 + exit "$original_status" + } + trap '_k3slr_password_pipe_signal INT 130' INT + trap '_k3slr_password_pipe_signal TERM 143' TERM + trap '_k3slr_password_pipe_exit_cleanup "$?"' EXIT + + exec {tty_fd}<&0 || return 1 + set -m + monitor_enabled=true + lifecycle_state='launching' + _k3slr_lifecycle_boundary coproc-launch-before + _k3slr_password_pipe_pending_checkpoint || return $? + coproc K3SLR_PASSWORD_PRODUCER { + producer_cancel='' + trap '[[ "$producer_cancel" == TERM ]] || producer_cancel=INT' INT + trap 'producer_cancel=TERM' TERM + kill -STOP "$BASHPID" + producer_worker_rc=143 + if [[ -z "$producer_cancel" ]]; then + if _k3slr_validated_keepass_stdout "$database" "$entry" <&"$tty_fd"; then + producer_worker_rc=0 + else + producer_worker_rc=$? + fi + fi + while [[ -n "$producer_cancel" ]]; do + _k3slr_command /usr/bin/sleep 0.01 || : + done + exit "$producer_worker_rc" + } + producer_pid="$K3SLR_PASSWORD_PRODUCER_PID" + lifecycle_state='published' + _k3slr_lifecycle_boundary pid-published "$producer_pid" + if ! _k3slr_process_identity producer_identity "$producer_pid"; then + _k3slr_password_pipe_cleanup || return 1 + if [[ -n "$pending_signal" ]]; then return "$pending_status"; fi + return 1 + fi + if ! _k3slr_wait_for_stopped_group producer_pgid stopped_identity "$producer_pid" || + [[ "$stopped_identity" != "$producer_identity" ]]; then + _k3slr_password_pipe_cleanup || return 1 + if [[ -n "$pending_signal" ]]; then return "$pending_status"; fi + return 1 + fi + lifecycle_state='stopped_pinned' + _k3slr_lifecycle_boundary stop-query-complete "$producer_pid" + _k3slr_password_pipe_pending_checkpoint || return $? + if [[ "$producer_pgid" == "$caller_pgid" ]]; then + _k3slr_password_pipe_cleanup || return 1 + return 1 + fi + set +m + monitor_enabled=false + _k3slr_lifecycle_boundary cont-before + _k3slr_password_pipe_pending_checkpoint || return $? + _k3slr_continue_owned_child "$producer_pid" "$producer_pgid" "$producer_identity" || return 1 + lifecycle_state='continued' + _k3slr_lifecycle_boundary cont-after + _k3slr_password_pipe_pending_checkpoint || return $? + producer_source_fd="${K3SLR_PASSWORD_PRODUCER[0]}" + exec {pipe_fd}<&"$producer_source_fd" || return 1 + exec {producer_source_fd}<&- + exec {tty_fd}<&- + tty_fd='' + + lifecycle_state='waiting' + if _k3slr_wait_child "$producer_pid"; then producer_rc=0; else producer_rc=$?; fi + _k3slr_lifecycle_boundary wait-reaped-before-clear + if _k3slr_process_identity current_identity "$producer_pid" 2>/dev/null && + [[ "$current_identity" == "$producer_identity" ]]; then + _k3slr_password_pipe_cleanup || return 1 + fi + lifecycle_state='reaped' producer_pid='' producer_pgid='' producer_identity='' + _k3slr_lifecycle_boundary state-cleared + if [[ -n "$pending_signal" ]]; then + return "$pending_status" + fi + (( producer_rc == 0 )) || return "$producer_rc" + _k3slr_keepass_database_lineage_matches "$database" "$database_snapshot" || return 1 + _k3slr_lifecycle_boundary pre-consumer + _k3slr_password_pipe_pending_checkpoint || return $? + + if [[ "$operation" == format ]]; then + if _k3slr_cryptsetup_format_stdin "$loop_device" <&"$pipe_fd"; then consumer_rc=0; else consumer_rc=$?; fi + elif [[ "$operation" == open ]]; then + if _k3slr_cryptsetup_open_stdin "$loop_device" "$mapping_name" <&"$pipe_fd"; then consumer_rc=0; else consumer_rc=$?; fi + elif [[ "$operation" == open-readonly ]]; then + if _k3slr_cryptsetup_open_readonly_stdin "$loop_device" "$mapping_name" <&"$pipe_fd"; then consumer_rc=0; else consumer_rc=$?; fi + else + return 1 + fi + exec {pipe_fd}<&- + pipe_fd='' + return "$consumer_rc" +) + +_k3slr_luks_format_from_keepass() { + (( $# == 3 )) || return 1 + _k3slr_run_password_pipe format "$1" "$2" "$3" +} + +_k3slr_luks_open_from_keepass() { + (( $# == 4 )) || return 1 + _k3slr_run_password_pipe open "$1" "$2" "$3" "$4" +} + +_k3slr_luks_open_readonly_from_keepass() { + (( $# == 4 )) || return 1 + _k3slr_run_password_pipe open-readonly "$1" "$2" "$3" "$4" +} diff --git a/scripts/lib/k3s-secret-encryption.sh b/scripts/lib/k3s-secret-encryption.sh new file mode 100644 index 0000000..6a6f551 --- /dev/null +++ b/scripts/lib/k3s-secret-encryption.sh @@ -0,0 +1,569 @@ +#!/usr/bin/env bash +# Source-only helpers for k3s Secret-encryption validation. They deliberately +# return classifications instead of printing service, config, or key material. + +_k3s_status_schema_valid() { + jq -e -s ' + length == 1 and + (.[0] | type == "object" and + (.stage | type == "string") and + (.activekey | type == "string") and + ((has("enable") | not) or (.enable | type == "boolean")) and + ((has("hashmatch") | not) or (.hashmatch | type == "boolean")) and + ((has("hasherror") | not) or (.hasherror | type == "string")) and + ((has("inactivekeys") | not) or (.inactivekeys | type == "array"))) + ' >/dev/null 2>&1 <<<"$1" +} + +classify_encryption_status() { + local status_json="$1" + + if ! _k3s_status_schema_valid "$status_json"; then + printf 'invalid\n' + return 0 + fi + + if jq -e ' + (has("enable") | not) and .stage == "" and .activekey == "" and + (has("hasherror") | not) + ' >/dev/null <<<"$status_json"; then + printf 'disabled_no_config\n' + return 0 + fi + + if jq -e ' + has("enable") and + ((.hashmatch != true) or (has("hasherror") and .hasherror != "")) + ' >/dev/null <<<"$status_json"; then + printf 'hash_mismatch\n' + return 0 + fi + + if jq -e ' + .enable == false and .stage == "start" and .activekey == "" and + .hashmatch == true and ((has("inactivekeys") | not) or (.inactivekeys | length == 0)) + ' >/dev/null <<<"$status_json"; then + printf 'transition_start\n' + return 0 + fi + + if jq -e ' + .enable == true and (.activekey | length > 0) and .hashmatch == true and + (.stage == "start" or .stage == "reencrypt_finished") + ' >/dev/null <<<"$status_json"; then + printf 'enabled_stable\n' + return 0 + fi + + if jq -e '.enable == true and .hashmatch == true' >/dev/null <<<"$status_json"; then + printf 'unsafe_transition\n' + else + printf 'invalid\n' + fi +} + +classify_encryption_provider() { + local status_json="$1" + local active_key + + if ! _k3s_status_schema_valid "$status_json"; then + printf 'invalid\n' + return 0 + fi + active_key="$(jq -r '.activekey' <<<"$status_json")" + case "$active_key" in + 'AES-CBC '*) + [[ "${active_key#AES-CBC }" != "$active_key" && -n "${active_key#AES-CBC }" ]] && printf 'aescbc\n' || printf 'invalid\n' + ;; + 'XSalsa20-POLY1305 '*) + [[ "${active_key#XSalsa20-POLY1305 }" != "$active_key" && -n "${active_key#XSalsa20-POLY1305 }" ]] && printf 'secretbox\n' || printf 'invalid\n' + ;; + *) + printf 'invalid\n' + ;; + esac +} + +version_supports_late_enable() { + local version="$1" + local minor patch + + [[ "$version" =~ ^v1\.([0-9]+)\.([0-9]+)\+k3s[0-9]+$ ]] || return 1 + minor="${BASH_REMATCH[1]}" + patch="${BASH_REMATCH[2]}" + case "$minor" in + 33) (( patch >= 10 )) ;; + 34) (( patch >= 6 )) ;; + 35) (( patch >= 3 )) ;; + *) (( minor >= 36 )) ;; + esac +} + +require_exact_encryption_state() { + local expected="$1" + local status_json="$2" + [[ "$(classify_encryption_status "$status_json")" == "$expected" ]] +} + +_k3s_secure_regular_file() { + local path="$1" + local metadata + + [[ -f "$path" && ! -L "$path" ]] || return 1 + metadata="$(stat --format='%u:%a:%F' -- "$path" 2>/dev/null)" || return 1 + [[ "$metadata" == '0:600:regular file' ]] +} + +_k3s_read_file() { + local path="$1" + [[ -f "$path" && ! -L "$path" ]] || return 1 + command cat -- "$path" +} + +_k3s_list_yaml_files() { + local config="$1" file + [[ -f "$config" && ! -L "$config" ]] && printf '%s\n' "$config" + if [[ -d "${config}.d" && ! -L "${config}.d" ]]; then + while IFS= read -r file; do printf '%s\n' "$file"; done < <( + find "${config}.d" -maxdepth 1 -type f -name '*.yaml' -print | sort + ) + fi +} + +verify_local_encryption_config_integrity() { + local config_path="$1" + local state_path="$2" + local annotation_value="$3" + local status_stage="$4" + local state_value config_hash config_metadata state_metadata + + if ! _k3s_secure_regular_file "$config_path" || ! _k3s_secure_regular_file "$state_path"; then + printf 'mismatch\n' + return 1 + fi + [[ "$status_stage" =~ ^[a-z_]+$ ]] || { + printf 'mismatch\n' + return 1 + } + state_value="$(<"$state_path")" + [[ "$(wc -c <"$state_path" | tr -d '[:space:]')" == "${#state_value}" ]] || { + printf 'mismatch\n' + return 1 + } + [[ "$state_value" =~ ^([a-z_]+)-([0-9a-f]{64})$ ]] || { + printf 'mismatch\n' + return 1 + } + config_hash="$(sha256sum -- "$config_path" | awk '{print $1}')" || { + printf 'mismatch\n' + return 1 + } + config_metadata="$(stat --format='%u:%a:%F' -- "$config_path" 2>/dev/null)" || config_metadata='' + state_metadata="$(stat --format='%u:%a:%F' -- "$state_path" 2>/dev/null)" || state_metadata='' + verify_local_encryption_config_integrity_evidence \ + "$config_metadata" "$state_metadata" "$config_hash" "$state_value" \ + "$annotation_value" "$status_stage" +} + +verify_local_encryption_config_integrity_evidence() { + local config_metadata="$1" state_metadata="$2" config_hash="$3" + local state_value="$4" annotation_value="$5" status_stage="$6" expected + + if [[ "$config_metadata" != '0:600:regular file' || + "$state_metadata" != '0:600:regular file' || + ! "$config_hash" =~ ^[0-9a-f]{64}$ || + ! "$status_stage" =~ ^[a-z_]+$ || + ! "$state_value" =~ ^[a-z_]+-[0-9a-f]{64}$ ]]; then + printf 'mismatch\n' + return 1 + fi + expected="${status_stage}-${config_hash}" + if [[ "$state_value" == "$expected" && "$annotation_value" == "$expected" ]]; then + printf 'match\n' + return 0 + fi + printf 'mismatch\n' + return 1 +} + +_k3s_systemctl_show() { + systemctl show k3s \ + --property=ExecStart \ + --property=Environment \ + --property=EnvironmentFiles 2>/dev/null +} + +_k3s_systemd_dropin_lines() { + local dropin_dir="${K3S_SYSTEMD_DROPIN_DIR:-/etc/systemd/system/k3s.service.d}" + local file + + [[ -e "$dropin_dir" ]] || return 0 + [[ -d "$dropin_dir" && ! -L "$dropin_dir" ]] || return 1 + while IFS= read -r file; do + [[ -r "$file" && ! -L "$file" ]] || return 1 + sed -n -E '/^[[:space:]]*(Environment|EnvironmentFile)=/p' "$file" + done < <(find "$dropin_dir" -maxdepth 1 -type f -name '*.conf' -print | sort) +} + +_k3s_is_dynamic_value() { + local value="$1" + [[ "$value" == *'$'* || "$value" == *'`'* || "$value" == *'\\'* ]] +} + +# Strict, non-evaluating readers for systemd's already-resolved show output and +# EnvironmentFile syntax. Unsupported quoting, resets, or expansions fail +# closed instead of being interpreted by a shell. +_k3s_plain_value() { + local value="$1" quote="'" + [[ -n "$value" && "$value" != *' '* && "$value" != *'"'* && "$value" != *"$quote"* && "$value" != *'$'* && "$value" != *'`'* && "$value" != *'\\'* ]] +} + +_k3s_safe_payload() { + local value="$1" quote="'" + [[ -n "$value" && "$value" != *'"'* && "$value" != *"$quote"* && "$value" != *'$'* && "$value" != *'`'* && "$value" != *'\\'* ]] +} + +_k3s_effective_service_show() { + local show dropins line has_environment=false + show="$(_k3s_systemctl_show)" || return 2 + while IFS= read -r line; do + [[ "$line" == Environment=* || "$line" == EnvironmentFiles=* ]] || continue + [[ -n "${line#*=}" ]] && has_environment=true + done <<<"$show" + if ! "$has_environment"; then + dropins="$(_k3s_systemd_dropin_lines)" || return 2 + [[ -z "$dropins" ]] || show+=$'\n'"$dropins" + fi + printf '%s\n' "$show" +} + +_k3s_envfile_values() { + local file="$1" key="$2" line trimmed name value text char quote='' content + local in_record=false target_record=false continued=false + local index length + content="$(_k3s_read_file "$file")" || return 2 + while IFS= read -r line || [[ -n "$line" ]]; do + if ! "$in_record"; then + trimmed="${line#"${line%%[![:space:]]*}"}" + [[ -z "$trimmed" || "$trimmed" == \#* || "$trimmed" == \;* ]] && continue + [[ "$trimmed" == *=* ]] || { [[ "$trimmed" == "$key" ]] && return 2; continue; } + name="${trimmed%%=*}" + value="${trimmed#*=}" + target_record=false + [[ "$name" == "$key" ]] && target_record=true + text="$value" + quote='' + else + text="$line" + fi + + continued=false + length=${#text} + for ((index=0; index= 2 )) || return 2 + [[ "${words[0]##*/}" == k3s && "${words[1]}" == server ]] || return 2 + for ((index=0; index<${#words[@]}; index++)); do + token="${words[index]}" + case "$token" in + "${option}" ) + if [[ "$option" == '--secrets-encryption' ]]; then printf 'true\n'; + elif (( index + 1 < ${#words[@]} )); then option_value="${words[index + 1]}"; _k3s_plain_value "$option_value" || return 2; printf '%s\n' "$option_value"; else return 2; fi + ;; + "${option}"=*) option_value="${token#*=}"; _k3s_plain_value "$option_value" || return 2; printf '%s\n' "$option_value" ;; + esac + done + done <<<"$show" + (( exec_count == 1 )) || return 2 + return 0 +} + +_k3s_unique_value() { + local values="$1" value count=0 chosen='' + while IFS= read -r value; do + [[ -n "$value" ]] || continue + ((count++)); chosen="$value" + done <<<"$values" + (( count <= 1 )) || return 2 + [[ -n "$chosen" ]] && printf '%s\n' "$chosen" + return 0 +} + +_k3s_selected_config() { + local show="$1" cli env default_root raw + raw="$(_k3s_cli_values "$show" --config)" || return 2 + cli="$(_k3s_unique_value "$raw")" || return 2 + raw="$(_k3s_service_env_values "$show" K3S_CONFIG_FILE)" || return 2 + env="$(_k3s_unique_value "$raw")" || return 2 + if _k3s_is_dynamic_value "$cli" || _k3s_is_dynamic_value "$env"; then + return 2 + fi + default_root="${K3S_CONFIG_DIR:-/etc/rancher/k3s}" + if [[ -n "$cli" ]]; then printf '%s\n' "$cli"; elif [[ -n "$env" ]]; then printf '%s\n' "$env"; else printf '%s\n' "${default_root}/config.yaml"; fi +} + +_k3s_yaml_value() { + local config="$1" key="$2" file line raw value selected='' selected_file='' content files + files="$(_k3s_list_yaml_files "$config")" || return 2 + while IFS= read -r file; do + [[ -n "$file" ]] || continue + content="$(_k3s_read_file "$file")" || return 2 + raw='' + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line#"${line%%[![:space:]]*}"}" + [[ "$line" == "${key}:"* ]] || continue + value="${line#"${key}:"}" + value="$(sed -E 's/[[:space:]]+#.*$//; s/^[[:space:]]+//; s/[[:space:]]+$//;' <<<"$value")" + _k3s_plain_value "$value" || return 2 + [[ -z "$raw" || "$raw" == "$value" ]] || return 2 + raw="$value" + done <<<"$content" + if [[ -n "$raw" ]]; then selected="$raw"; selected_file="$file"; fi + done <<<"$files" + [[ -n "$selected" ]] && printf '%s|%s\n' "$selected_file" "$selected" + return 0 +} + +_k3s_resolve_key() { + local show="$1" cli_option="$2" env_key="$3" yaml_key="$4" cli env config yaml raw + raw="$(_k3s_cli_values "$show" "$cli_option")" || return 2 + cli="$(_k3s_unique_value "$raw")" || return 2 + [[ -z "$cli" ]] || { printf 'command-line|%s\n' "$cli"; return 0; } + raw="$(_k3s_service_env_values "$show" "$env_key")" || return 2 + env="$(_k3s_unique_value "$raw")" || return 2 + [[ -z "$env" ]] || { printf 'environment|%s\n' "$env"; return 0; } + config="$(_k3s_selected_config "$show")" || return 2 + yaml="$(_k3s_yaml_value "$config" "$yaml_key")" || return 2 + [[ -z "$yaml" ]] || printf '%s\n' "$yaml" +} + +detect_effective_encryption_config_owner() { + local show enable provider enable_source enable_value provider_source provider_value + show="$(_k3s_effective_service_show)" || { printf 'ambiguous\n'; return 0; } + enable="$(_k3s_resolve_key "$show" --secrets-encryption K3S_SECRETS_ENCRYPTION secrets-encryption)" || { printf 'ambiguous\n'; return 0; } + provider="$(_k3s_resolve_key "$show" --secrets-encryption-provider K3S_SECRETS_ENCRYPTION_PROVIDER secrets-encryption-provider)" || { printf 'ambiguous\n'; return 0; } + [[ -n "$enable" ]] || { printf 'ambiguous\n'; return 0; } + IFS='|' read -r enable_source enable_value <<<"$enable" + [[ "$enable_value" == true ]] || { printf 'ambiguous\n'; return 0; } + if [[ -n "$provider" ]]; then IFS='|' read -r provider_source provider_value <<<"$provider"; [[ "$provider_value" == aescbc || "$provider_value" == secretbox ]] || { printf 'ambiguous\n'; return 0; }; printf '%s/%s\n' "$provider_value" "$provider_source"; else printf 'aescbc/implicit-default\n'; fi +} + +_k3s_datastore_local_evidence() { + local data_dir="$1" + local state_db="${data_dir}/server/db/state.db" + local etcd_dir="${data_dir}/server/db/etcd" + if [[ -f "$state_db" && -d "$etcd_dir" ]]; then printf 'ambiguous\n'; + elif [[ -f "$state_db" ]]; then printf 'sqlite\n'; + elif [[ -d "$etcd_dir" ]]; then printf 'embedded-etcd\n'; + else printf 'none\n'; fi +} + +detect_k3s_datastore() { + local show data endpoint source value evidence + show="$(_k3s_effective_service_show)" || { printf 'ambiguous\n'; return 0; } + data="$(_k3s_resolve_key "$show" --data-dir K3S_DATA_DIR data-dir)" || { printf 'ambiguous\n'; return 0; } + endpoint="$(_k3s_resolve_key "$show" --datastore-endpoint K3S_DATASTORE_ENDPOINT datastore-endpoint)" || { printf 'ambiguous\n'; return 0; } + if [[ -n "$data" ]]; then IFS='|' read -r source value <<<"$data"; data="$value"; else source=default; data=/var/lib/rancher/k3s; fi + _k3s_is_dynamic_value "$data" || [[ "$data" == /* ]] || { printf 'ambiguous\n'; return 0; } + [[ "$source" == default || "$data" == /var/lib/rancher/k3s ]] || { printf 'ambiguous\n'; return 0; } + evidence="$(_k3s_datastore_local_evidence "$data")" || { printf 'ambiguous\n'; return 0; } + if [[ -n "$endpoint" ]]; then [[ "$evidence" == none ]] && printf 'external\n' || printf 'ambiguous\n'; + elif [[ "$evidence" == sqlite || "$evidence" == embedded-etcd ]]; then printf '%s\n' "$evidence"; + else printf 'ambiguous\n'; fi +} + +_k3s_now_seconds() { + printf '%s\n' "$SECONDS" +} + +_k3s_timeout_before_deadline() { + local deadline="$1" now remaining command_timeout=9 + now="$(_k3s_now_seconds)" || return 1 + [[ "$now" =~ ^[0-9]+$ ]] || return 1 + remaining=$((deadline - now)) + (( remaining > 1 )) || return 1 + (( command_timeout < remaining )) || command_timeout=$((remaining - 1)) + printf '%s\n' "$command_timeout" +} + +_k3s_sleep_until_poll() { + local deadline="$1" next_poll="$2" now delay remaining + now="$(_k3s_now_seconds)" || return 1 + [[ "$now" =~ ^[0-9]+$ ]] || return 1 + (( now < deadline )) || return 1 + delay=$((next_poll - now)) + (( delay > 0 )) || return 0 + remaining=$((deadline - now)) + (( delay < remaining )) || delay="$remaining" + sleep "$delay" +} + +_k3s_read_encryption_status() { + local command_timeout="$1" + [[ "$command_timeout" =~ ^[1-9]$ ]] || return 1 + timeout --signal=TERM --kill-after=1s "${command_timeout}s" \ + sudo --non-interactive k3s secrets-encrypt status --output json 2>/dev/null +} + +wait_for_k3s_api() { + local attempt started_at deadline next_poll command_timeout + started_at="$(_k3s_now_seconds)" || return 1 + [[ "$started_at" =~ ^[0-9]+$ ]] || return 1 + deadline=$((started_at + 600)) + for ((attempt=1; attempt<=60; attempt++)); do + command_timeout="$(_k3s_timeout_before_deadline "$deadline")" || return 1 + if timeout --signal=TERM --kill-after=1s "${command_timeout}s" \ + sudo --non-interactive k3s kubectl get --raw=/readyz >/dev/null 2>&1; then + return 0 + fi + (( attempt < 60 )) || break + next_poll=$((started_at + attempt * 10)) + _k3s_sleep_until_poll "$deadline" "$next_poll" || return 1 + done + return 1 +} + +wait_for_reencrypt_finished() { + local rotate_rc="$1" + local attempt status_json status_class stage started_at deadline next_poll command_timeout + + # rotate-keys can return non-zero after the server accepted the operation; + # a later authoritative finished status is therefore allowed to succeed. + [[ "$rotate_rc" =~ ^[0-9]+$ ]] || return 1 + started_at="$(_k3s_now_seconds)" || return 1 + [[ "$started_at" =~ ^[0-9]+$ ]] || return 1 + deadline=$((started_at + 600)) + for ((attempt=1; attempt<=60; attempt++)); do + command_timeout="$(_k3s_timeout_before_deadline "$deadline")" || return 1 + status_json="$(_k3s_read_encryption_status "$command_timeout")" || return 1 + status_class="$(classify_encryption_status "$status_json")" + [[ "$status_class" != hash_mismatch && "$status_class" != invalid ]] || return 1 + stage="$(jq -r '.stage' <<<"$status_json" 2>/dev/null)" || return 1 + [[ "$stage" != start ]] || return 1 + [[ "$stage" == reencrypt_finished ]] && return 0 + [[ "$stage" == reencrypt_active ]] || return 1 + (( attempt < 60 )) || break + next_poll=$((started_at + attempt * 10)) + _k3s_sleep_until_poll "$deadline" "$next_poll" || return 1 + done + return 1 +} diff --git a/scripts/lib/slack-webhook-recovery.sh b/scripts/lib/slack-webhook-recovery.sh new file mode 100644 index 0000000..326a324 --- /dev/null +++ b/scripts/lib/slack-webhook-recovery.sh @@ -0,0 +1,1232 @@ +#!/usr/bin/bash + +set -Eeuo pipefail +set +x +umask 077 +ulimit -c 0 + +readonly SWR_WINDOWS_UUID='4EA0196C0C5FA27E' +readonly SWR_KDBX_RELATIVE='HyeonworksRecovery/vault/hyeonworks-recovery.kdbx' +readonly SWR_ENTRY_GROUP='/Platform/Observability/Slack' +readonly SWR_ENTRY_TITLE='Alertmanager webhook - desktop-infra-전체' +readonly SWR_WORKSPACE_NAME='desktop-infra' +readonly SWR_CHANNEL_NAME='desktop-infra-전체' +readonly SWR_ALLOWED_SHELLOPTS="${SHELLOPTS}" + +_SWR_VAULT_CLIENT_SOCKET='' +_SWR_VAULT_CLIENT_READ_FD='' +_SWR_VAULT_CLIENT_WRITE_FD='' +_SWR_VAULT_CLIENT_PID='' + +SWR_RECOVERY_DISK_BY_ID='' +SWR_RECOVERY_PARTITION_BY_ID='' +SWR_RECOVERY_FS_UUID='' +SWR_RECOVERY_PARTUUID='' +SWR_RECOVERY_MODEL='' +SWR_RECOVERY_SERIAL='' +SWR_RECOVERY_WWN='' +SWR_K3S_DISK_BY_ID='' +SWR_K3S_PARTITION_BY_ID='' +SWR_RECOVERY_DEVICE='' +SWR_TRANSACTION_STATE='' +SWR_PENDING_SIGNAL=0 + +plain_fail() { + builtin printf 'ERROR: %s\n' "$1" >&2 + return 1 +} + +_swr_has_imported_function_overrides() { + local exported_line + while IFS= read -r exported_line; do + [[ -z "$exported_line" ]] && continue + return 0 + done < <(builtin declare -Fx) + return 1 +} + +_swr_shellopts_are_safe() { + local shellopts="${1-}" shellopts_declaration + [[ "$shellopts" == "$SWR_ALLOWED_SHELLOPTS" ]] || return 1 + shellopts_declaration="$(builtin declare -p SHELLOPTS 2>/dev/null)" || return 1 + [[ "$shellopts_declaration" == 'declare -r SHELLOPTS='* ]] +} + +_swr_initial_guard() { + local effective_uid="${1-}" bash_env_value="${2-}" env_value="${3-}" shellopts_value="${4-}" script_path="${5-}" + + [[ "$effective_uid" =~ ^[0-9]+$ && "$effective_uid" != 0 ]] || return 1 + [[ -z "$bash_env_value" && -z "$env_value" ]] || return 1 + _swr_shellopts_are_safe "$shellopts_value" || return 1 + [[ -z "${CDPATH:-}" ]] || return 1 + ! _swr_has_imported_function_overrides || return 1 + [[ "$script_path" == /* ]] || return 1 + [[ "$script_path" == */scripts/bootstrap/backup-slack-webhook-recovery.sh ]] || return 1 + [[ -f "$script_path" && ! -L "$script_path" ]] || return 1 +} + +_swr_output_name_is_safe() { + local destination_name="${1-}" required_prefix="${2-}" + [[ "$destination_name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || return 1 + [[ "$destination_name" == "${required_prefix}"* ]] || return 1 +} + +_swr_parse_cli() { + local mode_name="${1-}" path_name="${2-}" mode='DRY_RUN' webhook_path='' + shift 2 || return 1 + _swr_output_name_is_safe "$mode_name" _swr_ || return 1 + _swr_output_name_is_safe "$path_name" _swr_ || return 1 + + case "$#" in + 0) + ;; + 3) + [[ "$1" == --execute && "$2" == --slack-webhook-file ]] || return 1 + [[ "$3" == /* ]] || return 1 + mode='EXECUTE' + webhook_path="$3" + ;; + *) + return 1 + ;; + esac + + builtin printf -v "$mode_name" '%s' "$mode" + builtin printf -v "$path_name" '%s' "$webhook_path" +} + +_swr_print_contract() { + builtin printf '%s\n' \ + 'SLACK_KEEPASS_RECOVERY_MODE=DRY_RUN' \ + "WINDOWS_SSD_UUID=${SWR_WINDOWS_UUID}" \ + "KDBX_RELATIVE_PATH=${SWR_KDBX_RELATIVE}" \ + 'MUTATION=NOT_REQUESTED' \ + 'OFF_HOST_RECOVERY_SATISFIED=NO' +} + +_swr_require_execute_tty() { + local tty_fd + [[ -t 0 ]] || return 1 + exec {tty_fd}<>/dev/tty || return 1 + [[ -t "$tty_fd" ]] || { + exec {tty_fd}>&- + return 1 + } + exec {tty_fd}>&- +} + +_swr_validate_runtime() { + local mode="${1-}" script_path="${2-}" + _swr_initial_guard "${EUID:-}" "${BASH_ENV-}" "${ENV-}" "${SHELLOPTS-}" "$script_path" || return 1 + case "$mode" in + DRY_RUN) return 0 ;; + EXECUTE) _swr_require_execute_tty ;; + *) return 1 ;; + esac +} + +_swr_command() { + command "$@" +} + +_swr_capture_command() { + local output_name="${1-}" framed='' captured='' rc_text rc + local frame_prefix=$'\036SWR_CAPTURE_RC=' frame_suffix + shift || return 1 + _swr_output_name_is_safe "$output_name" _swr_ || return 1 + framed="$( + set +e + _swr_command "$@" 2>/dev/null + _swr_framed_rc=$? + builtin printf '\036SWR_CAPTURE_RC=%03d\037' "$_swr_framed_rc" + )" || return 1 + local LC_ALL=C + (( ${#framed} >= ${#frame_prefix} + 4 )) || return 1 + frame_suffix="${framed: -(${#frame_prefix} + 4)}" + [[ "${frame_suffix:0:${#frame_prefix}}" == "$frame_prefix" && + "${frame_suffix: -1}" == $'\037' ]] || return 1 + rc_text="${frame_suffix:${#frame_prefix}:3}" + [[ "$rc_text" =~ ^[0-9]{3}$ ]] || return 1 + rc=$((10#$rc_text)) + (( rc >= 0 && rc <= 255 )) || return 1 + captured="${framed:0:${#framed}-${#frame_suffix}}" + if (( ${#captured} > 16384 )); then + builtin printf -v "$output_name" '%s' '' + return 90 + fi + builtin printf -v "$output_name" '%s' "$captured" + return "$rc" +} + +_swr_load_recovery_contract() { + local contract_path="${1-}" line index + local -a actual=() expected=( + 'K3SLR_SCHEMA_VERSION=1' + 'K3SLR_RECOVERY_DISK_BY_ID=/dev/disk/by-id/wwn-0x500a0751e6aa6254' + 'K3SLR_RECOVERY_PARTITION_BY_ID=/dev/disk/by-id/wwn-0x500a0751e6aa6254-part3' + 'K3SLR_RECOVERY_FS_UUID=4EA0196C0C5FA27E' + 'K3SLR_RECOVERY_PARTUUID=4670aa9f-9045-4bce-930d-9e84dfec9f38' + 'K3SLR_RECOVERY_MODEL=CT1000MX500SSD1' + 'K3SLR_RECOVERY_SERIAL=2306E6AA6254' + 'K3SLR_RECOVERY_WWN=0x500a0751e6aa6254' + 'K3SLR_K3S_DISK_BY_ID=/dev/disk/by-id/wwn-0x500a07512df28bd1' + 'K3SLR_K3S_PARTITION_BY_ID=/dev/disk/by-id/wwn-0x500a07512df28bd1-part1' + 'K3SLR_K3S_FS_UUID=b86086ef-2b3c-4638-abcf-fc7f137dcb97' + 'K3SLR_K3S_PARTUUID=b081b955-d6bc-442c-ac0f-db76560a5245' + 'K3SLR_K3S_MODEL=CT250MX500SSD1' + 'K3SLR_K3S_SERIAL=21132DF28BD1' + 'K3SLR_K3S_WWN=0x500a07512df28bd1' + 'K3SLR_OWNER_UID=1000' + 'K3SLR_OWNER_GID=1000' + 'K3SLR_OUTER_MOUNT=/mnt/k3s-recovery-ssd' + 'K3SLR_INNER_MOUNT=/srv/recovery/k3s' + 'K3SLR_ROOT_RELATIVE=HyeonworksRecovery' + 'K3SLR_DATABASE_RELATIVE=HyeonworksRecovery/vault/hyeonworks-recovery.kdbx' + 'K3SLR_CONTAINER_RELATIVE=HyeonworksRecovery/containers/k3s-recovery.luks' + 'K3SLR_RUNTIME_METADATA_RELATIVE=HyeonworksRecovery/volume.env' + 'K3SLR_MAPPING_NAME=k3s-recovery' + 'K3SLR_PROOF_MAPPING_NAME=k3s-recovery-proof' + 'K3SLR_INNER_LABEL=K3S_RECOVERY' + 'K3SLR_CONTAINER_SIZE_BYTES=34359738368' + 'K3SLR_MINIMUM_FREE_BYTES=10737418240' + 'K3SLR_OUTER_MIN_REMAINING_PERCENT=20' + ) + + [[ $# -eq 1 && "$contract_path" == /* && -f "$contract_path" && ! -L "$contract_path" ]] || return 1 + while IFS= builtin read -r line; do + actual+=("$line") + (( ${#actual[@]} <= ${#expected[@]} )) || return 1 + done <"$contract_path" + (( ${#actual[@]} == ${#expected[@]} )) || return 1 + for index in "${!expected[@]}"; do + [[ "${actual[index]}" == "${expected[index]}" ]] || return 1 + done + + SWR_RECOVERY_DISK_BY_ID='/dev/disk/by-id/wwn-0x500a0751e6aa6254' + SWR_RECOVERY_PARTITION_BY_ID='/dev/disk/by-id/wwn-0x500a0751e6aa6254-part3' + SWR_RECOVERY_FS_UUID='4EA0196C0C5FA27E' + SWR_RECOVERY_PARTUUID='4670aa9f-9045-4bce-930d-9e84dfec9f38' + SWR_RECOVERY_MODEL='CT1000MX500SSD1' + SWR_RECOVERY_SERIAL='2306E6AA6254' + SWR_RECOVERY_WWN='0x500a0751e6aa6254' + SWR_K3S_DISK_BY_ID='/dev/disk/by-id/wwn-0x500a07512df28bd1' + SWR_K3S_PARTITION_BY_ID='/dev/disk/by-id/wwn-0x500a07512df28bd1-part1' +} + +_swr_capture_single_line() { + local output_name="${1-}" captured='' + shift || return 1 + _swr_capture_command "$output_name" "$@" || return 1 + captured="${!output_name}" + [[ "$captured" == *$'\n' ]] || return 1 + captured="${captured%$'\n'}" + [[ -n "$captured" && "$captured" != *$'\n'* && "$captured" != *$'\r'* ]] || return 1 + builtin printf -v "$output_name" '%s' "$captured" +} + +_swr_resolve_device() { + local output_name="${1-}" + local _swr_recovery_disk='' _swr_recovery_partition='' + local _swr_k3s_disk='' _swr_k3s_partition='' + local _swr_recovery_parent='' _swr_k3s_parent='' + local _swr_uuid='' _swr_partuuid='' _swr_type='' + local _swr_model='' _swr_serial='' _swr_wwn='' + [[ $# -eq 1 ]] || return 1 + _swr_output_name_is_safe "$output_name" _swr_ || return 1 + [[ -n "$SWR_RECOVERY_DISK_BY_ID" && -n "$SWR_K3S_DISK_BY_ID" ]] || return 1 + + _swr_capture_single_line _swr_recovery_disk /usr/bin/readlink -e -- "$SWR_RECOVERY_DISK_BY_ID" || return 1 + _swr_capture_single_line _swr_recovery_partition /usr/bin/readlink -e -- "$SWR_RECOVERY_PARTITION_BY_ID" || return 1 + _swr_capture_single_line _swr_k3s_disk /usr/bin/readlink -e -- "$SWR_K3S_DISK_BY_ID" || return 1 + _swr_capture_single_line _swr_k3s_partition /usr/bin/readlink -e -- "$SWR_K3S_PARTITION_BY_ID" || return 1 + [[ "$_swr_recovery_disk" == /* && "$_swr_recovery_partition" == /* && + "$_swr_k3s_disk" == /* && "$_swr_k3s_partition" == /* ]] || return 1 + [[ "$_swr_recovery_disk" != "$_swr_recovery_partition" && + "$_swr_recovery_disk" != "$_swr_k3s_disk" ]] || return 1 + + _swr_capture_single_line _swr_recovery_parent /usr/bin/lsblk --noheadings --raw --paths --output PKNAME -- "$_swr_recovery_partition" || return 1 + _swr_capture_single_line _swr_k3s_parent /usr/bin/lsblk --noheadings --raw --paths --output PKNAME -- "$_swr_k3s_partition" || return 1 + [[ "$_swr_recovery_parent" == "$_swr_recovery_disk" && + "$_swr_k3s_parent" == "$_swr_k3s_disk" && + "$_swr_recovery_parent" != "$_swr_k3s_parent" ]] || return 1 + + _swr_capture_single_line _swr_uuid /usr/sbin/blkid --output value --match-tag UUID -- "$_swr_recovery_partition" || return 1 + _swr_capture_single_line _swr_partuuid /usr/sbin/blkid --output value --match-tag PARTUUID -- "$_swr_recovery_partition" || return 1 + _swr_capture_single_line _swr_type /usr/sbin/blkid --output value --match-tag TYPE -- "$_swr_recovery_partition" || return 1 + [[ "$_swr_uuid" == "$SWR_RECOVERY_FS_UUID" && + "$_swr_partuuid" == "$SWR_RECOVERY_PARTUUID" && "$_swr_type" == ntfs ]] || return 1 + + _swr_capture_single_line _swr_model /usr/bin/lsblk --noheadings --raw --paths --nodeps --output MODEL -- "$_swr_recovery_disk" || return 1 + _swr_capture_single_line _swr_serial /usr/bin/lsblk --noheadings --raw --paths --nodeps --output SERIAL -- "$_swr_recovery_disk" || return 1 + _swr_capture_single_line _swr_wwn /usr/bin/lsblk --noheadings --raw --paths --nodeps --output WWN -- "$_swr_recovery_disk" || return 1 + [[ "$_swr_model" == "$SWR_RECOVERY_MODEL" && + "$_swr_serial" == "$SWR_RECOVERY_SERIAL" && "$_swr_wwn" == "$SWR_RECOVERY_WWN" ]] || return 1 + + SWR_RECOVERY_DEVICE="$_swr_recovery_partition" + builtin printf -v "$output_name" '%s' "$_swr_recovery_partition" +} + +_swr_findmnt_capture() { + local output_name="${1-}" selector="${2-}" value="${3-}" _swr_findmnt_raw='' rc + [[ $# -eq 3 ]] || return 2 + _swr_output_name_is_safe "$output_name" _swr_ || return 2 + case "$selector" in --source|--mountpoint) ;; *) return 2 ;; esac + if _swr_capture_command _swr_findmnt_raw /usr/bin/findmnt --json "$selector" "$value" \ + --output SOURCE,TARGET,FSTYPE,VFS-OPTIONS,FS-OPTIONS; then + rc=0 + else + rc=$? + fi + builtin printf -v "$output_name" '%s' "$_swr_findmnt_raw" + return "$rc" +} + +_swr_require_unmounted() { + local device="${1-}" _swr_findmnt_output='' rc + [[ $# -eq 1 && "$device" == /* ]] || return 1 + if _swr_findmnt_capture _swr_findmnt_output --source "$device"; then + return 1 + else + rc=$? + fi + [[ "$rc" == 1 && -z "$_swr_findmnt_output" ]] +} + +_swr_require_mountpoint_unused() { + local mountpoint="${1-}" _swr_findmnt_output='' rc + [[ $# -eq 1 && "$mountpoint" == /* ]] || return 1 + if _swr_findmnt_capture _swr_findmnt_output --mountpoint "$mountpoint"; then + return 1 + else + rc=$? + fi + [[ "$rc" == 1 && -z "$_swr_findmnt_output" ]] +} + +_swr_runtime_root() { + local _swr_canonical='' + _swr_capture_single_line _swr_canonical /usr/bin/readlink -e -- /run/user/1000 || return 1 + [[ "$_swr_canonical" == /run/user/1000 ]] || return 1 + builtin printf '%s\n' "$_swr_canonical" +} + +_swr_mountpoint_is_safe() { + local mountpoint="${1-}" runtime_root metadata='' + [[ $# -eq 1 ]] || return 1 + runtime_root="$(_swr_runtime_root)" || return 1 + [[ "$runtime_root" == /* && -d "$runtime_root" && ! -L "$runtime_root" && -O "$runtime_root" ]] || return 1 + [[ "$mountpoint" == "$runtime_root/slack-webhook-recovery."* && + -d "$mountpoint" && ! -L "$mountpoint" && -O "$mountpoint" ]] || return 1 + _swr_capture_single_line _swr_metadata /usr/bin/stat --format=%u:%g:%a -- "$mountpoint" || return 1 + metadata="$_swr_metadata" + [[ "$metadata" == 1000:1000:700 ]] +} + +_swr_create_mountpoint() { + local output_name="${1-}" runtime_root _swr_mount_created='' + [[ $# -eq 1 ]] || return 1 + _swr_output_name_is_safe "$output_name" _swr_ || return 1 + runtime_root="$(_swr_runtime_root)" || return 1 + [[ "$runtime_root" == /* && -d "$runtime_root" && ! -L "$runtime_root" && -O "$runtime_root" ]] || return 1 + _swr_capture_single_line _swr_mount_created /usr/bin/mktemp -d -- "$runtime_root/slack-webhook-recovery.XXXXXXXX" || return 1 + [[ "$_swr_mount_created" == "$runtime_root/slack-webhook-recovery."* ]] || return 1 + builtin printf -v "$output_name" '%s' "$_swr_mount_created" + _swr_command /usr/bin/chmod 0700 -- "$_swr_mount_created" || return 1 + _swr_mountpoint_is_safe "$_swr_mount_created" || return 1 +} + +_swr_remove_mountpoint() { + local mountpoint="${1-}" + [[ $# -eq 1 ]] || return 1 + _swr_mountpoint_is_safe "$mountpoint" || return 1 + _swr_command /usr/bin/rmdir -- "$mountpoint" +} + +_swr_probe_rw() { + local device="${1-}" + [[ $# -eq 1 && "$device" == /* ]] || return 1 + _swr_command /usr/bin/sudo -n /usr/bin/timeout --signal=TERM --kill-after=5s 20s \ + /usr/bin/ntfs-3g.probe --readwrite "$device" +} + +_swr_mount_volume_command() { + local device="${1-}" mountpoint="${2-}" phase="${3-}" options + [[ $# -eq 3 && -n "$SWR_RECOVERY_DEVICE" && + "$device" == "$SWR_RECOVERY_DEVICE" ]] || return 1 + case "$phase" in ro|rw) ;; *) return 1 ;; esac + _swr_mountpoint_is_safe "$mountpoint" || return 1 + options="norecover,no_def_opts,allow_other,nodev,nosuid,noexec,uid=1000,gid=1000,fmask=0177,dmask=0077,${phase}" + _swr_command /usr/bin/sudo -n /usr/bin/timeout --signal=TERM --kill-after=10s 45s \ + /usr/bin/ntfs-3g -o "$options" "$device" "$mountpoint" +} + +_swr_mount_volume() { + local device="${1-}" mountpoint="${2-}" phase="${3-}" + [[ $# -eq 3 ]] || return 1 + _swr_mountpoint_is_safe "$mountpoint" || return 1 + _swr_mount_volume_command "$device" "$mountpoint" "$phase" +} + +_swr_parse_findmnt_json() { + local json="${1-}" source_name="${2-}" target_name="${3-}" + local fstype_name="${4-}" vfs_name="${5-}" fs_name="${6-}" parsed encoded decoded rc index + local frame_marker=$'\036' LC_ALL=C + local -a encoded_fields=() fields=() + [[ $# -eq 6 && ${#json} -le 16384 ]] || return 1 + _swr_output_name_is_safe "$source_name" _swr_ || return 1 + _swr_output_name_is_safe "$target_name" _swr_ || return 1 + _swr_output_name_is_safe "$fstype_name" _swr_ || return 1 + _swr_output_name_is_safe "$vfs_name" _swr_ || return 1 + _swr_output_name_is_safe "$fs_name" _swr_ || return 1 + if parsed="$(builtin printf '%s' "$json" | _swr_command /usr/bin/jq -er ' + if (type == "object" and (.filesystems | type) == "array" and (.filesystems | length) == 1) + then .filesystems[0] + | if (type == "object" and (.source | type) == "string" + and (.target | type) == "string" and (.fstype | type) == "string" + and (."vfs-options" | type) == "string" and (."fs-options" | type) == "string") + then [.source, .target, .fstype, ."vfs-options", ."fs-options"] + else error("invalid findmnt record") end + else error("ambiguous findmnt record") end + | if all(.[]; length > 0 and (test("[\\x00-\\x20\\x7f]") | not)) + then .[] | @base64 + else error("unsafe findmnt field") end + ' 2>/dev/null)"; then + (( ${#parsed} <= 16384 )) || return 1 + else + return 1 + fi + mapfile -t encoded_fields <<<"$parsed" + (( ${#encoded_fields[@]} == 5 )) || return 1 + for index in "${!encoded_fields[@]}"; do + encoded="${encoded_fields[index]}" + [[ -n "$encoded" && "$encoded" =~ ^[A-Za-z0-9+/]*={0,2}$ ]] || return 1 + decoded="$({ + set +e + builtin printf '%s' "$encoded" | _swr_command /usr/bin/base64 --decode + rc=$? + builtin printf '%s' "$frame_marker" + exit "$rc" + })" || return 1 + [[ "$decoded" == *"$frame_marker" ]] || return 1 + decoded="${decoded%"$frame_marker"}" + [[ -n "$decoded" && "$decoded" != *[$' \t\r\n\v\f']* && + "$decoded" != *$'\177'* ]] || return 1 + fields+=("$decoded") + done + [[ -n "${fields[0]}" && -n "${fields[1]}" && -n "${fields[2]}" ]] || return 1 + builtin printf -v "$source_name" '%s' "${fields[0]}" + builtin printf -v "$target_name" '%s' "${fields[1]}" + builtin printf -v "$fstype_name" '%s' "${fields[2]}" + builtin printf -v "$vfs_name" '%s' "${fields[3]}" + builtin printf -v "$fs_name" '%s' "${fields[4]}" +} + +_swr_option_present() { + local options="${1-}" expected="${2-}" + [[ ",$options," == *",$expected,"* ]] +} + +_swr_mount_options_are_safe() { + local phase="${1-}" vfs_options="${2-}" fs_options="${3-}" token + local vfs_phase=0 vfs_opposite=0 vfs_nosuid=0 vfs_nodev=0 vfs_noexec=0 + local vfs_relatime=0 + local fs_phase=0 fs_opposite=0 fs_allow_other=0 fs_default_permissions=0 + local fs_user_id=0 fs_group_id=0 fs_blksize=0 + local -a tokens=() + [[ $# -eq 3 ]] || return 1 + case "$phase" in ro|rw) ;; *) return 1 ;; esac + + [[ -n "$vfs_options" && "$vfs_options" != ,* && "$vfs_options" != *, && + "$vfs_options" != *,,* && "$vfs_options" != *[$' \t\r\n\v\f']* && + "$vfs_options" != *$'\177'* ]] || return 1 + IFS=, read -r -a tokens <<<"$vfs_options" + for token in "${tokens[@]}"; do + [[ -n "$token" && "$token" != *[$'\t\r\n ']* ]] || return 1 + case "$token" in + "$phase") vfs_phase=$((vfs_phase + 1)) ;; + ro|rw) vfs_opposite=$((vfs_opposite + 1)) ;; + nosuid) vfs_nosuid=$((vfs_nosuid + 1)) ;; + nodev) vfs_nodev=$((vfs_nodev + 1)) ;; + noexec) vfs_noexec=$((vfs_noexec + 1)) ;; + relatime) vfs_relatime=$((vfs_relatime + 1)) ;; + suid|dev|exec) return 1 ;; + uid=*|gid=*|fmask=*|dmask=*|umask|umask=*|mask|mask=*) return 1 ;; + *) return 1 ;; + esac + done + (( vfs_phase == 1 && vfs_opposite == 0 && vfs_nosuid == 1 && + vfs_nodev == 1 && vfs_noexec == 1 && vfs_relatime == 1 )) || return 1 + + tokens=() + [[ -n "$fs_options" && "$fs_options" != ,* && "$fs_options" != *, && + "$fs_options" != *,,* && "$fs_options" != *[$' \t\r\n\v\f']* && + "$fs_options" != *$'\177'* ]] || return 1 + IFS=, read -r -a tokens <<<"$fs_options" + for token in "${tokens[@]}"; do + [[ -n "$token" && "$token" != *[$'\t\r\n ']* ]] || return 1 + case "$token" in + "$phase") fs_phase=$((fs_phase + 1)) ;; + ro|rw) fs_opposite=$((fs_opposite + 1)) ;; + allow_other) fs_allow_other=$((fs_allow_other + 1)) ;; + default_permissions) fs_default_permissions=$((fs_default_permissions + 1)) ;; + user_id=*) fs_user_id=$((fs_user_id + 1)); [[ "$token" == user_id=0 ]] || return 1 ;; + group_id=*) fs_group_id=$((fs_group_id + 1)); [[ "$token" == group_id=0 ]] || return 1 ;; + blksize=*) fs_blksize=$((fs_blksize + 1)); [[ "$token" == blksize=4096 ]] || return 1 ;; + uid=*|gid=*|fmask=*|dmask=*) return 1 ;; + umask|umask=*|mask|mask=*) return 1 ;; + suid|dev|exec) return 1 ;; + *) return 1 ;; + esac + done + (( fs_phase == 1 && fs_opposite == 0 && fs_allow_other == 1 && + fs_default_permissions == 1 && fs_user_id == 1 && fs_group_id == 1 && + fs_blksize == 1 )) +} + +_swr_validate_mount() { + local device="${1-}" mountpoint="${2-}" phase="${3-}" + local _swr_findmnt_output='' _swr_source='' _swr_target='' + local _swr_fstype='' _swr_vfs='' _swr_fs='' _swr_metadata='' + [[ $# -eq 3 && "$device" == /* ]] || return 1 + case "$phase" in ro|rw) ;; *) return 1 ;; esac + _swr_mountpoint_is_safe "$mountpoint" || return 1 + _swr_findmnt_capture _swr_findmnt_output --mountpoint "$mountpoint" || return 1 + _swr_parse_findmnt_json "$_swr_findmnt_output" _swr_source _swr_target \ + _swr_fstype _swr_vfs _swr_fs || return 1 + [[ "$_swr_source" == "$device" && "$_swr_target" == "$mountpoint" && + "$_swr_fstype" == fuseblk ]] || return 1 + _swr_mount_options_are_safe "$phase" "$_swr_vfs" "$_swr_fs" || return 1 + _swr_capture_single_line _swr_metadata /usr/bin/stat --format=%u:%g:%a -- "$mountpoint" || return 1 + [[ "$_swr_metadata" == 1000:1000:700 ]] +} + +_swr_stably_unmounted() { + local device="${1-}" + _swr_require_unmounted "$device" || return 1 + _swr_require_unmounted "$device" +} + +_swr_unmount_volume() { + local device="${1-}" mountpoint="${2-}" unmount_rc=0 + [[ $# -eq 2 && "$device" == /* && "$mountpoint" == /* ]] || return 1 + _swr_command /usr/bin/sudo -n /usr/bin/timeout --signal=TERM --kill-after=10s 45s \ + /usr/bin/umount -- "$mountpoint" >/dev/null 2>&1 || unmount_rc=$? + if _swr_stably_unmounted "$device"; then + return 0 + fi + plain_fail "manual recovery required: mount state is present or ambiguous after unmount status ${unmount_rc}" +} + +_swr_volume_action() { + local result_name="${1-}" + [[ $# -eq 3 ]] || return 1 + _swr_output_name_is_safe "$result_name" _swr_ || return 1 + plain_fail 'encrypted database snapshot boundary is unavailable' +} + +_swr_restore_trap() { + local saved="${1-}" signal_name="${2-}" + if [[ -n "$saved" ]]; then + builtin eval "$saved" + else + builtin trap - "$signal_name" + fi +} + +_swr_run_volume_cycle() { + local device="${1-}" mountpoint="${2-}" result_name="${3-}" + local _swr_phase_result='' phase action_rc=0 cleanup_rc=0 mount_rc=0 final_rc=0 + local _swr_cycle_signal=0 mounted=0 attempted=0 + local old_hup old_int old_term + [[ $# -eq 2 || $# -eq 3 ]] || return 1 + [[ -z "$result_name" ]] || _swr_output_name_is_safe "$result_name" _swr_ || return 1 + old_hup="$(builtin trap -p HUP)" + old_int="$(builtin trap -p INT)" + old_term="$(builtin trap -p TERM)" + builtin trap '_swr_cycle_signal=129' HUP + builtin trap '_swr_cycle_signal=130' INT + builtin trap '_swr_cycle_signal=143' TERM + + if ! _swr_require_mountpoint_unused "$mountpoint"; then + final_rc=1 + elif ! _swr_mountpoint_is_safe "$mountpoint"; then + final_rc=1 + elif ! _swr_probe_rw "$device"; then + final_rc=1 + else + for phase in ro rw; do + if [[ "$phase" == rw ]]; then + [[ "$_swr_phase_result" == mutation ]] || break + if ! _swr_mountpoint_is_safe "$mountpoint"; then + final_rc=1 + break + elif ! _swr_probe_rw "$device"; then + final_rc=1 + break + fi + fi + attempted=1 + if _swr_mount_volume_command "$device" "$mountpoint" "$phase"; then + mounted=1 + else + mount_rc=$? + _swr_unmount_volume "$device" "$mountpoint" || cleanup_rc=$? + final_rc="$mount_rc" + (( cleanup_rc == 0 )) || final_rc="$cleanup_rc" + break + fi + if (( _swr_cycle_signal != 0 )); then + action_rc="$_swr_cycle_signal" + elif ! _swr_validate_mount "$device" "$mountpoint" "$phase"; then + action_rc=1 + elif ! _swr_volume_action _swr_phase_result "$phase" "$mountpoint"; then + action_rc=$? + (( action_rc == 0 )) && action_rc=1 + fi + _swr_unmount_volume "$device" "$mountpoint" || cleanup_rc=$? + mounted=0 + if (( cleanup_rc != 0 )); then + final_rc="$cleanup_rc" + break + fi + if (( _swr_cycle_signal != 0 )); then + final_rc="$_swr_cycle_signal" + break + fi + if (( action_rc != 0 )); then + final_rc="$action_rc" + break + fi + case "$phase:$_swr_phase_result" in + ro:exact-noop) break ;; + ro:mutation) ;; + rw:complete) break ;; + *) final_rc=1; break ;; + esac + done + fi + + if (( mounted != 0 )); then + _swr_unmount_volume "$device" "$mountpoint" || cleanup_rc=$? + (( cleanup_rc == 0 )) || final_rc="$cleanup_rc" + fi + _swr_restore_trap "$old_hup" HUP + _swr_restore_trap "$old_int" INT + _swr_restore_trap "$old_term" TERM + if (( final_rc == 0 )) && [[ -n "$result_name" ]]; then + builtin printf -v "$result_name" '%s' "$_swr_phase_result" + fi + return "$final_rc" +} + +_swr_open_recovery_boundary() { + local result_name="${1-}" contract_path="${2-}" + local _swr_device='' _swr_mountpoint='' _swr_boundary_result='' rc=0 + [[ $# -eq 2 ]] || return 1 + _swr_output_name_is_safe "$result_name" _swr_ || return 1 + _swr_load_recovery_contract "$contract_path" || return 1 + _swr_resolve_device _swr_device || return 1 + _swr_require_unmounted "$_swr_device" || return 1 + _swr_runtime_root >/dev/null || return 1 + _swr_create_mountpoint _swr_mountpoint || return 1 + _swr_require_mountpoint_unused "$_swr_mountpoint" || return 1 + if ! _swr_command /usr/bin/sudo -v; then + _swr_require_mountpoint_unused "$_swr_mountpoint" && + _swr_remove_mountpoint "$_swr_mountpoint" >/dev/null 2>&1 || true + return 1 + fi + if ! _swr_mountpoint_is_safe "$_swr_mountpoint" || + ! _swr_require_mountpoint_unused "$_swr_mountpoint"; then + _swr_require_mountpoint_unused "$_swr_mountpoint" && + _swr_remove_mountpoint "$_swr_mountpoint" >/dev/null 2>&1 || true + return 1 + fi + if ! _swr_run_volume_cycle "$_swr_device" "$_swr_mountpoint" _swr_boundary_result; then + rc=$? + (( rc == 0 )) && rc=1 + fi + if _swr_require_unmounted "$_swr_device" && + _swr_require_mountpoint_unused "$_swr_mountpoint"; then + _swr_remove_mountpoint "$_swr_mountpoint" || rc=1 + else + rc=1 + fi + if (( rc == 0 )); then + builtin printf -v "$result_name" '%s' "$_swr_boundary_result" + fi + return "$rc" +} + +_swr_terminate_vault_child() { + local child_pid="${1-}" requested_signal="${2-TERM}" + [[ ( $# -eq 1 || $# -eq 2 ) && "$child_pid" =~ ^[1-9][0-9]*$ ]] || return 1 + case "$requested_signal" in HUP|INT|TERM) ;; *) return 1 ;; esac + kill -"$requested_signal" "$child_pid" 2>/dev/null || true + if ! /usr/bin/timeout --signal=TERM --kill-after=0.1s 1s \ + /usr/bin/tail --pid="$child_pid" -f /dev/null >/dev/null 2>&1; then + kill -KILL "$child_pid" 2>/dev/null || true + fi + wait "$child_pid" 2>/dev/null || true + ! kill -0 "$child_pid" 2>/dev/null +} + +_swr_wait_vault_ready() { + local socket_path="${1-}" child_pid="${2-}" attempts="${3-}" attempt + [[ $# -eq 3 && "$socket_path" == /* && "$child_pid" =~ ^[1-9][0-9]*$ && + "$attempts" =~ ^[1-9][0-9]*$ ]] || return 1 + for (( attempt = 0; attempt < attempts; attempt++ )); do + (( SWR_PENDING_SIGNAL == 0 )) || return 2 + if [[ -S "$socket_path" && ! -L "$socket_path" ]]; then + return 0 + fi + [[ ! -e "$socket_path" && ! -L "$socket_path" ]] || return 1 + if ! kill -0 "$child_pid" 2>/dev/null; then + return 1 + fi + /usr/bin/sleep 0.05 + done + return 1 +} + +_swr_start_vault() { + local pid_name="${1-}" socket_name="${2-}" runtime_root="${3-}" webhook_file="${4-}" + local repository_root="${SWR_REPOSITORY_ROOT-}" helper socket_path child_pid readiness_rc=0 + [[ $# -eq 4 ]] || return 1 + _swr_output_name_is_safe "$pid_name" _swr_ || return 1 + _swr_output_name_is_safe "$socket_name" _swr_ || return 1 + [[ "$repository_root" == /* && "$runtime_root" == /* && "$webhook_file" == /* ]] || return 1 + helper="${repository_root}/scripts/libexec/slack-webhook-recovery-vault.py" + socket_path="${runtime_root}/vault.sock" + [[ -f "$helper" && ! -L "$helper" ]] || return 1 + [[ -d "$runtime_root" && ! -L "$runtime_root" ]] || return 1 + [[ ! -e "$socket_path" && ! -L "$socket_path" ]] || return 1 + + /usr/bin/env -i HOME=/nonexistent LC_ALL=C.UTF-8 PATH=/usr/bin:/bin \ + /usr/bin/python3 "$helper" --serve --runtime-root "$runtime_root" \ + --webhook-file "$webhook_file" & + child_pid=$! + _swr_wait_vault_ready "$socket_path" "$child_pid" 18000 || readiness_rc=$? + if (( readiness_rc != 0 || SWR_PENDING_SIGNAL != 0 )); then + case "$SWR_PENDING_SIGNAL" in + 129) _swr_terminate_vault_child "$child_pid" HUP || true ;; + 130) _swr_terminate_vault_child "$child_pid" INT || true ;; + 143) _swr_terminate_vault_child "$child_pid" TERM || true ;; + *) _swr_terminate_vault_child "$child_pid" TERM || true ;; + esac + return 1 + fi + builtin printf -v "$pid_name" '%s' "$child_pid" + builtin printf -v "$socket_name" '%s' "$socket_path" + return 0 +} + +_swr_vault_request() { + local result_name="${1-}" socket_path="${2-}" command="${3-}" reply + [[ $# -eq 3 ]] || return 1 + _swr_output_name_is_safe "$result_name" _swr_ || return 1 + [[ "$socket_path" == /* && -S "$socket_path" && ! -L "$socket_path" ]] || return 1 + case "$command" in + PREPARE|'BUILD ADD'|'BUILD UPDATE'|VERIFY_CANDIDATE|SHUTDOWN) + ;; + 'VERIFY_COMMITTED '/*) + [[ "${command#VERIFY_COMMITTED }" != *' '* ]] || return 1 + ;; + *) + return 1 + ;; + esac + if [[ -z "$_SWR_VAULT_CLIENT_SOCKET" ]]; then + coproc SWR_VAULT_CLIENT { + /usr/bin/timeout --signal=TERM --kill-after=1s 900s \ + /usr/bin/socat - "UNIX-CONNECT:${socket_path}" 2>/dev/null + } + _SWR_VAULT_CLIENT_PID="$SWR_VAULT_CLIENT_PID" + exec {_SWR_VAULT_CLIENT_READ_FD}<&"${SWR_VAULT_CLIENT[0]}" || return 1 + exec {_SWR_VAULT_CLIENT_WRITE_FD}>&"${SWR_VAULT_CLIENT[1]}" || return 1 + _SWR_VAULT_CLIENT_SOCKET="$socket_path" + fi + [[ "$_SWR_VAULT_CLIENT_SOCKET" == "$socket_path" ]] || return 1 + builtin printf '%s\n' "$command" >&"$_SWR_VAULT_CLIENT_WRITE_FD" || return 1 + IFS= builtin read -r -t 20 -u "$_SWR_VAULT_CLIENT_READ_FD" reply || return 1 + case "$reply" in + absent|exact-noop|mismatch|ambiguous|failed|candidate-ready|verified|stopped) + builtin printf -v "$result_name" '%s' "$reply" + ;; + *) + return 1 + ;; + esac +} + +_swr_stop_vault() { + local socket_path="${1-}" child_pid="${2-}" _swr_stop_reply='' + local shutdown_ok=1 + [[ $# -eq 2 && "$child_pid" =~ ^[1-9][0-9]*$ ]] || return 1 + if [[ -S "$socket_path" && ! -L "$socket_path" ]]; then + _swr_vault_request _swr_stop_reply "$socket_path" SHUTDOWN || shutdown_ok=0 + [[ "$_swr_stop_reply" == stopped ]] || shutdown_ok=0 + fi + if [[ -n "$_SWR_VAULT_CLIENT_WRITE_FD" ]]; then + exec {_SWR_VAULT_CLIENT_WRITE_FD}>&- + fi + if [[ -n "$_SWR_VAULT_CLIENT_READ_FD" ]]; then + exec {_SWR_VAULT_CLIENT_READ_FD}<&- + fi + if [[ -n "$_SWR_VAULT_CLIENT_PID" ]]; then + (( shutdown_ok != 0 )) || kill -TERM "$_SWR_VAULT_CLIENT_PID" 2>/dev/null || true + wait "$_SWR_VAULT_CLIENT_PID" 2>/dev/null || true + fi + _SWR_VAULT_CLIENT_SOCKET='' + _SWR_VAULT_CLIENT_READ_FD='' + _SWR_VAULT_CLIENT_WRITE_FD='' + _SWR_VAULT_CLIENT_PID='' + if (( shutdown_ok == 0 )); then + kill -TERM "$child_pid" 2>/dev/null || true + wait "$child_pid" 2>/dev/null || true + return 1 + fi + /usr/bin/timeout --signal=TERM --kill-after=1s 10s \ + /usr/bin/tail --pid="$child_pid" -f /dev/null >/dev/null 2>&1 || { + kill -TERM "$child_pid" 2>/dev/null || true + wait "$child_pid" 2>/dev/null || true + return 1 + } + wait "$child_pid" +} + +defer_signal() { + case "${1-}" in + HUP) SWR_PENDING_SIGNAL=129 ;; + INT) SWR_PENDING_SIGNAL=130 ;; + TERM) SWR_PENDING_SIGNAL=143 ;; + *) return 1 ;; + esac +} + +_swr_work_root_is_safe() { + local work_root="${1-}" _swr_metadata='' _swr_filesystem='' + [[ $# -eq 1 && "$work_root" == /run/user/1000/platform-slack-keepass.* && + -d "$work_root" && ! -L "$work_root" && -O "$work_root" ]] || return 1 + _swr_capture_single_line _swr_metadata /usr/bin/stat --format=%u:%g:%a -- "$work_root" || return 1 + _swr_capture_single_line _swr_filesystem /usr/bin/stat --file-system --format=%T -- "$work_root" || return 1 + [[ "$_swr_metadata" == 1000:1000:700 && "$_swr_filesystem" == tmpfs ]] +} + +_swr_prepare_transaction() { + local work_name="${1-}" device_name="${2-}" mount_name="${3-}" + local webhook_file="${4-}" contract_path="${5-}" + local _swr_prepared_device='' _swr_prepared_work_root='' _swr_prepared_mountpoint='' + local _swr_runtime_metadata='' _swr_runtime_filesystem='' + [[ $# -eq 5 && "$webhook_file" == /* && "$contract_path" == /* ]] || return 1 + _swr_output_name_is_safe "$work_name" _swr_ || return 1 + _swr_output_name_is_safe "$device_name" _swr_ || return 1 + _swr_output_name_is_safe "$mount_name" _swr_ || return 1 + _swr_load_recovery_contract "$contract_path" || return 1 + _swr_resolve_device _swr_prepared_device || return 1 + builtin printf -v "$device_name" '%s' "$_swr_prepared_device" + _swr_require_unmounted "$_swr_prepared_device" || return 1 + [[ "$(_swr_runtime_root)" == /run/user/1000 ]] || return 1 + _swr_capture_single_line _swr_runtime_metadata /usr/bin/stat --format=%u:%g:%a -- /run/user/1000 || return 1 + _swr_capture_single_line _swr_runtime_filesystem /usr/bin/stat --file-system --format=%T -- /run/user/1000 || return 1 + [[ "$_swr_runtime_metadata" == 1000:1000:700 && + "$_swr_runtime_filesystem" == tmpfs ]] || return 1 + _swr_capture_single_line _swr_prepared_work_root /usr/bin/mktemp -d --tmpdir=/run/user/1000 \ + platform-slack-keepass.XXXXXXXX || return 1 + builtin printf -v "$work_name" '%s' "$_swr_prepared_work_root" + if ! _swr_command /usr/bin/chmod 0700 -- "$_swr_prepared_work_root" || + ! _swr_work_root_is_safe "$_swr_prepared_work_root"; then + return 1 + fi + if _swr_create_mountpoint _swr_prepared_mountpoint; then + builtin printf -v "$mount_name" '%s' "$_swr_prepared_mountpoint" + else + builtin printf -v "$mount_name" '%s' "$_swr_prepared_mountpoint" + return 1 + fi + _swr_require_mountpoint_unused "$_swr_prepared_mountpoint" || return 1 +} + +_swr_refresh_sudo() { + _swr_command /usr/bin/sudo -v +} + +_swr_dirfd_helper() { + local repository_root="${SWR_REPOSITORY_ROOT-}" helper + helper="${repository_root}/scripts/libexec/slack-webhook-recovery-dirfd.py" + [[ "$repository_root" == /* && -f "$helper" && ! -L "$helper" ]] || return 1 + _swr_command /usr/bin/env -i LC_ALL=C.UTF-8 PATH=/usr/bin:/bin \ + /usr/bin/python3 "$helper" "$@" +} + +_swr_dirfd_snapshot() { + local mountpoint="${1-}" work_root="${2-}" + [[ $# -eq 2 ]] || return 1 + _swr_dirfd_helper snapshot --mount-root "$mountpoint" \ + --destination "$work_root/baseline.kdbx" --result "$work_root/baseline.json" +} + +_swr_audit_runtime_processes() { + local work_root="${1-}" + [[ $# -eq 1 && "$work_root" == /* ]] || return 1 + _swr_dirfd_helper audit-processes --runtime-root "$work_root" +} + +_swr_dirfd_snapshot_current() { + local mountpoint="${1-}" work_root="${2-}" + [[ $# -eq 2 ]] || return 1 + _swr_dirfd_helper snapshot-current --mount-root "$mountpoint" \ + --candidate "$work_root/candidate.kdbx" --destination "$work_root/current.kdbx" \ + --result "$work_root/current.json" +} + +_swr_dirfd_commit() { + local result_name="${1-}" mountpoint="${2-}" work_root="${3-}" + local backup_name="${4-}" _swr_fs_status='' command_rc=0 + [[ $# -eq 4 ]] || return 1 + _swr_output_name_is_safe "$result_name" _swr_ || return 1 + if _swr_dirfd_helper commit --mount-root "$mountpoint" \ + --baseline "$work_root/baseline.json" --candidate "$work_root/candidate.kdbx" \ + --backup-name "$backup_name" --result "$work_root/commit.json"; then + command_rc=0 + else + command_rc=$? + fi + if _swr_fs_status="$(_swr_dirfd_helper status --result "$work_root/commit.json")"; then + [[ -n "$_swr_fs_status" && "$_swr_fs_status" != *$'\n'* && + "$_swr_fs_status" != *$'\r'* ]] || return 1 + case "$_swr_fs_status" in + precommit-failure|rename-attempted|verified-commit|committed-but-uncertain) ;; + *) return 1 ;; + esac + builtin printf -v "$result_name" '%s' "$_swr_fs_status" + else + builtin printf -v "$result_name" '%s' missing + fi + [[ "$_swr_fs_status" == verified-commit ]] && command_rc=0 + return "$command_rc" +} + +_swr_confirm_token() { + local expected="${1-}" prompt tty_fd reply='' + case "$expected" in + UPDATE) prompt='Type UPDATE SLACK KDBX default: ' ;; + WRITE) prompt='Type WRITE SLACK KDBX default: ' ;; + *) return 1 ;; + esac + exec {tty_fd}<>/dev/tty || return 1 + builtin printf '%s' "$prompt" >&"$tty_fd" + IFS= builtin read -r -u "$tty_fd" reply || { + exec {tty_fd}>&- + return 1 + } + exec {tty_fd}>&- + [[ "$reply" == "$expected SLACK KDBX default" ]] +} + +_swr_inventory_directory() { + local output_name="${1-}" directory="${2-}" inventory_fd inventory_pid + local original_read_fd original_write_fd record='' producer_rc=0 framing_rc=0 + local record_bytes=0 total_bytes=0 next_total=0 LC_ALL=C + local -n output_ref="$output_name" + [[ $# -eq 2 && "$directory" == /* ]] || return 1 + _swr_output_name_is_safe "$output_name" _swr_ || return 1 + output_ref=() + coproc SWR_INVENTORY_PROCESS { + _swr_command /usr/bin/find "$directory" -mindepth 1 -maxdepth 1 -printf '%f\0' + } + inventory_pid="$SWR_INVENTORY_PROCESS_PID" + original_read_fd="${SWR_INVENTORY_PROCESS[0]}" + original_write_fd="${SWR_INVENTORY_PROCESS[1]}" + if ! exec {inventory_fd}<&"$original_read_fd"; then + kill -TERM "$inventory_pid" 2>/dev/null || true + wait "$inventory_pid" 2>/dev/null || true + unset SWR_INVENTORY_PROCESS SWR_INVENTORY_PROCESS_PID + return 1 + fi + exec {original_read_fd}<&- + exec {original_write_fd}>&- + while true; do + record='' + if IFS= builtin read -r -d '' -n 256 -u "$inventory_fd" record; then + record_bytes=${#record} + next_total=$((total_bytes + record_bytes)) + if (( record_bytes < 1 || record_bytes > 255 || next_total > 4096 || + ${#output_ref[@]} >= 16 )) || + [[ "$record" == */* || "$record" == . || "$record" == .. ]]; then + framing_rc=1 + break + fi + output_ref+=("$record") + total_bytes=$next_total + else + if [[ -n "$record" ]]; then + framing_rc=1 + fi + break + fi + done + exec {inventory_fd}<&- + if (( framing_rc != 0 )); then + kill -TERM "$inventory_pid" 2>/dev/null || true + fi + if wait "$inventory_pid"; then + producer_rc=0 + else + producer_rc=$? + fi + unset SWR_INVENTORY_PROCESS SWR_INVENTORY_PROCESS_PID + (( producer_rc == 0 && framing_rc == 0 )) +} + +_swr_remove_work_root() { + local work_root="${1-}" path name _swr_metadata='' + local -a _swr_inventory=() + local -a owned_files=( + baseline.kdbx baseline.json candidate.kdbx current.kdbx current.json commit.json + ) + [[ $# -eq 1 ]] || return 1 + _swr_work_root_is_safe "$work_root" || return 1 + [[ ! -e "$work_root/vault.sock" && ! -L "$work_root/vault.sock" ]] || return 1 + local -A expected=() + for name in "${owned_files[@]}"; do expected["$name"]=1; done + _swr_inventory_directory _swr_inventory "$work_root" || return 1 + for name in "${_swr_inventory[@]}"; do + [[ -n "${expected[$name]+x}" ]] || return 1 + done + for name in "${owned_files[@]}"; do + path="$work_root/$name" + [[ -e "$path" || -L "$path" ]] || continue + [[ -f "$path" && ! -L "$path" && -O "$path" ]] || return 1 + _swr_capture_single_line _swr_metadata /usr/bin/stat --format=%u:%g:%a:%h -- "$path" || return 1 + [[ "$_swr_metadata" == 1000:1000:600:1 ]] || return 1 + done + for path in "${owned_files[@]}"; do + path="$work_root/$path" + [[ -e "$path" || -L "$path" ]] || continue + [[ -f "$path" && ! -L "$path" && -O "$path" ]] || return 1 + _swr_capture_single_line _swr_metadata /usr/bin/stat --format=%u:%g:%a:%h -- "$path" || return 1 + [[ "$_swr_metadata" == 1000:1000:600:1 ]] || return 1 + _swr_command /usr/bin/rm -f -- "$path" || return 1 + done + _swr_command /usr/bin/rmdir -- "$work_root" +} + +cleanup_transaction() { + local work_root="${1-}" device="${2-}" mountpoint="${3-}" + local socket_path="${4-}" child_pid="${5-}" rc=0 unmounted=0 + [[ $# -eq 5 ]] || return 1 + if [[ -n "$child_pid" ]]; then + _swr_stop_vault "$socket_path" "$child_pid" || rc=1 + fi + if [[ -n "$work_root" ]]; then + _swr_audit_runtime_processes "$work_root" || rc=1 + fi + if [[ -n "$device" ]] && _swr_stably_unmounted "$device"; then + unmounted=1 + elif [[ -n "$device" ]]; then + rc=1 + fi + if (( unmounted != 0 )) && [[ -n "$mountpoint" && -d "$mountpoint" ]]; then + _swr_require_mountpoint_unused "$mountpoint" && _swr_remove_mountpoint "$mountpoint" || rc=1 + fi + if (( unmounted != 0 )) && [[ -n "$work_root" ]]; then + _swr_remove_work_root "$work_root" || rc=1 + fi + return "$rc" +} + +_swr_restore_transaction_traps() { + _swr_restore_trap "$1" HUP + _swr_restore_trap "$2" INT + _swr_restore_trap "$3" TERM +} + +_swr_unmount_transaction_once() { + local mounted_name="${1-}" device="${2-}" mountpoint="${3-}" + [[ $# -eq 3 && "$mounted_name" == mounted ]] || return 1 + # Prearm the no-retry state before the potentially ambiguous unmount call. + builtin printf -v "$mounted_name" '%s' 0 + _swr_unmount_volume "$device" "$mountpoint" +} + +execute_transaction() { + local webhook_file="${1-}" contract_path="${2-}" + local _swr_work_root='' _swr_device='' _swr_mountpoint='' + local _swr_vault_pid='' _swr_vault_socket='' _swr_reply='' _swr_fs_commit='' + local backup_name='' mounted=0 cleanup_rc=0 final_rc=1 TZ=UTC0 + local old_hup old_int old_term + [[ $# -eq 2 && "$webhook_file" == /* && "$contract_path" == /* ]] || return 1 + SWR_TRANSACTION_STATE=PRECOMMIT_FAILURE + SWR_PENDING_SIGNAL=0 + old_hup="$(builtin trap -p HUP)" + old_int="$(builtin trap -p INT)" + old_term="$(builtin trap -p TERM)" + builtin trap 'defer_signal HUP' HUP + builtin trap 'defer_signal INT' INT + builtin trap 'defer_signal TERM' TERM + + if _swr_prepare_transaction _swr_work_root _swr_device _swr_mountpoint \ + "$webhook_file" "$contract_path" && (( SWR_PENDING_SIGNAL == 0 )) && + _swr_start_vault _swr_vault_pid _swr_vault_socket "$_swr_work_root" "$webhook_file" && + (( SWR_PENDING_SIGNAL == 0 )) && _swr_refresh_sudo && + (( SWR_PENDING_SIGNAL == 0 )) && _swr_probe_rw "$_swr_device" && + (( SWR_PENDING_SIGNAL == 0 )); then + mounted=1 + if _swr_mount_volume_command "$_swr_device" "$_swr_mountpoint" ro && + (( SWR_PENDING_SIGNAL == 0 )) && + _swr_validate_mount "$_swr_device" "$_swr_mountpoint" ro && + (( SWR_PENDING_SIGNAL == 0 )) && + _swr_dirfd_snapshot "$_swr_mountpoint" "$_swr_work_root" && + (( SWR_PENDING_SIGNAL == 0 )) && + _swr_unmount_transaction_once mounted "$_swr_device" "$_swr_mountpoint" && + (( SWR_PENDING_SIGNAL == 0 )); then + if _swr_vault_request _swr_reply "$_swr_vault_socket" PREPARE; then + (( SWR_PENDING_SIGNAL == 0 )) || _swr_reply=signal-pending + case "$_swr_reply" in + exact-noop) + if (( SWR_PENDING_SIGNAL == 0 )); then + SWR_TRANSACTION_STATE=NOOP + final_rc=0 + fi + ;; + absent|mismatch) + if [[ "$_swr_reply" == mismatch ]] && ! _swr_confirm_token UPDATE; then + final_rc=1 + elif (( SWR_PENDING_SIGNAL == 0 )) && + _swr_vault_request _swr_reply "$_swr_vault_socket" \ + "BUILD $([[ "$_swr_reply" == absent ]] && builtin printf ADD || builtin printf UPDATE)" && + [[ "$_swr_reply" == candidate-ready ]] && + (( SWR_PENDING_SIGNAL == 0 )) && + _swr_vault_request _swr_reply "$_swr_vault_socket" VERIFY_CANDIDATE && + [[ "$_swr_reply" == verified ]] && + (( SWR_PENDING_SIGNAL == 0 )) && + _swr_confirm_token WRITE && (( SWR_PENDING_SIGNAL == 0 )) && + _swr_probe_rw "$_swr_device" && (( SWR_PENDING_SIGNAL == 0 )); then + mounted=1 + if _swr_mount_volume_command "$_swr_device" "$_swr_mountpoint" rw && + (( SWR_PENDING_SIGNAL == 0 )) && + _swr_validate_mount "$_swr_device" "$_swr_mountpoint" rw && + (( SWR_PENDING_SIGNAL == 0 )); then + builtin printf -v backup_name 'hyeonworks-recovery.pre-slack-%(%Y%m%dT%H%M%SZ)T.kdbx' -1 + if _swr_dirfd_commit _swr_fs_commit "$_swr_mountpoint" "$_swr_work_root" "$backup_name"; then + if [[ "$_swr_fs_commit" == verified-commit ]]; then + if (( SWR_PENDING_SIGNAL != 0 )); then + SWR_TRANSACTION_STATE=COMMITTED_BUT_UNCERTAIN + elif _swr_dirfd_snapshot_current "$_swr_mountpoint" "$_swr_work_root" && + _swr_vault_request _swr_reply "$_swr_vault_socket" \ + "VERIFY_COMMITTED $_swr_work_root/current.kdbx" && + [[ "$_swr_reply" == verified ]] && + (( SWR_PENDING_SIGNAL == 0 )); then + SWR_TRANSACTION_STATE=VERIFIED_COMMIT + else + SWR_TRANSACTION_STATE=COMMITTED_BUT_UNCERTAIN + fi + else + SWR_TRANSACTION_STATE=COMMITTED_BUT_UNCERTAIN + fi + else + case "$_swr_fs_commit" in + precommit-failure) SWR_TRANSACTION_STATE=PRECOMMIT_FAILURE ;; + rename-attempted|committed-but-uncertain|missing) + SWR_TRANSACTION_STATE=COMMITTED_BUT_UNCERTAIN + ;; + *) SWR_TRANSACTION_STATE=COMMITTED_BUT_UNCERTAIN ;; + esac + fi + if _swr_unmount_transaction_once mounted "$_swr_device" "$_swr_mountpoint"; then + [[ "$SWR_TRANSACTION_STATE" == VERIFIED_COMMIT ]] && final_rc=0 || final_rc=1 + else + SWR_TRANSACTION_STATE=MANUAL_RECOVERY_REQUIRED + final_rc=1 + fi + fi + fi + ;; + ambiguous|failed) final_rc=1 ;; + signal-pending) final_rc=1 ;; + *) final_rc=1 ;; + esac + fi + fi + fi + + if (( mounted != 0 )); then + if _swr_unmount_volume "$_swr_device" "$_swr_mountpoint"; then + mounted=0 + else + SWR_TRANSACTION_STATE=MANUAL_RECOVERY_REQUIRED + fi + fi + cleanup_transaction "$_swr_work_root" "$_swr_device" "$_swr_mountpoint" \ + "$_swr_vault_socket" "$_swr_vault_pid" || cleanup_rc=$? + if (( cleanup_rc != 0 )); then + SWR_TRANSACTION_STATE=MANUAL_RECOVERY_REQUIRED + final_rc=1 + fi + + if (( SWR_PENDING_SIGNAL != 0 )); then + final_rc="$SWR_PENDING_SIGNAL" + fi + + case "$SWR_TRANSACTION_STATE" in + NOOP) + (( final_rc == 0 && SWR_PENDING_SIGNAL == 0 )) && builtin printf '%s\n' \ + 'SLACK_KEEPASS_RECOVERY=NOOP' \ + 'WINDOWS_SSD_UNMOUNTED=PASS' \ + 'OFF_HOST_RECOVERY_SATISFIED=NO' + ;; + VERIFIED_COMMIT) + (( final_rc == 0 && SWR_PENDING_SIGNAL == 0 )) && builtin printf '%s\n' \ + 'SLACK_KEEPASS_RECOVERY=COMMITTED' \ + 'KDBX_PRECHANGE_BACKUP=CREATED' \ + 'WINDOWS_SSD_UNMOUNTED=PASS' \ + 'OFF_HOST_RECOVERY_SATISFIED=NO' + ;; + COMMITTED_BUT_UNCERTAIN|MANUAL_RECOVERY_REQUIRED) + builtin printf '%s\n' 'SLACK_KEEPASS_RECOVERY=MANUAL_RECOVERY_REQUIRED' + final_rc=1 + ;; + esac + (( SWR_PENDING_SIGNAL == 0 )) || final_rc="$SWR_PENDING_SIGNAL" + _swr_restore_transaction_traps "$old_hup" "$old_int" "$old_term" + return "$final_rc" +} + +_swr_main() { + local script_path="$0" _swr_mode _swr_webhook_path + + if [[ "$script_path" == 'scripts/bootstrap/backup-slack-webhook-recovery.sh' ]]; then + script_path="${SWR_REPOSITORY_ROOT}/scripts/bootstrap/backup-slack-webhook-recovery.sh" + fi + + _swr_parse_cli _swr_mode _swr_webhook_path "$@" || { + plain_fail 'expected no arguments or exactly --execute --slack-webhook-file ABSOLUTE_PATH' + return 1 + } + _swr_validate_runtime "$_swr_mode" "$script_path" || { + plain_fail 'runtime guard rejected the invocation' + return 1 + } + + case "$_swr_mode" in + DRY_RUN) + _swr_print_contract + ;; + EXECUTE) + [[ -n "$_swr_webhook_path" ]] || { + plain_fail 'absolute webhook path is required' + return 1 + } + execute_transaction "$_swr_webhook_path" "${SWR_RECOVERY_CONTRACT-}" + ;; + *) + plain_fail 'unsupported mode' + return 1 + ;; + esac +} + +if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then + return 0 +fi +_swr_main "$@" diff --git a/scripts/libexec/k3s-local-recovery-a1-check.sh b/scripts/libexec/k3s-local-recovery-a1-check.sh new file mode 100644 index 0000000..2211439 --- /dev/null +++ b/scripts/libexec/k3s-local-recovery-a1-check.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash + +# Direct-exec root helper for one staged A1 transaction. + +_k3slra1h_command() { + "$@" +} + +_k3slra1h_euid_is_root() { + (( EUID == 0 )) +} + +_k3slra1h_output_name_is_safe() { + local output_name="${1-}" + (( $# == 1 )) || return 1 + [[ "$output_name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || return 1 + [[ "$output_name" != _k3slra1h_* ]] +} + +_k3slra1h_capture_metadata() { + local target="${1-}" output_name="${2-}" line='' + (( $# == 2 )) || return 1 + _k3slra1h_output_name_is_safe "$output_name" || return 1 + case "$output_name" in target|output_name|line|record) return 1 ;; esac + _k3slra1h_packet_line metadata "$target" line || return 1 + printf -v "$output_name" '%s' "$line" +} + +_k3slra1h_capture_canonical() { + local target="${1-}" output_name="${2-}" line='' + (( $# == 2 )) || return 1 + _k3slra1h_output_name_is_safe "$output_name" || return 1 + case "$output_name" in target|output_name|line) return 1 ;; esac + _k3slra1h_packet_line readlink "$target" line || return 1 + [[ "$line" == "$target" ]] || return 1 + printf -v "$output_name" '%s' "$line" +} + +_k3slra1h_stderr_marker() { + local chunk='' read_rc=0 + if IFS= read -r -d '' -n 1 chunk; then + read_rc=0 + else + read_rc=$? + fi + if (( read_rc == 0 )); then + printf '\0STDERR\0' + while IFS= read -r -d '' -n 4096 chunk; do + : + done + else + if [[ -n "$chunk" ]]; then + printf '\0STDERR\0' + fi + fi +} + +_k3slra1h_metadata_packet() { + local target="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1h_command /usr/bin/stat --format='%d:%i|%h|%s|%y|%F|%u:%g|%a' -- "$target" 2> >(_k3slra1h_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1h_readlink_packet() { + local target="${1-}" rc=0 + (( $# == 1 )) || return 1 + if _k3slra1h_command /usr/bin/readlink --canonicalize-existing -- "$target" 2> >(_k3slra1h_stderr_marker); then + rc=0 + else + rc=$? + fi + printf '\0RC=%s\0' "$rc" +} + +_k3slra1h_packet_value() { + local producer="${1-}" argument="${2-}" output_name="${3-}" + local -a records=() + (( $# == 3 )) || return 1 + _k3slra1h_output_name_is_safe "$output_name" || return 1 + case "$output_name" in producer|argument|output_name|records) return 1 ;; esac + case "$producer" in + metadata) mapfile -d '' -t records < <(_k3slra1h_metadata_packet "$argument") ;; + readlink) mapfile -d '' -t records < <(_k3slra1h_readlink_packet "$argument") ;; + *) return 1 ;; + esac + (( ${#records[@]} == 2 )) || return 1 + [[ "${records[1]}" == RC=0 ]] || return 1 + printf -v "$output_name" '%s' "${records[0]}" +} + +_k3slra1h_packet_line() { + local producer="${1-}" argument="${2-}" output_name="${3-}" value='' + (( $# == 3 )) || return 1 + _k3slra1h_output_name_is_safe "$output_name" || return 1 + case "$output_name" in producer|argument|output_name|value) return 1 ;; esac + _k3slra1h_packet_value "$producer" "$argument" value || return 1 + [[ "$value" == *$'\n' ]] || return 1 + [[ "$value" != *$'\r'* ]] || return 1 + value="${value%$'\n'}" + [[ -n "$value" ]] || return 1 + [[ "$value" != *$'\n'* ]] || return 1 + printf -v "$output_name" '%s' "$value" +} + +_k3slra1h_output_names_are_distinct() { + local name + local -A seen=() + (( $# >= 1 )) || return 1 + for name in "$@"; do + _k3slra1h_output_name_is_safe "$name" || return 1 + [[ "${seen[$name]+set}" != set ]] || return 1 + printf -v 'seen[$name]' '%s' 1 + done +} + +_k3slra1h_metadata_fields() { + local target="${1-}" out_identity="${2-}" out_nlink="${3-}" out_size="${4-}" + local out_mtime="${5-}" out_type="${6-}" out_owner="${7-}" out_mode="${8-}" + local metadata='' normalized_type='' + (( $# == 8 )) || return 1 + _k3slra1h_output_names_are_distinct "$out_identity" "$out_nlink" "$out_size" "$out_mtime" "$out_type" "$out_owner" "$out_mode" || return 1 + case "$out_identity|$out_nlink|$out_size|$out_mtime|$out_type|$out_owner|$out_mode" in + *'|target|'*|target\|*|*\|target|*'|out_identity|'*|out_identity\|*|*\|out_identity|*'|out_nlink|'*|out_nlink\|*|*\|out_nlink|*'|out_size|'*|out_size\|*|*\|out_size|*'|out_mtime|'*|out_mtime\|*|*\|out_mtime|*'|out_type|'*|out_type\|*|*\|out_type|*'|out_owner|'*|out_owner\|*|*\|out_owner|*'|out_mode|'*|out_mode\|*|*\|out_mode|*'|metadata|'*|metadata\|*|*\|metadata|*'|normalized_type|'*|normalized_type\|*|*\|normalized_type) return 1 ;; + esac + _k3slra1h_capture_metadata "$target" metadata || return 1 + [[ "$metadata" =~ ^([0-9]+:[0-9]+)\|([0-9]+)\|([0-9]+)\|([^|]+)\|(directory|regular[[:space:]]file|regular[[:space:]]empty[[:space:]]file|symbolic[[:space:]]link)\|([0-9]+:[0-9]+)\|([0-9]+)$ ]] || return 1 + normalized_type="${BASH_REMATCH[5]}" + if [[ "$normalized_type" == 'regular empty file' ]]; then + printf -v normalized_type '%s' 'regular file' + fi + printf -v "$out_identity" '%s' "${BASH_REMATCH[1]}" + printf -v "$out_nlink" '%s' "${BASH_REMATCH[2]}" + printf -v "$out_size" '%s' "${BASH_REMATCH[3]}" + printf -v "$out_mtime" '%s' "${BASH_REMATCH[4]}" + printf -v "$out_type" '%s' "$normalized_type" + printf -v "$out_owner" '%s' "${BASH_REMATCH[6]}" + printf -v "$out_mode" '%s' "${BASH_REMATCH[7]}" +} + +_k3slra1h_regular_root_file_is_safe() { + local target="${1-}" expected_device="${2-}" canonical='' + local identity='' nlink='' size='' mtime='' type='' owner='' mode='' + (( $# == 2 )) || return 1 + _k3slra1h_capture_canonical "$target" canonical || return 1 + [[ "$canonical" == "$target" ]] || return 1 + _k3slra1h_metadata_fields "$target" identity nlink size mtime type owner mode || return 1 + [[ "${identity%%:*}" == "$expected_device" ]] || return 1 + [[ "$nlink" == 1 ]] || return 1 + [[ "$type" == 'regular file' ]] || return 1 + [[ "$owner" == 0:0 ]] || return 1 + [[ "$mode" == 600 ]] +} + +_k3slra1h_root_directory_is_safe() { + local target="${1-}" expected_device="${2-}" canonical='' + local identity='' nlink='' size='' mtime='' type='' owner='' mode='' + (( $# == 2 )) || return 1 + _k3slra1h_capture_canonical "$target" canonical || return 1 + [[ "$canonical" == "$target" ]] || return 1 + _k3slra1h_metadata_fields "$target" identity nlink size mtime type owner mode || return 1 + [[ "${identity%%:*}" == "$expected_device" ]] || return 1 + [[ "$type" == directory ]] || return 1 + [[ "$owner" == 0:0 ]] || return 1 + [[ "$mode" == 700 ]] +} + +_k3slra1h_staged_root_is_safe() { + local capture_root="${1-}" canonical='' device='' + local identity='' nlink='' size='' mtime='' type='' owner='' mode='' + (( $# == 1 )) || return 1 + [[ "$capture_root" == /* ]] || return 1 + [[ "$capture_root" != */ ]] || return 1 + [[ "$capture_root" != *//* ]] || return 1 + _k3slra1h_capture_canonical "$capture_root" canonical || return 1 + [[ "$canonical" == "$capture_root" ]] || return 1 + _k3slra1h_metadata_fields "$capture_root" identity nlink size mtime type owner mode || return 1 + [[ "$type" == directory ]] || return 1 + [[ "$owner" == 0:0 ]] || return 1 + [[ "$mode" == 700 ]] || return 1 + device="${identity%%:*}" + [[ "$device" =~ ^[0-9]+$ ]] || return 1 + _k3slra1h_root_directory_is_safe "${capture_root}/lib" "$device" || return 1 + _k3slra1h_root_directory_is_safe "${capture_root}/libexec" "$device" || return 1 + _k3slra1h_root_directory_is_safe "${capture_root}/contract" "$device" || return 1 + _k3slra1h_regular_root_file_is_safe "${capture_root}/lib/k3s-local-recovery.sh" "$device" || return 1 + _k3slra1h_regular_root_file_is_safe "${capture_root}/lib/k3s-local-recovery-a1.sh" "$device" || return 1 + _k3slra1h_regular_root_file_is_safe "${capture_root}/contract/local-recovery.env" "$device" || return 1 + _k3slra1h_regular_root_file_is_safe "${capture_root}/libexec/k3s-local-recovery-a1-check.sh" "$device" +} + +_k3slra1h_load_staged_dependencies() { + local capture_root="${1-}" shared_library='' a1_library='' contract_file='' + (( $# == 1 )) || return 1 + _k3slra1h_staged_root_is_safe "$capture_root" || return 1 + shared_library="${capture_root}/lib/k3s-local-recovery.sh" + a1_library="${capture_root}/lib/k3s-local-recovery-a1.sh" + contract_file="${capture_root}/contract/local-recovery.env" + source "$shared_library" || return 1 + source "$a1_library" || return 1 + _k3slr_load_contract "$contract_file" || return 1 + [[ "${K3SLR_MAPPING_NAME-}" == "$K3SLRA1_MAIN_MAPPING_NAME" ]] || return 1 + [[ "${K3SLR_PROOF_MAPPING_NAME-}" == k3s-recovery-proof ]] +} + +_k3slra1h_check_main_impl() { + local mount_root="${1-}" role="${2-}" capture_root="${3-}" + (( $# == 3 )) || return 1 + _k3slra1h_euid_is_root || return 1 + [[ "$-" != *x* ]] || return 1 + case "$role" in + original-baseline) [[ "$mount_root" == "${capture_root}/original-baseline-mount" ]] || return 1 ;; + proof) [[ "$mount_root" == "${capture_root}/proof-mount" ]] || return 1 ;; + original-final) [[ "$mount_root" == "${capture_root}/original-final-mount" ]] || return 1 ;; + *) return 1 ;; + esac + _k3slra1h_load_staged_dependencies "$capture_root" || return 1 + _k3slra1_verify_payload_root "$mount_root" "$role" "$capture_root" +} + +k3slra1_check_main() { + _k3slra1h_check_main_impl "$@" >/dev/null 2>&1 +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + _k3slra1h_direct_signal_rc=0 + _k3slra1h_direct_rc=0 + trap '_k3slra1h_direct_signal_rc=130; if (( _k3slra1h_direct_rc == 0 )); then _k3slra1h_direct_rc=130; fi' INT + trap '_k3slra1h_direct_signal_rc=143; if (( _k3slra1h_direct_rc == 0 )); then _k3slra1h_direct_rc=143; fi' TERM + if k3slra1_check_main "$@"; then + _k3slra1h_direct_rc=0 + else + _k3slra1h_direct_rc=$? + fi + if (( _k3slra1h_direct_signal_rc != 0 && _k3slra1h_direct_rc == 0 )); then + _k3slra1h_direct_rc=$_k3slra1h_direct_signal_rc + fi + exit "$_k3slra1h_direct_rc" +fi diff --git a/scripts/libexec/slack-webhook-recovery-dirfd.py b/scripts/libexec/slack-webhook-recovery-dirfd.py new file mode 100644 index 0000000..faf613f --- /dev/null +++ b/scripts/libexec/slack-webhook-recovery-dirfd.py @@ -0,0 +1,1095 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import ctypes +import hashlib +import json +import os +import pathlib +import signal +import stat +import tempfile +import uuid +from contextlib import ExitStack +from typing import Any, Callable, Sequence + + +SCHEMA = "platform-slack-keepass-fs-v1" +VAULT_PARTS = ("HyeonworksRecovery", "vault") +BACKUPS_PARTS = VAULT_PARTS + ("backups",) +MAIN_NAME = "hyeonworks-recovery.kdbx" +MAIN_RELATIVE = VAULT_PARTS + (MAIN_NAME,) + +OPEN_DIR_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC +OPEN_FILE_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC +CREATE_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC +COPY_CHUNK_SIZE = 1024 * 1024 +MAX_BASELINE_JSON_BYTES = 64 * 1024 +BLOCKED_SIGNALS = {signal.SIGHUP, signal.SIGINT, signal.SIGTERM} +MOUNT_UID = 1000 +MOUNT_GID = 1000 + + +class FsContractError(RuntimeError): + pass + + +def _require_absolute(path: str) -> str: + if not os.path.isabs(path): + raise FsContractError(f"absolute path required: {path!r}") + return path + + +def _validate_component(part: str) -> None: + if not part or part in {".", ".."} or "/" in part: + raise FsContractError(f"unsafe path component: {part!r}") + + +def _close_fd(fd: int) -> None: + try: + os.close(fd) + except OSError: + pass + + +def _mode_string(mode: int) -> str: + return f"{stat.S_IMODE(mode):04o}" + + +def open_physical_absolute(path: str, flags: int, mode: int | None = None) -> int: + absolute = _require_absolute(path) + if mode is None: + return os.open(absolute, flags) + return os.open(absolute, flags, mode) + + +def attest_directory_fd( + fd: int, + *, + expected_uid: int | None = None, + expected_gid: int | None = None, +) -> os.stat_result: + if expected_uid is None: + expected_uid = os.getuid() + if expected_gid is None: + expected_gid = os.getgid() + st = os.fstat(fd) + if not stat.S_ISDIR(st.st_mode): + raise FsContractError("expected directory") + if st.st_uid != expected_uid: + raise FsContractError("directory owner mismatch") + if st.st_gid != expected_gid: + raise FsContractError("directory group mismatch") + if stat.S_IMODE(st.st_mode) != 0o700: + raise FsContractError("directory mode mismatch") + return st + + +def _attest_regular_fd( + fd: int, + *, + expected_uid: int | None = None, + expected_gid: int | None = None, +) -> os.stat_result: + if expected_uid is None: + expected_uid = os.getuid() + if expected_gid is None: + expected_gid = os.getgid() + st = os.fstat(fd) + if not stat.S_ISREG(st.st_mode): + raise FsContractError("expected regular file") + if st.st_uid != expected_uid: + raise FsContractError("file owner mismatch") + if st.st_gid != expected_gid: + raise FsContractError("file group mismatch") + if stat.S_IMODE(st.st_mode) != 0o600: + raise FsContractError("file mode mismatch") + if st.st_nlink != 1: + raise FsContractError("file link count mismatch") + return st + + +def open_directory_chain(root_fd: int, parts: Sequence[str]) -> int: + current = os.dup(root_fd) + try: + for part in parts: + _validate_component(part) + next_fd = os.open(part, OPEN_DIR_FLAGS, dir_fd=current) + os.close(current) + current = next_fd + return current + except BaseException: + os.close(current) + raise + + +def _attest_retained_directory_entry(parent_fd: int, name: str, child_fd: int) -> None: + _validate_component(name) + retained = attest_directory_fd( + child_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if not stat.S_ISDIR(current.st_mode): + raise FsContractError("mount directory entry type mismatch") + if (current.st_dev, current.st_ino) != (retained.st_dev, retained.st_ino): + raise FsContractError("mount directory entry identity mismatch") + if current.st_uid != MOUNT_UID or current.st_gid != MOUNT_GID: + raise FsContractError("mount directory entry owner mismatch") + if stat.S_IMODE(current.st_mode) != 0o700: + raise FsContractError("mount directory entry mode mismatch") + + +def _attest_mount_bindings( + mount_fd: int, + recovery_fd: int, + vault_fd: int, + backups_fd: int | None = None, +) -> None: + attest_directory_fd( + mount_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + _attest_retained_directory_entry(mount_fd, VAULT_PARTS[0], recovery_fd) + _attest_retained_directory_entry(recovery_fd, VAULT_PARTS[1], vault_fd) + if backups_fd is not None: + _attest_retained_directory_entry(vault_fd, "backups", backups_fd) + + +def _open_mount_tree( + stack: ExitStack, + mount_root: str, + *, + include_backups: bool, +) -> tuple[int, int, int, int | None]: + mount_fd = stack.enter_context(_fd_context(_open_absolute_directory(mount_root))) + attest_directory_fd( + mount_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + recovery_fd = stack.enter_context( + _fd_context(open_directory_chain(mount_fd, (VAULT_PARTS[0],))) + ) + attest_directory_fd( + recovery_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + vault_fd = stack.enter_context( + _fd_context(open_directory_chain(recovery_fd, (VAULT_PARTS[1],))) + ) + attest_directory_fd( + vault_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + backups_fd = None + if include_backups: + backups_fd = stack.enter_context( + _fd_context(open_directory_chain(vault_fd, ("backups",))) + ) + attest_directory_fd( + backups_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + _attest_mount_bindings(mount_fd, recovery_fd, vault_fd, backups_fd) + return mount_fd, recovery_fd, vault_fd, backups_fd + + +def _require_same_identity(left_fd: int, right_fd: int, description: str) -> None: + left = os.fstat(left_fd) + right = os.fstat(right_fd) + if (left.st_dev, left.st_ino) != (right.st_dev, right.st_ino): + raise FsContractError(f"{description} identity drift") + + +def _attest_canonical_mount_tree( + mount_root: str, + mount_fd: int, + recovery_fd: int, + vault_fd: int, + backups_fd: int | None = None, + main_fd: int | None = None, +) -> None: + with ExitStack() as canonical_stack: + canonical_mount_fd, canonical_recovery_fd, canonical_vault_fd, canonical_backups_fd = ( + _open_mount_tree( + canonical_stack, + mount_root, + include_backups=backups_fd is not None, + ) + ) + _require_same_identity(mount_fd, canonical_mount_fd, "mount root") + _require_same_identity(recovery_fd, canonical_recovery_fd, "recovery directory") + _require_same_identity(vault_fd, canonical_vault_fd, "vault directory") + if backups_fd is not None: + if canonical_backups_fd is None: + raise FsContractError("canonical backups directory unavailable") + _require_same_identity(backups_fd, canonical_backups_fd, "backups directory") + if main_fd is not None: + canonical_main_fd = canonical_stack.enter_context( + _fd_context( + open_regular_at( + canonical_vault_fd, + MAIN_NAME, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + ) + ) + _attest_retained_regular_entry( + canonical_vault_fd, + MAIN_NAME, + canonical_main_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + _require_same_identity(main_fd, canonical_main_fd, "canonical main") + if not files_equal_fd(main_fd, canonical_main_fd): + raise FsContractError("canonical main content drift") + _attest_mount_bindings( + canonical_mount_fd, + canonical_recovery_fd, + canonical_vault_fd, + canonical_backups_fd, + ) + _require_same_identity(mount_fd, canonical_mount_fd, "mount root") + _require_same_identity(recovery_fd, canonical_recovery_fd, "recovery directory") + _require_same_identity(vault_fd, canonical_vault_fd, "vault directory") + if backups_fd is not None and canonical_backups_fd is not None: + _require_same_identity(backups_fd, canonical_backups_fd, "backups directory") + + +def _absolute_parts(path: str) -> tuple[str, ...]: + pure = pathlib.PurePosixPath(_require_absolute(path)) + return tuple(pure.parts[1:]) + + +def _open_absolute_directory(path: str) -> int: + root_fd = open_physical_absolute("/", OPEN_DIR_FLAGS) + try: + return open_directory_chain(root_fd, _absolute_parts(path)) + finally: + _close_fd(root_fd) + + +def _open_absolute_parent(path: str) -> tuple[int, str]: + pure = pathlib.PurePosixPath(_require_absolute(path)) + if pure.name in {"", "/", ".", ".."}: + raise FsContractError("path must name a file") + root_fd = open_physical_absolute("/", OPEN_DIR_FLAGS) + try: + parent_fd = open_directory_chain(root_fd, tuple(pure.parts[1:-1])) + finally: + _close_fd(root_fd) + return parent_fd, pure.name + + +def open_regular_at( + dir_fd: int, + name: str, + *, + expected_uid: int | None = None, + expected_gid: int | None = None, +) -> int: + _validate_component(name) + fd = os.open(name, OPEN_FILE_FLAGS, dir_fd=dir_fd) + try: + _attest_regular_fd( + fd, + expected_uid=expected_uid, + expected_gid=expected_gid, + ) + return fd + except BaseException: + _close_fd(fd) + raise + + +def _open_absolute_regular(path: str) -> int: + parent_fd, name = _open_absolute_parent(path) + try: + attest_directory_fd(parent_fd) + return open_regular_at(parent_fd, name) + finally: + _close_fd(parent_fd) + + +def _sha256_fd(fd: int) -> str: + os.lseek(fd, 0, os.SEEK_SET) + digest = hashlib.sha256() + while True: + chunk = os.read(fd, COPY_CHUNK_SIZE) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def stable_copy_fd( + src_fd: int, + dst_fd: int, + *, + expected_uid: int | None = None, + expected_gid: int | None = None, +) -> dict[str, Any]: + os.lseek(src_fd, 0, os.SEEK_SET) + os.lseek(dst_fd, 0, os.SEEK_SET) + digest = hashlib.sha256() + size = 0 + while True: + chunk = os.read(src_fd, COPY_CHUNK_SIZE) + if not chunk: + break + digest.update(chunk) + size += len(chunk) + view = memoryview(chunk) + while view: + written = os.write(dst_fd, view) + view = view[written:] + os.fsync(dst_fd) + dst_stat = _attest_regular_fd( + dst_fd, + expected_uid=expected_uid, + expected_gid=expected_gid, + ) + if dst_stat.st_size != size: + raise FsContractError("copied size mismatch") + return {"size": size, "sha256": digest.hexdigest()} + + +def files_equal_fd(left_fd: int, right_fd: int) -> bool: + left_stat = os.fstat(left_fd) + right_stat = os.fstat(right_fd) + if left_stat.st_size != right_stat.st_size: + return False + os.lseek(left_fd, 0, os.SEEK_SET) + os.lseek(right_fd, 0, os.SEEK_SET) + while True: + left_chunk = os.read(left_fd, COPY_CHUNK_SIZE) + right_chunk = os.read(right_fd, COPY_CHUNK_SIZE) + if left_chunk != right_chunk: + return False + if not left_chunk: + return True + + +def sync_filesystem_fd(fd: int) -> None: + libc = ctypes.CDLL(None, use_errno=True) + syncfs = libc.syncfs + syncfs.argtypes = [ctypes.c_int] + syncfs.restype = ctypes.c_int + rc = syncfs(fd) + if rc != 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + +def _build_state(operation: str, mount_root: str, source_stat: os.stat_result, source_sha256: str) -> dict[str, Any]: + return { + "schema": SCHEMA, + "operation": operation, + "mount": { + "realpath": os.path.realpath(mount_root), + "relative_vault": "/".join(VAULT_PARTS), + }, + "source": { + "relative_path": "/".join(MAIN_RELATIVE), + "size": source_stat.st_size, + "mode": _mode_string(source_stat.st_mode), + "uid": source_stat.st_uid, + "gid": source_stat.st_gid, + "inode": source_stat.st_ino, + "sha256": source_sha256, + }, + } + + +def _read_json_fd(fd: int, *, max_bytes: int) -> dict[str, Any]: + st = os.fstat(fd) + if not stat.S_ISREG(st.st_mode): + raise FsContractError("expected regular file") + if st.st_size > max_bytes: + raise FsContractError("baseline json too large") + os.lseek(fd, 0, os.SEEK_SET) + chunks: list[bytes] = [] + total = 0 + while True: + chunk = os.read(fd, min(COPY_CHUNK_SIZE, max_bytes - total + 1)) + if not chunk: + break + total += len(chunk) + if total > max_bytes: + raise FsContractError("baseline json too large") + chunks.append(chunk) + try: + parsed = json.loads(b"".join(chunks).decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise FsContractError("baseline json invalid") from exc + if not isinstance(parsed, dict): + raise FsContractError("baseline json payload mismatch") + return parsed + + +def _read_result(path: str) -> dict[str, Any]: + parent_fd, name = _open_absolute_parent(path) + try: + attest_directory_fd(parent_fd) + baseline_fd = open_regular_at(parent_fd, name) + try: + return _read_json_fd(baseline_fd, max_bytes=MAX_BASELINE_JSON_BYTES) + finally: + _close_fd(baseline_fd) + finally: + _close_fd(parent_fd) + + +def _validate_snapshot_state(state: dict[str, Any], mount_root: str) -> None: + if set(state) != {"schema", "operation", "mount", "source"}: + raise FsContractError("unexpected baseline keys") + if state["schema"] != SCHEMA: + raise FsContractError("baseline schema mismatch") + if state["operation"] != "snapshot": + raise FsContractError("baseline operation mismatch") + mount = state["mount"] + source = state["source"] + if not isinstance(mount, dict) or set(mount) != {"realpath", "relative_vault"}: + raise FsContractError("baseline mount payload mismatch") + if not isinstance(source, dict) or set(source) != { + "relative_path", + "size", + "mode", + "uid", + "gid", + "inode", + "sha256", + }: + raise FsContractError("baseline source payload mismatch") + if mount["realpath"] != os.path.realpath(mount_root): + raise FsContractError("baseline mount root mismatch") + if mount["relative_vault"] != "/".join(VAULT_PARTS): + raise FsContractError("baseline vault mismatch") + if source["relative_path"] != "/".join(MAIN_RELATIVE): + raise FsContractError("baseline source path mismatch") + if source["mode"] != "0600": + raise FsContractError("baseline source mode mismatch") + if source["uid"] != MOUNT_UID: + raise FsContractError("baseline source owner mismatch") + if source["gid"] != MOUNT_GID: + raise FsContractError("baseline source group mismatch") + if not isinstance(source["size"], int) or source["size"] < 0: + raise FsContractError("baseline size mismatch") + if not isinstance(source["inode"], int) or source["inode"] <= 0: + raise FsContractError("baseline inode mismatch") + if not isinstance(source["sha256"], str) or len(source["sha256"]) != 64: + raise FsContractError("baseline hash mismatch") + + +def _assert_matches_baseline(main_fd: int, baseline_state: dict[str, Any]) -> os.stat_result: + source = baseline_state["source"] + main_stat = _attest_regular_fd( + main_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + if main_stat.st_size != source["size"]: + raise FsContractError("baseline size drift") + if _mode_string(main_stat.st_mode) != source["mode"]: + raise FsContractError("baseline mode drift") + if main_stat.st_uid != source["uid"]: + raise FsContractError("baseline owner drift") + if main_stat.st_gid != source["gid"]: + raise FsContractError("baseline group drift") + if main_stat.st_ino != source["inode"]: + raise FsContractError("baseline inode drift") + if _sha256_fd(main_fd) != source["sha256"]: + raise FsContractError("baseline content drift") + return main_stat + + +def _create_private_file( + dir_fd: int, + name: str, + *, + expected_uid: int | None = None, + expected_gid: int | None = None, +) -> int: + _validate_component(name) + fd = os.open(name, CREATE_FILE_FLAGS, 0o600, dir_fd=dir_fd) + try: + _attest_regular_fd( + fd, + expected_uid=expected_uid, + expected_gid=expected_gid, + ) + return fd + except BaseException: + _close_fd(fd) + raise + + +def _attest_retained_regular_entry( + dir_fd: int, + name: str, + file_fd: int, + *, + expected_uid: int, + expected_gid: int, +) -> os.stat_result: + _validate_component(name) + retained = _attest_regular_fd( + file_fd, + expected_uid=expected_uid, + expected_gid=expected_gid, + ) + current = os.stat(name, dir_fd=dir_fd, follow_symlinks=False) + if not stat.S_ISREG(current.st_mode): + raise FsContractError("file entry type mismatch") + if (current.st_dev, current.st_ino) != (retained.st_dev, retained.st_ino): + raise FsContractError("file entry identity mismatch") + if current.st_uid != expected_uid or current.st_gid != expected_gid: + raise FsContractError("file entry owner mismatch") + if stat.S_IMODE(current.st_mode) != 0o600: + raise FsContractError("file entry mode mismatch") + if current.st_nlink != 1: + raise FsContractError("file entry link count mismatch") + return retained + + +def _unlink_if_same_file(dir_fd: int, name: str, expected_stat: os.stat_result) -> None: + try: + current_fd = os.open(name, OPEN_FILE_FLAGS, dir_fd=dir_fd) + except FileNotFoundError: + return + try: + current_stat = os.fstat(current_fd) + if ( + current_stat.st_dev == expected_stat.st_dev + and current_stat.st_ino == expected_stat.st_ino + ): + os.unlink(name, dir_fd=dir_fd) + finally: + _close_fd(current_fd) + + +def _snapshot_from_main(operation: str, mount_root: str, destination: str, candidate: str | None = None) -> dict[str, Any]: + with ExitStack() as stack: + mount_fd, recovery_fd, vault_fd, _ = _open_mount_tree( + stack, + mount_root, + include_backups=False, + ) + main_fd = stack.enter_context( + _fd_context( + open_regular_at( + vault_fd, + MAIN_NAME, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + ) + ) + _attest_retained_regular_entry( + vault_fd, + MAIN_NAME, + main_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + if candidate is not None: + candidate_fd = stack.enter_context(_fd_context(_open_absolute_regular(candidate))) + if not files_equal_fd(main_fd, candidate_fd): + raise FsContractError("candidate does not match installed database") + dest_parent_fd, dest_name = _open_absolute_parent(destination) + stack.callback(_close_fd, dest_parent_fd) + attest_directory_fd(dest_parent_fd) + dest_fd = stack.enter_context(_fd_context(_create_private_file(dest_parent_fd, dest_name))) + dest_stat = os.fstat(dest_fd) + try: + _attest_mount_bindings(mount_fd, recovery_fd, vault_fd) + _attest_canonical_mount_tree( + mount_root, + mount_fd, + recovery_fd, + vault_fd, + main_fd=main_fd, + ) + copied = stable_copy_fd( + main_fd, + dest_fd, + expected_uid=os.getuid(), + expected_gid=os.getgid(), + ) + _attest_retained_regular_entry( + vault_fd, + MAIN_NAME, + main_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + _attest_canonical_mount_tree( + mount_root, + mount_fd, + recovery_fd, + vault_fd, + main_fd=main_fd, + ) + state = _build_state(operation, mount_root, os.fstat(main_fd), copied["sha256"]) + _attest_canonical_mount_tree( + mount_root, + mount_fd, + recovery_fd, + vault_fd, + main_fd=main_fd, + ) + return state + except BaseException: + _unlink_if_same_file(dest_parent_fd, dest_name, dest_stat) + raise + + +def snapshot_database(mount_root: str, destination: str) -> dict[str, Any]: + return _snapshot_from_main("snapshot", mount_root, destination) + + +def snapshot_committed_database(mount_root: str, candidate: str, destination: str) -> dict[str, Any]: + return _snapshot_from_main("snapshot-current", mount_root, destination, candidate=candidate) + + +def commit_database( + mount_root: str, + baseline: str, + candidate: str, + backup_name: str, + *, + status_hook: Callable[[str], None] | None = None, +) -> dict[str, Any]: + _validate_component(backup_name) + baseline_state = _read_result(baseline) + _validate_snapshot_state(baseline_state, mount_root) + result = { + "schema": SCHEMA, + "operation": "commit", + "mount": { + "realpath": os.path.realpath(mount_root), + "relative_vault": "/".join(VAULT_PARTS), + }, + "backup_name": backup_name, + } + with ExitStack() as stack: + mount_fd, recovery_fd, vault_fd, backups_fd = _open_mount_tree( + stack, + mount_root, + include_backups=True, + ) + if backups_fd is None: + raise FsContractError("backups directory unavailable") + original_fd = stack.enter_context( + _fd_context( + open_regular_at( + vault_fd, + MAIN_NAME, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + ) + ) + _assert_matches_baseline(original_fd, baseline_state) + candidate_fd = stack.enter_context(_fd_context(_open_absolute_regular(candidate))) + + _attest_mount_bindings(mount_fd, recovery_fd, vault_fd, backups_fd) + backup_fd = stack.enter_context( + _fd_context( + _create_private_file( + backups_fd, + backup_name, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + ) + ) + stable_copy_fd( + original_fd, + backup_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + backup_stat = _attest_retained_regular_entry( + backups_fd, + backup_name, + backup_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + os.fsync(backups_fd) + sync_filesystem_fd(backups_fd) + + reopened_backup_fd = stack.enter_context( + _fd_context( + open_regular_at( + backups_fd, + backup_name, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + ) + ) + reopened_stat = _attest_retained_regular_entry( + backups_fd, + backup_name, + reopened_backup_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + if (reopened_stat.st_dev, reopened_stat.st_ino) != ( + backup_stat.st_dev, + backup_stat.st_ino, + ): + raise FsContractError("backup identity proof failed") + if not files_equal_fd(original_fd, reopened_backup_fd): + raise FsContractError("backup proof failed") + + stage_name = f".{MAIN_NAME}.stage.{os.getpid()}.{uuid.uuid4().hex}" + _attest_mount_bindings(mount_fd, recovery_fd, vault_fd, backups_fd) + _attest_retained_regular_entry( + backups_fd, + backup_name, + backup_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + stage_fd = stack.enter_context( + _fd_context( + _create_private_file( + vault_fd, + stage_name, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + ) + ) + stage_stat = os.fstat(stage_fd) + stage_replaced = False + try: + stable_copy_fd( + candidate_fd, + stage_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + os.fsync(vault_fd) + sync_filesystem_fd(vault_fd) + + current_main_fd = stack.enter_context( + _fd_context( + open_regular_at( + vault_fd, + MAIN_NAME, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + ) + ) + _assert_matches_baseline(current_main_fd, baseline_state) + _attest_retained_regular_entry( + vault_fd, + MAIN_NAME, + current_main_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + _attest_mount_bindings(mount_fd, recovery_fd, vault_fd, backups_fd) + _attest_retained_regular_entry( + backups_fd, + backup_name, + backup_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + _attest_retained_regular_entry( + vault_fd, + stage_name, + stage_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + _attest_canonical_mount_tree( + mount_root, + mount_fd, + recovery_fd, + vault_fd, + backups_fd, + current_main_fd, + ) + + if status_hook is not None: + status_hook("rename-attempted") + previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, BLOCKED_SIGNALS) + try: + os.replace(stage_name, MAIN_NAME, src_dir_fd=vault_fd, dst_dir_fd=vault_fd) + stage_replaced = True + installed_fd = stack.enter_context( + _fd_context( + open_regular_at( + vault_fd, + MAIN_NAME, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + ) + ) + _attest_retained_regular_entry( + vault_fd, + MAIN_NAME, + installed_fd, + expected_uid=MOUNT_UID, + expected_gid=MOUNT_GID, + ) + if not files_equal_fd(installed_fd, candidate_fd): + raise FsContractError("installed file mismatch") + os.fsync(installed_fd) + os.fsync(vault_fd) + sync_filesystem_fd(vault_fd) + _attest_canonical_mount_tree( + mount_root, + mount_fd, + recovery_fd, + vault_fd, + backups_fd, + installed_fd, + ) + _attest_canonical_mount_tree( + mount_root, + mount_fd, + recovery_fd, + vault_fd, + backups_fd, + installed_fd, + ) + result["fs_commit"] = "verified-commit" + return result + except BaseException: + result["fs_commit"] = "committed-but-uncertain" + return result + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + except BaseException: + if not stage_replaced: + _unlink_if_same_file(vault_fd, stage_name, stage_stat) + raise + + return result + + +def write_result(path: str, state: dict[str, Any]) -> None: + parent_fd, name = _open_absolute_parent(path) + payload = json.dumps(state, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n" + temp_name = f".{name}.tmp.{os.getpid()}.{uuid.uuid4().hex}" + temp_stat = None + replaced = False + try: + attest_directory_fd(parent_fd) + temp_fd = _create_private_file(parent_fd, temp_name) + try: + temp_stat = os.fstat(temp_fd) + view = memoryview(payload) + while view: + written = os.write(temp_fd, view) + view = view[written:] + os.fsync(temp_fd) + _attest_retained_regular_entry( + parent_fd, + temp_name, + temp_fd, + expected_uid=os.getuid(), + expected_gid=os.getgid(), + ) + finally: + _close_fd(temp_fd) + os.replace(temp_name, name, src_dir_fd=parent_fd, dst_dir_fd=parent_fd) + replaced = True + os.fsync(parent_fd) + finally: + if not replaced and temp_stat is not None: + _unlink_if_same_file(parent_fd, temp_name, temp_stat) + _close_fd(parent_fd) + + +class _fd_context: + def __init__(self, fd: int): + self.fd = fd + + def __enter__(self) -> int: + return self.fd + + def __exit__(self, exc_type, exc, tb) -> None: + _close_fd(self.fd) + + +def _cmd_snapshot(args: argparse.Namespace) -> dict[str, Any]: + state = snapshot_database(args.mount_root, args.destination) + write_result(args.result, state) + return state + + +def _cmd_commit(args: argparse.Namespace) -> dict[str, Any]: + def persist_status(fs_commit: str) -> None: + if fs_commit not in {"precommit-failure", "rename-attempted"}: + raise FsContractError("invalid commit prearm status") + write_result( + args.result, + { + "schema": SCHEMA, + "operation": "commit-status", + "fs_commit": fs_commit, + }, + ) + + persist_status("precommit-failure") + state = commit_database( + args.mount_root, + args.baseline, + args.candidate, + args.backup_name, + status_hook=persist_status, + ) + write_result(args.result, state) + return state + + +def _cmd_snapshot_current(args: argparse.Namespace) -> dict[str, Any]: + state = snapshot_committed_database(args.mount_root, args.candidate, args.destination) + write_result(args.result, state) + return state + + +def read_commit_status(path: str) -> str: + state = _read_result(path) + if state.get("schema") != SCHEMA: + raise FsContractError("commit status schema mismatch") + operation = state.get("operation") + fs_commit = state.get("fs_commit") + if operation == "commit-status": + if set(state) != {"schema", "operation", "fs_commit"} or fs_commit not in { + "precommit-failure", + "rename-attempted", + }: + raise FsContractError("commit prearm payload mismatch") + elif operation == "commit": + if set(state) != {"schema", "operation", "mount", "backup_name", "fs_commit"} or fs_commit not in { + "verified-commit", + "committed-but-uncertain", + }: + raise FsContractError("commit result payload mismatch") + else: + raise FsContractError("commit status operation mismatch") + return fs_commit + + +def _cmd_status(args: argparse.Namespace) -> dict[str, Any]: + status = read_commit_status(args.result) + os.write(1, status.encode("ascii") + b"\n") + return {"fs_commit": status} + + +def audit_runtime_processes(runtime_root: str) -> None: + canonical = os.path.realpath(_require_absolute(runtime_root)) + runtime_info = os.stat(canonical, follow_symlinks=False) + for entry in os.listdir("/proc"): + if not entry.isdigit() or int(entry) == os.getpid(): + continue + try: + process_info = os.stat(f"/proc/{entry}", follow_symlinks=False) + command_line = pathlib.Path(f"/proc/{entry}/cmdline").read_bytes() + except OSError: + continue + if process_info.st_uid != os.getuid() or not any( + marker in command_line + for marker in ( + b"slack-webhook-recovery-vault.py", + b"slack-webhook-recovery-dirfd.py", + b"keepassxc-cli", + b"socat", + ) + ): + continue + retained_runtime = canonical.encode() in command_line + try: + fd_names = os.listdir(f"/proc/{entry}/fd") + except OSError: + fd_names = () + for fd_name in fd_names: + try: + fd_info = os.stat(f"/proc/{entry}/fd/{fd_name}") + fd_target = os.readlink(f"/proc/{entry}/fd/{fd_name}") + except OSError: + continue + if (fd_info.st_dev, fd_info.st_ino) == ( + runtime_info.st_dev, runtime_info.st_ino + ) or fd_target == canonical or fd_target.startswith(canonical + "/"): + retained_runtime = True + break + if retained_runtime: + raise FsContractError("runtime-owned process remains") + + +def _cmd_audit_processes(args: argparse.Namespace) -> dict[str, Any]: + audit_runtime_processes(args.runtime_root) + return {"processes": "absent"} + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + snapshot = subparsers.add_parser("snapshot") + snapshot.add_argument("--mount-root", required=True) + snapshot.add_argument("--destination", required=True) + snapshot.add_argument("--result", required=True) + snapshot.set_defaults(func=_cmd_snapshot) + + commit = subparsers.add_parser("commit") + commit.add_argument("--mount-root", required=True) + commit.add_argument("--baseline", required=True) + commit.add_argument("--candidate", required=True) + commit.add_argument("--backup-name", required=True) + commit.add_argument("--result", required=True) + commit.set_defaults(func=_cmd_commit) + + snapshot_current = subparsers.add_parser("snapshot-current") + snapshot_current.add_argument("--mount-root", required=True) + snapshot_current.add_argument("--candidate", required=True) + snapshot_current.add_argument("--destination", required=True) + snapshot_current.add_argument("--result", required=True) + snapshot_current.set_defaults(func=_cmd_snapshot_current) + + status_parser = subparsers.add_parser("status") + status_parser.add_argument("--result", required=True) + status_parser.set_defaults(func=_cmd_status) + + audit_processes = subparsers.add_parser("audit-processes") + audit_processes.add_argument("--runtime-root", required=True) + audit_processes.set_defaults(func=_cmd_audit_processes) + + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + try: + args.func(args) + return 0 + except (OSError, FsContractError, ValueError, KeyError, TypeError): + os.write(2, b"dirfd helper failed\n") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/libexec/slack-webhook-recovery-vault.py b/scripts/libexec/slack-webhook-recovery-vault.py new file mode 100644 index 0000000..553fc09 --- /dev/null +++ b/scripts/libexec/slack-webhook-recovery-vault.py @@ -0,0 +1,947 @@ +#!/usr/bin/python3 + +from __future__ import annotations + +import hmac +import ctypes +import os +import re +import signal +import socket +import stat +import struct +import subprocess +import sys +import termios +from typing import Callable, Sequence + + +KEEPASSXC_CLI = "/usr/bin/keepassxc-cli" +PYTHON3 = "/usr/bin/python3" +SOCKET_NAME = "vault.sock" +BASELINE_NAME = "baseline.kdbx" +CANDIDATE_NAME = "candidate.kdbx" +ENTRY_GROUP = "/Platform/Observability/Slack" +ENTRY_TITLE = "Alertmanager webhook - desktop-infra-전체" +ENTRY_PATH = ENTRY_GROUP + "/" + ENTRY_TITLE +ENTRY_USERNAME = "desktop-infra" +ENTRY_URL = "" +MAX_WEBHOOK_BYTES = 4096 +MAX_INPUT_BYTES = 4096 +MAX_PROTOCOL_BYTES = 4096 +MAX_CHILD_OUTPUT_BYTES = 64 * 1024 +MAX_CHILD_STDERR_BYTES = 8192 +DEFAULT_ACCEPT_TIMEOUT = 900.0 +DEFAULT_IO_TIMEOUT = 15.0 +DEFAULT_CHILD_TIMEOUT = 30.0 +WEBHOOK_RE = re.compile( + rb"https://hooks[.]slack[.]com/services/" + rb"[A-Za-z0-9_-]+/[A-Za-z0-9_-]+/[A-Za-z0-9_-]+" +) + + +class VaultContractError(RuntimeError): + pass + + +class VaultProtocolError(VaultContractError): + pass + + +class VaultSignal(BaseException): + def __init__(self, signum: int) -> None: + super().__init__(signum) + self.signum = signum + + +HANDLED_SIGNALS = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM) + + +def install_process_signal_handlers() -> None: + def raise_first_signal(signum: int, _frame: object) -> None: + for handled in HANDLED_SIGNALS: + signal.signal(handled, signal.SIG_IGN) + raise VaultSignal(signum) + + for signum in HANDLED_SIGNALS: + signal.signal(signum, raise_first_signal) + + +class KeepassResult: + def __init__(self, returncode: int, stdout: bytes, stderr: bytes) -> None: + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +def wipe_mutable_buffer(buffer: bytearray) -> None: + for index in range(len(buffer)): + buffer[index] = 0 + + +def _regular_metadata(fd: int, *, allow_unlinked: bool = False) -> tuple[int, ...]: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode): + raise VaultContractError("expected regular file") + if info.st_uid != os.getuid(): + raise VaultContractError("file owner mismatch") + if stat.S_IMODE(info.st_mode) != 0o600: + raise VaultContractError("file mode mismatch") + if info.st_nlink != 1 and not (allow_unlinked and info.st_nlink == 0): + raise VaultContractError("file link count mismatch") + return ( + info.st_dev, + info.st_ino, + info.st_uid, + stat.S_IMODE(info.st_mode), + info.st_nlink, + info.st_size, + ) + + +class RetainedWebhook: + def __init__(self, fd: int, metadata: tuple[int, ...], content: bytearray) -> None: + self._fd = fd + self._metadata = metadata + self._content = bytearray(content) + self._closed = False + + @property + def fd(self) -> int: + if self._closed: + raise VaultContractError("webhook descriptor is closed") + return self._fd + + def read_bytes(self) -> bytes: + if self._closed: + raise VaultContractError("webhook descriptor is closed") + before = _regular_metadata(self._fd) + if before != self._metadata: + raise VaultContractError("webhook metadata drift") + collected = bytearray() + confirmation = bytearray() + try: + for target in (collected, confirmation): + os.lseek(self._fd, 0, os.SEEK_SET) + while len(target) <= MAX_WEBHOOK_BYTES: + chunk = os.read( + self._fd, + min(1024, MAX_WEBHOOK_BYTES + 1 - len(target)), + ) + if not chunk: + break + target.extend(chunk) + after = _regular_metadata(self._fd) + if after != before: + raise VaultContractError("webhook metadata drift") + if len(collected) > MAX_WEBHOOK_BYTES or len(confirmation) > MAX_WEBHOOK_BYTES: + raise VaultContractError("webhook content is oversized") + if not hmac.compare_digest(bytes(collected), bytes(confirmation)) or not hmac.compare_digest( + bytes(confirmation), self._content + ): + raise VaultContractError("webhook content drift") + return bytes(collected) + finally: + wipe_mutable_buffer(collected) + wipe_mutable_buffer(confirmation) + + def close(self) -> None: + if not self._closed: + try: + os.close(self._fd) + finally: + wipe_mutable_buffer(self._content) + self._closed = True + + +def _validate_webhook_content(content: bytes) -> None: + if not content or len(content) > MAX_WEBHOOK_BYTES: + raise VaultContractError("invalid webhook content") + if b"\r" in content or b"\n" in content or b"\0" in content: + raise VaultContractError("invalid webhook content") + if content.strip() != content: + raise VaultContractError("invalid webhook content") + if WEBHOOK_RE.fullmatch(content) is None: + raise VaultContractError("invalid webhook content") + + +def open_validated_webhook(path: str) -> RetainedWebhook: + if not os.path.isabs(path): + raise VaultContractError("absolute webhook path required") + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC + fd = os.open(path, flags) + try: + before = _regular_metadata(fd) + content_buffer = bytearray() + confirmation_buffer = bytearray() + try: + for target in (content_buffer, confirmation_buffer): + if _regular_metadata(fd) != before: + raise VaultContractError("webhook metadata drift") + os.lseek(fd, 0, os.SEEK_SET) + while len(target) <= MAX_WEBHOOK_BYTES: + chunk = os.read( + fd, + min(1024, MAX_WEBHOOK_BYTES + 1 - len(target)), + ) + if not chunk: + break + target.extend(chunk) + if _regular_metadata(fd) != before: + raise VaultContractError("webhook metadata drift") + after = _regular_metadata(fd) + if before != after: + raise VaultContractError("webhook metadata drift") + if not hmac.compare_digest( + bytes(content_buffer), bytes(confirmation_buffer) + ): + raise VaultContractError("webhook content drift") + _validate_webhook_content(confirmation_buffer) + return RetainedWebhook(fd, before, confirmation_buffer) + finally: + wipe_mutable_buffer(content_buffer) + wipe_mutable_buffer(confirmation_buffer) + except BaseException: + os.close(fd) + raise + + +def _read_line_fd(fd: int, limit: int) -> bytearray: + value = bytearray() + try: + while len(value) <= limit: + chunk = os.read(fd, 1) + if not chunk: + raise VaultContractError("operator input ended early") + if chunk == b"\n": + return value + if chunk in {b"\r", b"\0"}: + raise VaultContractError("invalid operator input") + value.extend(chunk) + raise VaultContractError("operator input is oversized") + except BaseException: + wipe_mutable_buffer(value) + raise + + +def _validate_app_name(raw: bytes) -> str: + try: + app = raw.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise VaultContractError("invalid app name") from exc + if not 1 <= len(app) <= 80 or app.strip() != app: + raise VaultContractError("invalid app name") + if any(character in ";=\\\r\n" or not character.isprintable() for character in app): + raise VaultContractError("invalid app name") + return app + + +def read_operator_secrets( + *, input_fd: int | None = None, output_fd: int | None = None +) -> tuple[str, bytearray]: + owned_fd = False + if input_fd is None: + input_fd = os.open("/dev/tty", os.O_RDWR | os.O_CLOEXEC) + output_fd = input_fd + owned_fd = True + if output_fd is None: + raise VaultContractError("operator output descriptor required") + master = bytearray() + old_termios = None + try: + os.write(output_fd, b"Slack app name: ") + raw_app = _read_line_fd(input_fd, MAX_INPUT_BYTES) + try: + app = _validate_app_name(bytes(raw_app)) + finally: + wipe_mutable_buffer(raw_app) + if os.isatty(input_fd): + previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + try: + old_termios = termios.tcgetattr(input_fd) + new_termios = list(old_termios) + new_termios[3] &= ~termios.ECHO + termios.tcsetattr(input_fd, termios.TCSAFLUSH, new_termios) + os.write(output_fd, b"KeePassXC master password: ") + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + else: + os.write(output_fd, b"KeePassXC master password: ") + master = _read_line_fd(input_fd, MAX_INPUT_BYTES) + if not master: + raise VaultContractError("empty master password") + if old_termios is not None: + previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + try: + termios.tcsetattr(input_fd, termios.TCSAFLUSH, old_termios) + old_termios = None + os.write(output_fd, b"\n") + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + return app, master + except BaseException: + wipe_mutable_buffer(master) + raise + finally: + if old_termios is not None: + previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + try: + termios.tcsetattr(input_fd, termios.TCSAFLUSH, old_termios) + os.write(output_fd, b"\n") + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + if owned_fd: + os.close(input_fd) + + +def _validate_child_stderr(stderr: bytes) -> None: + if len(stderr) > MAX_CHILD_STDERR_BYTES or b"\0" in stderr or b"\r" in stderr: + raise VaultContractError("invalid KeePass diagnostic") + try: + text = stderr.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise VaultContractError("invalid KeePass diagnostic") from exc + if any(character not in "\t\n" and not character.isprintable() for character in text): + raise VaultContractError("invalid KeePass diagnostic") + + +def run_keepass( + arguments: Sequence[str], + master_buffer: bytearray, + private_home: str, + *, + webhook: RetainedWebhook | None = None, + timeout: float = DEFAULT_CHILD_TIMEOUT, + check: bool = True, +) -> KeepassResult: + if not arguments or any(not isinstance(value, str) or "\0" in value for value in arguments): + raise VaultContractError("invalid KeePass arguments") + if not os.path.isabs(private_home): + raise VaultContractError("private home must be absolute") + child_input = bytearray(master_buffer) + child_input.extend(b"\n") + if webhook is not None: + protected = bytearray(webhook.read_bytes()) + try: + child_input.extend(protected) + child_input.extend(b"\n") + finally: + wipe_mutable_buffer(protected) + environment = { + "HOME": os.devnull, + "XDG_CONFIG_HOME": os.devnull, + "LC_ALL": "C.UTF-8", + "PATH": "/usr/bin:/bin", + } + expected_parent_pid = os.getpid() + def prepare_child() -> None: + os.umask(0o077) + arm_parent_death_signal(expected_parent_pid) + process = subprocess.Popen( + [KEEPASSXC_CLI, *arguments], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + close_fds=True, + start_new_session=True, + preexec_fn=prepare_child, + ) + try: + try: + stdout, stderr = process.communicate(bytes(child_input), timeout=timeout) + except subprocess.TimeoutExpired as exc: + try: + os.killpg(process.pid, 15) + except ProcessLookupError: + pass + try: + process.communicate(timeout=1.0) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, 9) + except ProcessLookupError: + pass + process.communicate() + raise VaultContractError("KeePass child timeout") from exc + if len(stdout) > MAX_CHILD_OUTPUT_BYTES: + raise VaultContractError("KeePass output is oversized") + _validate_child_stderr(stderr) + result = KeepassResult(process.returncode, stdout, stderr) + if check and process.returncode != 0: + raise VaultContractError("KeePass command failed") + return result + finally: + wipe_mutable_buffer(child_input) + + +def arm_parent_death_signal(expected_parent_pid: int) -> None: + """Ensure a vault death cannot orphan a secret-bearing KeePass child.""" + if expected_parent_pid <= 1: + raise VaultContractError("invalid expected parent PID") + libc = ctypes.CDLL(None, use_errno=True) + prctl = libc.prctl + rc = prctl(1, 9, 0, 0, 0) # PR_SET_PDEATHSIG, SIGKILL + if rc != 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + if os.getppid() != expected_parent_pid: + os.kill(os.getpid(), 9) + + +def expected_notes(app_name: str) -> str: + _validate_app_name(app_name.encode("utf-8")) + return ( + "channel=desktop-infra-전체;" + f"app={app_name};" + "recovery=revoke-and-reissue-in-slack" + ) + + +def _entry_count( + database: str, + master: bytearray, + private_home: str, + child_timeout: float = DEFAULT_CHILD_TIMEOUT, +) -> tuple[int, set[str]]: + result = run_keepass( + ["ls", "-q", "-R", "-f", database], master, private_home, + check=False, timeout=child_timeout, + ) + if result.returncode != 0: + raise VaultContractError("KeePass database unlock failed") + try: + text = result.stdout.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise VaultContractError("invalid KeePass listing") from exc + lines = text.splitlines() + exact = ENTRY_PATH.lstrip("/") + return sum(line == exact for line in lines), set(lines) + + +def _shown_attributes( + database: str, + master: bytearray, + private_home: str, + child_timeout: float = DEFAULT_CHILD_TIMEOUT, +) -> tuple[str, str, str, str]: + result = run_keepass( + [ + "show", "-q", "-a", "Title", "-a", "UserName", "-a", "URL", + "-a", "Notes", database, ENTRY_PATH, + ], + master, + private_home, + timeout=child_timeout, + ) + try: + text = result.stdout.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise VaultContractError("invalid KeePass attributes") from exc + lines = text.split("\n") + if len(lines) != 5 or lines[-1] != "" or any("\r" in line for line in lines[:-1]): + raise VaultContractError("invalid KeePass attribute framing") + return lines[0], lines[1], lines[2], lines[3] + + +def protected_value_matches(producer_rc: int, shown: bytes, expected: bytes) -> bool: + comparison = bytearray(expected) + comparison.extend(b"\n") + try: + return producer_rc == 0 and hmac.compare_digest(shown, bytes(comparison)) + finally: + wipe_mutable_buffer(comparison) + + +def _password_matches( + database: str, + master: bytearray, + webhook: RetainedWebhook, + private_home: str, + child_timeout: float = DEFAULT_CHILD_TIMEOUT, +) -> bool: + result = run_keepass( + ["show", "-q", "-s", "-a", "Password", database, ENTRY_PATH], + master, + private_home, + check=False, + timeout=child_timeout, + ) + expected = bytearray(webhook.read_bytes()) + shown = bytearray(result.stdout) + try: + return protected_value_matches(result.returncode, bytes(shown), bytes(expected)) + finally: + wipe_mutable_buffer(expected) + wipe_mutable_buffer(shown) + + +def classify_database( + database: str, + master_buffer: bytearray, + webhook: RetainedWebhook, + app_name: str, + private_home: str, + child_timeout: float = DEFAULT_CHILD_TIMEOUT, +) -> str: + try: + count, _ = _entry_count(database, master_buffer, private_home, child_timeout) + if count == 0: + return "absent" + if count > 1: + return "ambiguous" + expected = (ENTRY_TITLE, ENTRY_USERNAME, ENTRY_URL, expected_notes(app_name)) + if _shown_attributes(database, master_buffer, private_home, child_timeout) != expected: + return "mismatch" + if not _password_matches( + database, master_buffer, webhook, private_home, child_timeout + ): + return "mismatch" + return "exact-noop" + except (OSError, VaultContractError): + return "failed" + + +def _copy_private_database(source: str, destination: str) -> None: + source_fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + destination_fd = -1 + try: + _regular_metadata(source_fd) + destination_fd = os.open( + destination, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, + ) + while True: + chunk = os.read(source_fd, 1024 * 1024) + if not chunk: + break + view = memoryview(chunk) + while view: + written = os.write(destination_fd, view) + view = view[written:] + os.fsync(destination_fd) + _regular_metadata(destination_fd) + finally: + if destination_fd >= 0: + os.close(destination_fd) + os.close(source_fd) + + +def _ensure_groups( + database: str, + master: bytearray, + private_home: str, + existing_lines: set[str], + child_timeout: float = DEFAULT_CHILD_TIMEOUT, +) -> None: + for group in ("/Platform", "/Platform/Observability", ENTRY_GROUP): + flattened = group.lstrip("/") + "/" + if flattened not in existing_lines: + run_keepass( + ["mkdir", "-q", database, group], master, private_home, + timeout=child_timeout, + ) + existing_lines.add(flattened) + + +def build_candidate( + baseline: str, + candidate: str, + mode: str, + master_buffer: bytearray, + webhook: RetainedWebhook, + app_name: str, + private_home: str, + child_timeout: float = DEFAULT_CHILD_TIMEOUT, +) -> None: + if mode not in {"ADD", "UPDATE"}: + raise VaultContractError("invalid candidate mode") + classification = classify_database( + baseline, master_buffer, webhook, app_name, private_home, child_timeout + ) + if (mode == "ADD" and classification != "absent") or ( + mode == "UPDATE" and classification != "mismatch" + ): + raise VaultContractError("candidate mode does not match classification") + _copy_private_database(baseline, candidate) + try: + _, existing_lines = _entry_count( + candidate, master_buffer, private_home, child_timeout + ) + notes = expected_notes(app_name) + if mode == "ADD": + _ensure_groups( + candidate, master_buffer, private_home, existing_lines, child_timeout + ) + arguments = [ + "add", "-q", "-p", "-u", ENTRY_USERNAME, "--url", ENTRY_URL, + "--notes", notes, candidate, ENTRY_PATH, + ] + else: + arguments = [ + "edit", "-q", "-p", "-t", ENTRY_TITLE, "-u", ENTRY_USERNAME, + "--url", ENTRY_URL, "--notes", notes, candidate, ENTRY_PATH, + ] + run_keepass( + arguments, master_buffer, private_home, webhook=webhook, + timeout=child_timeout, + ) + if not verify_database( + candidate, master_buffer, webhook, app_name, private_home, child_timeout + ): + raise VaultContractError("candidate verification failed") + except BaseException: + try: + os.unlink(candidate) + except OSError: + pass + raise + + +def verify_database( + database: str, + master_buffer: bytearray, + webhook: RetainedWebhook, + app_name: str, + private_home: str, + child_timeout: float = DEFAULT_CHILD_TIMEOUT, +) -> bool: + return classify_database( + database, master_buffer, webhook, app_name, private_home, child_timeout + ) == "exact-noop" + + +def _attest_runtime_root(runtime_root: str) -> str: + if not os.path.isabs(runtime_root): + raise VaultContractError("absolute runtime root required") + canonical = os.path.realpath(runtime_root) + if canonical != os.path.abspath(runtime_root): + raise VaultContractError("runtime root must be canonical") + expected_parent = f"/run/user/{os.getuid()}" + if os.path.commonpath((canonical, expected_parent)) != expected_parent or canonical == expected_parent: + raise VaultContractError("runtime root is outside the private runtime") + fd = os.open(canonical, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(fd) + if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise VaultContractError("runtime root metadata mismatch") + finally: + os.close(fd) + mount_type = None + best_length = -1 + with open("/proc/self/mountinfo", "r", encoding="utf-8") as stream: + for line in stream: + left, right = line.rstrip("\n").split(" - ", 1) + mount_point = left.split()[4].replace("\\040", " ") + if canonical == mount_point or canonical.startswith(mount_point.rstrip("/") + "/"): + if len(mount_point) > best_length: + mount_type = right.split()[0] + best_length = len(mount_point) + if mount_type != "tmpfs": + raise VaultContractError("runtime root is not on tmpfs") + return canonical + + +def _retained_runtime_path(runtime_fd: int) -> str: + info = os.fstat(runtime_fd) + if not stat.S_ISDIR(info.st_mode): + raise VaultContractError("retained runtime is not a directory") + proc_path = f"/proc/{os.getpid()}/fd/{runtime_fd}" + link_info = os.lstat(proc_path) + if not stat.S_ISLNK(link_info.st_mode): + raise VaultContractError("retained runtime proc anchor is unavailable") + resolved = os.stat(proc_path) + if (resolved.st_dev, resolved.st_ino) != (info.st_dev, info.st_ino): + raise VaultContractError("retained runtime proc anchor mismatch") + return proc_path + + +def _open_snapshot_below_runtime( + path: str, runtime_root: str, runtime_fd: int +) -> int: + if not os.path.isabs(path) or os.path.normpath(path) != path: + raise VaultProtocolError("snapshot must be canonical") + relative = os.path.relpath(path, runtime_root) + parts = relative.split(os.sep) + if not parts or any( + not part or part in {".", ".."} or os.sep in part for part in parts + ): + raise VaultProtocolError("snapshot is outside runtime root") + current_fd = os.dup(runtime_fd) + try: + for part in parts[:-1]: + next_fd = os.open( + part, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, + dir_fd=current_fd, + ) + os.close(current_fd) + current_fd = next_fd + directory_info = os.fstat(current_fd) + if ( + directory_info.st_uid != os.getuid() + or stat.S_IMODE(directory_info.st_mode) != 0o700 + ): + raise VaultProtocolError("snapshot directory metadata mismatch") + return os.open( + parts[-1], + os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, + dir_fd=current_fd, + ) + finally: + os.close(current_fd) + + +def _attest_snapshot( + path: str, runtime_root: str, runtime_fd: int | None = None +) -> None: + if runtime_fd is not None: + fd = _open_attested_snapshot(path, runtime_root, runtime_fd) + try: + return + finally: + os.close(fd) + if not os.path.isabs(path) or os.path.realpath(path) != os.path.abspath(path): + raise VaultProtocolError("snapshot must be canonical") + if os.path.commonpath((path, runtime_root)) != runtime_root or path == runtime_root: + raise VaultProtocolError("snapshot is outside runtime root") + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + _regular_metadata(fd) + finally: + os.close(fd) + + +def _open_attested_snapshot(path: str, runtime_root: str, runtime_fd: int) -> int: + fd = _open_snapshot_below_runtime(path, runtime_root, runtime_fd) + try: + _regular_metadata(fd) + return fd + except BaseException: + os.close(fd) + raise + + +def read_protocol_line(connection: socket.socket, *, timeout: float) -> str: + connection.settimeout(timeout) + data = bytearray() + try: + while len(data) <= MAX_PROTOCOL_BYTES: + chunk = connection.recv(MAX_PROTOCOL_BYTES + 2 - len(data)) + if not chunk: + raise VaultProtocolError("partial protocol EOF") + data.extend(chunk) + if b"\n" in data: + break + if len(data) > MAX_PROTOCOL_BYTES + 1: + raise VaultProtocolError("protocol request is oversized") + if not data.endswith(b"\n") or data.count(b"\n") != 1: + raise VaultProtocolError("invalid protocol framing") + line = data[:-1] + if b"\r" in line or b"\0" in line: + raise VaultProtocolError("invalid protocol bytes") + try: + return line.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise VaultProtocolError("invalid protocol UTF-8") from exc + finally: + wipe_mutable_buffer(data) + + +def parse_protocol_command(line: str) -> tuple[str, str | None]: + if line in {"PREPARE", "VERIFY_CANDIDATE", "SHUTDOWN"}: + return line, None + if line in {"BUILD ADD", "BUILD UPDATE"}: + return "BUILD", line.split(" ", 1)[1] + prefix = "VERIFY_COMMITTED " + if line.startswith(prefix): + path = line[len(prefix):] + if not path or " " in path or not os.path.isabs(path): + raise VaultProtocolError("invalid committed snapshot command") + return "VERIFY_COMMITTED", path + raise VaultProtocolError("unknown protocol command") + + +def peer_uid(connection: socket.socket) -> int: + credentials = connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12) + _, uid, _ = struct.unpack("3i", credentials) + return uid + + +def _write_reply(connection: socket.socket, reply: str, timeout: float) -> None: + payload = reply.encode("ascii") + b"\n" + connection.settimeout(timeout) + connection.sendall(payload) + + +def serve_private_socket( + runtime_root: str, + webhook_file: str, + *, + input_fd: int | None = None, + output_fd: int | None = None, + accept_timeout: float = DEFAULT_ACCEPT_TIMEOUT, + io_timeout: float = DEFAULT_IO_TIMEOUT, + child_timeout: float = DEFAULT_CHILD_TIMEOUT, + wipe_hook: Callable[[bytearray], None] | None = None, +) -> None: + if child_timeout <= 0: + raise VaultContractError("invalid child timeout") + runtime = "" + runtime_fd = -1 + runtime_access = "" + webhook = None + master = bytearray() + listener = None + socket_identity = None + try: + runtime = _attest_runtime_root(runtime_root) + runtime_fd = os.open( + runtime, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC + ) + runtime_access = _retained_runtime_path(runtime_fd) + webhook = open_validated_webhook(webhook_file) + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + socket_path = os.path.join(runtime_access, SOCKET_NAME) + private_home = os.devnull + baseline = os.path.join(runtime_access, BASELINE_NAME) + candidate = os.path.join(runtime_access, CANDIDATE_NAME) + app_name, master = read_operator_secrets(input_fd=input_fd, output_fd=output_fd) + previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + try: + listener.bind(socket_path) + os.chmod(SOCKET_NAME, 0o600, dir_fd=runtime_fd, follow_symlinks=False) + socket_info = os.stat( + SOCKET_NAME, dir_fd=runtime_fd, follow_symlinks=False + ) + socket_identity = (socket_info.st_dev, socket_info.st_ino) + listener.listen(1) + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + listener.settimeout(accept_timeout) + connection, _ = listener.accept() + listener.close() + if peer_uid(connection) != os.getuid(): + connection.close() + raise VaultContractError("socket peer UID mismatch") + state = "initial" + classification = None + with connection: + while True: + try: + command, argument = parse_protocol_command( + read_protocol_line(connection, timeout=io_timeout) + ) + if command == "PREPARE": + if state != "initial": + raise VaultProtocolError("PREPARE out of state") + classification = classify_database( + baseline, master, webhook, app_name, private_home, + child_timeout, + ) + state = "prepared" + _write_reply(connection, classification, io_timeout) + elif command == "BUILD": + if state != "prepared" or ( + argument == "ADD" and classification != "absent" + ) or ( + argument == "UPDATE" and classification != "mismatch" + ): + raise VaultProtocolError("BUILD out of state") + try: + build_candidate( + baseline, candidate, argument or "", master, + webhook, app_name, private_home, child_timeout, + ) + except (OSError, VaultContractError): + _write_reply(connection, "failed", io_timeout) + continue + state = "candidate-ready" + _write_reply(connection, "candidate-ready", io_timeout) + elif command == "VERIFY_CANDIDATE": + if state != "candidate-ready": + raise VaultProtocolError("VERIFY_CANDIDATE out of state") + verified = verify_database( + candidate, master, webhook, app_name, private_home, + child_timeout, + ) + if verified: + state = "candidate-verified" + _write_reply(connection, "verified" if verified else "failed", io_timeout) + elif command == "VERIFY_COMMITTED": + if state != "candidate-verified" or argument is None: + raise VaultProtocolError("VERIFY_COMMITTED out of state") + snapshot_fd = _open_attested_snapshot( + argument, runtime, runtime_fd + ) + try: + verified = verify_database( + f"/proc/{os.getpid()}/fd/{snapshot_fd}", + master, + webhook, + app_name, + private_home, + child_timeout, + ) + finally: + os.close(snapshot_fd) + _write_reply(connection, "verified" if verified else "failed", io_timeout) + elif command == "SHUTDOWN": + _write_reply(connection, "stopped", io_timeout) + break + except VaultProtocolError: + try: + _write_reply(connection, "failed", io_timeout) + except OSError: + pass + raise + finally: + try: + if listener is not None: + listener.close() + finally: + try: + if webhook is not None: + webhook.close() + finally: + try: + wipe_mutable_buffer(master) + if wipe_hook is not None: + wipe_hook(master) + finally: + try: + if socket_identity is not None: + try: + current = os.stat( + SOCKET_NAME, dir_fd=runtime_fd, + follow_symlinks=False, + ) + if (current.st_dev, current.st_ino) == socket_identity: + os.unlink(SOCKET_NAME, dir_fd=runtime_fd) + except FileNotFoundError: + pass + finally: + if runtime_fd >= 0: + os.close(runtime_fd) + + +def _main(argv: Sequence[str]) -> int: + if len(argv) != 5 or argv[0] != "--serve" or argv[1] != "--runtime-root" or argv[3] != "--webhook-file": + return 2 + runtime_root, webhook_file = argv[2], argv[4] + if not os.path.isabs(runtime_root) or not os.path.isabs(webhook_file): + return 2 + install_process_signal_handlers() + try: + serve_private_socket(runtime_root, webhook_file) + return 0 + except VaultSignal as exc: + return 128 + exc.signum + except (OSError, VaultContractError): + os.write(2, b"vault helper failed\n") + return 1 + + +if __name__ == "__main__": + raise SystemExit(_main(sys.argv[1:])) diff --git a/scripts/validate/README.md b/scripts/validate/README.md new file mode 100644 index 0000000..6c6df6e --- /dev/null +++ b/scripts/validate/README.md @@ -0,0 +1,561 @@ +# Phase 1·2 검증 + +`render-phase1.sh`와 `render-phase2.sh`는 클러스터를 변경하지 않고 각 단계의 +구성을 렌더링한 뒤 핵심 불변 조건을 확인한다. Phase 2 검증은 먼저 Phase 1 +검증을 재실행하므로 두 단계의 경계도 함께 확인한다. + +```sh +bash scripts/validate/render-phase1.sh +bash scripts/validate/render-phase2.sh +``` + +`helm`이 `PATH`에 없거나 검증용 바이너리를 별도로 내려받았다면 절대 경로를 +지정할 수 있다. + +```sh +PLATFORM_HELM_BIN=/tmp/helm-v3.19.4/helm \ + bash scripts/validate/render-phase2.sh +``` + +`PLATFORM_HELM_BIN`은 실행 가능한 일반 파일의 절대 경로여야 하며, 지정하지 +않으면 `PATH`에서 `helm`을 찾는다. 어느 경로를 사용하든 버전은 정확히 +`v3.19.4`여야 한다. + +`render-phase1.sh --verified-output-dir`는 일반 사용자가 직접 호출하는 출력 옵션이 +아니라 `apply-phase1-gitea.sh`와 `apply-gitea-oidc.sh`가 함께 사용하는 내부 +handoff 전용이다. 기존의 비어 있는 `/tmp/platform-phase1-apply.*` 디렉터리만 +허용하며, 경로가 심볼릭 링크이거나 현재 사용자 소유가 아니거나 mode `0700`이 +아니면 중단한다. 검증을 모두 통과한 경우에만 다음 여섯 manifest를 mode +`0600`으로 복사하고 원본과 `cmp`로 다시 비교한다. + +- `namespaces.yaml` +- `ssd-local-pv.yaml` +- `cnpg-operator.yaml` +- `platform-postgres.yaml` +- `gitea.yaml` +- `gitea-oidc.yaml` + +`apply-phase1-gitea.sh`는 여섯 산출물의 SHA-256을 메모리에 고정하고 확인 후, +실제 적용 대상인 `namespaces.yaml`부터 `gitea.yaml`까지 다섯 manifest를 각각 +apply하기 직전에 다시 검사한다. `gitea-oidc.yaml`은 적용하지 않는다. + +`apply-gitea-oidc.sh`는 같은 handoff의 파일 수·권한·소유권을 확인하되, +`gitea-oidc.yaml`의 SHA-256만 적용 대상으로 고정하고 재확인한 뒤 그 하나만 +적용한다. 두 경로 모두 검증 뒤 Chart를 다시 내려받거나 Kustomize를 다시 실행하지 +않는다. 취소·오류·`INT`·`TERM` 종료 시 각 적용 스크립트가 자신의 handoff +디렉터리 전체를 삭제한다. + +공통 검증 기준은 다음과 같다. + +- 로컬 `kubectl`의 Kustomize가 Argo CD 3.4.2와 같은 `v5.8.1`인지 확인 +- Helm이 Argo CD 3.4.2와 같은 `v3.19.4`인지 확인 +- `sha256sum`과 `tar`가 설치되어 있는지 확인 +- `helm pull`로 고정 버전 패키지를 공식 저장소에서 임시 디렉터리로 받은 뒤 + SHA-256으로 바이트 단위 무결성 확인 +- CloudNativePG Chart `0.29.0`의 SHA-256이 + `668e065ff53508d58238788fd35b355a925060843629a951df0e6a9362e6d32f`인지 확인 +- Gitea Chart `12.7.0`의 SHA-256이 + `5881ef9c59400bee2d5547e77c4cd0efb925143c2f5d93fb4f38446db76b0167`인지 확인 +- Helm Chart는 반드시 `--enable-helm`과 기본 `LoadRestrictionsRootOnly`로 렌더링 + +Phase 1은 CloudNativePG CRD, Gitea용 `Cluster`·`DatabaseRole`·`Database`, Gitea +PVC·Ingress를 확인한다. Ingress의 `git.learn.hyeonworks.com` host, 내부 TLS 부재, +외부 HTTPS `ROOT_URL`, 애플리케이션 `NodePort`·`LoadBalancer`와 Gitea SSH Service +부재도 검증한다. Phase 1 PostgreSQL 빌드 루트에는 Keycloak 리소스·Secret 참조· +namespace ingress 허용이 없어야 한다. + +Gitea는 두 프로필을 모두 렌더하되 적용 경계를 분리해 검증한다. + +- `services/gitea` → `gitea.yaml`: 신규 설치용 baseline. OIDC Secret·ID host + alias·Keycloak egress·브랜딩이 없어야 하고 자체 가입을 닫은 로컬 비상 로그인만 + 유지한다. +- `services/gitea/profiles/oidc` → `gitea-oidc.yaml`: 공통 + `values/baseline.yaml`에 `values/oidc.yaml`을 병합한다. 정확한 OIDC Secret 참조, + public issuer, 외부 인증 전용 가입, CoreDNS 기반 public issuer 접근, `/32` egress와 해시된 브랜딩 + ConfigMap·read-only mount를 확인한다. + +공유 Cluster의 `pg_hba`도 네 규칙의 순서와 값을 검증한다. `gitea` Role은 `gitea` +DB, `keycloak` Role은 `keycloak` DB에만 SCRAM으로 접속을 허용하고, 각 Role이 다른 +DB에 접속하는 경우에는 바로 다음 규칙에서 `reject`해야 한다. + +Phase 2는 다음을 추가로 검증한다. + +- 공식 Keycloak `26.7.0` 원격 리소스와 Keycloak 단일 인스턴스·외부 HTTPS + hostname·내부 HTTP·Traefik Ingress +- Keycloak PostgreSQL 증분 루트에 `DatabaseRole`·`Database`·추가형 + NetworkPolicy만 있고 `Cluster` 사본은 없는지 확인 +- AIStor Operator Chart `5.10.0` SHA-256 `e5534f5ae4f6f12a3528a8cda5954d73280cba1a8e979e1628fbf3aac76babd1` +- AIStor ObjectStore Chart `1.0.16` SHA-256 `50ffa6a4e014cdc48566b237593baab41d039f103cdc50c1b1ac173b8d8bf71e` +- AIStor XFS Local PV의 `Retain`·`WaitForFirstConsumer`·`900Gi`·node affinity +- ObjectStore 서버 1개·볼륨 1개, PVC 보호, S3·Console `ClusterIP`, 외부 + Ingress·NodePort·LoadBalancer 부재 +- Phase 2 소스와 렌더 결과의 자격 증명·라이선스 `Secret` 부재 + +Secret 검사는 관리 대상 YAML 전체에 적용하되, 렌더 과정에서 생성되고 Git에서 +무시되는 업스트림 Chart 캐시인 `**/.helm/**`과 `**/charts/**`는 제외한다. + +스크립트는 `/tmp/platform-phase1-render.*` 아래에만 중간 산출물을 만들고 종료 시 +삭제한다. 따라서 `helm pull`로 받은 임시 `.tgz`도 종료 시 제거된다. 각 Helm +렌더 직전에는 다이제스트를 통과한 패키지만 아래의 정확한 생성 캐시 버전 +디렉터리에 압축 해제한다. + +- `infrastructure/controllers/cloudnative-pg/.helm/charts/cloudnative-pg-0.29.0/cloudnative-pg/Chart.yaml` +- `services/gitea/profiles/oidc/.helm/charts/gitea-12.7.0/gitea/Chart.yaml` + +Phase 2도 같은 방식으로 AIStor Operator와 ObjectStore의 정확한 버전 캐시만 +일시 생성한다. 스크립트는 해당 버전 디렉터리가 이미 존재하거나 캐시 상위 디렉터리가 +심볼릭 링크면 이를 임의 삭제·덮어쓰지 않고 중단한다. 패키지 다이제스트가 일치한 뒤에만 +정확한 디렉터리를 생성하며, 자신이 생성한 버전 디렉터리는 `EXIT` 종료 정리에서 +제거한다. `.helm/charts`와 `.helm`도 이번 실행에서 생성했고 비어 있을 때만 +`rmdir`로 정리한다. 기존의 다른 캐시는 삭제하지 않는다. + +정상 종료와 처리되는 `INT`·`TERM`에서는 생성한 압축 해제 캐시와 `/tmp` 렌더 +산출물이 제거된다. 강제 종료 등으로 정확한 캐시가 남으면 다음 실행은 안전하게 +거부하므로 운영자가 경로와 내용을 확인한 뒤에만 수동 정리한다. 패키지가 없거나 +다이제스트가 다르거나 압축 해제 후 `Chart.yaml`이 없으면 렌더 전에 즉시 실패한다. +실제 적용, namespace 생성, Secret 변경, Nginx 변경은 수행하지 않는다. + +내부 handoff 모드를 사용해도 검증 스크립트 자신의 Chart package, generated cache와 +작업용 render 디렉터리는 동일하게 정리된다. 보존되는 것은 적용 스크립트가 만든 +제한된 임시 디렉터리의 검증 완료 manifest 여섯 개뿐이며, 적용 스크립트 종료 시 +함께 제거된다. + +## 관측성 코어 정적 render 검증 + +`render-observability-core.sh`는 Helm `v3.19.4`, Kustomize `v5.8.1`, Kubernetes +render target `1.36.2`와 계획에 기록된 여섯 chart archive/name/version/appVersion, +열두 image digest를 고정한다. 기본 실행은 정적 render만 수행하며 Kubernetes +resource를 적용하지 않는다. live 관측성 mutation gate가 닫힌 동안에는 fixture 시험만 +실행한다. + +```sh +bash -n scripts/validate/render-observability-core.sh +bash -n scripts/validate/test-render-observability-core.sh +bash scripts/validate/test-render-observability-core.sh +``` + +fixture 시험은 chart를 다운로드하지 않고 source된 assertion 함수에 변형 YAML을 +전달한다. 성공 표식은 `OBSERVABILITY CORE RENDER ASSERTION TEST PASS`다. 전체 renderer는 +Task 2 이후 child root가 존재할 때 다음과 같이 사용한다. + +```sh +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/validate/render-observability-core.sh +``` + +내부 `--verified-output-dir`는 비어 있는 current-user 소유 mode `0700`의 +`/tmp/platform-observability-core-apply.*`만 허용한다. 모든 검증과 child/aggregate +canonical resource equivalence가 끝난 뒤에만 manifest 열네 개와 +`resource-index.tsv`를 regular mode `0600` 파일로 넘긴다. Grafana와 Blackbox child +manifest도 각각 `grafana.yaml`, `blackbox.yaml`로 포함한다. handoff directory의 +device/inode를 고정하고 쓰기 직전에 identity와 empty 상태를 다시 확인하며, 각 파일은 +완성된 mode `0600` staging inode를 exact name에 no-clobber link하는 방식으로 공개한다. +최종 entry set은 열일곱 exact name뿐이어야 한다. 각 chart cache는 모든 render root의 +physical lineage를 먼저 확인한 뒤 directory FD 기준 exclusive mkdir로 이번 실행의 정확한 +version 경로만 만들고, 기존 경로를 덮어쓰지 않으며, 종료 시 실제로 생성·추적한 경로만 +제거한다. 실행 전부터 존재한 빈 `.helm`이나 `.helm/charts`는 제거하지 않는다. +cache cleanup은 tracked directory를 random no-replace quarantine name으로 옮긴 뒤 identity를 +다시 확인하고, 모든 child를 directory FD 기준으로 같은 방식으로 격리해 제거한다. cleanup +도중 원래 path에 replacement가 생겨도 이를 순회하거나 삭제하지 않으며 identity mismatch나 +unexpected entry는 성공 종료로 숨기지 않는다. + +Alertmanager Slack source 계약은 Secret payload를 읽지 않는 focused validator로 +검사한다. 기본 모드는 source와 Kustomize/KPS wiring만 검사하며, 선택적 server dry-run도 +두 non-Secret source만 admission에 제출하고 live object를 변경하지 않는다. + +```sh +bash scripts/validate/test-observability-alerting.sh +bash scripts/validate/test-observability-alerting.sh --server-dry-run +``` + +`render-observability-access.sh`는 core renderer의 검증된 Grafana·Blackbox·targets· +dashboard·rule·Alertmanager bytes와 private DNS를 access 단계의 일곱 manifest로 +분리한다. rules-alerts와 complete mode는 두 metric inventory phase가 있는 direct +`/tmp/platform-observability-metrics.XXXXXX` mode `0700` 디렉터리만 허용한다. 두 +inventory directory는 mode `0700`, JSON/checksum은 owner mode `0600`, link count 1과 +exact checksum이어야 한다. 게시 전 모든 destination absence와 root inode를 고정하고, +모든 source를 mode `0600` staging inode에 동기화한 뒤 no-clobber link한다. 한 파일이라도 +실패하면 이번 실행이 생성한 파일을 전부 제거해 부분 handoff를 남기지 않는다. + +```sh +bash -n scripts/validate/render-observability-access.sh +bash scripts/validate/test-render-observability-access.sh +bash scripts/validate/render-observability-access.sh \ + --component complete \ + --verified-output-dir /tmp/platform-observability-metrics.XXXXXX +``` + +출력에는 Secret kind나 credential payload가 없으며, 이미 게시된 output root를 재사용하면 +no-clobber로 즉시 거부한다. live 적용은 별도 apply transaction과 Slack Secret, +HTTPS runbook URL, source-proof/acceptance evidence가 모두 준비된 뒤에만 수행한다. + +## Slack webhook KeePass 복구 사본 + +`backup-slack-webhook-recovery.sh`는 Alertmanager Slack webhook의 재해 복구용 암호화 +사본을 같은 host의 별도 내장 Windows SSD에 있는 기존 KDBX에 보관한다. 이 사본은 일반 +K3s 재시작이나 host reboot에 필요하지 않으며, datastore·Secret·bootstrap state를 잃었을 +때를 위한 것이다. 기본 no-argument 실행은 고정 contract만 출력하고 SSD, KDBX, webhook, +sudo에 접근하지 않는다. 지원되는 public interface는 다음 두 형식뿐이다. + +```sh +bash scripts/bootstrap/backup-slack-webhook-recovery.sh +bash scripts/bootstrap/backup-slack-webhook-recovery.sh \ + --execute \ + --slack-webhook-file /home/donghyeon/.secrets/alertmanager/slack-webhook +``` + +execute는 전용 TTY에서 Slack app name과 KeePassXC master password를 받고, write 또는 +mismatch update 전에 정확한 확인 문자열을 요구한다. webhook payload, master password, +그 hash·encoding·size·URL component, KDBX protected output은 argv, environment, stdout, +log 또는 plaintext 파일에 출력하거나 기록하지 않는다. + +검증된 read-only exact match는 `SLACK_KEEPASS_RECOVERY=NOOP`, 검증된 변경은 +`SLACK_KEEPASS_RECOVERY=COMMITTED`를 출력한다. 두 성공 분기 모두 SSD source가 실제로 +unmount된 뒤 `WINDOWS_SSD_UNMOUNTED=PASS`와 +`OFF_HOST_RECOVERY_SATISFIED=NO`를 출력한다. 변경 분기는 durable non-clobbering +pre-change backup을 만든 뒤 `KDBX_PRECHANGE_BACKUP=CREATED`도 출력한다. rename 시도 뒤 +응답 손실, post-commit 검증 실패, 또는 cleanup/unmount 불명은 재시도하지 않고 +`SLACK_KEEPASS_RECOVERY=MANUAL_RECOVERY_REQUIRED`로 중단하며 main과 backup을 보존한다. +private tmpfs work root, socket, helper process와 mount source의 제거·부재 증명 전에는 성공 +token을 출력하지 않는다. + +이 same-host encrypted copy는 off-host escrow가 아니며 Phase 4의 off-host recovery gate를 +충족하거나 승인하지 않는다. + +## 관측성 Secret create-only bootstrap + +`create-observability-secrets.sh`는 기본 실행에서 고정 contract만 출력하며 cluster, sudo, +payload file에 접근하지 않는다. execute는 선택한 `grafana-admin`과 +`alertmanager-slack-webhook`의 exact `Opaque` key set과 기존 payload 일치를 확인하며, +기존 payload가 다르면 rotation 없이 중단한다. 둘을 함께 선택하면 한 create-only +transaction으로 처리한다. + +```sh +bash scripts/bootstrap/create-observability-secrets.sh +bash scripts/bootstrap/create-observability-secrets.sh \ + --execute --grafana-admin \ + --grafana-admin-user-file /absolute/current-user-0600/admin-user \ + --grafana-admin-password-file /absolute/current-user-0600/admin-password +bash scripts/bootstrap/create-observability-secrets.sh \ + --execute --slack-webhook \ + --slack-webhook-file /absolute/current-user-0600/slack-webhook +bash scripts/bootstrap/create-observability-secrets.sh \ + --check-grafana-recovery-evidence +bash scripts/bootstrap/create-observability-secrets.sh \ + --check-slack-recovery-evidence +``` + +execute는 confirmation 전과 create 직전에 k3s encryption/restore validator를 각각 새 +process로 실행한다. 입력은 no-follow private snapshot으로만 create에 전달하고 값이나 hash를 +출력하지 않는다. transaction 실패 시 이번 호출이 생성하고 UID를 캡처한 Secret만 Kubernetes +API UID precondition으로 삭제한다. create 결과 또는 ownership이 모호하면 해당 object는 +삭제하지 않고 `MANUAL_RECOVERY_REQUIRED=YES`로 중단한다. recovery marker는 root-owned +mode `0600` 네 필드만 허용하며 standalone check는 current-user kube context와 30일 age를 +확인하고 marker stat/read에만 좁은 sudo를 사용한다. + +fixture 회귀는 실제 production 함수와 private Unix-socket API precondition 경계를 실행하되 +live Secret이나 실제 payload에는 접근하지 않는다. + +```sh +bash -n scripts/bootstrap/create-observability-secrets.sh +bash scripts/validate/test-create-observability-secrets.sh +``` + +## Grafana–Keycloak OIDC bootstrap 검증 + +`configure-keycloak-grafana-oidc.sh`는 인자 없이 실행하면 고정 client/group/mapper/Secret +계획만 출력하며 Kubernetes, Keycloak, sudo와 payload에 접근하지 않는다. execute는 정확히 +`default` context, `https://127.0.0.1:6443` API, Ready +`donghyeon-system-product-name` node와 bounded authorization을 다시 고정한 뒤 loopback-only +Keycloak Admin API와 private Unix-socket Kubernetes Secret API만 사용한다. + +```sh +bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh +bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh --execute +bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh \ + --execute --admin OBS_ADMIN_USER --viewer OBS_VIEWER_USER +bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh \ + --check-recovery-evidence +``` + +`--admin`과 `--viewer`는 정확히 한 명의 기존 Keycloak user만 각 고정 group에 추가한다. +옵션이 없으면 임의 사용자를 만들거나 membership을 바꾸지 않는다. 기존 exact client, +full-path `groups` mapper, 두 group과 matching `observability/grafana-keycloak-oidc` Secret은 +credential rotation과 client/Secret rewrite 없이 재사용한다. declarative update가 필요한 +기존 client는 현재 credential을 private file로 보존해 PUT과 rollback에 명시적으로 넣고, +전후 client-secret 관계를 payload 출력 없이 비교한다. + +execute는 confirmation 전과 첫 mutation 직전에 encryption/restore validator를 각각 fresh +`/usr/bin/env -i` process로 실행한다. `APPLY default`와 +`RECOVERY KEYCLOAK default` 뒤에만 mutation을 시작한다. 실패나 `HUP`/`INT`/`TERM`은 exact +snapshot과 transaction ledger로 이번 실행이 추가한 membership, mapper/client/group/Secret만 +역순 복구한다. 생성 Secret 삭제는 UID와 resourceVersion precondition을 모두 사용한다. +ownership 또는 response 결과가 모호하면 대상 삭제를 시도하지 않고 +`MANUAL_RECOVERY_REQUIRED=YES`로 중단한다. + +성공 시 `/etc/hyeonworks/platform/recovery-evidence/keycloak.env`에는 schema, context, +resource, checked-at UTC 네 non-secret field만 `root:root 0600`으로 atomic 기록한다. +standalone check는 current-user kube context와 exact marker schema, no-follow/link metadata와 +30일 age를 검사하고 marker 접근에만 좁은 sudo를 사용한다. + +focused fixture는 stateful fake Keycloak Admin API와 raw Kubernetes Secret API를 통해 실제 +production state machine을 실행한다. create/update/no-op, duplicate cardinality, membership +ownership, response loss, conflict, timeout, rollback과 signals를 검증하되 live cluster, sudo, +실제 Secret을 읽거나 변경하지 않는다. + +```sh +bash -n scripts/bootstrap/configure-keycloak-grafana-oidc.sh +bash -n scripts/validate/test-configure-keycloak-grafana-oidc.sh +bash scripts/validate/test-configure-keycloak-grafana-oidc.sh +``` + +## Phase 3 비공개 관리 UI 검증 + +`render-admin-services.sh`는 pgAdmin OCI Chart `9.16.0`과 AIStor ObjectStore +Chart `1.0.16` archive SHA-256을 다시 검증한다. pgAdmin main과 두 init image가 +모두 `9.16` amd64 digest로 바뀌었는지, 렌더 결과에 Secret kind가 없는지, +Recreate·2Gi Retain Local PV·두 admin Ingress와 namespace 간 NetworkPolicy가 +유지되는지 확인한다. + +```sh +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/validate/render-admin-services.sh +``` + +실제 적용 뒤 `admin-ui-smoke.sh --execute --run-s3`로 공개 admin DNS 부재, +LAN·Tailscale·Pod DNS, 인증서 SAN, 외부 403, loopback NodePort, pgAdmin 비밀번호 +저장 차단과 AIStor S3 회귀를 확인한다. 허용·비허용 Keycloak 사용자 브라우저 +시험은 수동 수용 시험으로 남긴다. + +임시 network smoke Pod는 BusyBox 1.37.0의 amd64 manifest digest를 고정하고 +non-root UID, RuntimeDefault seccomp, 전체 capability drop, +allowPrivilegeEscalation=false, read-only root filesystem과 service account token +미마운트를 적용한다. `platform-admin`의 Pod Security restricted 정책을 +우회하지 않는다. + +## 로컬 K3s recovery 저장소 읽기 전용 검증 + +`k3s-local-recovery.sh`는 recovery SSD의 고정 hardware identity, SMART/NTFS 상태와 +mount → container → loop → LUKS2 → ext4 lineage를 읽기 전용으로 판정한다. 실행 +전에 같은 terminal에서 sudo credential을 미리 준비해야 하며, validator 자체가 암호를 +요구하거나 package를 설치하지 않는다. + +```sh +sudo -v +bash scripts/validate/k3s-local-recovery.sh --expect-device-ready +bash scripts/validate/k3s-local-recovery.sh --expect-closed +bash scripts/validate/k3s-local-recovery.sh --expect-open +bash scripts/validate/k3s-local-recovery.sh --expect-open --check-latest-bundle +``` + +기대 상태는 정확히 하나만 지정한다. `--expect-device-ready`와 `--expect-closed`는 두 +platform mount, 운영·proof mapping과 container 관련 loop가 모두 없는 상태만 허용한다. +device-ready 검사는 canonical recovery partition이 다른 mountpoint에도 source로 쓰이지 +않고 `ntfs-3g.probe --readwrite`가 성공해야 한다. `--expect-open`은 그 partition이 승인된 +outer mount 한 곳에만 연결되고, `ntfs3`의 `nodev,nosuid,noexec`와 안전한 +`umask=077` 또는 동등한 `dmask=0077,fmask=0077`을 가져야 한다. 또한 canonical parent +chain과 container inode, loop backing inode/device, zero offset/size limit, LUKS2 mapper와 +두 mount의 major:minor가 하나의 lineage여야 하며 마지막에 같은 snapshot을 다시 확인한다. +완전 할당된 고정 크기 container, ext4 label과 `root:root 0700` inner root도 모두 +필수다. dirty/hibernated NTFS를 고치거나 force mount하는 동작은 없다. root EUID나 +상속된 xtrace 상태에서는 workspace path/config를 읽기 전에 거부한다. + +`--check-latest-bundle`은 open 상태에서만 쓸 수 있다. inner root의 root-owned mode +`0600` `.latest-post-bundle.env`는 다음 세 key를 정확히 한 번씩 가져야 한다. + +```text +schema=k3slr-latest-post-bundle-v1 +relative_path=k3s-secrets-encryption-YYYYMMDDTHHMMSSZ/post +directory_identity=DEVICE:INODE +``` + +validator는 이 제한된 relative path와 directory identity를 전후로 재확인하고, +root-owned post `bundle.env` schema와 안전한 relative-name manifest를 검사한다. manifest의 +각 parent와 leaf는 symlink가 아닌 pinned directory 내부 object여야 하며, regular-file +device/inode를 hash 전후에 확인하면서 각 target을 개별 `sha256sum`으로 검증한다. 성공 +출력은 다음 네 분류뿐이며 stable ID, UUID, serial, WWN, loop/KDBX/bundle path와 payload를 +표시하지 않는다. + +```text +Recovery device: match +Recovery state: device_ready|closed|open +Lineage: match +Latest bundle: verified|not_checked +``` + +fixture 회귀는 system command 경계만 argv log를 남기는 fake로 바꾸고 실제 collector와 +parser를 호출한다. latest verifier는 임시 일반 directory의 valid, malformed, duplicate, +symlink escape, hash mismatch bundle을 직접 검사하며 live mount나 block device를 만들지 +않는다. + +```sh +bash scripts/validate/test-k3s-local-recovery.sh +``` + +### KeePassXC → cryptsetup anonymous-pipe feasibility + +`k3s-local-recovery-feasibility.sh`는 recovery 저장소를 만들기 전에 고정 package와 +KeePassXC CLI의 synthetic KDBX 동작을 확인한다. 요구 version은 `keepassxc` +`2.7.6+dfsg.1-1build3`, `cryptsetup-bin` `2:2.7.0-1ubuntu4.2`이며 executable도 +root-owned regular non-symlink, group/other non-writable 조건을 만족해야 한다. + +```sh +bash scripts/validate/k3s-local-recovery-feasibility.sh +sudo -v +bash scripts/validate/k3s-local-recovery-feasibility.sh --execute +``` + +기본 실행은 package/executable prerequisite만 판정한다. `--execute`는 interactive stdin과 +같은 terminal의 cached sudo credential을 요구하며, inherited xtrace나 root EUID에서는 +workspace library를 읽기 전에 거부한다. 이 mode가 만드는 것은 current-user `0700` +`/tmp/k3slr-feasibility.*` 아래 synthetic KDBX와 attachment round-trip fixture뿐이다. +실제 KeePass DB, LUKS file, loop device, mapping, mount와 K3s 상태는 읽거나 변경하지 않는다. +성공·실패·`INT`·`TERM` 모두 exact fixture directory를 정리한다. + +실제 recovery open/format API는 고정 entry의 Password attribute를 shell 변수, command +substitution, argv, environment, file, log 또는 `tee`에 넣지 않는다. producer stdout은 +anonymous kernel pipe FD로만 전달된다. 검증된 caller TTY FD를 producer stdin에 명시적으로 +연결하고 producer 내부에서도 TTY를 재검증한다. producer output은 streaming `od`와 bounded +`mawk` validator가 전부 drain하며 non-shell process 안에 최대 40 byte만 유지한다. producer, +EOF, exact 40자+LF/class 검증과 KDBX 전체 parent lineage 재검증이 모두 성공한 뒤에만 정확한 +41 byte가 parent pipe로 전달되고 다음 두 고정 consumer 중 하나가 읽기 시작한다. +producer supervisor는 시작 직후 STOP handshake를 수행하고 caller와 다른 실제 PGID를 trusted +`ps`로 확인한 뒤에만 pipeline을 시작한다. 이때 `/proc/PID/stat`의 start time, direct parent, +observed PGID를 함께 고정하며, negative group signal 직전마다 동일 identity와 caller PGID를 +다시 확인한다. `INT`/`TERM` trap은 pending state만 기록하고 lifecycle checkpoint가 TTY/pipe +FD를 닫은 뒤 bounded TERM, 필요 시 KILL, direct-child wait/reap과 group 소멸 확인을 수행한다. +`/proc` reader는 정확히 LF 하나로 끝나는 단일 record와 EOF를 요구하고 CR, 추가 record, +non-canonical PID/PPID/PGID/starttime을 거부한다. PID publication 직후 STOP query보다 먼저 +direct-child identity를 pin하며, 이 최초 pin 자체가 실패하면 worker를 CONT하지 않고 freshly +published direct PID에만 positive KILL한 뒤 wait한다. +same-caller/query-failure에서는 group signal 없이 STOP된 exact direct child만 positive +TERM/CONT/KILL한다. wait/reap과 PID/PGID/identity clear는 pending signal이 stale group state를 +소비할 수 없는 하나의 transition으로 처리한다. coprocess PID가 PGID라고 가정하지 않는다. +supervisor는 initial STOP 전에 child-side INT/TERM cancellation deferral을 설치한다. cleanup +TERM을 받으면 worker를 시작하거나 종료하지 않고 ownership anchor로 남아, TERM-ignore +descendant가 있더라도 fresh authority로 group KILL과 direct wait를 완료할 수 있게 한다. + +```text +/usr/bin/sudo --non-interactive -- /usr/sbin/cryptsetup luksFormat --batch-mode --type luks2 --key-file=- LOOP +/usr/bin/sudo --non-interactive -- /usr/sbin/cryptsetup open --type luks2 --key-file=- LOOP ALLOWED_MAPPING +``` + +production helper는 outer mount부터 exact current-user `0600` regular non-symlink KDBX까지 +모든 component가 canonical non-symlink인 것, library-owned entry, canonical loop block +device, allowlisted mapping과 cached sudo를 요구한다. fixture suite는 partial/zero producer +failure의 consumer dispatch 0, xtrace 선차단, non-TTY, exact KeePassXC argv, process +argv/environment와 stdout/stderr/tmp/run 누출, real blocking-child signal reap, exclusive +synthetic lifecycle과 cleanup failure 전파를 low command-boundary fake로 검사한다. +signal fixture는 launch, PID publish, STOP/query, CONT 전후, wait/reap-clear, consumer 직전 +경계에서 `INT`/`TERM`을 결정적으로 주입하고 supervisor, process-substitution shell, +KeePassXC, `od`, `mawk`와 feasibility nested roles의 종료, caller/unrelated sentinel 생존, +consumer dispatch 0과 반복-run leak 0을 확인한다. +synthetic KDBX의 `add`와 `attachment-import`는 atomic-save inode 교체를 허용하되, 각 mutation +직후 exact path/owner/mode/non-symlink를 다시 확인하여 secure 새 inode를 baseline으로 삼는다. +read-only 단계에서는 baseline과 fixture parent identity가 바뀌면 즉시 실패한다. +고정 package가 없는 host에서는 `--execute`를 실행하거나 임의 version을 설치하지 않고 +명시적으로 SKIP한다. + +## k3s Secret 암호화 live 읽기 전용 검증 + +`k3s-secret-encryption.sh`는 k3s 서비스나 Kubernetes 리소스를 변경하지 않고 현재 +Secret 암호화 상태, 서버 annotation 일치 여부, local config 무결성과 API readiness를 +분류한다. 일반 사용자 shell에서 실행하면 제한 시간과 `sudo --non-interactive`가 적용된 +개별 read-only 명령만 root 전용 systemd 환경, datastore 증거와 credential 파일을 읽는다. +workspace의 validator나 library 자체를 root shell에서 실행하거나 source하지 않는다. +`/usr/local/bin/k3s`, `/usr/bin/systemctl`, `/usr/bin/stat` 등 허용된 절대경로의 +root-owned·non-group/other-writable system binary만 검증한 뒤 실행한다. raw status, 환경 변수 +원문, annotation, hash, active key, token, password 또는 config 내용은 사용자 terminal, +명령 인자나 handoff에 출력하지 않고 현재 사용자 프로세스의 메모리에서만 분류한다. + +validator는 비대화형 sudo만 사용하므로 실행 직전에 **같은 terminal**에서 credential을 +먼저 갱신해야 한다. 다른 terminal에서 실행한 `sudo -v`는 이 실행의 prerequisite를 +충족한다고 가정하지 않는다. + +```sh +sudo -v +``` + +credential validation이 실패하면 validator는 root-only evidence를 하나도 읽지 않고 +`sudo -v`를 같은 terminal에서 실행한 뒤 다시 시도하라는 오류로 즉시 종료한다. + +```sh +bash scripts/validate/k3s-secret-encryption.sh +bash scripts/validate/k3s-secret-encryption.sh --expect-disabled +bash scripts/validate/k3s-secret-encryption.sh --expect-transition-start +bash scripts/validate/k3s-secret-encryption.sh --expect-enabled +bash scripts/validate/k3s-secret-encryption.sh --expect-reencrypted +``` + +기대 상태 옵션은 최대 하나만 지정한다. 옵션이 없으면 inventory만 수행하며 status가 +`unsafe_transition`, `hash_mismatch`, `invalid`로 분류되면 실패한다. +모든 성공 경로는 version이 정확히 `v1.36.2+k3s1`, server가 정확히 1개, server +node가 정확히 `donghyeon-system-product-name`이고 Ready인 것을 요구한다. +`--expect-enabled`는 운영 진단용으로 `Enabled/start`와 +`Enabled/reencrypt_finished`를 모두 허용한다. Phase 4 완료 gate인 +`--expect-reencrypted`는 `Enabled/reencrypt_finished`이면서 local config, state, +server annotation hash가 모두 일치할 때만 성공한다. `--expect-transition-start`도 +status의 hash match가 확인된 exact transition만 허용한다. 기대 상태 불일치와 분류할 수 +없는 결과, API readyz 실패는 non-zero로 종료한다. + +검증된 비민감 handoff가 필요하면 caller가 먼저 제한된 임시 디렉터리를 만든다. + +```sh +VERIFY_DIR="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" +chmod 0700 "$VERIFY_DIR" +bash scripts/validate/k3s-secret-encryption.sh --expect-enabled \ + --verified-output-dir "$VERIFY_DIR" +``` + +validator는 handoff 디렉터리를 만들지 않는다. slash가 없는 nonempty suffix를 가진 +`/tmp/platform-k3s-encryption.` physical direct child 중 기존 empty, +non-symlink, 현재 사용자 소유 mode `0700` 디렉터리만 허용한다. 검증과 write는 같은 +열린 directory FD에 묶는다. +inventory와 기대 상태 검증이 성공한 뒤에만 noclobber와 `umask 077`로 정확히 +`inventory.env`, `status.sha256` 두 파일을 만들며, 두 파일은 현재 사용자 소유 regular +file mode `0600`이어야 한다. status 원문과 active key 이름은 어느 handoff에도 쓰지 +않는다. `status.sha256`은 status stdout의 공백이나 trailing newline이 아니라 `jq -cS`로 +검증·정렬·압축한 single JSON value의 UTF-8 bytes(끝 newline 없음)에 대한 SHA-256이다. +기존 파일, 추가 파일, nested/symlink parent, 다른 owner 또는 group/other 권한이 있으면 +실패한다. + +## k3s Secret 복구 증거 검증 + +`k3s-secret-encryption-restore-evidence.sh`는 다음 세 mode를 제공한다. + +```sh +bash scripts/validate/k3s-secret-encryption-restore-evidence.sh \ + --emit-result --bundle-metadata BUNDLE_METADATA_FILE --output RESULT_FILE +bash scripts/validate/k3s-secret-encryption-restore-evidence.sh \ + --record --bundle-metadata BUNDLE_METADATA_FILE --result-file RESULT_FILE +bash scripts/validate/k3s-secret-encryption-restore-evidence.sh --check +``` + +`--emit-result`는 격리 복구 host에서 current-user kube context와 live API를 확인하고 +`Enabled/reencrypt_finished`, server hash, local integrity, node Ready, version/backend와 +복구 전후 Secret object count가 모두 맞을 때만 non-sensitive result를 만든다. operator가 +default route, upstream DNS, 운영 API/datastore route와 모든 egress 차단 시험까지 완료했다고 +현재 context로 확인해야 한다. 결과에는 `destroyed`가 없다. + +`--record`는 외부 post metadata/result, 24시간 age와 운영 host의 권위 post metadata +일곱 field 전체를 비교한다. 일회용 복구 환경과 bundle 복사본 파기 후 정확한 확인 입력을 +받아야 하며 기존 evidence를 덮어쓰지 않는다. `--check`는 현재 권위 bundle-id, +version/backend, 30일 age, `destroyed=confirmed`, live re-encryption과 local integrity를 +읽기 전용으로 다시 검사한다. 2026-08-09 live 전환은 `Enabled/reencrypt_finished`, +hash·integrity·API·node 검사, post bundle 기록과 최신 marker 검증까지 통과했다. 다만 +격리 restore·파기 evidence는 아직 없으므로 `--check` 성공을 기록하거나 관측성 Phase 4 +gate를 열지 않는다. + +외부 입력은 현재 사용자 소유 regular non-symlink, mode `0600`, non-empty여야 한다. exact +allowlist parser가 duplicate/unknown/empty key, control character, malformed/trailing data, +`$(`와 backtick을 sudo 전에 거부한다. 전체 script를 sudo로 실행하지 않고 root 전용 파일의 +stat/read/install만 좁게 승격한다. result/evidence target은 같은 directory의 `0600` +temporary file을 검증한 뒤 기존 파일을 덮어쓰지 않는 atomic install로 만든다. + +전체 수동 절차와 격리 checklist는 +[k3s Secret 복구 drill](../../bootstrap/manual/k3s-secret-encryption-restore-drill.md)을 따른다. +parser와 세 mode의 fixture 회귀는 live cluster에 연결하지 않고 다음 명령에 포함된다. + +```sh +bash scripts/validate/test-k3s-secret-encryption-status.sh +``` diff --git a/scripts/validate/admin-ui-smoke.sh b/scripts/validate/admin-ui-smoke.sh new file mode 100755 index 0000000..568d568 --- /dev/null +++ b/scripts/validate/admin-ui-smoke.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +set +x + +readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)" +readonly LAN_IP=192.168.0.107 +readonly TAIL_IP=100.92.240.34 +readonly STORAGE_HOST=storage-admin.learn.hyeonworks.com +readonly DB_HOST=db-admin.learn.hyeonworks.com +readonly SMOKE_IMAGE="busybox@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0" +execute=false +run_s3=false + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'USAGE' +사용법: + bash scripts/validate/admin-ui-smoke.sh + bash scripts/validate/admin-ui-smoke.sh --execute [--run-s3] + +기본 실행은 공개 DNS, 인증서, 리소스, Secret key 이름만 읽습니다. +--execute는 LAN/Tailscale/Pod DNS와 실제 HTTPS·NetworkPolicy를 검사하는 +임시 Pod를 만들었다가 삭제합니다. --run-s3는 기존 인증 S3 회귀 검사도 +연결하며 그 스크립트의 APPLY 확인을 한 번 더 요구합니다. +USAGE +} + +while (( $# > 0 )); do + case "$1" in + --execute) execute=true; shift ;; + --run-s3) run_s3=true; shift ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; fail "지원하지 않는 인자: $1" ;; + esac +done +[[ "$run_s3" == false || "$execute" == true ]] || fail "--run-s3에는 --execute가 필요합니다" +[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요" +for cmd in curl dig jq kubectl nc openssl rg sort tr; do + command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다" +done + +for host in "$STORAGE_HOST" "$DB_HOST"; do + [[ -z "$(dig +short @1.1.1.1 A "$host" | tr -d '[:space:]')" ]] || fail "$host 공개 A가 있습니다" + [[ -z "$(dig +short @1.1.1.1 AAAA "$host" | tr -d '[:space:]')" ]] || fail "$host 공개 AAAA가 있습니다" +done + +kubectl -n object-storage get objectstore minio-aistor -o json | \ + jq -e '.status.healthStatus == "green"' >/dev/null || fail "AIStor가 green이 아닙니다" +kubectl -n platform-admin get deployment pgadmin -o json | \ + jq -e '.status.availableReplicas == 1 and .spec.strategy.type == "Recreate"' >/dev/null || \ + fail "pgAdmin Deployment가 available/Recreate 상태가 아닙니다" +kubectl -n platform-admin get pvc pgadmin -o json | \ + jq -e '.status.phase == "Bound" and .spec.resources.requests.storage == "2Gi"' >/dev/null || \ + fail "pgAdmin PVC가 2Gi Bound가 아닙니다" +kubectl get pv pgadmin-data-local-pv -o json | \ + jq -e '.spec.persistentVolumeReclaimPolicy == "Retain"' >/dev/null || fail "pgAdmin PV가 Retain이 아닙니다" + +for contract in "object-storage aistor-keycloak-oidc" "platform-admin pgadmin-keycloak-oidc"; do + set -- $contract + keys="$(kubectl -n "$1" get "secret/$2" \ + -o go-template='{{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}' | LC_ALL=C sort)" + [[ "$keys" == $'client-id\nclient-secret' ]] || fail "$1/$2 Secret 계약이 다릅니다" +done +keys="$(kubectl -n platform-admin get secret pgadmin-bootstrap \ + -o go-template='{{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}')" +[[ "$keys" == password ]] || fail "pgAdmin bootstrap Secret 계약이 다릅니다" + +config="$(kubectl -n platform-admin get configmap pgadmin-config -o jsonpath='{.data.config_local\.py}')" +printf '%s\n' "$config" | rg -q -F 'ALLOW_SAVE_PASSWORD = False' || fail "비밀번호 저장 차단이 없습니다" +printf '%s\n' "$config" | rg -q -F "'groups': ['/platform-db-admins']" || fail "pgAdmin 그룹 gate가 없습니다" +kubectl -n platform-data exec platform-postgres-1 -c postgres -- \ + psql -Atqc "SELECT count(*) FROM pg_roles WHERE rolname IN ('pgadmin','platform-dba') OR (rolsuper AND rolname NOT IN ('postgres','streaming_replica'));" \ + | rg -q '^0$' || fail "예상하지 못한 pgAdmin/DBA/superuser 역할이 있습니다" + +cert="$(openssl s_client -connect 127.0.0.1:443 -servername "$STORAGE_HOST" /dev/null | \ + openssl x509 -noout -text)" +printf '%s\n' "$cert" | rg -q "DNS:$STORAGE_HOST" || fail "admin 인증서 SAN이 없습니다" +printf '%s\n' "$cert" | rg -q "DNS:$DB_HOST" || fail "admin 인증서 SAN이 없습니다" + +printf 'READ-ONLY CHECK PASS: 공개 admin DNS 없음, 리소스·Secret 계약·인증서 정상\n' +if [[ "$execute" == false ]]; then + printf '%s\n' '--execute를 지정하지 않아 임시 Pod와 네트워크 요청은 만들지 않았습니다.' + exit 0 +fi + +[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다" +printf 'Type APPLY to run transient DNS and network smoke tests: ' +read -r answer +[[ "$answer" == APPLY ]] || fail "취소했습니다" + +for pair in "$LAN_IP $STORAGE_HOST" "$LAN_IP $DB_HOST" "$TAIL_IP $STORAGE_HOST" "$TAIL_IP $DB_HOST"; do + set -- $pair + [[ "$(dig +short "@$1" A "$2" | tail -n1)" == "$1" ]] || fail "$2 private DNS 실패" +done + +for host in "$STORAGE_HOST" "$DB_HOST"; do + code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --noproxy '*' --interface "$LAN_IP" --resolve "$host:443:$LAN_IP" "https://$host/")" + [[ "$code" == 200 || "$code" == 302 || "$code" == 303 ]] || fail "$host LAN HTTPS 실패: $code" + denied="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --noproxy '*' --resolve "$host:443:127.0.0.1" "https://$host/")" + [[ "$denied" == 403 ]] || fail "$host 허용 목록 밖 요청이 403이 아닙니다" +done + +if nc -z -w 2 "$LAN_IP" 30080 || nc -z -w 2 "$LAN_IP" 30443; then + fail "Traefik NodePort가 LAN 주소에 노출됐습니다" +fi +nc -z -w 2 127.0.0.1 30080 || fail "Traefik HTTP NodePort loopback이 닫혔습니다" + +pod="admin-ui-smoke-$(date +%H%M%S)" +cleanup_pod() { + kubectl -n platform-admin delete "pod/$pod" --ignore-not-found --wait=false >/dev/null 2>&1 || true +} +trap cleanup_pod EXIT INT TERM +kubectl apply -f - <&2 + exit 1 +} + +cleanup() { + if [[ -n "$pod_name" ]]; then + kubectl --namespace "$NAMESPACE" delete pod "$pod_name" \ + --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || true + fi +} + +usage() { + cat <<'USAGE' +Usage: + bash scripts/validate/aistor-s3-smoke.sh --execute + +Creates one restricted, short-lived AIStor Client Pod. It authenticates with +object-storage/aistor-root-configuration and performs: + bucket create -> object write -> SHA-256 read check + -> object delete -> bucket delete + +The script never prints credentials. It removes the temporary bucket and Pod. +Type APPLY only when prompted because this performs temporary S3 writes. +USAGE +} + +while (( $# > 0 )); do + case "$1" in + --execute) + execute_requested=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage >&2 + fail "unsupported argument: $1" + ;; + esac +done + +[[ "$execute_requested" == true ]] || { + usage + fail "--execute is required" +} + +for command_name in kubectl grep jq; do + command -v "$command_name" >/dev/null 2>&1 || \ + fail "${command_name} is required" +done + +[[ "$( + kubectl --namespace "$NAMESPACE" get objectstore "$OBJECTSTORE" \ + --output=jsonpath='{.status.currentState}' +)" == "Initialized" ]] || fail "ObjectStore is not Initialized" +[[ "$( + kubectl --namespace "$NAMESPACE" get objectstore "$OBJECTSTORE" \ + --output=jsonpath='{.status.healthStatus}' +)" == "green" ]] || fail "ObjectStore health is not green" +kubectl --namespace "$NAMESPACE" get secret "$ROOT_SECRET" >/dev/null + +printf '%s\n' \ + "Target: ${NAMESPACE}/${OBJECTSTORE}" \ + "Endpoint: ${S3_ENDPOINT}" \ + 'Temporary resources: one bucket, one object, and one restricted client Pod' \ + 'No credential value will be printed.' +printf 'Type APPLY to run the authenticated S3 smoke test: ' +IFS= read -r confirmation +[[ "$confirmation" == "APPLY" ]] || fail "confirmation did not match APPLY" + +pod_name="aistor-s3-smoke-$(date +%H%M%S)-$$" +trap cleanup EXIT INT TERM + +smoke_command='set -eu +. /run/aistor-root/config.env +bucket="platform-smoke-$(date +%s)" +cleanup_bucket() { mc rb --force "local/${bucket}" >/dev/null 2>&1 || true; } +trap cleanup_bucket EXIT +mc alias set local "'"$S3_ENDPOINT"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null +payload="aistor-authenticated-smoke-$(date +%s)" +expected="$(printf "%s" "$payload" | sha256sum | cut -d " " -f 1)" +mc mb "local/${bucket}" >/dev/null +printf "%s" "$payload" | mc pipe "local/${bucket}/probe.txt" >/dev/null +actual="$(mc cat "local/${bucket}/probe.txt" | sha256sum | cut -d " " -f 1)" +[ "$expected" = "$actual" ] +mc stat "local/${bucket}/probe.txt" >/dev/null +mc rm "local/${bucket}/probe.txt" >/dev/null +mc rb "local/${bucket}" >/dev/null +trap - EXIT +printf "%s\n" "'"$PASS_LINE"'"' + +overrides="$( + jq -nc --arg command "$smoke_command" --arg image "$MC_IMAGE" '{ + spec: { + automountServiceAccountToken: false, + securityContext: { + runAsNonRoot: true, + runAsUser: 1000, + runAsGroup: 1000, + fsGroup: 1000, + seccompProfile: {type: "RuntimeDefault"} + }, + containers: [{ + name: "aistor-s3-smoke", + image: $image, + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "-ec"], + args: [$command], + securityContext: { + allowPrivilegeEscalation: false, + capabilities: {drop: ["ALL"]}, + readOnlyRootFilesystem: true + }, + env: [{name: "MC_CONFIG_DIR", value: "/tmp/mc"}], + volumeMounts: [ + {name: "root-config", mountPath: "/run/aistor-root", readOnly: true}, + {name: "tmp", mountPath: "/tmp"} + ] + }], + volumes: [ + { + name: "root-config", + secret: {secretName: "'"$ROOT_SECRET"'", defaultMode: 256} + }, + {name: "tmp", emptyDir: {}} + ] + } + }' +)" + +kubectl --namespace "$NAMESPACE" run "$pod_name" \ + --image="$MC_IMAGE" \ + --restart=Never \ + --labels='platform.hyeonworks.com/aistor-client=true,app.kubernetes.io/name=aistor-s3-smoke' \ + --overrides="$overrides" + +succeeded=false +for _ in $(seq 1 90); do + phase="$( + kubectl --namespace "$NAMESPACE" get pod "$pod_name" \ + --output=jsonpath='{.status.phase}' + )" + if [[ "$phase" == "Succeeded" ]]; then + succeeded=true + break + fi + [[ "$phase" != "Failed" ]] || break + + waiting_reason="$( + kubectl --namespace "$NAMESPACE" get pod "$pod_name" \ + --output=jsonpath='{.status.containerStatuses[0].state.waiting.reason}' \ + 2>/dev/null || true + )" + case "$waiting_reason" in + CreateContainerConfigError|CrashLoopBackOff|ErrImagePull|ImagePullBackOff) + break + ;; + esac + sleep 2 +done + +if [[ "$succeeded" != true ]]; then + kubectl --namespace "$NAMESPACE" logs "$pod_name" || true + kubectl --namespace "$NAMESPACE" describe pod "$pod_name" | tail -n 60 + fail "authenticated S3 smoke test failed" +fi + +log_output="$(kubectl --namespace "$NAMESPACE" logs "$pod_name")" +grep --fixed-strings --line-regexp --quiet "$PASS_LINE" <<<"$log_output" || \ + fail "smoke Pod did not emit the exact success contract" +printf '%s\n' "$PASS_LINE" +printf '%s\n' \ + "Client image: ${MC_IMAGE}" \ + 'Temporary bucket, object, and client Pod cleanup: complete' diff --git a/scripts/validate/capture-observability-metric-inventory.sh b/scripts/validate/capture-observability-metric-inventory.sh new file mode 100644 index 0000000..38f9c2f --- /dev/null +++ b/scripts/validate/capture-observability-metric-inventory.sh @@ -0,0 +1,812 @@ +#!/usr/bin/env bash + +# Repository-owned, read-only Prometheus metric inventory capture. This file is +# sourceable so its parsers can be exercised without contacting a cluster. + +readonly INVENTORY_REJECTION_STATUS=23 +readonly INVENTORY_SCHEMA='platform-observability-metric-inventory/v1' +readonly INVENTORY_CONTEXT='default' +readonly INVENTORY_API_SERVER='https://127.0.0.1:6443' +readonly INVENTORY_PROMETHEUS_NAMESPACE='observability' +readonly INVENTORY_PROMETHEUS_NAME='observability-core-kube-pr-prometheus' +readonly INVENTORY_PROMETHEUS_SERVICE='observability-core-kube-pr-prometheus' +readonly INVENTORY_MAX_RESPONSE_BYTES=8388608 + +inventory_reject() { + printf 'REJECT: target-initial inventory precondition failed\n' >&2 + return "$INVENTORY_REJECTION_STATUS" +} + +inventory_expected_pools() { + local phase="${1-target-initial}" + [[ "$phase" == target-initial || "$phase" == post-substrate ]] || { + inventory_reject + return + } + cat <<'POOLS' +podMonitor/platform-data/platform-postgres/0 +serviceMonitor/gitea/gitea/0 +serviceMonitor/keycloak/keycloak/0 +serviceMonitor/kube-system/traefik/0 +serviceMonitor/object-storage/aistor-bucket-usage/0 +serviceMonitor/observability-agent/alloy/0 +serviceMonitor/observability-agent/node-exporter/0 +serviceMonitor/observability/grafana/0 +serviceMonitor/observability/loki/0 +serviceMonitor/observability/observability-core-kube-pr-alertmanager/0 +serviceMonitor/observability/observability-core-kube-pr-alertmanager/1 +serviceMonitor/observability/observability-core-kube-pr-apiserver/0 +serviceMonitor/observability/observability-core-kube-pr-coredns/0 +serviceMonitor/observability/observability-core-kube-pr-kubelet/0 +serviceMonitor/observability/observability-core-kube-pr-kubelet/1 +serviceMonitor/observability/observability-core-kube-pr-kubelet/2 +serviceMonitor/observability/observability-core-kube-pr-operator/0 +serviceMonitor/observability/observability-core-kube-pr-prometheus/0 +serviceMonitor/observability/observability-core-kube-pr-prometheus/1 +serviceMonitor/observability/observability-core-kube-state-metrics/0 +serviceMonitor/observability/tempo/0 +POOLS + if [[ "$phase" == post-substrate ]]; then + cat <<'POOLS' +probe/observability/platform-private-edge +probe/observability/platform-private-edge +probe/observability/platform-private-edge +probe/observability/platform-private-internal +probe/observability/platform-private-internal +probe/observability/platform-private-internal +probe/observability/platform-public-edge +probe/observability/platform-public-edge +serviceMonitor/observability/blackbox-exporter/0 +POOLS + fi +} + +post_substrate_expected_probe_targets() { + cat <<'JSON' +[ + {"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fdb-admin.learn.hyeonworks.com%2F","labels":{"instance":"https://db-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}}, + {"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fgrafana.learn.hyeonworks.com%2F","labels":{"instance":"https://grafana.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}}, + {"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fstorage-admin.learn.hyeonworks.com%2F","labels":{"instance":"https://storage-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}}, + {"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=db-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fpgadmin.platform-admin.svc.cluster.local%2Fmisc%2Fping","labels":{"instance":"http://pgadmin.platform-admin.svc.cluster.local/misc/ping","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}}, + {"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=grafana.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fgrafana.observability.svc.cluster.local%2Fapi%2Fhealth","labels":{"instance":"http://grafana.observability.svc.cluster.local/api/health","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}}, + {"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=storage-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fminio-aistor-console.object-storage.svc.cluster.local%3A9090%2F","labels":{"instance":"http://minio-aistor-console.object-storage.svc.cluster.local:9090/","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}}, + {"scrapePool":"probe/observability/platform-public-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fgit.learn.hyeonworks.com%2Fapi%2Fhealthz","labels":{"instance":"https://git.learn.hyeonworks.com/api/healthz","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}}, + {"scrapePool":"probe/observability/platform-public-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fid.learn.hyeonworks.com%2Frealms%2Fhyeonworks%2F.well-known%2Fopenid-configuration","labels":{"instance":"https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}} +] +JSON +} + +sanitize_scrape_url() { + local url="${1-}" mode="${2-strict}" remainder authority + if [[ -z "$url" || "$url" == *[$' \t\r\n']* || "$url" == *'#'* ]]; then + inventory_reject + return + fi + if [[ "$mode" == strip-query ]]; then + [[ "$url" == *'?'* ]] || { inventory_reject; return; } + url="${url%%\?*}" + elif [[ "$mode" != strict || "$url" == *'?'* ]]; then + inventory_reject + return + fi + case "$url" in + http://*|https://*) ;; + *) inventory_reject; return ;; + esac + remainder="${url#*://}" + authority="${remainder%%/*}" + if [[ -z "$authority" || "$authority" == *'@'* ]]; then + inventory_reject + return + fi + printf '%s\n' "$url" +} + +validate_targets_json() { + local path="${1--}" phase="${2-target-initial}" expected_json post_probes='[]' + expected_json="$(inventory_expected_pools "$phase" | jq -Rsc 'split("\n") | map(select(length > 0)) | sort')" || { + inventory_reject + return + } + if [[ "$phase" == post-substrate ]]; then + post_probes="$(post_substrate_expected_probe_targets | jq -cS 'sort_by(.scrapePool,.scrapeUrl)')" || { + inventory_reject + return + } + fi + if ! jq -e --arg phase "$phase" --argjson expected "$expected_json" --argjson postProbes "$post_probes" ' + type == "object" and .status == "success" and + (.data | type == "object") and + (.data.activeTargets | type == "array") and + (.data.droppedTargets | type == "array") and + ((.data.activeTargets | map(.scrapePool) | sort) == $expected) and + all(.data.activeTargets[]; + type == "object" and (.scrapePool | type == "string") and + (.scrapeUrl | type == "string") and (.labels | type == "object") and + (.scrapeUrl | test("^https?://[^/?#@[:space:]]+(?:/[^#[:space:]]*)?$")) and + ($phase == "post-substrate" or + (.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not)) and + (.health == "up") and (.lastError == "")) and + all(.data.droppedTargets[]?; + (.scrapePool | type == "string") and + ($phase == "post-substrate" or + (.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not))) and + ([.data.activeTargets[] | + select(.scrapePool == "serviceMonitor/object-storage/aistor-bucket-usage/0") | + select(.labels.job == "minio-aistor" and + .labels.namespace == "object-storage" and + .labels.service == "minio" and + .labels.endpoint == "http-minio" and + .labels.container == "minio")] | length == 1) and + ($phase == "target-initial" or ( + ([.data.activeTargets[] | + select(.scrapePool | startswith("probe/observability/")) | + {scrapePool,scrapeUrl,labels}] | sort_by(.scrapePool,.scrapeUrl)) == $postProbes and + ([.data.activeTargets[] | + select(.scrapePool == "serviceMonitor/observability/blackbox-exporter/0") | + select(.labels.job == "blackbox-exporter" and + .labels.namespace == "observability" and + .labels.service == "blackbox-exporter" and + .labels.endpoint == "http" and + .labels.container == "blackbox-exporter" and + (.labels.pod | test("^blackbox-exporter-[a-z0-9]+-[a-z0-9]+$")) and + (.labels.instance | test("^[0-9.]+:9115$")) and + (.scrapeUrl | test("^http://[0-9.]+:9115/metrics$")))] | length == 1) + )) + ' "$path" >/dev/null 2>&1; then + inventory_reject + return + fi +} + +validate_dropped_targets_json() { + local path="${1--}" phase="${2-target-initial}" + if ! jq -e --arg phase "$phase" ' + type == "object" and .status == "success" and + (.data | type == "object") and + (.data.activeTargets | type == "array") and + (.data.droppedTargets | type == "array") and + all(.data.droppedTargets[]?; + (.scrapePool | type == "string") and + (if $phase == "target-initial" + then (.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not) + else (.scrapePool | test("(?:^|/)probe(?:/|$)"; "i") | not) + end)) + ' "$path" >/dev/null 2>&1; then + inventory_reject + return + fi +} + +validate_probe_series_phase() { + local path="${1--}" phase="${2-target-initial}" + if ! jq -e --arg phase "$phase" ' + type == "object" and .status == "success" and (.data | type == "array") and + (if $phase == "target-initial" then (.data | length) == 0 + elif $phase == "post-substrate" then + (.data | length) > 0 and + all(.data[]; + type == "object" and (."__name__" | type == "string") and + (."__name__" | startswith("probe_")) and + .namespace == "observability") + else false end) + ' "$path" >/dev/null 2>&1; then + inventory_reject + return + fi +} + +extract_metric_names() { + local path="${1--}" names + if ! names="$(jq -er ' + if type == "object" and .status == "success" then + if (.data | type == "object") and (.data | length > 0) and + all(.data | keys[]; test("^[a-zA-Z_:][a-zA-Z0-9_:]*$")) + then .data | keys | sort | .[] + elif (.data | type == "array") and (.data | length > 0) and + all(.data[]; + type == "object" and (.target | type == "object") and + (.metric | type == "string") and + (.metric | test("^[a-zA-Z_:][a-zA-Z0-9_:]*$")) and + (.type | type == "string") and (.help | type == "string") and + (.unit | type == "string")) + then [.data[].metric] | unique | sort | .[] + else error("invalid metadata") end + else error("invalid metadata") end + ' "$path" 2>/dev/null)"; then + inventory_reject + return + fi + printf '%s\n' "$names" +} + +extract_recording_rule_names() { + local path="${1--}" names + if ! names="$(jq -er ' + if type == "object" and .status == "success" and + (.data | type == "object") and (.data.groups | type == "array") and + all(.data.groups[]; (.rules | type == "array")) + then [.data.groups[].rules[] | select(.type == "recording") | .name] | + if all(.[]; type == "string" and test("^[a-zA-Z_:][a-zA-Z0-9_:]*$")) + then unique | sort | join("\n") else error("invalid recording rule") end + else error("invalid rules") end + ' "$path" 2>/dev/null)"; then + inventory_reject + return + fi + [[ -z "$names" ]] || printf '%s\n' "$names" +} + +reject_recording_rule_collisions() { + local rules_path="${1-}" metadata_path="${2-}" rules metrics name + if ! rules="$(extract_recording_rule_names "$rules_path")" || + ! metrics="$(extract_metric_names "$metadata_path")"; then + inventory_reject + return + fi + reject_metric_name_collisions "$rules" "$metrics" +} + +reject_metric_name_collisions() { + local rules="${1-}" metrics="${2-}" name + while IFS= read -r name; do + [[ -z "$name" ]] && continue + if grep -Fqx -- "$name" <<<"$rules"; then + inventory_reject + return + fi + done <<<"$metrics" +} + +validate_metric_names_for_pool() { + local metrics="${1-}" pool="${2-}" phase="${3-target-initial}" + if [[ "$phase" == post-substrate && "$pool" == probe/observability/* ]]; then + grep -Fqx probe_success <<<"$metrics" || { inventory_reject; return; } + elif grep -Eq '^probe_' <<<"$metrics"; then + inventory_reject + return + fi +} + +extract_series_label_names() { + local path="${1--}" result + if ! result="$(jq -ceS ' + if type == "object" and .status == "success" and + (.data | type == "array") and (.data | length > 0) and + all(.data[]; type == "object") and + all([.data[] | keys[]][]; test("^[a-zA-Z_][a-zA-Z0-9_./-]*$")) + then [.data[] | keys[]] | unique | sort + else error("invalid series") end + ' "$path" 2>/dev/null)"; then + inventory_reject + return + fi + printf '%s\n' "$result" +} + +prometheus_selector_for_labels() { + local labels_json="${1-}" + jq -cer ' + if type == "object" and length > 0 and + all(keys[]; test("^[a-zA-Z_][a-zA-Z0-9_./-]*$")) and + all(.[]; type == "string" and (explode | all(. >= 32 and . != 127))) + then "{" + (to_entries | sort_by(.key) | + map((if (.key | test("^[a-zA-Z_][a-zA-Z0-9_]*$")) + then .key else (.key | @json) end) + "=" + (.value | @json)) | join(",")) + "}" + else error("invalid target labels") end + ' <<<"$labels_json" 2>/dev/null || inventory_reject +} + +inventory_provenance() { + case "${1-}" in + podMonitor/platform-data/platform-postgres/0|\ + serviceMonitor/gitea/gitea/0|\ + serviceMonitor/keycloak/keycloak/0|\ + serviceMonitor/kube-system/traefik/0|\ + serviceMonitor/object-storage/aistor-bucket-usage/0) + printf '%s\n' 'task6-owned' + ;; + serviceMonitor/observability/grafana/0) + printf '%s\n' 'preexisting-verify-only' + ;; + serviceMonitor/observability/blackbox-exporter/0|probe/observability/platform-*) + printf '%s\n' 'substrate-owned' + ;; + *) + printf '%s\n' 'core' + ;; + esac +} + +validate_output_directory() { + local directory="${1-}" canonical uid mode current component + if [[ -z "$directory" || "$directory" != /* || "$directory" == *[$'\t\r\n']* || + ! -d "$directory" || -L "$directory" ]]; then + inventory_reject + return + fi + if ! canonical="$(realpath -e -- "$directory" 2>/dev/null)" || [[ "$canonical" != "$directory" ]]; then + inventory_reject + return + fi + current='' + IFS='/' read -r -a components <<<"${directory#/}" + for component in "${components[@]}"; do + [[ -n "$component" && "$component" != '.' && "$component" != '..' ]] || { + inventory_reject + return + } + current="$current/$component" + if [[ -L "$current" ]]; then + inventory_reject + return + fi + done + if ! uid="$(stat -c %u -- "$directory" 2>/dev/null)" || + ! mode="$(stat -c %a -- "$directory" 2>/dev/null)" || + [[ "$uid" != "$(id -u)" || "$mode" != 700 ]]; then + inventory_reject + return + fi + if [[ -n "$(find "$directory" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]]; then + inventory_reject + return + fi +} + +inventory_kubectl() { + case "${1-}" in + config) + case "${2-}" in current-context|view) ;; *) inventory_reject; return ;; esac + ;; + get) ;; + auth) + [[ "${2-}" == can-i ]] || { inventory_reject; return; } + ;; + *) + inventory_reject + return + ;; + esac + timeout --signal=TERM --kill-after=2s 15s \ + kubectl --request-timeout=10s "$@" 2>/dev/null +} + +inventory_process_snapshot() { + local pid="${1-}" stat_line remainder state start_time + [[ "$pid" =~ ^[1-9][0-9]*$ && -r "/proc/$pid/stat" ]] || return 1 + IFS= read -r stat_line <"/proc/$pid/stat" || return 1 + remainder="${stat_line##*) }" + state="${remainder%% *}" + start_time="$(awk '{print $20}' <<<"$remainder")" || return 1 + [[ "$state" =~ ^[A-Z]$ && "$start_time" =~ ^[0-9]+$ ]] || return 1 + printf '%s:%s\n' "$state" "$start_time" +} + +inventory_port_forward_is_owned() { + local pid="${INVENTORY_PORT_FORWARD_PID:-}" expected="${INVENTORY_PORT_FORWARD_START:-}" snapshot + [[ "$pid" =~ ^[1-9][0-9]*$ && "$expected" =~ ^[0-9]+$ ]] || return 1 + snapshot="$(inventory_process_snapshot "$pid")" || return 1 + [[ "${snapshot#*:}" == "$expected" ]] +} + +inventory_stop_port_forward() { + local pid="${INVENTORY_PORT_FORWARD_PID:-}" snapshot state attempt + [[ -n "$pid" ]] || return 0 + if ! inventory_port_forward_is_owned; then + INVENTORY_PORT_FORWARD_PID='' + INVENTORY_PORT_FORWARD_START='' + return 0 + fi + + kill -TERM "$pid" 2>/dev/null || true + for ((attempt = 0; attempt < 40; attempt++)); do + if ! inventory_port_forward_is_owned; then break; fi + snapshot="$(inventory_process_snapshot "$pid")" || break + state="${snapshot%%:*}" + [[ "$state" == Z || "$state" == X ]] && break + sleep 0.025 + done + + if inventory_port_forward_is_owned; then + snapshot="$(inventory_process_snapshot "$pid")" || snapshot='' + state="${snapshot%%:*}" + if [[ "$state" != Z && "$state" != X ]]; then + kill -KILL "$pid" 2>/dev/null || true + fi + fi + for ((attempt = 0; attempt < 40; attempt++)); do + if ! inventory_port_forward_is_owned; then break; fi + snapshot="$(inventory_process_snapshot "$pid")" || break + state="${snapshot%%:*}" + [[ "$state" == Z || "$state" == X ]] && break + sleep 0.025 + done + if inventory_port_forward_is_owned; then + snapshot="$(inventory_process_snapshot "$pid")" || snapshot='' + state="${snapshot%%:*}" + if [[ "$state" != Z && "$state" != X ]]; then return 1; fi + fi + INVENTORY_PORT_FORWARD_PID='' + INVENTORY_PORT_FORWARD_START='' +} + +build_inventory_json() { + local target_rows="${1-}" phase="${2-}" timestamp="${3-}" + [[ -n "$target_rows" && -f "$target_rows" && ! -L "$target_rows" && + -n "$phase" && -n "$timestamp" ]] || { inventory_reject; return; } + jq -cnS --slurpfile targets "$target_rows" \ + --arg schema "$INVENTORY_SCHEMA" --arg phase "$phase" --arg captured_at_utc "$timestamp" \ + --arg context "$INVENTORY_CONTEXT" --arg api_server "$INVENTORY_API_SERVER" \ + --arg prometheus "$INVENTORY_PROMETHEUS_NAMESPACE/$INVENTORY_PROMETHEUS_NAME" ' + {schema:$schema,phase:$phase,captured_at_utc:$captured_at_utc,context:$context, + api_server:$api_server,prometheus:$prometheus,preexisting_verify_only:["grafana"], + targets:($targets | sort_by(.scrape_pool))} + ' +} + +validate_post_substrate_query_body() { + local path="${1--}" kind="${2-}" expected expected_tls now + expected="$(post_substrate_expected_probe_targets | jq -cS '[.[].labels] | sort_by(.job,.instance)')" || { + inventory_reject + return + } + expected_tls="$(post_substrate_expected_probe_targets | jq -cS '[.[].labels | select(.instance | startswith("https://"))] | sort_by(.job,.instance)')" || { + inventory_reject + return + } + now="$(date -u +%s)" || { inventory_reject; return; } + if ! jq -e --arg kind "$kind" --argjson expected "$expected" \ + --argjson expectedTls "$expected_tls" --argjson now "$now" ' + type == "object" and .status == "success" and .data.resultType == "vector" and + (.data.result | type == "array") and + (if $kind == "tls" then + ([.data.result[].metric | del(.__name__)] | sort_by(.job,.instance)) == $expectedTls and + all(.data.result[]; (.value[1] | tonumber) > $now) + else + ([.data.result[].metric | del(.__name__)] | sort_by(.job,.instance)) == $expected and + (if $kind == "success" then all(.data.result[]; .value[1] == "1") + elif $kind == "duration" then all(.data.result[]; (.value[1] | tonumber) >= 0) + elif $kind == "status" then + all(.data.result[]; + if .metric.job == "blackbox-private-edge" then .value[1] == "403" + else .value[1] == "200" end) + else false end) + end) + ' "$path" >/dev/null 2>&1; then + inventory_reject + return + fi +} + +inventory_validate_post_substrate_metrics() { + local body spec + for spec in \ + 'success:probe_success{namespace="observability"}' \ + 'status:probe_http_status_code{namespace="observability"}' \ + 'duration:probe_duration_seconds{namespace="observability"}' \ + 'tls:probe_ssl_earliest_cert_expiry{namespace="observability"}'; do + if ! inventory_prometheus_json body '/api/v1/query' --get \ + --data-urlencode "query=${spec#*:}" || + ! printf '%s' "$body" | validate_post_substrate_query_body - "${spec%%:*}"; then + inventory_reject + return + fi + body='' + done +} + +inventory_capture_main() { + local output_dir phase context api prometheus_json pods_json permission + local port='' attempt start_attempt forward_ready=0 timestamp snapshot + local targets_raw='' dropped_raw='' rules_raw='' probe_raw='' metadata_raw='' series_raw='' recording_names='' + local row pool scrape_url labels selector metric_names metric + local metric_labels series_selector series_count metrics_path metrics_json provenance + local target_rows counter=0 inventory hash actual_files + + if (($# != 5)) || [[ "$1" != --execute || "$2" != --phase || + ("$3" != target-initial && "$3" != post-substrate) || + "$4" != --output-dir || -z "$5" ]]; then + inventory_reject + return + fi + phase="$3" + output_dir="$5" + + for permission in jq kubectl curl timeout realpath stat find mktemp sha256sum sync grep head; do + command -v "$permission" >/dev/null 2>&1 || { inventory_reject; return; } + done + validate_output_directory "$output_dir" || return + + INVENTORY_WORK_DIR="$(mktemp -d /tmp/platform-observability-inventory-capture.XXXXXX)" || { + inventory_reject + return + } + chmod 0700 "$INVENTORY_WORK_DIR" || { inventory_reject; return; } + INVENTORY_STAGE_JSON='' + INVENTORY_STAGE_SHA='' + INVENTORY_OUTPUT_DIR="$output_dir" + INVENTORY_CREATED_JSON=0 + INVENTORY_CREATED_SHA=0 + INVENTORY_JSON_ID='' + INVENTORY_SHA_ID='' + INVENTORY_SUCCESS=0 + INVENTORY_PORT_FORWARD_PID='' + INVENTORY_PORT_FORWARD_START='' + + trap inventory_capture_cleanup EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + + if ! context="$(inventory_kubectl config current-context)" || [[ "$context" != "$INVENTORY_CONTEXT" ]]; then + inventory_reject + return + fi + if ! api="$(inventory_kubectl config view --minify -o json | jq -er '.clusters | if length == 1 then .[0].cluster.server else error("cluster count") end' 2>/dev/null)" || + [[ "$api" != "$INVENTORY_API_SERVER" ]]; then + inventory_reject + return + fi + if ! inventory_kubectl get --raw=/readyz >/dev/null; then inventory_reject; return; fi + + for permission in \ + 'get pods' \ + 'get services' \ + 'get prometheuses.monitoring.coreos.com' \ + 'create pods/portforward'; do + if ! inventory_kubectl auth can-i $permission --namespace "$INVENTORY_PROMETHEUS_NAMESPACE" | grep -Fxq yes; then + inventory_reject + return + fi + done + + if ! prometheus_json="$(inventory_kubectl get prometheus "$INVENTORY_PROMETHEUS_NAME" \ + --namespace "$INVENTORY_PROMETHEUS_NAMESPACE" -o json)" || + ! jq -e --arg namespace "$INVENTORY_PROMETHEUS_NAMESPACE" --arg name "$INVENTORY_PROMETHEUS_NAME" ' + .kind == "Prometheus" and .metadata.namespace == $namespace and .metadata.name == $name + ' <<<"$prometheus_json" >/dev/null 2>&1; then + inventory_reject + return + fi + if ! pods_json="$(inventory_kubectl get pods --namespace "$INVENTORY_PROMETHEUS_NAMESPACE" \ + -l "prometheus=$INVENTORY_PROMETHEUS_NAME" -o json)" || + ! jq -e ' + (.items | length == 1) and .items[0].status.phase == "Running" and + any(.items[0].status.conditions[]; .type == "Ready" and .status == "True") and + any(.items[0].status.containerStatuses[]; .name == "prometheus" and .ready == true) + ' <<<"$pods_json" >/dev/null 2>&1; then + inventory_reject + return + fi + + : >"$INVENTORY_WORK_DIR/port-forward.log" + for ((start_attempt = 0; start_attempt < 10; start_attempt++)); do + port=$((20000 + ((RANDOM + $$ + start_attempt * 7919) % 40000))) + kubectl --request-timeout=10s -n "$INVENTORY_PROMETHEUS_NAMESPACE" \ + port-forward --address 127.0.0.1 \ + "service/$INVENTORY_PROMETHEUS_SERVICE" "$port:9090" \ + >/dev/null 2>"$INVENTORY_WORK_DIR/port-forward.log" & + INVENTORY_PORT_FORWARD_PID=$! + snapshot="$(inventory_process_snapshot "$INVENTORY_PORT_FORWARD_PID")" || { + inventory_reject + return + } + INVENTORY_PORT_FORWARD_START="${snapshot#*:}" + INVENTORY_PROMETHEUS_BASE="http://127.0.0.1:$port" + for ((attempt = 0; attempt < 40; attempt++)); do + if ! kill -0 "$INVENTORY_PORT_FORWARD_PID" 2>/dev/null; then break; fi + if inventory_prometheus_ready; then + forward_ready=1 + break 2 + fi + sleep 0.05 + done + inventory_stop_port_forward || { inventory_reject; return; } + done + if [[ "$forward_ready" != 1 || -z "$INVENTORY_PORT_FORWARD_PID" ]]; then + inventory_reject + return + fi + + if ! inventory_prometheus_json targets_raw '/api/v1/targets' --get --data 'state=active' || + ! printf '%s' "$targets_raw" | validate_targets_json - "$phase" || + ! inventory_prometheus_json dropped_raw '/api/v1/targets' --get --data 'state=dropped' || + ! printf '%s' "$dropped_raw" | validate_dropped_targets_json - "$phase" || + ! inventory_prometheus_json rules_raw '/api/v1/rules' || + ! recording_names="$(printf '%s' "$rules_raw" | extract_recording_rule_names -)" || + ! inventory_prometheus_json probe_raw '/api/v1/series' --get \ + --data-urlencode 'match[]={__name__=~"probe_.+"}' || + ! printf '%s' "$probe_raw" | validate_probe_series_phase - "$phase"; then + inventory_reject + return + fi + if [[ "$phase" == post-substrate ]] && ! inventory_validate_post_substrate_metrics; then + inventory_reject + return + fi + dropped_raw='' + rules_raw='' + probe_raw='' + + target_rows="$INVENTORY_WORK_DIR/target-rows.jsonl" + : >"$target_rows" + while IFS= read -r row; do + pool="$(jq -er '.scrapePool' <<<"$row" 2>/dev/null)" || { inventory_reject; return; } + scrape_url="$(jq -er '.scrapeUrl' <<<"$row" 2>/dev/null)" || { inventory_reject; return; } + if [[ "$phase" == post-substrate && "$pool" == probe/observability/* ]]; then + scrape_url="$(sanitize_scrape_url "$scrape_url" strip-query)" || return + else + scrape_url="$(sanitize_scrape_url "$scrape_url")" || return + fi + labels="$(jq -ceS '.labels' <<<"$row" 2>/dev/null)" || { inventory_reject; return; } + selector="$(prometheus_selector_for_labels "$labels")" || return + counter=$((counter + 1)) + if ! inventory_prometheus_json metadata_raw '/api/v1/targets/metadata' --get \ + --data-urlencode "match_target=$selector" --data 'limit=10000'; then + inventory_reject + return + fi + metric_names="$(printf '%s' "$metadata_raw" | extract_metric_names -)" || return + metadata_raw='' + reject_metric_name_collisions "$recording_names" "$metric_names" || return + validate_metric_names_for_pool "$metric_names" "$pool" "$phase" || return + + metrics_path="$INVENTORY_WORK_DIR/metrics-$counter.jsonl" + : >"$metrics_path" + while IFS= read -r metric; do + series_selector="$(prometheus_selector_for_labels "$( + jq -ce --arg metric "$metric" '. + {"__name__":$metric}' <<<"$labels" 2>/dev/null + )")" || return + if ! inventory_prometheus_json series_raw '/api/v1/series' --get \ + --data-urlencode "match[]=$series_selector"; then + inventory_reject + return + fi + series_count="$(printf '%s' "$series_raw" | jq -er ' + if type == "object" and .status == "success" and + (.data | type == "array") and all(.data[]; type == "object") + then .data | length else error("invalid series") end + ' 2>/dev/null)" || { inventory_reject; return; } + if (( series_count == 0 )); then + series_raw='' + continue + fi + metric_labels="$(printf '%s' "$series_raw" | extract_series_label_names -)" || return + series_raw='' + jq -cnS --arg name "$metric" --argjson label_names "$metric_labels" \ + '{name:$name,label_names:$label_names}' >>"$metrics_path" || { inventory_reject; return; } + done <<<"$metric_names" + [[ -s "$metrics_path" ]] || { inventory_reject; return; } + metrics_json="$(jq -csS 'sort_by(.name)' "$metrics_path" 2>/dev/null)" || { inventory_reject; return; } + provenance="$(inventory_provenance "$pool")" + jq -cnS --arg scrape_pool "$pool" --arg scrape_url "$scrape_url" \ + --arg provenance "$provenance" --argjson metrics "$metrics_json" \ + '{scrape_pool:$scrape_pool,scrape_url:$scrape_url,health:"up",last_error:"", + provenance:$provenance,metrics:$metrics}' >>"$target_rows" || { inventory_reject; return; } + done < <(printf '%s' "$targets_raw" | jq -c '.data.activeTargets | sort_by(.scrapePool)[]') + targets_raw='' + + timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" || { inventory_reject; return; } + inventory="$(build_inventory_json "$target_rows" "$phase" "$timestamp")" || return + + validate_output_directory "$output_dir" || return + INVENTORY_STAGE_JSON="$(mktemp "$output_dir/.inventory.json.tmp.XXXXXX")" || { inventory_reject; return; } + chmod 0600 "$INVENTORY_STAGE_JSON" || { inventory_reject; return; } + printf '%s\n' "$inventory" >"$INVENTORY_STAGE_JSON" || { inventory_reject; return; } + sync -f "$INVENTORY_STAGE_JSON" >/dev/null 2>&1 || { inventory_reject; return; } + INVENTORY_JSON_ID="$(stat -c %d:%i -- "$INVENTORY_STAGE_JSON")" || { inventory_reject; return; } + hash="$(sha256sum -- "$INVENTORY_STAGE_JSON" | awk '{print $1}')" || { inventory_reject; return; } + INVENTORY_STAGE_SHA="$(mktemp "$output_dir/.inventory.sha256.tmp.XXXXXX")" || { inventory_reject; return; } + chmod 0600 "$INVENTORY_STAGE_SHA" || { inventory_reject; return; } + printf '%s inventory.json\n' "$hash" >"$INVENTORY_STAGE_SHA" || { inventory_reject; return; } + sync -f "$INVENTORY_STAGE_SHA" >/dev/null 2>&1 || { inventory_reject; return; } + INVENTORY_SHA_ID="$(stat -c %d:%i -- "$INVENTORY_STAGE_SHA")" || { inventory_reject; return; } + + if ! ln -- "$INVENTORY_STAGE_JSON" "$output_dir/inventory.json" 2>/dev/null; then inventory_reject; return; fi + INVENTORY_CREATED_JSON=1 + rm -- "$INVENTORY_STAGE_JSON" || { inventory_reject; return; } + INVENTORY_STAGE_JSON='' + if ! ln -- "$INVENTORY_STAGE_SHA" "$output_dir/inventory.sha256" 2>/dev/null; then inventory_reject; return; fi + INVENTORY_CREATED_SHA=1 + rm -- "$INVENTORY_STAGE_SHA" || { inventory_reject; return; } + INVENTORY_STAGE_SHA='' + sync -f "$output_dir/inventory.json" >/dev/null 2>&1 || { inventory_reject; return; } + sync -f "$output_dir/inventory.sha256" >/dev/null 2>&1 || { inventory_reject; return; } + sync -f "$output_dir" >/dev/null 2>&1 || { inventory_reject; return; } + + actual_files="$(find "$output_dir" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" + if [[ "$actual_files" != $'inventory.json\ninventory.sha256' ]] || + [[ ! -f "$output_dir/inventory.json" || -L "$output_dir/inventory.json" ]] || + [[ ! -f "$output_dir/inventory.sha256" || -L "$output_dir/inventory.sha256" ]] || + [[ "$(stat -c %a -- "$output_dir/inventory.json")" != 600 ]] || + [[ "$(stat -c %a -- "$output_dir/inventory.sha256")" != 600 ]]; then + inventory_reject + return + fi + INVENTORY_SUCCESS=1 + printf 'OK: %s inventory captured\n' "$phase" +} + +inventory_prometheus_json() { + local destination_name="$1" api_path="$2" response + local LC_ALL=C + local -n destination="$destination_name" + shift 2 + if ! response="$( + set -o pipefail + curl --disable --silent --show-error --fail --connect-timeout 3 --max-time 10 \ + --max-filesize "$INVENTORY_MAX_RESPONSE_BYTES" --output - \ + "$@" "$INVENTORY_PROMETHEUS_BASE$api_path" 2>/dev/null | + LC_ALL=C head -c "$((INVENTORY_MAX_RESPONSE_BYTES + 1))" + pipeline_status=$? + printf '\034' + exit "$pipeline_status" + )"; then + response='' + inventory_reject + return + fi + [[ "$response" == *$'\034' ]] || { response=''; inventory_reject; return; } + response="${response%?}" + if (( ${#response} > INVENTORY_MAX_RESPONSE_BYTES )); then + response='' + inventory_reject + return + fi + destination="$response" +} + +inventory_prometheus_ready() { + curl --disable --silent --show-error --fail --connect-timeout 1 --max-time 1 \ + --max-filesize 1024 --output /dev/null "$INVENTORY_PROMETHEUS_BASE/-/ready" 2>/dev/null +} + +inventory_capture_cleanup() { + local rc=$? + trap - EXIT HUP INT TERM + if [[ -n "${INVENTORY_PORT_FORWARD_PID:-}" ]]; then + inventory_stop_port_forward || true + fi + if [[ -n "${INVENTORY_STAGE_JSON:-}" && -f "$INVENTORY_STAGE_JSON" && ! -L "$INVENTORY_STAGE_JSON" ]]; then + rm -- "$INVENTORY_STAGE_JSON" + fi + if [[ -n "${INVENTORY_STAGE_SHA:-}" && -f "$INVENTORY_STAGE_SHA" && ! -L "$INVENTORY_STAGE_SHA" ]]; then + rm -- "$INVENTORY_STAGE_SHA" + fi + if [[ "${INVENTORY_SUCCESS:-0}" != 1 ]]; then + if [[ "${INVENTORY_CREATED_SHA:-0}" == 1 ]]; then + inventory_remove_created_file "${INVENTORY_OUTPUT_DIR:-}" inventory.sha256 "${INVENTORY_SHA_ID:-}" + fi + if [[ "${INVENTORY_CREATED_JSON:-0}" == 1 ]]; then + inventory_remove_created_file "${INVENTORY_OUTPUT_DIR:-}" inventory.json "${INVENTORY_JSON_ID:-}" + fi + if [[ -n "${INVENTORY_OUTPUT_DIR:-}" && -d "$INVENTORY_OUTPUT_DIR" && ! -L "$INVENTORY_OUTPUT_DIR" ]]; then + sync -f "$INVENTORY_OUTPUT_DIR" >/dev/null 2>&1 || true + fi + fi + if [[ -n "${INVENTORY_WORK_DIR:-}" && -d "$INVENTORY_WORK_DIR" && ! -L "$INVENTORY_WORK_DIR" ]]; then + case "$INVENTORY_WORK_DIR" in + /tmp/platform-observability-inventory-capture.*) rm -rf -- "$INVENTORY_WORK_DIR" ;; + esac + fi + exit "$rc" +} + +inventory_remove_created_file() { + local directory="${1-}" basename="${2-}" expected_id="${3-}" path actual_id canonical + [[ -n "$directory" && -n "$basename" && -n "$expected_id" && -d "$directory" && ! -L "$directory" ]] || return 0 + canonical="$(realpath -e -- "$directory" 2>/dev/null)" || return 0 + [[ "$canonical" == "$directory" ]] || return 0 + path="$directory/$basename" + [[ -f "$path" && ! -L "$path" ]] || return 0 + actual_id="$(stat -c %d:%i -- "$path" 2>/dev/null)" || return 0 + [[ "$actual_id" == "$expected_id" ]] || return 0 + rm -- "$path" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + set -Eeuo pipefail + umask 077 + inventory_capture_main "$@" +fi diff --git a/scripts/validate/k3s-local-recovery-feasibility.sh b/scripts/validate/k3s-local-recovery-feasibility.sh new file mode 100644 index 0000000..56ac440 --- /dev/null +++ b/scripts/validate/k3s-local-recovery-feasibility.sh @@ -0,0 +1,431 @@ +#!/usr/bin/env bash + +PATH=/usr/sbin:/usr/bin:/sbin:/bin +export PATH +LC_ALL=C +export LC_ALL +umask 077 +set -Eeuo pipefail + +_k3slrf_initial_guard() { + local candidate_euid="${1-}" shell_flags="${2-}" + [[ "$candidate_euid" =~ ^[0-9]+$ ]] || return 1 + (( candidate_euid != 0 )) || return 1 + [[ "$shell_flags" != *x* ]] +} + +if ! _k3slrf_initial_guard "$EUID" "$-"; then + printf 'Local recovery feasibility refused\n' >&2 + if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then return 1; else exit 1; fi +fi + +readonly K3SLRF_KEEPASSXC='/usr/bin/keepassxc-cli' +readonly K3SLRF_CRYPTSETUP='/usr/sbin/cryptsetup' +readonly K3SLRF_KEEPASS_PACKAGE='keepassxc' +readonly K3SLRF_KEEPASS_VERSION='2.7.6+dfsg.1-1build3' +readonly K3SLRF_CRYPTSETUP_PACKAGE='cryptsetup-bin' +readonly K3SLRF_CRYPTSETUP_VERSION='2:2.7.0-1ubuntu4.2' +readonly K3SLRF_SCRIPT_DIR="$(cd -P -- "${BASH_SOURCE[0]%/*}" && pwd -P)" +readonly K3SLRF_REPOSITORY_ROOT="$(cd -P -- "${K3SLRF_SCRIPT_DIR}/../.." && pwd -P)" +readonly K3SLRF_LIBRARY_PATH="${K3SLRF_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery.sh" +# shellcheck source=../lib/k3s-local-recovery.sh +builtin source -- "$K3SLRF_LIBRARY_PATH" || { + printf 'Local recovery feasibility refused\n' >&2 + if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then return 1; else exit 1; fi +} + +_k3slrf_usage() { + printf 'Usage: bash scripts/validate/k3s-local-recovery-feasibility.sh [--execute]\n' +} + +_k3slrf_fail() { + printf 'Local recovery feasibility failed\n' >&2 + return 1 +} + +_k3slrf_require_interactive_stdin() { + [[ -t 0 ]] +} + +_k3slrf_require_cached_sudo() { + _k3slr_cached_sudo_is_available +} + +_k3slrf_package_version_is_exact() { + local package="$1" expected="$2" output + output="$( + _k3slr_command /usr/bin/dpkg-query --show --showformat='${db:Status-Abbrev}|${Version}\n' -- "$package" 2>/dev/null + printf '\034%s' "$?" + )" || return 1 + [[ "$output" == "ii |${expected}"$'\n'$'\0340' ]] +} + +_k3slrf_verify_prerequisites() { + _k3slr_trusted_root_executable /usr/bin/dpkg-query || return 1 + _k3slr_trusted_root_executable /usr/bin/mktemp || return 1 + _k3slr_trusted_root_executable /usr/bin/rm || return 1 + _k3slr_trusted_root_executable /usr/bin/cmp || return 1 + _k3slr_trusted_root_executable /usr/bin/mawk || return 1 + _k3slr_trusted_root_executable /usr/bin/od || return 1 + _k3slr_trusted_root_executable /usr/bin/ps || return 1 + _k3slr_trusted_root_executable /usr/bin/sleep || return 1 + _k3slr_trusted_root_executable /usr/bin/sha256sum || return 1 + _k3slr_trusted_root_executable "$K3SLRF_KEEPASSXC" || return 1 + _k3slr_trusted_root_executable "$K3SLRF_CRYPTSETUP" || return 1 + _k3slrf_package_version_is_exact "$K3SLRF_KEEPASS_PACKAGE" "$K3SLRF_KEEPASS_VERSION" || return 1 + _k3slrf_package_version_is_exact "$K3SLRF_CRYPTSETUP_PACKAGE" "$K3SLRF_CRYPTSETUP_VERSION" +} + +_k3slrf_create_fixture_dir() { + local destination_name="$1" identity_name="$2" owned_name="$3" created physical metadata uid + created="$(_k3slr_command /usr/bin/mktemp -d /tmp/k3slr-feasibility.XXXXXX)" || return 1 + printf -v "$destination_name" '%s' "$created" + printf -v "$owned_name" '%s' true + [[ "$created" =~ ^/tmp/k3slr-feasibility\.[A-Za-z0-9]+$ ]] || return 1 + _k3slr_command /usr/bin/test -d "$created" || return 1 + _k3slr_command /usr/bin/test ! -L "$created" || return 1 + physical="$(cd -P -- "$created" && pwd -P)" || return 1 + [[ "$physical" == "$created" ]] || return 1 + _k3slr_capture_one_line uid /usr/bin/id -u || return 1 + _k3slr_capture_one_line metadata /usr/bin/stat --format='%u|%a|%F|%d:%i' -- "$created" || return 1 + [[ "$metadata" == "${uid}|700|directory|"* ]] || return 1 + printf -v "$identity_name" '%s' "${metadata##*|}" +} + +_k3slrf_cleanup_fixture_dir() { + local fixture_dir="$1" expected_identity="$2" metadata uid + [[ "$fixture_dir" =~ ^/tmp/k3slr-feasibility\.[A-Za-z0-9]+$ ]] || return 1 + _k3slr_command /usr/bin/test -d "$fixture_dir" || return 1 + _k3slr_command /usr/bin/test ! -L "$fixture_dir" || return 1 + _k3slr_capture_one_line uid /usr/bin/id -u || return 1 + _k3slr_capture_one_line metadata /usr/bin/stat --format='%u|%a|%F|%d:%i' -- "$fixture_dir" 2>/dev/null || return 1 + [[ "$metadata" == "${uid}|700|directory|"* ]] || return 1 + [[ -z "$expected_identity" || "${metadata##*|}" == "$expected_identity" ]] || return 1 + _k3slr_command /usr/bin/rm --recursive --one-file-system -- "$fixture_dir" +} + +_k3slrf_synthetic_master_once_stdout() { + printf '%s' 'k3slr-package-feasibility-master-v1' | + _k3slr_command /usr/bin/sha256sum | + _k3slr_command /usr/bin/mawk '{ print $1 }' +} + +_k3slrf_synthetic_master_twice_stdout() { + _k3slrf_synthetic_master_once_stdout + _k3slrf_synthetic_master_once_stdout +} + +_k3slrf_validate_generated_password_stdin() { + _k3slr_command /usr/bin/od -An -v -tx1 | + _k3slr_command /usr/bin/mawk ' + { + for (i=1; i<=NF; i++) { + n++; byte=$i + if (n == 41) { if (byte != "0a") bad=1 } + else if (byte ~ /^3[0-9]$/) digit=1 + else if (byte ~ /^(4[1-9a-f]|5[0-9a])$/) upper=1 + else if (byte ~ /^(6[1-9a-f]|7[0-9a])$/) lower=1 + else bad=1 + } + } + END { exit !(n == 41 && !bad && lower && upper && digit) } + ' +} + +_k3slrf_file_identity() { + local destination_name="$1" path="$2" metadata uid + _k3slr_command /usr/bin/test -f "$path" || return 1 + _k3slr_command /usr/bin/test ! -L "$path" || return 1 + _k3slr_capture_one_line uid /usr/bin/id -u || return 1 + _k3slr_capture_one_line metadata /usr/bin/stat --format='%u|%a|%F|%d:%i' -- "$path" || return 1 + [[ "$metadata" == "${uid}|600|regular file|"* || + "$metadata" == "${uid}|600|regular empty file|"* ]] || return 1 + printf -v "$destination_name" '%s' "${metadata##*|}" +} + +_k3slrf_file_identity_matches() { + local current='' + _k3slrf_file_identity current "$1" || return 1 + [[ "$current" == "$2" ]] +} + +_k3slrf_directory_identity() { + local destination_name="$1" path="$2" physical metadata uid + [[ "$path" =~ ^/tmp/k3slr-feasibility\.[A-Za-z0-9]+$ ]] || return 1 + _k3slr_command /usr/bin/test -d "$path" || return 1 + _k3slr_command /usr/bin/test ! -L "$path" || return 1 + physical="$(cd -P -- "$path" && pwd -P)" || return 1 + [[ "$physical" == "$path" ]] || return 1 + _k3slr_capture_one_line uid /usr/bin/id -u || return 1 + _k3slr_capture_one_line metadata /usr/bin/stat --format='%u|%a|%F|%d:%i' -- "$path" || return 1 + [[ "$metadata" == "${uid}|700|directory|"* ]] || return 1 + printf -v "$destination_name" '%s' "${metadata##*|}" +} + +_k3slrf_directory_identity_matches() { + local current='' + _k3slrf_directory_identity current "$1" || return 1 + [[ "$current" == "$2" ]] +} + +_k3slrf_run_tracked() { + local rc caller_group='' current_identity='' stopped_identity='' + _k3slr_process_group_for_pid caller_group "$BASHPID" || return 1 + K3SLRF_ACTIVE_CHILD_STATE='launching' + K3SLRF_LAST_CLEANUP_OK=false + _k3slr_lifecycle_boundary feasibility-launch-before + _k3slrf_pending_checkpoint || return $? + set -m + ( + tracked_cancel='' + trap '[[ "$tracked_cancel" == TERM ]] || tracked_cancel=INT' INT + trap 'tracked_cancel=TERM' TERM + kill -STOP "$BASHPID" + tracked_worker_rc=143 + if [[ -z "$tracked_cancel" ]]; then + if "$@"; then tracked_worker_rc=0; else tracked_worker_rc=$?; fi + fi + while [[ -n "$tracked_cancel" ]]; do + _k3slr_command /usr/bin/sleep 0.01 || : + done + exit "$tracked_worker_rc" + ) & + K3SLRF_ACTIVE_CHILD_PID=$! + K3SLRF_ACTIVE_CHILD_STATE='published' + _k3slr_lifecycle_boundary feasibility-pid-published "$K3SLRF_ACTIVE_CHILD_PID" + if ! _k3slr_process_identity K3SLRF_ACTIVE_CHILD_IDENTITY "$K3SLRF_ACTIVE_CHILD_PID"; then + set +m + _k3slrf_cleanup_active_child || return 1 + if [[ -n "$K3SLRF_PENDING_SIGNAL" ]]; then return "$K3SLRF_PENDING_STATUS"; fi + return 1 + fi + if ! _k3slr_wait_for_stopped_group K3SLRF_ACTIVE_CHILD_PGID stopped_identity "$K3SLRF_ACTIVE_CHILD_PID" || + [[ "$stopped_identity" != "$K3SLRF_ACTIVE_CHILD_IDENTITY" ]]; then + set +m + _k3slrf_cleanup_active_child || return 1 + if [[ -n "$K3SLRF_PENDING_SIGNAL" ]]; then return "$K3SLRF_PENDING_STATUS"; fi + return 1 + fi + K3SLRF_ACTIVE_CHILD_STATE='stopped_pinned' + _k3slr_lifecycle_boundary feasibility-stop-query-complete + _k3slrf_pending_checkpoint || { rc=$?; set +m; return "$rc"; } + if [[ "$K3SLRF_ACTIVE_CHILD_PGID" == "$caller_group" ]]; then + set +m + _k3slrf_cleanup_active_child || return 1 + return 1 + fi + set +m + _k3slr_lifecycle_boundary feasibility-cont-before + _k3slrf_pending_checkpoint || return $? + _k3slr_continue_owned_child "$K3SLRF_ACTIVE_CHILD_PID" "$K3SLRF_ACTIVE_CHILD_PGID" \ + "$K3SLRF_ACTIVE_CHILD_IDENTITY" || return 1 + K3SLRF_ACTIVE_CHILD_STATE='continued' + _k3slr_lifecycle_boundary feasibility-cont-after + _k3slrf_pending_checkpoint || return $? + K3SLRF_ACTIVE_CHILD_STATE='waiting' + if _k3slr_wait_child "$K3SLRF_ACTIVE_CHILD_PID"; then rc=0; else rc=$?; fi + _k3slr_lifecycle_boundary feasibility-wait-reaped-before-clear + if _k3slr_process_identity current_identity "$K3SLRF_ACTIVE_CHILD_PID" 2>/dev/null && + [[ "$current_identity" == "$K3SLRF_ACTIVE_CHILD_IDENTITY" ]]; then + _k3slrf_cleanup_active_child || return 1 + fi + K3SLRF_ACTIVE_CHILD_STATE='reaped' K3SLRF_ACTIVE_CHILD_PID='' \ + K3SLRF_ACTIVE_CHILD_PGID='' K3SLRF_ACTIVE_CHILD_IDENTITY='' + K3SLRF_LAST_CLEANUP_OK=true + _k3slr_lifecycle_boundary feasibility-state-cleared + if [[ -n "$K3SLRF_PENDING_SIGNAL" ]]; then + rc="$K3SLRF_PENDING_STATUS" + fi + return "$rc" +} + +_k3slrf_db_create() { + _k3slrf_synthetic_master_twice_stdout | + _k3slr_command "$K3SLRF_KEEPASSXC" db-create --quiet --set-password "$1" >/dev/null 2>&1 +} + +_k3slrf_entry_add() { + _k3slrf_synthetic_master_once_stdout | + _k3slr_command "$K3SLRF_KEEPASSXC" add --quiet --generate --length 40 \ + --lower --upper --numeric --every-group "$1" "$K3SLR_KEEPASS_ENTRY" >/dev/null 2>&1 +} + +_k3slrf_password_check() { + _k3slrf_synthetic_master_once_stdout | + _k3slr_command "$K3SLRF_KEEPASSXC" show --show-protected \ + --attributes Password "$1" "$K3SLR_KEEPASS_ENTRY" 2>/dev/null | + _k3slrf_validate_generated_password_stdin +} + +_k3slrf_attachment_listing_stdout() { + (( $# == 1 )) || return 1 + _k3slrf_synthetic_master_once_stdout | + _k3slr_command "$K3SLRF_KEEPASSXC" show --quiet --attributes Title \ + --show-attachments "$1" "$K3SLR_KEEPASS_ENTRY" +} + +_k3slrf_attachment_state_is() { + local database="${1-}" fixture_dir="${2-}" expected_state="${3-}" + local capture_base='' record='' state='' + (( $# == 3 )) || return 1 + [[ "$expected_state" == absent || "$expected_state" == present ]] || return 1 + capture_base="${fixture_dir}/keepass-listing.${BASHPID}" + _k3slr_capture_keepass_attachment_listing record \ + "${capture_base}.stdout" "${capture_base}.stderr" \ + _k3slrf_attachment_listing_stdout "$database" || return 1 + _k3slr_parse_keepass_attachment_listing "$record" fixture.bin state || return 1 + [[ "$state" == "$expected_state" ]] +} + +_k3slrf_attachment_import() { + _k3slrf_synthetic_master_once_stdout | + _k3slr_command "$K3SLRF_KEEPASSXC" attachment-import --quiet "$1" "$K3SLR_KEEPASS_ENTRY" \ + fixture.bin "$2" >/dev/null 2>&1 +} + +_k3slrf_attachment_export() { + _k3slrf_synthetic_master_once_stdout | + _k3slr_command "$K3SLRF_KEEPASSXC" attachment-export --quiet "$1" "$K3SLR_KEEPASS_ENTRY" \ + fixture.bin "$2" >/dev/null 2>&1 +} + +_k3slrf_run_synthetic() { + local fixture_dir="$1" database input_file output_file database_identity input_identity output_identity fixture_identity export_fd + database="${fixture_dir}/synthetic.kdbx" + input_file="${fixture_dir}/input.bin" + output_file="${fixture_dir}/output.bin" + [[ ! -e "$database" && ! -L "$database" && ! -e "$input_file" && ! -L "$input_file" && + ! -e "$output_file" && ! -L "$output_file" ]] || return 1 + _k3slrf_directory_identity fixture_identity "$fixture_dir" || return 1 + ( set -o noclobber; printf '%s\n' 'k3slr-attachment-roundtrip-v1' >"$input_file" ) || return 1 + _k3slrf_file_identity input_identity "$input_file" || return 1 + _k3slrf_run_tracked _k3slrf_db_create "$database" || return 1 + _k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1 + _k3slrf_file_identity database_identity "$database" || return 1 + _k3slrf_run_tracked _k3slrf_entry_add "$database" || return 1 + _k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1 + _k3slrf_file_identity database_identity "$database" || return 1 + _k3slrf_run_tracked _k3slrf_password_check "$database" || return 1 + _k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1 + _k3slrf_file_identity_matches "$database" "$database_identity" || return 1 + _k3slrf_file_identity_matches "$input_file" "$input_identity" || return 1 + _k3slrf_run_tracked _k3slrf_attachment_state_is "$database" "$fixture_dir" absent || return 1 + _k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1 + _k3slrf_file_identity_matches "$database" "$database_identity" || return 1 + _k3slrf_file_identity_matches "$input_file" "$input_identity" || return 1 + _k3slrf_run_tracked _k3slrf_attachment_import "$database" "$input_file" || return 1 + _k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1 + _k3slrf_file_identity database_identity "$database" || return 1 + _k3slrf_run_tracked _k3slrf_attachment_state_is "$database" "$fixture_dir" present || return 1 + _k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1 + _k3slrf_file_identity_matches "$database" "$database_identity" || return 1 + set -o noclobber + if ! exec {export_fd}>"$output_file"; then + set +o noclobber + return 1 + fi + set +o noclobber + _k3slrf_file_identity output_identity "$output_file" || return 1 + _k3slrf_run_tracked _k3slrf_attachment_export "$database" "/proc/${BASHPID}/fd/${export_fd}" || return 1 + exec {export_fd}>&- + _k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1 + _k3slrf_file_identity_matches "$database" "$database_identity" || return 1 + _k3slrf_file_identity_matches "$output_file" "$output_identity" || return 1 + _k3slr_command /usr/bin/cmp --silent -- "$input_file" "$output_file" +} + +k3slr_local_recovery_feasibility_main() ( + local execute=false fixture_dir='' fixture_identity='' fixture_owned=false + local K3SLRF_ACTIVE_CHILD_PID='' K3SLRF_ACTIVE_CHILD_PGID='' K3SLRF_ACTIVE_CHILD_IDENTITY='' + local K3SLRF_ACTIVE_CHILD_STATE='empty' K3SLRF_PENDING_SIGNAL='' K3SLRF_PENDING_STATUS='' + local K3SLRF_LAST_CLEANUP_OK=false + case "$#" in + 0) ;; + 1) [[ "$1" == --execute ]] || { _k3slrf_usage >&2; return 2; }; execute=true ;; + *) _k3slrf_usage >&2; return 2 ;; + esac + + if "$execute"; then + _k3slrf_require_interactive_stdin || { _k3slrf_fail; return 1; } + fi + _k3slrf_verify_prerequisites || { _k3slrf_fail; return 1; } + if ! "$execute"; then + printf 'Local recovery feasibility prerequisites: pass\n' + return 0 + fi + _k3slrf_require_cached_sudo || { _k3slrf_fail; return 1; } + + _k3slrf_cleanup_active_child() { + local cleanup_rc=0 + if [[ -n "$K3SLRF_ACTIVE_CHILD_PID" ]]; then + if [[ "$K3SLRF_ACTIVE_CHILD_STATE" == published ]]; then + _k3slr_terminate_published_child "$K3SLRF_ACTIVE_CHILD_PID" || cleanup_rc=$? + elif [[ -n "$K3SLRF_ACTIVE_CHILD_IDENTITY" ]]; then + _k3slr_terminate_and_reap "$K3SLRF_ACTIVE_CHILD_PID" "$K3SLRF_ACTIVE_CHILD_PGID" \ + "$K3SLRF_ACTIVE_CHILD_IDENTITY" || cleanup_rc=$? + else + cleanup_rc=1 + fi + if (( cleanup_rc == 0 )); then + K3SLRF_ACTIVE_CHILD_PID='' + K3SLRF_ACTIVE_CHILD_PGID='' + K3SLRF_ACTIVE_CHILD_IDENTITY='' + K3SLRF_ACTIVE_CHILD_STATE='reaped' + K3SLRF_LAST_CLEANUP_OK=true + fi + fi + return "$cleanup_rc" + } + _k3slrf_signal() { + if [[ "$1" == TERM || "$K3SLRF_PENDING_SIGNAL" != TERM ]]; then + K3SLRF_PENDING_SIGNAL="$1" + K3SLRF_PENDING_STATUS="$2" + fi + } + _k3slrf_pending_checkpoint() { + local signal_status cleanup_rc=0 + [[ -n "$K3SLRF_PENDING_SIGNAL" ]] || return 0 + signal_status="$K3SLRF_PENDING_STATUS" + if [[ -n "$K3SLRF_ACTIVE_CHILD_PID" ]]; then + [[ -n "$K3SLRF_ACTIVE_CHILD_IDENTITY" ]] || return 1 + _k3slrf_cleanup_active_child || cleanup_rc=$? + else + K3SLRF_LAST_CLEANUP_OK=true + fi + (( cleanup_rc == 0 )) || return 1 + return "$signal_status" + } + _k3slrf_exit_cleanup() { + local original_status="$1" cleanup_rc=0 + trap - EXIT + _k3slrf_cleanup_active_child || cleanup_rc=$? + if "$fixture_owned"; then + _k3slrf_cleanup_fixture_dir "$fixture_dir" "$fixture_identity" >/dev/null 2>&1 || cleanup_rc=1 + fi + (( cleanup_rc == 0 )) || exit 1 + exit "$original_status" + } + trap '_k3slrf_exit_cleanup "$?"' EXIT + trap '_k3slrf_signal INT 130' INT + trap '_k3slrf_signal TERM 143' TERM + _k3slrf_create_fixture_dir fixture_dir fixture_identity fixture_owned || { _k3slrf_fail; return 1; } + if ! _k3slrf_run_synthetic "$fixture_dir"; then + if [[ -n "$K3SLRF_PENDING_STATUS" && -z "$K3SLRF_ACTIVE_CHILD_PID" && + "$K3SLRF_LAST_CLEANUP_OK" == true ]]; then + _k3slrf_cleanup_fixture_dir "$fixture_dir" "$fixture_identity" || return 1 + fixture_owned=false + trap - EXIT INT TERM + return "$K3SLRF_PENDING_STATUS" + fi + _k3slrf_fail + return 1 + fi + _k3slrf_cleanup_fixture_dir "$fixture_dir" "$fixture_identity" || { _k3slrf_fail; return 1; } + fixture_owned=false + trap - EXIT INT TERM + printf 'Local recovery feasibility: pass\n' +) + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + k3slr_local_recovery_feasibility_main "$@" +fi diff --git a/scripts/validate/k3s-local-recovery.sh b/scripts/validate/k3s-local-recovery.sh new file mode 100644 index 0000000..fec719f --- /dev/null +++ b/scripts/validate/k3s-local-recovery.sh @@ -0,0 +1,718 @@ +#!/usr/bin/env bash + +PATH=/usr/sbin:/usr/bin:/sbin:/bin +export PATH +LC_ALL=C +export LC_ALL +umask 077 +set -Eeuo pipefail +shopt -s extglob + +_k3slrv_initial_guard() { + local candidate_euid="${1-}" shell_flags="${2-}" + [[ "$candidate_euid" =~ ^[0-9]+$ ]] || return 1 + (( candidate_euid != 0 )) || return 1 + [[ "$shell_flags" != *x* ]] +} + +if ! _k3slrv_initial_guard "$EUID" "$-"; then + printf 'Recovery validation refused\n' >&2 + if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then return 1; else exit 1; fi +fi + +readonly K3SLRV_SECURE_PATH='/usr/sbin:/usr/bin:/sbin:/bin' +readonly K3SLRV_TRUSTED_BINARIES=( + /usr/bin/findmnt /usr/bin/lsblk /usr/bin/stat /usr/bin/readlink + /usr/sbin/blkid /usr/bin/ntfs-3g.probe /usr/sbin/smartctl + /usr/sbin/losetup /usr/sbin/cryptsetup /usr/bin/du + /usr/bin/sha256sum /usr/bin/test /usr/bin/sudo /usr/bin/mawk +) +readonly K3SLRV_SCRIPT_DIR="$(cd -P -- "${BASH_SOURCE[0]%/*}" && pwd -P)" +readonly K3SLRV_REPOSITORY_ROOT="$(cd -P -- "${K3SLRV_SCRIPT_DIR}/../.." && pwd -P)" +readonly K3SLRV_LIBRARY_PATH="${K3SLRV_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery.sh" +readonly K3SLRV_CONTRACT_PATH="${K3SLRV_REPOSITORY_ROOT}/infrastructure/security/k3s/local-recovery.env" + +_k3slrv_usage() { + printf 'Usage: bash scripts/validate/k3s-local-recovery.sh [--expect-device-ready|--expect-closed|--expect-open] [--check-latest-bundle]\n' +} + +_k3slrv_fail() { + printf 'Recovery validation failed\n' >&2 + return 1 +} + +_k3slrv_exec() { + local scope="$1" + shift + case "$scope" in + user) "$@" ;; + root) /usr/bin/sudo --non-interactive -- "$@" ;; + *) return 1 ;; + esac +} + +# Bash command substitution normally discards every trailing newline. Append a +# non-newline sentinel inside the substitution, then remove only that sentinel +# so record parsers can distinguish one record from a record plus blank lines. +_k3slrv_capture() { + local destination="$1" captured command_rc sentinel=$'\036' + shift + if captured="$( + set +e + "$@" + command_rc=$? + printf '\036' + exit "$command_rc" + )"; then + command_rc=0 + else + command_rc=$? + fi + [[ "$captured" == *"$sentinel" ]] || return 125 + captured="${captured%"$sentinel"}" + [[ "$captured" != *"$sentinel"* ]] || return 125 + printf -v "$destination" '%s' "$captured" + return "$command_rc" +} + +_k3slrv_normalize_one_record() { + local destination="$1" value="$2" + [[ -n "$value" && "$value" != *$'\r'* ]] || return 1 + value="${value%$'\n'}" + [[ -n "$value" && "$value" != *$'\n'* ]] || return 1 + printf -v "$destination" '%s' "$value" +} + +_k3slrv_verify_binary() { + local binary="$1" metadata uid mode kind group_digit other_digit + [[ "$binary" == /* ]] || return 1 + _k3slrv_exec user /usr/bin/test -f "$binary" >/dev/null 2>&1 || return 1 + _k3slrv_exec user /usr/bin/test ! -L "$binary" >/dev/null 2>&1 || return 1 + _k3slrv_capture metadata _k3slrv_exec user /usr/bin/stat --format='%u|%a|%F' -- "$binary" 2>/dev/null || return 1 + _k3slrv_normalize_one_record metadata "$metadata" || return 1 + IFS='|' read -r uid mode kind <<<"$metadata" + [[ "$uid" == 0 && "$mode" =~ ^[0-7]{3,4}$ && "$kind" == 'regular file' ]] || return 1 + group_digit="${mode: -2:1}" + other_digit="${mode: -1}" + (( (10#$group_digit & 2) == 0 && (10#$other_digit & 2) == 0 )) +} + +_k3slrv_verify_trusted_binaries() { + local binary + [[ "$PATH" == "$K3SLRV_SECURE_PATH" ]] || return 1 + for binary in "${K3SLRV_TRUSTED_BINARIES[@]}"; do + _k3slrv_verify_binary "$binary" || return 1 + done +} + +_k3slrv_require_cached_sudo() { + _k3slrv_exec user /usr/bin/sudo --non-interactive --validate >/dev/null 2>&1 +} + +_k3slrv_root() { + _k3slrv_exec root "$@" +} + +_k3slrv_trim() { + local value="$1" + value="${value##+([[:space:]])}" + value="${value%%+([[:space:]])}" + printf '%s\n' "$value" +} + +_k3slrv_one_line() { + local output + _k3slrv_capture output _k3slrv_exec user "$@" 2>/dev/null || return 1 + _k3slrv_normalize_one_record output "$output" || return 1 + _k3slrv_trim "$output" +} + +_k3slrv_one_line_root() { + local output + _k3slrv_capture output _k3slrv_root "$@" 2>/dev/null || return 1 + _k3slrv_normalize_one_record output "$output" || return 1 + _k3slrv_trim "$output" +} + +_k3slrv_blkid() { + _k3slrv_one_line_root /usr/sbin/blkid --output value --match-tag "$1" -- "$2" +} + +_k3slrv_lsblk() { + _k3slrv_one_line /usr/bin/lsblk --noheadings --nodeps --paths --output "$1" -- "$2" +} + +_k3slrv_smart_evidence() { + local disk="$1" output line id raw + local -a fields=() lines=() + local health='' reallocated='' pending='' uncorrectable='' + _k3slrv_capture output _k3slrv_root /usr/sbin/smartctl -H -A -- "$disk" 2>/dev/null || return 1 + mapfile -t lines < <(printf '%s' "$output") + (( ${#lines[@]} > 0 )) || return 1 + for line in "${lines[@]}"; do + [[ "$line" != *$'\r'* ]] || return 1 + if [[ "$line" =~ ^SMART[[:space:]]overall-health[[:space:]]self-assessment[[:space:]]test[[:space:]]result:[[:space:]]*PASSED[[:space:]]*$ ]]; then + [[ -z "$health" ]] || return 1 + health=PASSED + continue + fi + [[ "$line" =~ ^[[:space:]]*(5|197|198)[[:space:]] ]] || continue + id="${BASH_REMATCH[1]}" + read -r -a fields <<<"$line" + (( ${#fields[@]} >= 2 )) || return 1 + raw="${fields[${#fields[@]} - 1]}" + [[ "$raw" =~ ^[0-9]+$ ]] || return 1 + case "$id" in + 5) [[ -z "$reallocated" ]] || return 1; reallocated="$raw" ;; + 197) [[ -z "$pending" ]] || return 1; pending="$raw" ;; + 198) [[ -z "$uncorrectable" ]] || return 1; uncorrectable="$raw" ;; + esac + done + [[ -n "$health" && -n "$reallocated" && -n "$pending" && -n "$uncorrectable" ]] || return 1 + printf 'smart_health=%s\nsmart_reallocated=%s\nsmart_pending=%s\nsmart_uncorrectable=%s\n' \ + "$health" "$reallocated" "$pending" "$uncorrectable" +} + +_k3slrv_findmnt_line() { + local output + _k3slrv_capture output _k3slrv_root /usr/bin/findmnt --noheadings --raw --output SOURCE,FSTYPE,OPTIONS,ID,MAJ:MIN --mountpoint "$1" 2>/dev/null || return 1 + _k3slrv_normalize_one_record output "$output" || return 1 + output="$(_k3slrv_trim "$output")" + [[ -n "$output" ]] || return 1 + printf '%s\n' "$output" +} + +_k3slrv_findmnt_optional() { + local output rc=0 + _k3slrv_capture output _k3slrv_root /usr/bin/findmnt --noheadings --raw --output SOURCE,FSTYPE,OPTIONS,ID,MAJ:MIN --mountpoint "$1" 2>/dev/null || rc=$? + case "$rc" in + 0) + _k3slrv_normalize_one_record output "$output" || return 1 + output="$(_k3slrv_trim "$output")" + [[ -n "$output" ]] || return 1 + printf '%s\n' "$output" + ;; + 1) return 0 ;; + *) return 1 ;; + esac +} + +_k3slrv_findmnt_source_optional() { + local output rc=0 + _k3slrv_capture output _k3slrv_root /usr/bin/findmnt --noheadings --raw --output TARGET --source "$1" 2>/dev/null || rc=$? + case "$rc" in + 0) + _k3slrv_normalize_one_record output "$output" || return 1 + output="$(_k3slrv_trim "$output")" + [[ -n "$output" ]] || return 1 + printf '%s\n' "$output" + ;; + 1) return 0 ;; + *) return 1 ;; + esac +} + +_k3slrv_collect_device_evidence() { + local mode="$1" recovery_partition recovery_disk recovery_parent k3s_partition k3s_disk k3s_parent + local recovery_model recovery_serial recovery_wwn recovery_mm k3s_model k3s_serial k3s_wwn k3s_mm + local recovery_uuid recovery_partuuid recovery_type k3s_uuid k3s_partuuid k3s_type outer_mount source_mounts + recovery_partition="$(_k3slrv_one_line /usr/bin/readlink -f -- "$K3SLR_RECOVERY_PARTITION_BY_ID")" || return 1 + recovery_disk="$(_k3slrv_one_line /usr/bin/readlink -f -- "$K3SLR_RECOVERY_DISK_BY_ID")" || return 1 + recovery_parent="$(_k3slrv_lsblk PKNAME "$recovery_partition")" || return 1 + k3s_partition="$(_k3slrv_one_line /usr/bin/readlink -f -- "$K3SLR_K3S_PARTITION_BY_ID")" || return 1 + k3s_disk="$(_k3slrv_one_line /usr/bin/readlink -f -- "$K3SLR_K3S_DISK_BY_ID")" || return 1 + k3s_parent="$(_k3slrv_lsblk PKNAME "$k3s_partition")" || return 1 + recovery_model="$(_k3slrv_lsblk MODEL "$recovery_disk")" || return 1 + recovery_serial="$(_k3slrv_lsblk SERIAL "$recovery_disk")" || return 1 + recovery_wwn="$(_k3slrv_lsblk WWN "$recovery_disk")" || return 1 + recovery_mm="$(_k3slrv_lsblk MAJ:MIN "$recovery_disk")" || return 1 + k3s_model="$(_k3slrv_lsblk MODEL "$k3s_disk")" || return 1 + k3s_serial="$(_k3slrv_lsblk SERIAL "$k3s_disk")" || return 1 + k3s_wwn="$(_k3slrv_lsblk WWN "$k3s_disk")" || return 1 + k3s_mm="$(_k3slrv_lsblk MAJ:MIN "$k3s_disk")" || return 1 + recovery_uuid="$(_k3slrv_blkid UUID "$recovery_partition")" || return 1 + recovery_partuuid="$(_k3slrv_blkid PARTUUID "$recovery_partition")" || return 1 + recovery_type="$(_k3slrv_blkid TYPE "$recovery_partition")" || return 1 + k3s_uuid="$(_k3slrv_blkid UUID "$k3s_partition")" || return 1 + k3s_partuuid="$(_k3slrv_blkid PARTUUID "$k3s_partition")" || return 1 + k3s_type="$(_k3slrv_blkid TYPE "$k3s_partition")" || return 1 + printf 'recovery_partition=%s\nrecovery_disk=%s\nrecovery_partition_parent=%s\n' \ + "$recovery_partition" "$recovery_disk" "$recovery_parent" + printf 'recovery_fs_uuid=%s\nrecovery_partuuid=%s\nrecovery_type=%s\n' \ + "$recovery_uuid" "$recovery_partuuid" "$recovery_type" + printf 'recovery_model=%s\nrecovery_serial=%s\nrecovery_wwn=%s\nrecovery_major_minor=%s\n' \ + "$recovery_model" "$recovery_serial" "$recovery_wwn" "$recovery_mm" + printf 'k3s_partition=%s\nk3s_disk=%s\nk3s_partition_parent=%s\n' \ + "$k3s_partition" "$k3s_disk" "$k3s_parent" + printf 'k3s_fs_uuid=%s\nk3s_partuuid=%s\nk3s_type=%s\n' \ + "$k3s_uuid" "$k3s_partuuid" "$k3s_type" + printf 'k3s_model=%s\nk3s_serial=%s\nk3s_wwn=%s\nk3s_major_minor=%s\n' \ + "$k3s_model" "$k3s_serial" "$k3s_wwn" "$k3s_mm" + _k3slrv_smart_evidence "$recovery_disk" || return 1 + if [[ "$mode" == open ]]; then + printf 'ntfs_probe=not_applicable\n' + else + outer_mount="$(_k3slrv_findmnt_optional "$K3SLR_OUTER_MOUNT")" || return 1 + [[ -z "$outer_mount" ]] || return 1 + source_mounts="$(_k3slrv_findmnt_source_optional "$recovery_partition")" || return 1 + [[ -z "$source_mounts" ]] || return 1 + if _k3slrv_root /usr/bin/ntfs-3g.probe --readwrite "$recovery_partition" >/dev/null 2>&1; then + printf 'ntfs_probe=pass\n' + else + printf 'ntfs_probe=fail\n' + fi + fi +} + +_k3slrv_mapping_evidence() { + local name="$1" prefix="$2" output type='' device='' rc=0 + local line + local -a lines=() + _k3slrv_capture output _k3slrv_root /usr/sbin/cryptsetup status -- "$name" 2>/dev/null || rc=$? + if (( rc == 4 )); then + printf '%s_present=0\n' "$prefix" + return 0 + fi + (( rc == 0 )) || return 1 + mapfile -t lines < <(printf '%s' "$output") + (( ${#lines[@]} > 0 )) || return 1 + for line in "${lines[@]}"; do + [[ "$line" != *$'\r'* ]] || return 1 + line="$(_k3slrv_trim "$line")" + case "$line" in + type:*) [[ -z "$type" ]] || return 1; type="$(_k3slrv_trim "${line#type:}")" ;; + device:*) [[ -z "$device" ]] || return 1; device="$(_k3slrv_trim "${line#device:}")" ;; + esac + done + [[ -n "$type" && -n "$device" ]] || return 1 + printf '%s_present=1\n%s_type=%s\n%s_device=%s\n' "$prefix" "$prefix" "$type" "$prefix" "$device" +} + +_k3slrv_pin_container_chain() { + local container="$1" current="$K3SLR_OUTER_MOUNT" canonical metadata component index=0 snapshot='' + local -a components=() + IFS='/' read -r -a components <<<"$K3SLR_CONTAINER_RELATIVE" + for component in '' "${components[@]}"; do + if (( index > 0 )); then current="${current}/${component}"; fi + _k3slrv_root /usr/bin/test ! -L "$current" >/dev/null 2>&1 || return 1 + canonical="$(_k3slrv_one_line_root /usr/bin/readlink -e -- "$current")" || return 1 + [[ "$canonical" == "$current" ]] || return 1 + metadata="$(_k3slrv_one_line_root /usr/bin/stat --format='%d:%i|%F' -- "$current")" || return 1 + if (( index < ${#components[@]} )); then + [[ "$metadata" == *'|directory' ]] || return 1 + else + [[ "$current" == "$container" && "$metadata" == *'|regular file' ]] || return 1 + fi + snapshot+="${metadata%%|*}," + index=$((index + 1)) + done + printf '%s\n' "${snapshot%,}" +} + +_k3slrv_mapping_value() { + local evidence="$1" key="$2" line found='' + while IFS= read -r line; do + [[ "$line" == "$key="* ]] || continue + [[ -z "$found" ]] || return 1 + found="${line#*=}" + done <<<"$evidence" + [[ -n "$found" ]] || return 1 + printf '%s\n' "$found" +} + +_k3slrv_collect_state_evidence() { + local mode="$1" outer='' inner='' loops='' outer_source outer_fstype outer_options outer_mount_id outer_major_minor + local inner_source inner_fstype inner_options inner_mount_id inner_major_minor container metadata kind uid gid file_mode size blocks allocated inode container_device + local inner_type inner_label recovery_partition source_mounts chain_snapshot loop_device loop_back_inode loop_back_major_minor + local loop_major_minor loop_offset loop_sizelimit mapping_snapshot proof_snapshot mapping_device mapping_loop_major_minor mapping_major_minor + local outer_after inner_after source_after chain_after loops_after mapping_after proof_after mapping_mm_after + local container_metadata_after inner_type_after inner_label_after inner_root_metadata inner_root_after + local mapping_device_after mapping_loop_mm_after + local -a loop_records=() + outer="$(_k3slrv_findmnt_optional "$K3SLR_OUTER_MOUNT")" || return 1 + inner="$(_k3slrv_findmnt_optional "$K3SLR_INNER_MOUNT")" || return 1 + container="${K3SLR_OUTER_MOUNT}/${K3SLR_CONTAINER_RELATIVE}" + if [[ "$mode" != open ]]; then + _k3slrv_capture loops _k3slrv_root /usr/sbin/losetup --associated "$container" --noheadings --output NAME 2>/dev/null || return 1 + printf 'outer_mounted=%s\ninner_mounted=%s\n' "$([[ -n "$outer" ]] && printf 1 || printf 0)" "$([[ -n "$inner" ]] && printf 1 || printf 0)" + if [[ -z "$loops" ]]; then + printf 'loop_count=0\n' + else + mapfile -t loop_records < <(printf '%s' "$loops") + if (( ${#loop_records[@]} == 1 )) && [[ -n "${loop_records[0]}" && "${loop_records[0]}" != *$'\r'* ]]; then + printf 'loop_count=1\n' + else + printf 'loop_count=2\n' + fi + fi + _k3slrv_mapping_evidence "$K3SLR_MAPPING_NAME" mapping || return 1 + _k3slrv_mapping_evidence "$K3SLR_PROOF_MAPPING_NAME" proof_mapping || return 1 + return 0 + fi + _k3slrv_capture loops _k3slrv_root /usr/sbin/losetup --list --associated "$container" --noheadings --raw \ + --output NAME,BACK-INO,BACK-MAJ:MIN,MAJ:MIN,OFFSET,SIZELIMIT 2>/dev/null || return 1 + _k3slrv_normalize_one_record loops "$loops" || return 1 + [[ -n "$outer" && -n "$inner" ]] || return 1 + read -r outer_source outer_fstype outer_options outer_mount_id outer_major_minor <<<"$outer" + read -r inner_source inner_fstype inner_options inner_mount_id inner_major_minor <<<"$inner" + read -r loop_device loop_back_inode loop_back_major_minor loop_major_minor loop_offset loop_sizelimit <<<"$loops" + [[ -n "$outer_mount_id" && -n "$outer_major_minor" && -n "$inner_mount_id" && -n "$inner_major_minor" && + -n "$loop_device" && -n "$loop_back_inode" && -n "$loop_back_major_minor" && -n "$loop_major_minor" && + -n "$loop_offset" && -n "$loop_sizelimit" ]] || return 1 + outer_source="$(_k3slrv_one_line /usr/bin/readlink -f -- "$outer_source")" || return 1 + recovery_partition="$(_k3slrv_one_line /usr/bin/readlink -f -- "$K3SLR_RECOVERY_PARTITION_BY_ID")" || return 1 + source_mounts="$(_k3slrv_findmnt_source_optional "$recovery_partition")" || return 1 + [[ "$source_mounts" == "$K3SLR_OUTER_MOUNT" ]] || return 1 + chain_snapshot="$(_k3slrv_pin_container_chain "$container")" || return 1 + metadata="$(_k3slrv_one_line_root /usr/bin/stat --format='%F|%u|%g|%a|%s|%b|%i|%Hd:%Ld' -- "$container")" || return 1 + IFS='|' read -r kind uid gid file_mode size blocks inode container_device <<<"$metadata" + [[ "$container_device" =~ ^[0-9]+:[0-9]+$ ]] || return 1 + [[ "$blocks" =~ ^[0-9]+$ && "$blocks" -le 18014398509481983 ]] || return 1 + allocated=$((blocks * 512)) + printf 'outer_mounted=1\nouter_source=%s\nouter_fstype=%s\nouter_options=%s\n' "$outer_source" "$outer_fstype" "$outer_options" + printf 'inner_mounted=1\ninner_source=%s\ninner_fstype=%s\ninner_options=%s\n' "$inner_source" "$inner_fstype" "$inner_options" + printf 'container_kind=%s\ncontainer_symlink=%s\ncontainer_uid=%s\ncontainer_gid=%s\ncontainer_mode=%s\n' \ + "$([[ "$kind" == 'regular file' ]] && printf regular || printf other)" \ + "$(_k3slrv_root /usr/bin/test -L "$container" >/dev/null 2>&1 && printf 1 || printf 0)" "$uid" "$gid" "$file_mode" + printf 'container_size=%s\ncontainer_allocated=%s\nloop_count=1\nloop_device=%s\n' "$size" "$allocated" "$loop_device" + _k3slrv_capture mapping_snapshot _k3slrv_mapping_evidence "$K3SLR_MAPPING_NAME" mapping || return 1 + _k3slrv_capture proof_snapshot _k3slrv_mapping_evidence "$K3SLR_PROOF_MAPPING_NAME" proof_mapping || return 1 + printf '%s%s' "$mapping_snapshot" "$proof_snapshot" + mapping_device="$(_k3slrv_mapping_value "$mapping_snapshot" mapping_device)" || return 1 + mapping_loop_major_minor="$(_k3slrv_lsblk MAJ:MIN "$mapping_device")" || return 1 + mapping_major_minor="$(_k3slrv_lsblk MAJ:MIN "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1 + inner_type="$(_k3slrv_blkid TYPE "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1 + inner_label="$(_k3slrv_blkid LABEL "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1 + printf 'inner_type=%s\ninner_label=%s\n' "$inner_type" "$inner_label" + inner_root_metadata="$(_k3slrv_one_line_root /usr/bin/stat --format='%u|%g|%a|%F' -- "$K3SLR_INNER_MOUNT")" || return 1 + IFS='|' read -r uid gid file_mode kind <<<"$inner_root_metadata" + printf 'inner_root_uid=%s\ninner_root_gid=%s\ninner_root_mode=%s\ninner_root_kind=%s\n' \ + "$uid" "$gid" "$file_mode" "$kind" + outer_after="$(_k3slrv_findmnt_optional "$K3SLR_OUTER_MOUNT")" || return 1 + inner_after="$(_k3slrv_findmnt_optional "$K3SLR_INNER_MOUNT")" || return 1 + source_after="$(_k3slrv_findmnt_source_optional "$recovery_partition")" || return 1 + chain_after="$(_k3slrv_pin_container_chain "$container")" || return 1 + _k3slrv_capture loops_after _k3slrv_root /usr/sbin/losetup --list --associated "$container" --noheadings --raw \ + --output NAME,BACK-INO,BACK-MAJ:MIN,MAJ:MIN,OFFSET,SIZELIMIT 2>/dev/null || return 1 + _k3slrv_normalize_one_record loops_after "$loops_after" || return 1 + _k3slrv_capture mapping_after _k3slrv_mapping_evidence "$K3SLR_MAPPING_NAME" mapping || return 1 + _k3slrv_capture proof_after _k3slrv_mapping_evidence "$K3SLR_PROOF_MAPPING_NAME" proof_mapping || return 1 + container_metadata_after="$(_k3slrv_one_line_root /usr/bin/stat --format='%F|%u|%g|%a|%s|%b|%i|%Hd:%Ld' -- "$container")" || return 1 + inner_type_after="$(_k3slrv_blkid TYPE "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1 + inner_label_after="$(_k3slrv_blkid LABEL "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1 + inner_root_after="$(_k3slrv_one_line_root /usr/bin/stat --format='%u|%g|%a|%F' -- "$K3SLR_INNER_MOUNT")" || return 1 + mapping_device_after="$(_k3slrv_mapping_value "$mapping_after" mapping_device)" || return 1 + mapping_loop_mm_after="$(_k3slrv_lsblk MAJ:MIN "$mapping_device_after")" || return 1 + mapping_mm_after="$(_k3slrv_lsblk MAJ:MIN "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1 + [[ "$outer_after" == "$outer" && "$inner_after" == "$inner" && "$source_after" == "$source_mounts" && + "$chain_after" == "$chain_snapshot" && "$loops_after" == "$loops" && "$mapping_after" == "$mapping_snapshot" && + "$proof_after" == "$proof_snapshot" && "$container_metadata_after" == "$metadata" && + "$inner_type_after" == "$inner_type" && "$inner_label_after" == "$inner_label" && + "$inner_root_after" == "$inner_root_metadata" && "$mapping_loop_mm_after" == "$mapping_loop_major_minor" && + "$mapping_mm_after" == "$mapping_major_minor" ]] || return 1 + printf '%s\n' \ + 'container_chain_pinned=1' \ + "container_inode=${inode}" \ + "container_device=${container_device}" \ + "loop_back_inode=${loop_back_inode}" \ + "loop_back_major_minor=${loop_back_major_minor}" \ + "outer_major_minor=${outer_major_minor}" \ + "loop_major_minor=${loop_major_minor}" \ + "mapping_loop_major_minor=${mapping_loop_major_minor}" \ + "mapping_major_minor=${mapping_major_minor}" \ + "inner_major_minor=${inner_major_minor}" \ + "loop_offset=${loop_offset}" \ + "loop_sizelimit=${loop_sizelimit}" \ + 'snapshot_stable=1' +} + +_k3slrv_parse_evidence() { + local input="$1" destination_name="$2" + shift 2 + local -n destination="$destination_name" + local line key value expected + local -A allowed=() seen=() + local -a lines=() + destination=() + for expected in "$@"; do allowed["$expected"]=1; done + mapfile -t lines < <(printf '%s' "$input") + (( ${#lines[@]} > 0 )) || return 1 + for line in "${lines[@]}"; do + [[ "$line" =~ ^([a-z][a-z0-9_]*)=([A-Za-z0-9._:/,+=-]*)$ ]] || return 1 + key="${BASH_REMATCH[1]}"; value="${BASH_REMATCH[2]}" + [[ "${allowed[$key]+yes}" == yes && "${seen[$key]+yes}" != yes ]] || return 1 + seen["$key"]=1; destination["$key"]="$value" + done + for expected in "${!allowed[@]}"; do [[ "${seen[$expected]+yes}" == yes ]] || return 1; done +} + +_k3slrv_validate_device() { + local mode="$1" input="$2" + local -A e=() + local keys=(recovery_partition recovery_disk recovery_partition_parent recovery_fs_uuid recovery_partuuid recovery_type + recovery_model recovery_serial recovery_wwn recovery_major_minor k3s_partition k3s_disk k3s_partition_parent + k3s_fs_uuid k3s_partuuid k3s_type k3s_model k3s_serial k3s_wwn k3s_major_minor smart_health + smart_reallocated smart_pending smart_uncorrectable ntfs_probe) + _k3slrv_parse_evidence "$input" e "${keys[@]}" || return 1 + [[ "${e[recovery_partition_parent]}" == "${e[recovery_disk]}" && "${e[k3s_partition_parent]}" == "${e[k3s_disk]}" ]] || return 1 + [[ "${e[recovery_fs_uuid]}" == "$K3SLR_RECOVERY_FS_UUID" && "${e[recovery_partuuid]}" == "$K3SLR_RECOVERY_PARTUUID" && "${e[recovery_type]}" == ntfs ]] || return 1 + [[ "${e[recovery_model]}" == "$K3SLR_RECOVERY_MODEL" && "${e[recovery_serial]}" == "$K3SLR_RECOVERY_SERIAL" && "${e[recovery_wwn]}" == "$K3SLR_RECOVERY_WWN" ]] || return 1 + [[ "${e[k3s_fs_uuid]}" == "$K3SLR_K3S_FS_UUID" && "${e[k3s_partuuid]}" == "$K3SLR_K3S_PARTUUID" && "${e[k3s_type]}" == ext4 ]] || return 1 + [[ "${e[k3s_model]}" == "$K3SLR_K3S_MODEL" && "${e[k3s_serial]}" == "$K3SLR_K3S_SERIAL" && "${e[k3s_wwn]}" == "$K3SLR_K3S_WWN" ]] || return 1 + [[ "${e[recovery_major_minor]}" != "${e[k3s_major_minor]}" && "${e[smart_health]}" == PASSED ]] || return 1 + [[ "${e[smart_reallocated]}" == 0 && "${e[smart_pending]}" == 0 && "${e[smart_uncorrectable]}" == 0 ]] || return 1 + if [[ "$mode" == open ]]; then [[ "${e[ntfs_probe]}" == not_applicable || "${e[ntfs_probe]}" == pass ]]; + else [[ "${e[ntfs_probe]}" == pass ]]; fi +} + +_k3slrv_option_present() { + [[ ",${1}," == *",${2},"* ]] +} + +_k3slrv_outer_masks_secure() { + local options="$1" option umask_count=0 dmask_count=0 fmask_count=0 invalid=false + local -a option_list=() + IFS=',' read -r -a option_list <<<"$options" + for option in "${option_list[@]}"; do + case "$option" in + umask=077|umask=0077) umask_count=$((umask_count + 1)) ;; + dmask=077|dmask=0077) dmask_count=$((dmask_count + 1)) ;; + fmask=077|fmask=0077|fmask=177|fmask=0177) fmask_count=$((fmask_count + 1)) ;; + umask=*|dmask=*|fmask=*) invalid=true ;; + esac + done + "$invalid" && return 1 + if (( umask_count == 1 && dmask_count == 0 && fmask_count == 0 )); then return 0; fi + (( umask_count == 0 && dmask_count == 1 && fmask_count == 1 )) +} + +_k3slrv_validate_open_snapshot() { + local input="$1" + local -A snapshot=() + _k3slrv_parse_evidence "$input" snapshot \ + container_chain_pinned container_inode container_device loop_back_inode loop_back_major_minor outer_major_minor \ + loop_major_minor mapping_loop_major_minor mapping_major_minor inner_major_minor \ + loop_offset loop_sizelimit snapshot_stable || return 1 + [[ "${snapshot[container_chain_pinned]}" == 1 && "${snapshot[snapshot_stable]}" == 1 ]] || return 1 + [[ "${snapshot[container_inode]}" =~ ^[0-9]+$ && + "${snapshot[loop_back_inode]}" == "${snapshot[container_inode]}" ]] || return 1 + [[ "${snapshot[container_device]}" =~ ^[0-9]+:[0-9]+$ && + "${snapshot[loop_back_major_minor]}" == "${snapshot[container_device]}" && + "${snapshot[container_device]}" == "${snapshot[outer_major_minor]}" ]] || return 1 + [[ "${snapshot[mapping_loop_major_minor]}" == "${snapshot[loop_major_minor]}" ]] || return 1 + [[ "${snapshot[mapping_major_minor]}" == "${snapshot[inner_major_minor]}" ]] || return 1 + [[ "${snapshot[loop_offset]}" == 0 && "${snapshot[loop_sizelimit]}" == 0 ]] +} + +_k3slrv_validate_state() { + local mode="$1" input="$2" device_input="$3" approved_partition='' line snapshot_raw + local -A e=() + if [[ "$mode" != open ]]; then + _k3slrv_parse_evidence "$input" e outer_mounted inner_mounted loop_count mapping_present proof_mapping_present || return 1 + [[ "${e[outer_mounted]}" == 0 && "${e[inner_mounted]}" == 0 && "${e[loop_count]}" == 0 && + "${e[mapping_present]}" == 0 && "${e[proof_mapping_present]}" == 0 ]] + return + fi + local keys=(outer_mounted outer_source outer_fstype outer_options inner_mounted inner_source inner_fstype inner_options + container_kind container_symlink container_uid container_gid container_mode container_size container_allocated + loop_count loop_device mapping_present mapping_type mapping_device proof_mapping_present inner_type inner_label + inner_root_uid inner_root_gid inner_root_mode inner_root_kind container_chain_pinned container_inode + container_device loop_back_inode loop_back_major_minor outer_major_minor loop_major_minor mapping_loop_major_minor + mapping_major_minor inner_major_minor loop_offset loop_sizelimit snapshot_stable) + _k3slrv_parse_evidence "$input" e "${keys[@]}" || return 1 + while IFS= read -r line; do + [[ "$line" == recovery_partition=* ]] || continue + [[ -z "$approved_partition" ]] || return 1 + approved_partition="${line#recovery_partition=}" + done <<<"$device_input" + [[ -n "$approved_partition" && "${e[outer_mounted]}" == 1 && + "${e[outer_source]}" == "$approved_partition" && "${e[outer_fstype]}" == ntfs3 ]] || return 1 + for option in rw nodev nosuid noexec "uid=${K3SLR_OWNER_UID}" "gid=${K3SLR_OWNER_GID}"; do + _k3slrv_option_present "${e[outer_options]}" "$option" || return 1 + done + _k3slrv_outer_masks_secure "${e[outer_options]}" || return 1 + [[ "${e[inner_mounted]}" == 1 && "${e[inner_source]}" == "/dev/mapper/${K3SLR_MAPPING_NAME}" && "${e[inner_fstype]}" == ext4 ]] || return 1 + for option in rw nodev nosuid noexec; do _k3slrv_option_present "${e[inner_options]}" "$option" || return 1; done + [[ "${e[container_kind]}" == regular && "${e[container_symlink]}" == 0 && + "${e[container_uid]}" == "$K3SLR_OWNER_UID" && "${e[container_gid]}" == "$K3SLR_OWNER_GID" ]] || return 1 + [[ "${e[container_mode]}" =~ ^[0-7]{3,4}$ ]] || return 1 + (( (10#${e[container_mode]: -2:1} & 7) == 0 && (10#${e[container_mode]: -1:1} & 7) == 0 )) || return 1 + [[ "${e[container_size]}" == "$K3SLR_CONTAINER_SIZE_BYTES" && "${e[container_allocated]}" =~ ^[0-9]+$ ]] || return 1 + (( e[container_allocated] >= K3SLR_CONTAINER_SIZE_BYTES )) || return 1 + [[ "${e[loop_count]}" == 1 && "${e[mapping_present]}" == 1 && "${e[mapping_type]}" == LUKS2 && + "${e[mapping_device]}" == "${e[loop_device]}" && "${e[proof_mapping_present]}" == 0 ]] || return 1 + [[ "${e[inner_type]}" == ext4 && "${e[inner_label]}" == "$K3SLR_INNER_LABEL" ]] || return 1 + [[ "${e[inner_root_uid]}" == 0 && "${e[inner_root_gid]}" == 0 && "${e[inner_root_mode]}" == 700 && "${e[inner_root_kind]}" == directory ]] || return 1 + printf -v snapshot_raw '%s\n' \ + "container_chain_pinned=${e[container_chain_pinned]}" \ + "container_inode=${e[container_inode]}" \ + "container_device=${e[container_device]}" \ + "loop_back_inode=${e[loop_back_inode]}" \ + "loop_back_major_minor=${e[loop_back_major_minor]}" \ + "outer_major_minor=${e[outer_major_minor]}" \ + "loop_major_minor=${e[loop_major_minor]}" \ + "mapping_loop_major_minor=${e[mapping_loop_major_minor]}" \ + "mapping_major_minor=${e[mapping_major_minor]}" \ + "inner_major_minor=${e[inner_major_minor]}" \ + "loop_offset=${e[loop_offset]}" \ + "loop_sizelimit=${e[loop_sizelimit]}" \ + "snapshot_stable=${e[snapshot_stable]}" + _k3slrv_validate_open_snapshot "${snapshot_raw%$'\n'}" +} + +_k3slrv_verify_manifest_targets() { + local bundle="$1" manifest="$2" records record expected_hash relative target component current canonical + local before after hash_line actual_hash count=0 + local -a components=() manifest_records=() + _k3slrv_capture records _k3slrv_root /usr/bin/mawk ' + NF!=2 || length($1)!=64 || $1 !~ /^[0-9a-f]+$/ || $2 !~ /^\.\/[A-Za-z0-9._\/-]+$/ || + $2 ~ /\.\./ || $2 ~ /\/\// || $2 ~ /\/\.\// || $2 ~ /\/\.$/ || $2 ~ /\/$/ { bad=1; exit } + { print $1 "|" $2 } + END { exit (bad || NR==0) } + ' "$manifest" 2>/dev/null || return 1 + mapfile -t manifest_records < <(printf '%s' "$records") + (( ${#manifest_records[@]} > 0 )) || return 1 + for record in "${manifest_records[@]}"; do + [[ "$record" == *'|'* ]] || return 1 + expected_hash="${record%%|*}" + relative="${record#*|}" + [[ "$expected_hash" =~ ^[0-9a-f]{64}$ && "$relative" =~ ^\./[A-Za-z0-9._/-]+$ ]] || return 1 + target="${bundle}/${relative#./}" + current="$bundle" + IFS='/' read -r -a components <<<"${relative#./}" + for component in "${components[@]}"; do + [[ -n "$component" && "$component" != . && "$component" != .. ]] || return 1 + current="${current}/${component}" + _k3slrv_root /usr/bin/test ! -L "$current" >/dev/null 2>&1 || return 1 + canonical="$(_k3slrv_one_line_root /usr/bin/readlink -e -- "$current")" || return 1 + [[ "$canonical" == "$current" ]] || return 1 + done + _k3slrv_root /usr/bin/test -f "$target" >/dev/null 2>&1 || return 1 + before="$(_k3slrv_one_line_root /usr/bin/stat --format='%d:%i|%F' -- "$target")" || return 1 + [[ "$before" == *'|regular file' ]] || return 1 + hash_line="$(_k3slrv_one_line_root /usr/bin/sha256sum -- "$target")" || return 1 + actual_hash="${hash_line%% *}" + [[ "$actual_hash" == "$expected_hash" ]] || return 1 + after="$(_k3slrv_one_line_root /usr/bin/stat --format='%d:%i|%F' -- "$target")" || return 1 + [[ "$after" == "$before" ]] || return 1 + count=$((count + 1)) + done + (( count > 0 )) +} + +_k3slrv_collect_latest_bundle_evidence() { + local metadata="${K3SLR_INNER_MOUNT}/.latest-post-bundle.env" parsed relative expected_identity bundle canonical_bundle actual_identity after_identity + local bundle_metadata manifest stat_value + _k3slrv_root /usr/bin/test -f "$metadata" >/dev/null 2>&1 || return 1 + _k3slrv_root /usr/bin/test ! -L "$metadata" >/dev/null 2>&1 || return 1 + stat_value="$(_k3slrv_one_line_root /usr/bin/stat --format='%u:%g:%a:%F' -- "$metadata")" || return 1 + [[ "$stat_value" == '0:0:600:regular file' ]] || return 1 + _k3slrv_capture parsed _k3slrv_root /usr/bin/mawk -F= ' + BEGIN { ok=1 } + !/^[a-z_]+=[A-Za-z0-9._:\/-]+$/ { ok=0; next } + $1=="schema" && !s++ && $2=="k3slr-latest-post-bundle-v1" { schema=$2; next } + $1=="relative_path" && !p++ && $2 ~ /^k3s-secrets-encryption-[0-9]{8}T[0-9]{6}Z\/post$/ { path=$2; next } + $1=="directory_identity" && !i++ && $2 ~ /^[0-9]+:[0-9]+$/ { identity=$2; next } + { ok=0 } + END { if (ok && NR==3 && s==1 && p==1 && i==1) print path "|" identity; else exit 1 } + ' "$metadata" 2>/dev/null || return 1 + _k3slrv_normalize_one_record parsed "$parsed" || return 1 + [[ "$parsed" == *'|'* ]] || return 1 + relative="${parsed%%|*}"; expected_identity="${parsed#*|}" + [[ "$relative" =~ ^k3s-secrets-encryption-[0-9]{8}T[0-9]{6}Z/post$ && "$relative" != *..* ]] || return 1 + bundle="${K3SLR_INNER_MOUNT}/${relative}" + canonical_bundle="$(_k3slrv_one_line_root /usr/bin/readlink -f -- "$bundle")" || return 1 + [[ "$canonical_bundle" == "$bundle" ]] || return 1 + actual_identity="$(_k3slrv_one_line_root /usr/bin/stat --dereference --format='%d:%i' -- "$bundle")" || return 1 + [[ "$actual_identity" == "$expected_identity" ]] || return 1 + stat_value="$(_k3slrv_one_line_root /usr/bin/stat --dereference --format='%u:%g:%a:%F' -- "$bundle")" || return 1 + [[ "$stat_value" == '0:0:700:directory' ]] || return 1 + bundle_metadata="${bundle}/bundle.env"; manifest="${bundle}/verification.manifest" + for file in "$bundle_metadata" "$manifest"; do + _k3slrv_root /usr/bin/test -f "$file" >/dev/null 2>&1 || return 1 + _k3slrv_root /usr/bin/test ! -L "$file" >/dev/null 2>&1 || return 1 + stat_value="$(_k3slrv_one_line_root /usr/bin/stat --format='%u:%g:%a:%F' -- "$file")" || return 1 + [[ "$stat_value" == '0:0:600:regular file' ]] || return 1 + done + _k3slrv_root /usr/bin/mawk -F= ' + BEGIN { ok=1 } + !/^[a-z][a-z0-9_]*=[A-Za-z0-9:+._-]+$/ { ok=0; next } + $1=="schema" && !a++ && $2=="platform-k3s-bundle-v1" { next } + $1=="bundle_id" && !b++ && $2 ~ /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ { next } + $1=="phase" && !c++ && $2=="post" { next } + $1=="k3s_version" && !d++ && $2=="v1.36.2+k3s1" { next } + $1=="datastore" && !e++ && ($2=="sqlite" || $2=="embedded-etcd") { next } + $1=="created_at_utc" && !f++ && $2 ~ /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$/ { next } + $1=="secret_count" && !g++ && $2 ~ /^[0-9]+$/ { next } + { ok=0 } + END { exit !(ok && NR==7 && a==1 && b==1 && c==1 && d==1 && e==1 && f==1 && g==1) } + ' "$bundle_metadata" >/dev/null 2>&1 || return 1 + _k3slrv_verify_manifest_targets "$bundle" "$manifest" || return 1 + after_identity="$(_k3slrv_one_line_root /usr/bin/stat --dereference --format='%d:%i' -- "$bundle")" || return 1 + [[ "$after_identity" == "$expected_identity" ]] || return 1 + printf '%s\n' \ + 'metadata_schema=k3slr-latest-post-bundle-v1' \ + 'relative_path_valid=1' \ + 'directory_identity_match=1' \ + 'directory_secure=1' \ + 'bundle_schema=platform-k3s-bundle-v1' \ + 'bundle_phase=post' \ + 'manifest_safe=1' \ + 'manifest_verified=1' \ + 'directory_identity_stable=1' +} + +_k3slrv_verify_latest_bundle() { + local raw + local -A evidence=() + _k3slrv_capture raw _k3slrv_collect_latest_bundle_evidence || return 1 + _k3slrv_parse_evidence "$raw" evidence \ + metadata_schema relative_path_valid directory_identity_match directory_secure \ + bundle_schema bundle_phase manifest_safe manifest_verified directory_identity_stable || return 1 + [[ "${evidence[metadata_schema]}" == k3slr-latest-post-bundle-v1 && + "${evidence[relative_path_valid]}" == 1 && + "${evidence[directory_identity_match]}" == 1 && + "${evidence[directory_secure]}" == 1 && + "${evidence[bundle_schema]}" == platform-k3s-bundle-v1 && + "${evidence[bundle_phase]}" == post && + "${evidence[manifest_safe]}" == 1 && + "${evidence[manifest_verified]}" == 1 && + "${evidence[directory_identity_stable]}" == 1 ]] +} + +k3slr_local_recovery_main() { + local mode='' check_latest=false argument device_raw state_raw latest=not_checked + for argument in "$@"; do + case "$argument" in + --expect-device-ready) [[ -z "$mode" ]] || { _k3slrv_usage >&2; return 2; }; mode=device_ready ;; + --expect-closed) [[ -z "$mode" ]] || { _k3slrv_usage >&2; return 2; }; mode=closed ;; + --expect-open) [[ -z "$mode" ]] || { _k3slrv_usage >&2; return 2; }; mode=open ;; + --check-latest-bundle) "$check_latest" && { _k3slrv_usage >&2; return 2; }; check_latest=true ;; + --help|-h) _k3slrv_usage; return 0 ;; + *) _k3slrv_usage >&2; return 2 ;; + esac + done + [[ -n "$mode" ]] || { _k3slrv_usage >&2; return 2; } + if "$check_latest" && [[ "$mode" != open ]]; then _k3slrv_usage >&2; return 2; fi + PATH="$K3SLRV_SECURE_PATH"; export PATH + _k3slrv_verify_trusted_binaries || { _k3slrv_fail; return 1; } + # Direct execution always replaces any inherited/predefined parser with the + # repository library in this unprivileged process. + # shellcheck source=/dev/null + builtin source -- "$K3SLRV_LIBRARY_PATH" || { _k3slrv_fail; return 1; } + _k3slr_load_contract "$K3SLRV_CONTRACT_PATH" || { _k3slrv_fail; return 1; } + _k3slrv_require_cached_sudo || { _k3slrv_fail; return 1; } + _k3slrv_capture device_raw _k3slrv_collect_device_evidence "$mode" || { _k3slrv_fail; return 1; } + _k3slrv_validate_device "$mode" "$device_raw" || { _k3slrv_fail; return 1; } + _k3slrv_capture state_raw _k3slrv_collect_state_evidence "$mode" || { _k3slrv_fail; return 1; } + _k3slrv_validate_state "$mode" "$state_raw" "$device_raw" || { _k3slrv_fail; return 1; } + if "$check_latest"; then + _k3slrv_verify_latest_bundle || { _k3slrv_fail; return 1; } + latest=verified + fi + printf 'Recovery device: match\nRecovery state: %s\nLineage: match\nLatest bundle: %s\n' "$mode" "$latest" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + k3slr_local_recovery_main "$@" +fi diff --git a/scripts/validate/k3s-secret-encryption-restore-evidence.sh b/scripts/validate/k3s-secret-encryption-restore-evidence.sh new file mode 100755 index 0000000..4930e08 --- /dev/null +++ b/scripts/validate/k3s-secret-encryption-restore-evidence.sh @@ -0,0 +1,636 @@ +#!/usr/bin/env bash + +# 격리 복구 결과를 만들고 운영 host의 복구 증거를 등록·검사한다. source-safe +# 함수 경계는 fixture가 live/root side effect만 치환할 수 있게 유지한다. +set -Eeuo pipefail + +if [[ "${BASH_SOURCE[0]}" == */* ]]; then + readonly KSRE_SCRIPT_DIR="${BASH_SOURCE[0]%/*}" +else + readonly KSRE_SCRIPT_DIR='.' +fi +readonly KSRE_ROOT="$(cd -- "${KSRE_SCRIPT_DIR}/../.." && pwd -P)" +readonly KSRE_STATUS_VALIDATOR="${KSRE_ROOT}/scripts/validate/k3s-secret-encryption.sh" +readonly KSRE_POST_BUNDLE='/etc/rancher/k3s/platform-post-bundle.env' +readonly KSRE_EVIDENCE='/etc/rancher/k3s/platform-restore-evidence.env' +readonly KSRE_VERSION='v1.36.2+k3s1' +readonly KSRE_KUBECTL='/usr/local/bin/kubectl' +readonly KSRE_LOCAL_API_SERVER='https://127.0.0.1:6443' +readonly KSRE_SECURE_PATH='/usr/sbin:/usr/bin:/sbin:/bin' +readonly KSRE_BROKER_RUNTIME_TIMEOUT='2s' +readonly KSRE_BROKER_KILL_AFTER='0.5s' +readonly KSRE_BROKER_READY_TIMEOUT='3' +readonly KSRE_BROKER_ACK_TIMEOUT='1.0' +readonly KSRE_BROKER_REAP_POLLS='300' +readonly KSRE_BROKER_SIGNAL_POLLS='50' +readonly KSRE_BROKER_POLL_INTERVAL='0.01' + +_ksre_fail() { + printf 'ERROR: 복구 증거 검증에 실패했습니다. 수동 절차를 확인하세요.\n' >&2 + return 1 +} + +_ksre_usage() { + cat <<'USAGE' +사용법: + bash scripts/validate/k3s-secret-encryption-restore-evidence.sh \ + --emit-result --bundle-metadata FILE --output FILE + bash scripts/validate/k3s-secret-encryption-restore-evidence.sh \ + --record --bundle-metadata FILE --result-file FILE + bash scripts/validate/k3s-secret-encryption-restore-evidence.sh --check + +--emit-result는 격리 복구 host의 live 상태를 검사한 뒤 결과 파일을 만듭니다. +--record는 복구 환경과 bundle 복사본 파기 확인 뒤 운영 host에 증거를 등록합니다. +--check는 권위 post bundle, live 암호화 상태와 등록된 증거를 읽기 전용 검사합니다. +USAGE +} + +_ksre_allowed_key() { + local kind="$1" key="$2" + case "$kind:$key" in + bundle:schema|bundle:bundle_id|bundle:phase|bundle:k3s_version|bundle:datastore|bundle:created_at_utc|bundle:secret_count) + return 0 + ;; + result:schema|result:bundle_id|result:k3s_version|result:datastore|result:status_class|result:rotation_stage|result:hashes|result:local_integrity|result:api_ready|result:node_ready|result:isolation|result:secret_count_match|result:tested_at_utc) + return 0 + ;; + evidence:schema|evidence:bundle_id|evidence:k3s_version|evidence:datastore|evidence:rotation_stage|evidence:local_integrity|evidence:node_ready|evidence:isolation|evidence:tested_at_utc|evidence:recorded_at_utc|evidence:destroyed) + return 0 + ;; + *) return 1 ;; + esac +} + +_ksre_expected_field_count() { + case "$1" in bundle) printf '7\n' ;; result) printf '13\n' ;; evidence) printf '11\n' ;; *) return 1 ;; esac +} + +_ksre_values_valid() { + local kind="$1" destination="$2" + local -n values="$destination" + [[ "${values[bundle_id]:-}" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ ]] || return 1 + [[ "${values[k3s_version]:-}" == "$KSRE_VERSION" ]] || return 1 + [[ "${values[datastore]:-}" == sqlite || "${values[datastore]:-}" == embedded-etcd ]] || return 1 + case "$kind" in + bundle) + [[ "${values[schema]:-}" == platform-k3s-bundle-v1 && + ( "${values[phase]:-}" == pre || "${values[phase]:-}" == post ) && + "${values[created_at_utc]:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && + "${values[secret_count]:-}" =~ ^[0-9]+$ ]] + ;; + result) + [[ "${values[schema]:-}" == platform-k3s-restore-result-v1 && + "${values[status_class]:-}" == enabled_stable && + "${values[rotation_stage]:-}" == reencrypt_finished && + "${values[hashes]:-}" == match && "${values[local_integrity]:-}" == match && + "${values[api_ready]:-}" == pass && "${values[node_ready]:-}" == pass && + "${values[isolation]:-}" == pass && "${values[secret_count_match]:-}" == pass && + "${values[tested_at_utc]:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] + ;; + evidence) + [[ "${values[schema]:-}" == platform-k3s-restore-evidence-v1 && + "${values[rotation_stage]:-}" == reencrypt_finished && + "${values[local_integrity]:-}" == match && "${values[node_ready]:-}" == pass && + "${values[isolation]:-}" == pass && "${values[destroyed]:-}" == confirmed && + "${values[tested_at_utc]:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && + "${values[recorded_at_utc]:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] + ;; + *) return 1 ;; + esac +} + +_ksre_parse_stream() { + local kind="$1" destination="$2" line key value count=0 expected + local -n output_fields="$destination" + output_fields=() + expected="$(_ksre_expected_field_count "$kind")" || return 1 + while IFS= read -r line || [[ -n "$line" ]]; do + [[ "$line" =~ ^([a-z0-9_]+)=([^=[:cntrl:]]+)$ ]] || return 1 + key="${BASH_REMATCH[1]}"; value="${BASH_REMATCH[2]}" + _ksre_allowed_key "$kind" "$key" || return 1 + [[ -z "${output_fields[$key]+present}" ]] || return 1 + output_fields["$key"]="$value" + count=$((count + 1)) + done + [[ "$count" == "$expected" ]] || return 1 + _ksre_values_valid "$kind" "$destination" +} + +_ksre_raw_stream_has_no_nul() { + /usr/bin/od -An -v -t u1 -- "$1" 2>/dev/null | + /usr/bin/awk '{ for (field = 1; field <= NF; field++) if ($field == 0) found = 1 } + END { exit(found ? 1 : 0) }' +} + +_ksre_parse_external_fd() { + local kind="$1" input_fd="$2" destination="$3" handle metadata + [[ "$input_fd" =~ ^[0-9]+$ ]] || return 1 + handle="/proc/self/fd/${input_fd}" + metadata="$(/usr/bin/stat --dereference --format='%u:%a:%F' -- "$handle" 2>/dev/null)" || return 1 + [[ "$metadata" == "${EUID}:600:regular file" && -s "$handle" ]] || return 1 + _ksre_raw_stream_has_no_nul "$handle" || return 1 + _ksre_parse_stream "$kind" "$destination" <"$handle" +} + +_ksre_broker_job_running() { + local expected_pid="$1" active_pid active_jobs + active_jobs="$(jobs -pr)" + while IFS= read -r active_pid; do + [[ "$active_pid" == "$expected_pid" ]] && return 0 + done <<<"$active_jobs" + return 1 +} + +_ksre_reap_broker_bounded() { + local broker_pid="$1" attempt + for ((attempt = 0; attempt < KSRE_BROKER_REAP_POLLS; attempt++)); do + if ! _ksre_broker_job_running "$broker_pid"; then + if wait "$broker_pid"; then return 0; else return $?; fi + fi + /usr/bin/sleep "$KSRE_BROKER_POLL_INTERVAL" + done + /bin/kill -TERM "$broker_pid" 2>/dev/null || : + for ((attempt = 0; attempt < KSRE_BROKER_SIGNAL_POLLS; attempt++)); do + if ! _ksre_broker_job_running "$broker_pid"; then + wait "$broker_pid" 2>/dev/null || : + return 1 + fi + /usr/bin/sleep "$KSRE_BROKER_POLL_INTERVAL" + done + /bin/kill -KILL "$broker_pid" 2>/dev/null || : + for ((attempt = 0; attempt < KSRE_BROKER_SIGNAL_POLLS; attempt++)); do + if ! _ksre_broker_job_running "$broker_pid"; then + wait "$broker_pid" 2>/dev/null || : + return 1 + fi + /usr/bin/sleep "$KSRE_BROKER_POLL_INTERVAL" + done + return 1 +} + +_ksre_open_external_nofollow() { + local path="$1" destination="$2" broker_pid broker_read broker_write + local ready opener_pid opener_fd opened_descriptor='' rc=0 + coproc KSRE_OPEN_BROKER { + exec /usr/bin/timeout --signal=TERM --kill-after="$KSRE_BROKER_KILL_AFTER" \ + "$KSRE_BROKER_RUNTIME_TIMEOUT" /usr/bin/python3 -I -S -c ' +import os +import select +import stat +import sys + +required_flags = ("O_PATH", "O_NOFOLLOW", "O_CLOEXEC", "O_NONBLOCK") +if any(not hasattr(os, flag) for flag in required_flags): + sys.exit(1) +try: + path_descriptor = os.open( + sys.argv[1], os.O_PATH | os.O_NOFOLLOW | os.O_CLOEXEC + ) +except OSError: + sys.exit(1) +try: + path_status = os.fstat(path_descriptor) + if not stat.S_ISREG(path_status.st_mode): + sys.exit(1) + try: + descriptor = os.open( + f"/proc/self/fd/{path_descriptor}", + os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK, + ) + except OSError: + sys.exit(1) + try: + content_status = os.fstat(descriptor) + if ( + not stat.S_ISREG(content_status.st_mode) + or content_status.st_dev != path_status.st_dev + or content_status.st_ino != path_status.st_ino + ): + sys.exit(1) + print(f"{os.getpid()}:{descriptor}", flush=True) + readable, _, _ = select.select( + [sys.stdin.buffer], [], [], float(sys.argv[2]) + ) + if not readable or sys.stdin.buffer.read(1) != b"x": + sys.exit(1) + finally: + os.close(descriptor) +finally: + os.close(path_descriptor) +' "$path" "$KSRE_BROKER_ACK_TIMEOUT" + } + broker_pid="$KSRE_OPEN_BROKER_PID" + broker_read="${KSRE_OPEN_BROKER[0]}" + broker_write="${KSRE_OPEN_BROKER[1]}" + if IFS= read -r -t "$KSRE_BROKER_READY_TIMEOUT" ready <&"$broker_read"; then + if [[ "$ready" =~ ^([0-9]+):([0-9]+)$ ]]; then + opener_pid="${BASH_REMATCH[1]}" + opener_fd="${BASH_REMATCH[2]}" + exec {opened_descriptor}<"/proc/${opener_pid}/fd/${opener_fd}" || rc=1 + else + rc=1 + fi + else + rc=1 + fi + if (( rc == 0 )); then + printf 'x' >&"$broker_write" || rc=1 + fi + exec {broker_write}>&- + exec {broker_read}<&- + if (( rc != 0 )); then /bin/kill -TERM "$broker_pid" 2>/dev/null || :; fi + if ! _ksre_reap_broker_bounded "$broker_pid"; then rc=1; fi + if (( rc != 0 )); then + [[ -z "$opened_descriptor" ]] || exec {opened_descriptor}<&- + return 1 + fi + printf -v "$destination" '%s' "$opened_descriptor" +} + +_ksre_parse_external_file() { + local kind="$1" path="$2" destination="$3" input_fd rc=0 + [[ -n "$path" ]] || return 1 + _ksre_open_external_nofollow "$path" input_fd || return 1 + if (( rc == 0 )); then _ksre_parse_external_fd "$kind" "$input_fd" "$destination" || rc=$?; fi + exec {input_fd}<&- + return "$rc" +} + +_ksre_parse_trusted_content() { + local kind="$1" content="$2" destination="$3" + _ksre_parse_stream "$kind" "$destination" <<<"$content" +} + +_ksre_current_context() { + /usr/bin/timeout --signal=TERM --kill-after=1s 9s \ + "$KSRE_KUBECTL" config current-context 2>/dev/null +} + +_ksre_require_same_context() { + local expected="$1" current + current="$(_ksre_current_context)" || return 1 + [[ -n "$current" && "$current" == "$expected" && "$current" != *$'\n'* ]] +} + +_ksre_validate_current_connection() { + [[ "$1" == "${KSRE_LOCAL_API_SERVER}|" ]] +} + +_ksre_current_connection() { + local context="$1" + /usr/bin/timeout --signal=TERM --kill-after=1s 9s \ + "$KSRE_KUBECTL" --context="$context" config view --minify \ + -o 'jsonpath={.clusters[0].cluster.server}{"|"}{.clusters[0].cluster.proxy-url}' \ + 2>/dev/null +} + +_ksre_require_same_connection() { + local context="$1" expected="$2" current + _ksre_validate_current_connection "$expected" || return 1 + current="$(_ksre_current_connection "$context")" || return 1 + [[ "$current" == "$expected" ]] +} + +_ksre_now_rfc3339() { /usr/bin/date -u +%Y-%m-%dT%H:%M:%SZ; } +_ksre_now_epoch() { /usr/bin/date -u +%s; } + +_ksre_timestamp_epoch() { + local timestamp="$1" normalized + normalized="$(/usr/bin/date -u -d "$timestamp" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" || return 1 + [[ "$normalized" == "$timestamp" ]] || return 1 + /usr/bin/date -u -d "$timestamp" +%s +} + +_ksre_age_within() { + local timestamp="$1" maximum="$2" now epoch age + now="$(_ksre_now_epoch)" || return 1 + epoch="$(_ksre_timestamp_epoch "$timestamp")" || return 1 + [[ "$now" =~ ^[0-9]+$ && "$epoch" =~ ^[0-9]+$ && "$maximum" =~ ^[0-9]+$ ]] || return 1 + age=$((now - epoch)) + (( age >= 0 && age <= maximum )) +} + +_ksre_bundle_equal() { + local left_name="$1" right_name="$2" key + local -n left="$left_name" right="$right_name" + for key in schema bundle_id phase k3s_version datastore created_at_utc secret_count; do + [[ "${left[$key]}" == "${right[$key]}" ]] || return 1 + done +} + +_ksre_read_root_file() { + local path="$1" kind="$2" destination="$3" metadata temporary rc=0 + /usr/bin/sudo --non-interactive -- /usr/bin/test ! -L "$path" || return 1 + /usr/bin/sudo --non-interactive -- /usr/bin/test -f "$path" || return 1 + /usr/bin/sudo --non-interactive -- /usr/bin/test -s "$path" || return 1 + metadata="$(/usr/bin/sudo --non-interactive -- /usr/bin/stat \ + --format='%u:%g:%a:%F' -- "$path" 2>/dev/null)" || return 1 + [[ "$metadata" == '0:0:600:regular file' ]] || return 1 + temporary="$(/usr/bin/mktemp /tmp/platform-k3s-restore-read.XXXXXX)" || return 1 + /usr/bin/chmod 0600 "$temporary" || { /bin/rm -f -- "$temporary"; return 1; } + /usr/bin/sudo --non-interactive -- /bin/cat -- "$path" >"$temporary" || rc=$? + if (( rc == 0 )); then _ksre_parse_external_file "$kind" "$temporary" "$destination" || rc=$?; fi + /bin/rm -f -- "$temporary" + return "$rc" +} + +_ksre_read_authoritative_bundle() { _ksre_read_root_file "$KSRE_POST_BUNDLE" bundle "$1"; } +_ksre_read_installed_evidence() { _ksre_read_root_file "$KSRE_EVIDENCE" evidence "$1"; } + +_ksre_inventory_value() { + local file="$1" key="$2" line found='' + while IFS= read -r line; do + [[ "$line" =~ ^([a-z_]+)=([A-Za-z0-9_.+-]+)$ ]] || return 1 + if [[ "${BASH_REMATCH[1]}" == "$key" ]]; then + [[ -z "$found" ]] || return 1 + found="${BASH_REMATCH[2]}" + fi + done <"$file" + [[ -n "$found" ]] || return 1 + printf '%s\n' "$found" +} + +_ksre_secret_count_from_stream() { + /usr/bin/jq -er '.items | length' 2>/dev/null +} + +_ksre_require_same_ready_node() { + local local_node="$1" local_node_uid="$2" nodes_json="$3" + [[ "$local_node" =~ ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ ]] || return 1 + [[ "$local_node_uid" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ ]] || return 1 + /usr/bin/jq -e --arg local_node "$local_node" --arg local_node_uid "$local_node_uid" ' + [.items[] | + select(any(.status.conditions[]?; .type == "Ready" and .status == "True"))] | + length == 1 and + .[0].metadata.name == $local_node and + .[0].metadata.uid == $local_node_uid + ' >/dev/null 2>&1 <<<"$nodes_json" +} + +_ksre_run_restore_status_validator() { + /usr/bin/bash "$KSRE_STATUS_VALIDATOR" --expect-reencrypted-restore \ + --verified-output-dir "$1" >/dev/null +} + +_ksre_current_user_api_ready() { + local context="$1" server="$2" + /usr/bin/timeout --signal=TERM --kill-after=1s 9s \ + /usr/bin/env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u NO_PROXY \ + -u http_proxy -u https_proxy -u all_proxy -u no_proxy \ + "$KSRE_KUBECTL" --context="$context" --server="$server" \ + get --raw=/readyz >/dev/null 2>&1 +} + +_ksre_current_user_nodes_json() { + local context="$1" server="$2" + /usr/bin/timeout --signal=TERM --kill-after=1s 15s \ + /usr/bin/env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u NO_PROXY \ + -u http_proxy -u https_proxy -u all_proxy -u no_proxy \ + "$KSRE_KUBECTL" --context="$context" --server="$server" \ + get nodes -o json 2>/dev/null +} + +_ksre_current_user_secret_count() { + local context="$1" server="$2" + /usr/bin/timeout --signal=TERM --kill-after=1s 30s \ + /usr/bin/env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u NO_PROXY \ + -u http_proxy -u https_proxy -u all_proxy -u no_proxy \ + "$KSRE_KUBECTL" --context="$context" --server="$server" \ + get secrets --all-namespaces -o json 2>/dev/null | + _ksre_secret_count_from_stream +} + +_ksre_live_restore_inventory() { + local context="$1" connection="$2" server + local handoff inventory nodes version datastore encryption rotation hashes integrity + local local_node local_node_uid local_node_ready + local api_ready node_ready secret_count rc=0 + _ksre_validate_current_connection "$connection" || return 1 + server="${connection%|}" + handoff="$(/usr/bin/mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" || return 1 + /usr/bin/chmod 0700 "$handoff" || { /bin/rm -rf -- "$handoff"; return 1; } + if ! _ksre_run_restore_status_validator "$handoff"; then + /bin/rm -rf -- "$handoff"; return 1 + fi + inventory="$handoff/inventory.env" + version="$(_ksre_inventory_value "$inventory" version)" || rc=1 + datastore="$(_ksre_inventory_value "$inventory" datastore)" || rc=1 + encryption="$(_ksre_inventory_value "$inventory" encryption)" || rc=1 + rotation="$(_ksre_inventory_value "$inventory" rotation)" || rc=1 + hashes="$(_ksre_inventory_value "$inventory" server_hashes)" || rc=1 + integrity="$(_ksre_inventory_value "$inventory" integrity)" || rc=1 + local_node="$(_ksre_inventory_value "$inventory" node_name)" || rc=1 + local_node_uid="$(_ksre_inventory_value "$inventory" node_uid)" || rc=1 + local_node_ready="$(_ksre_inventory_value "$inventory" node_ready)" || rc=1 + /bin/rm -rf -- "$handoff" + (( rc == 0 )) || return 1 + [[ "$encryption" == Enabled && "$local_node_ready" == Ready ]] || return 1 + if _ksre_current_user_api_ready "$context" "$server"; then api_ready=pass; else return 1; fi + nodes="$(_ksre_current_user_nodes_json "$context" "$server")" || return 1 + _ksre_require_same_ready_node "$local_node" "$local_node_uid" "$nodes" || return 1 + node_ready=pass + secret_count="$(_ksre_current_user_secret_count "$context" "$server")" || return 1 + [[ "$secret_count" =~ ^[0-9]+$ ]] || return 1 + _ksre_require_same_connection "$context" "$connection" || return 1 + printf '%s|%s|enabled_stable|%s|%s|%s|%s|%s|%s\n' \ + "$version" "$datastore" "$rotation" "$hashes" "$integrity" \ + "$api_ready" "$node_ready" "$secret_count" +} + +_ksre_isolation_attested() { + local context="$1" answer + [[ -t 0 ]] || return 1 + printf '격리·egress 차단 시험 완료 후 Type ISOLATED %s: ' "$context" >&2 + IFS= read -r answer + [[ "$answer" == "ISOLATED $context" ]] +} + +_ksre_output_target_absent() { + local output="$1" parent base + [[ -n "$output" && ! -e "$output" && ! -L "$output" ]] || return 1 + if [[ "$output" == */* ]]; then parent="${output%/*}"; base="${output##*/}" + else parent='.'; base="$output"; fi + [[ -n "$parent" && -n "$base" && "$base" != . && "$base" != .. ]] || return 1 + [[ -d "$parent" && ! -L "$parent" ]] || return 1 +} + +_ksre_write_output() { + local output="$1" content="$2" kind="$3" parent base temporary + _ksre_output_target_absent "$output" || return 1 + if [[ "$output" == */* ]]; then parent="${output%/*}"; base="${output##*/}" + else parent='.'; base="$output"; fi + temporary="$(/usr/bin/mktemp "${parent}/.${base}.tmp.XXXXXX")" || return 1 + /usr/bin/chmod 0600 "$temporary" || { /bin/rm -f -- "$temporary"; return 1; } + printf '%s\n' "$content" >"$temporary" || { /bin/rm -f -- "$temporary"; return 1; } + declare -A verification=() + _ksre_parse_external_file "$kind" "$temporary" verification || { /bin/rm -f -- "$temporary"; return 1; } + /bin/ln -- "$temporary" "$output" 2>/dev/null || { /bin/rm -f -- "$temporary"; return 1; } + /bin/rm -f -- "$temporary" +} + +_ksre_evidence_target_absent() { + /usr/bin/sudo --non-interactive -- /usr/bin/test ! -e "$KSRE_EVIDENCE" && + /usr/bin/sudo --non-interactive -- /usr/bin/test ! -L "$KSRE_EVIDENCE" +} + +_ksre_prompt_destroyed() { + local answer + [[ -t 0 ]] || return 1 + printf '일회용 환경과 bundle 복사본 파기 후 Type DESTROYED default: ' >&2 + IFS= read -r answer + [[ "$answer" == 'DESTROYED default' ]] +} + +_ksre_install_evidence() { + local content="$1" local_file root_temporary + local_file="$(/usr/bin/mktemp /tmp/platform-k3s-restore-evidence.XXXXXX)" || return 1 + /usr/bin/chmod 0600 "$local_file" || { /bin/rm -f -- "$local_file"; return 1; } + printf '%s\n' "$content" >"$local_file" || { /bin/rm -f -- "$local_file"; return 1; } + declare -A verification=() + _ksre_parse_external_file evidence "$local_file" verification || { /bin/rm -f -- "$local_file"; return 1; } + root_temporary="${KSRE_EVIDENCE}.new.$$" + /usr/bin/sudo --non-interactive -- /usr/bin/test ! -e "$KSRE_EVIDENCE" || { /bin/rm -f -- "$local_file"; return 1; } + /usr/bin/sudo --non-interactive -- /usr/bin/install -o root -g root -m 0600 -- \ + "$local_file" "$root_temporary" || { /bin/rm -f -- "$local_file"; return 1; } + /usr/bin/sudo --non-interactive -- /usr/bin/cmp --silent -- "$local_file" "$root_temporary" || { + /usr/bin/sudo --non-interactive -- /bin/rm -f -- "$root_temporary" >/dev/null 2>&1 || : + /bin/rm -f -- "$local_file"; return 1 + } + if ! /usr/bin/sudo --non-interactive -- /bin/ln -- "$root_temporary" "$KSRE_EVIDENCE" 2>/dev/null; then + /usr/bin/sudo --non-interactive -- /bin/rm -f -- "$root_temporary" >/dev/null 2>&1 || : + /bin/rm -f -- "$local_file"; return 1 + fi + /usr/bin/sudo --non-interactive -- /bin/rm -f -- "$root_temporary" || return 1 + /bin/rm -f -- "$local_file" +} + +_ksre_render_result() { + local -n bundle_fields="$1" + local tested_at="$2" + printf 'schema=platform-k3s-restore-result-v1\nbundle_id=%s\nk3s_version=%s\ndatastore=%s\nstatus_class=enabled_stable\nrotation_stage=reencrypt_finished\nhashes=match\nlocal_integrity=match\napi_ready=pass\nnode_ready=pass\nisolation=pass\nsecret_count_match=pass\ntested_at_utc=%s\n' \ + "${bundle_fields[bundle_id]}" "${bundle_fields[k3s_version]}" "${bundle_fields[datastore]}" "$tested_at" +} + +_ksre_render_evidence() { + local -n bundle_fields="$1" result_fields="$2" + local recorded_at="$3" + printf 'schema=platform-k3s-restore-evidence-v1\nbundle_id=%s\nk3s_version=%s\ndatastore=%s\nrotation_stage=reencrypt_finished\nlocal_integrity=match\nnode_ready=pass\nisolation=pass\ntested_at_utc=%s\nrecorded_at_utc=%s\ndestroyed=confirmed\n' \ + "${bundle_fields[bundle_id]}" "${bundle_fields[k3s_version]}" "${bundle_fields[datastore]}" \ + "${result_fields[tested_at_utc]}" "$recorded_at" +} + +_ksre_emit_result() { + local metadata="$1" output="$2" context connection live tested_at content + local version datastore status rotation hashes integrity api node count + declare -A bundle=() + context="$(_ksre_current_context)" || return 1 + [[ -n "$context" && "$context" != *$'\n'* ]] || return 1 + connection="$(_ksre_current_connection "$context")" || return 1 + _ksre_validate_current_connection "$connection" || return 1 + _ksre_parse_external_file bundle "$metadata" bundle || return 1 + [[ "${bundle[phase]}" == post ]] || return 1 + _ksre_output_target_absent "$output" || return 1 + live="$(_ksre_live_restore_inventory "$context" "$connection")" || return 1 + IFS='|' read -r version datastore status rotation hashes integrity api node count <<<"$live" + [[ "$version" == "${bundle[k3s_version]}" && "$datastore" == "${bundle[datastore]}" && + "$status" == enabled_stable && "$rotation" == reencrypt_finished && "$hashes" == match && + "$integrity" == match && "$api" == pass && "$node" == pass && + "$count" == "${bundle[secret_count]}" ]] || return 1 + _ksre_isolation_attested "$context" || return 1 + _ksre_require_same_connection "$context" "$connection" || return 1 + _ksre_require_same_context "$context" || return 1 + tested_at="$(_ksre_now_rfc3339)" || return 1 + content="$(_ksre_render_result bundle "$tested_at")" || return 1 + _ksre_write_output "$output" "$content" result +} + +_ksre_record() { + local metadata="$1" result_file="$2" context recorded_at content + declare -A bundle=() result=() authority=() verification=() + context="$(_ksre_current_context)" || return 1 + [[ -n "$context" && "$context" != *$'\n'* ]] || return 1 + _ksre_parse_external_file bundle "$metadata" bundle || return 1 + _ksre_parse_external_file result "$result_file" result || return 1 + [[ "${bundle[phase]}" == post && "${result[bundle_id]}" == "${bundle[bundle_id]}" && + "${result[k3s_version]}" == "${bundle[k3s_version]}" && + "${result[datastore]}" == "${bundle[datastore]}" ]] || return 1 + _ksre_age_within "${result[tested_at_utc]}" 86400 || return 1 + _ksre_read_authoritative_bundle authority || return 1 + _ksre_bundle_equal bundle authority || return 1 + _ksre_evidence_target_absent || return 1 + _ksre_prompt_destroyed || return 1 + _ksre_require_same_context "$context" || return 1 + recorded_at="$(_ksre_now_rfc3339)" || return 1 + content="$(_ksre_render_evidence bundle result "$recorded_at")" || return 1 + _ksre_parse_trusted_content evidence "$content" verification || return 1 + _ksre_install_evidence "$content" +} + +_ksre_check() { + local context connection live version datastore status rotation hashes integrity api node count + declare -A authority=() evidence=() + context="$(_ksre_current_context)" || return 1 + [[ -n "$context" && "$context" != *$'\n'* ]] || return 1 + connection="$(_ksre_current_connection "$context")" || return 1 + _ksre_validate_current_connection "$connection" || return 1 + _ksre_read_authoritative_bundle authority || return 1 + _ksre_read_installed_evidence evidence || return 1 + [[ "${authority[phase]}" == post && + "${evidence[bundle_id]}" == "${authority[bundle_id]}" && + "${evidence[k3s_version]}" == "${authority[k3s_version]}" && + "${evidence[datastore]}" == "${authority[datastore]}" ]] || return 1 + _ksre_age_within "${evidence[tested_at_utc]}" 2592000 || return 1 + live="$(_ksre_live_restore_inventory "$context" "$connection")" || return 1 + IFS='|' read -r version datastore status rotation hashes integrity api node count <<<"$live" + [[ "$version" == "${evidence[k3s_version]}" && "$datastore" == "${evidence[datastore]}" && + "$status" == enabled_stable && "$rotation" == reencrypt_finished && + "$hashes" == match && "$integrity" == match && "$api" == pass && "$node" == pass ]] || return 1 + _ksre_require_same_context "$context" +} + +k3s_secret_encryption_restore_evidence_main() { + local mode='' metadata='' result_file='' output='' + while (( $# > 0 )); do + case "$1" in + --emit-result|--record|--check) + [[ -z "$mode" ]] || { _ksre_usage >&2; return 2; } + mode="$1"; shift + ;; + --bundle-metadata) + [[ $# -ge 2 && -z "$metadata" ]] || { _ksre_usage >&2; return 2; } + metadata="$2"; shift 2 + ;; + --result-file) + [[ $# -ge 2 && -z "$result_file" ]] || { _ksre_usage >&2; return 2; } + result_file="$2"; shift 2 + ;; + --output) + [[ $# -ge 2 && -z "$output" ]] || { _ksre_usage >&2; return 2; } + output="$2"; shift 2 + ;; + -h|--help) _ksre_usage; return 0 ;; + *) _ksre_usage >&2; return 2 ;; + esac + done + case "$mode" in + --emit-result) + [[ -n "$metadata" && -n "$output" && -z "$result_file" ]] || { _ksre_usage >&2; return 2; } + _ksre_emit_result "$metadata" "$output" || _ksre_fail + ;; + --record) + [[ -n "$metadata" && -n "$result_file" && -z "$output" ]] || { _ksre_usage >&2; return 2; } + _ksre_record "$metadata" "$result_file" || _ksre_fail + ;; + --check) + [[ -z "$metadata" && -z "$result_file" && -z "$output" ]] || { _ksre_usage >&2; return 2; } + _ksre_check || _ksre_fail + ;; + *) _ksre_usage >&2; return 2 ;; + esac +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + PATH="$KSRE_SECURE_PATH" + export PATH + k3s_secret_encryption_restore_evidence_main "$@" +fi diff --git a/scripts/validate/k3s-secret-encryption.sh b/scripts/validate/k3s-secret-encryption.sh new file mode 100644 index 0000000..3eaaf51 --- /dev/null +++ b/scripts/validate/k3s-secret-encryption.sh @@ -0,0 +1,344 @@ +#!/usr/bin/env bash + +# Read-only k3s Secret-encryption inventory. Privileged execution is limited to +# bounded invocations of validated, root-owned system binaries; workspace code +# is never executed or sourced by sudo. +set -Eeuo pipefail + +readonly KSE_SECURE_PATH='/usr/sbin:/usr/bin:/sbin:/bin' +readonly KSE_EXPECTED_VERSION='v1.36.2+k3s1' +readonly KSE_EXPECTED_NODE='donghyeon-system-product-name' +readonly KSE_K3S='/usr/local/bin/k3s' +readonly KSE_TIMEOUT='/usr/bin/timeout' +readonly KSE_SUDO='/usr/bin/sudo' +readonly KSE_SYSTEMCTL='/usr/bin/systemctl' +readonly KSE_STAT='/usr/bin/stat' +readonly KSE_SHA256SUM='/usr/bin/sha256sum' +readonly KSE_CAT='/usr/bin/cat' +readonly KSE_JQ='/usr/bin/jq' +readonly KSE_FIND='/usr/bin/find' +readonly KSE_ID='/usr/bin/id' +readonly KSE_TEST='/usr/bin/test' +if [[ "${BASH_SOURCE[0]}" == */* ]]; then + readonly KSE_SCRIPT_DIR="${BASH_SOURCE[0]%/*}" +else + readonly KSE_SCRIPT_DIR='.' +fi +readonly KSE_ROOT="$(cd -- "${KSE_SCRIPT_DIR}/../.." && pwd -P)" +readonly KSE_LIBRARY_PATH="${KSE_ROOT}/scripts/lib/k3s-secret-encryption.sh" +readonly KSE_CONFIG_PATH='/var/lib/rancher/k3s/server/cred/encryption-config.json' +readonly KSE_STATE_PATH='/var/lib/rancher/k3s/server/cred/encryption-state.json' + +kse_fail() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +kse_usage() { + cat <<'USAGE' +사용법: + bash scripts/validate/k3s-secret-encryption.sh [기대 상태] [--verified-output-dir DIR] + +기대 상태(하나만 지정): + --expect-disabled + --expect-transition-start + --expect-enabled + --expect-reencrypted + --expect-reencrypted-restore # 격리 복구 host의 다른 node 이름 허용 +USAGE +} + +_kse_trusted_executable() { + local path="$1" metadata uid mode file_type + [[ -f "$path" && ! -L "$path" && -x "$path" ]] || return 1 + metadata="$($KSE_STAT --format='%u:%a:%F' -- "$path" 2>/dev/null)" || return 1 + IFS=: read -r uid mode file_type <<<"$metadata" + [[ "$uid" == 0 && "$mode" =~ ^[0-7]{3,4}$ && "$file_type" == 'regular file' ]] || return 1 + (( (8#$mode & 8#022) == 0 )) +} + +_kse_validate_runtime_dependencies() { + local binary + for binary in \ + "$KSE_TIMEOUT" "$KSE_SUDO" "$KSE_K3S" "$KSE_SYSTEMCTL" \ + "$KSE_STAT" "$KSE_SHA256SUM" "$KSE_CAT" "$KSE_JQ" \ + "$KSE_FIND" "$KSE_ID" "$KSE_TEST" \ + /usr/bin/sed /usr/bin/sort; do + _kse_trusted_executable "$binary" || return 1 + done +} + +_kse_privileged_capture() { + local binary="$1" + shift + "$KSE_TIMEOUT" --signal=TERM --kill-after=1s 9s \ + "$KSE_SUDO" --non-interactive -- "$binary" "$@" 2>/dev/null +} + +_kse_validate_sudo_credentials() { + "$KSE_TIMEOUT" --signal=TERM --kill-after=1s 9s \ + "$KSE_SUDO" --non-interactive --validate >/dev/null 2>&1 +} + +_kse_privileged_quiet() { + _kse_privileged_capture "$@" >/dev/null 2>&1 +} + +_kse_read_version() { + local raw version + raw="$(_kse_privileged_capture "$KSE_K3S" --version)" || return 1 + version="$(/usr/bin/sed -n -E 's/^k3s version (v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+)([[:space:]].*)?$/\1/p' <<<"$raw")" + [[ "$version" == "$KSE_EXPECTED_VERSION" ]] || return 1 + printf '%s\n' "$version" +} + +_kse_read_status() { + _kse_privileged_capture "$KSE_K3S" secrets-encrypt status --output json +} + +_kse_read_nodes() { + _kse_privileged_capture "$KSE_K3S" kubectl get nodes -o json +} + +_kse_read_systemctl_show() { + _kse_privileged_capture "$KSE_SYSTEMCTL" show k3s \ + --property=ExecStart --property=Environment --property=EnvironmentFiles +} + +_kse_api_ready() { + _kse_privileged_quiet "$KSE_K3S" kubectl get --raw=/readyz +} + +_kse_allowed_root_text_path() { + local path="$1" + [[ "$path" == /etc/rancher/k3s/config.yaml || + "$path" == /etc/systemd/system/k3s.service.env || + "$path" =~ ^/etc/rancher/k3s/config\.yaml\.d/[A-Za-z0-9._-]+\.yaml$ ]] +} + +_kse_read_root_text() { + local path="$1" metadata uid mode + _kse_allowed_root_text_path "$path" || return 1 + _kse_privileged_quiet "$KSE_TEST" -f "$path" || return 1 + _kse_privileged_quiet "$KSE_TEST" ! -L "$path" || return 1 + metadata="$(_kse_privileged_capture "$KSE_STAT" --format='%u:%a' -- "$path")" || return 1 + IFS=: read -r uid mode <<<"$metadata" + [[ "$uid" == 0 && "$mode" =~ ^[0-7]{3,4}$ ]] || return 1 + (( (8#$mode & 8#022) == 0 )) || return 1 + _kse_privileged_capture "$KSE_CAT" -- "$path" +} + +_kse_root_path_kind() { + local path="$1" + if _kse_privileged_quiet "$KSE_TEST" -f "$path"; then + printf 'file\n' + elif _kse_privileged_quiet "$KSE_TEST" -d "$path"; then + printf 'directory\n' + else + printf 'absent\n' + fi +} + +_kse_detect_datastore() { + local service_show="$1" + _k3s_systemctl_show() { printf '%s\n' "$service_show"; } + _k3s_systemd_dropin_lines() { return 0; } + _k3s_read_file() { _kse_read_root_text "$1"; } + _k3s_list_yaml_files() { + local config="$1" file listing='' + [[ "$config" == /etc/rancher/k3s/config.yaml ]] || return 2 + if _kse_privileged_quiet "$KSE_TEST" -f "$config"; then printf '%s\n' "$config"; fi + if _kse_privileged_quiet "$KSE_TEST" -d "${config}.d"; then + listing="$(_kse_privileged_capture "$KSE_FIND" "${config}.d" -maxdepth 1 -type f -name '*.yaml' -print)" || return 2 + while IFS= read -r file; do + [[ -z "$file" ]] || _kse_allowed_root_text_path "$file" || return 2 + done <<<"$listing" + [[ -z "$listing" ]] || printf '%s\n' "$listing" | /usr/bin/sort + fi + } + _k3s_datastore_local_evidence() { + local data_dir="$1" state_kind etcd_kind + [[ "$data_dir" == /var/lib/rancher/k3s ]] || { printf 'ambiguous\n'; return 0; } + state_kind="$(_kse_root_path_kind "${data_dir}/server/db/state.db")" || return 1 + etcd_kind="$(_kse_root_path_kind "${data_dir}/server/db/etcd")" || return 1 + if [[ "$state_kind" == file && "$etcd_kind" == directory ]]; then printf 'ambiguous\n'; + elif [[ "$state_kind" == file ]]; then printf 'sqlite\n'; + elif [[ "$etcd_kind" == directory ]]; then printf 'embedded-etcd\n'; + else printf 'none\n'; fi + } + detect_k3s_datastore +} + +_kse_root_metadata() { + local path="$1" metadata + metadata="$(_kse_privileged_capture "$KSE_STAT" --format='%u:%a:%F' -- "$path")" || return 1 + [[ "$metadata" == '0:600:regular file' ]] || return 1 + printf '%s\n' "$metadata" +} + +_kse_integrity_from_root_evidence() { + local annotation="$1" stage="$2" config_metadata state_metadata + local config_hash_line config_hash state_with_sentinel state_value result + config_metadata="$(_kse_root_metadata "$KSE_CONFIG_PATH")" || { printf 'mismatch\n'; return 1; } + state_metadata="$(_kse_root_metadata "$KSE_STATE_PATH")" || { printf 'mismatch\n'; return 1; } + config_hash_line="$(_kse_privileged_capture "$KSE_SHA256SUM" -- "$KSE_CONFIG_PATH")" || { printf 'mismatch\n'; return 1; } + config_hash="${config_hash_line%% *}" + [[ "$config_hash_line" == "${config_hash} ${KSE_CONFIG_PATH}" && "$config_hash" =~ ^[0-9a-f]{64}$ ]] || { printf 'mismatch\n'; return 1; } + state_with_sentinel="$(_kse_privileged_capture "$KSE_CAT" -- "$KSE_STATE_PATH"; printf .)" || { printf 'mismatch\n'; return 1; } + state_value="${state_with_sentinel%.}" + result="$(verify_local_encryption_config_integrity_evidence \ + "$config_metadata" "$state_metadata" "$config_hash" "$state_value" \ + "$annotation" "$stage" 2>/dev/null || true)" + [[ "$result" == match ]] || { printf 'mismatch\n'; return 1; } + printf 'match\n' +} + +_kse_validate_output_dir() { + local output_dir="$1" physical metadata uid + [[ "$output_dir" =~ ^/tmp/platform-k3s-encryption\.[A-Za-z0-9]+$ ]] || return 1 + [[ -d /tmp && ! -L /tmp && -d "$output_dir" && ! -L "$output_dir" ]] || return 1 + [[ "$(cd -P -- /tmp && pwd -P)" == /tmp ]] || return 1 + physical="$(cd -P -- "$output_dir" && pwd -P)" || return 1 + [[ "$physical" == "$output_dir" ]] || return 1 + uid="$($KSE_ID -u)" || return 1 + metadata="$($KSE_STAT --format='%u:%a:%F' -- "$output_dir" 2>/dev/null)" || return 1 + [[ "$metadata" == "${uid}:700:directory" ]] || return 1 + [[ -z "$($KSE_FIND "$output_dir" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]] +} + +_kse_write_verified_output() { + local output_dir="$1" invoking_uid metadata physical entry + local -a entries=() + local directory_fd + _kse_validate_output_dir "$output_dir" || return 1 + invoking_uid="$($KSE_ID -u)" || return 1 + exec {directory_fd}<"$output_dir" || return 1 + physical="$(cd -P -- "/proc/self/fd/${directory_fd}" && pwd -P)" || { exec {directory_fd}<&-; return 1; } + [[ "$physical" == "$output_dir" ]] || { exec {directory_fd}<&-; return 1; } + metadata="$($KSE_STAT -L --format='%u:%a:%F' -- "/proc/self/fd/${directory_fd}" 2>/dev/null)" || { exec {directory_fd}<&-; return 1; } + [[ "$metadata" == "${invoking_uid}:700:directory" ]] || { exec {directory_fd}<&-; return 1; } + ( + umask 077 + set -C + printf 'version=%s\nserver_count=%s\nnode_name=%s\nnode_uid=%s\nnode_ready=%s\ndatastore=%s\nencryption=%s\nrotation=%s\nserver_hashes=%s\nintegrity=%s\napi_readyz=%s\n' \ + "$version" "$server_count" "$node_name" "$node_uid" "$node_ready" "$datastore" "$encryption" "$rotation" "$server_hashes" "$integrity" "$api_readyz" \ + >"/proc/self/fd/${directory_fd}/inventory.env" + printf '%s\n' "$status_sha256" >"/proc/self/fd/${directory_fd}/status.sha256" + ) || { exec {directory_fd}<&-; return 1; } + metadata="$($KSE_STAT --format='%u:%a:%F' -- "/proc/self/fd/${directory_fd}/inventory.env" "/proc/self/fd/${directory_fd}/status.sha256" 2>/dev/null)" || { exec {directory_fd}<&-; return 1; } + [[ "$metadata" == "${invoking_uid}:600:regular file"$'\n'"${invoking_uid}:600:regular file" ]] || { exec {directory_fd}<&-; return 1; } + while IFS= read -r -d '' entry; do entries+=("${entry##*/}"); done < <( + "$KSE_FIND" -H "/proc/self/fd/${directory_fd}" -mindepth 1 -maxdepth 1 -print0 2>/dev/null + ) + exec {directory_fd}<&- + (( ${#entries[@]} == 2 )) || return 1 + [[ " ${entries[*]} " == *' inventory.env '* && " ${entries[*]} " == *' status.sha256 '* ]] +} + +k3s_secret_encryption_main() { + local expectation='' output_dir='' output_dir_seen=false + local raw_status canonical_status status_class stage status_hash_line + local nodes_json service_show annotations='' annotation='' first_annotation='' + local version server_count node_name node_uid node_ready datastore encryption rotation + local server_hashes=\mismatch integrity=mismatch api_readyz status_sha256 + + PATH="$KSE_SECURE_PATH" + export PATH + unset K3S_CONFIG_DIR K3S_SYSTEMD_DROPIN_DIR + + while (( $# > 0 )); do + case "$1" in + --expect-disabled|--expect-transition-start|--expect-enabled|--expect-reencrypted|--expect-reencrypted-restore) + [[ -z "$expectation" ]] || { kse_usage >&2; return 2; } + expectation="$1" + ;; + --verified-output-dir) + (( $# >= 2 )) || { kse_usage >&2; return 2; } + ! "$output_dir_seen" && [[ -n "$2" ]] || { kse_usage >&2; return 2; } + output_dir="$2"; output_dir_seen=true; shift + ;; + --help|-h) kse_usage; return 0 ;; + *) kse_usage >&2; return 2 ;; + esac + shift + done + + _kse_validate_runtime_dependencies || { kse_fail 'trusted runtime dependency validation failed'; return 1; } + ! "$output_dir_seen" || _kse_validate_output_dir "$output_dir" || { kse_fail 'verified output directory is unsafe'; return 1; } + _kse_validate_sudo_credentials || { + kse_fail 'noninteractive sudo authorization is unavailable; run sudo -v in the same terminal, then rerun the validator' + return 1 + } + + # shellcheck source=/dev/null + source "$KSE_LIBRARY_PATH" + version="$(_kse_read_version)" || { kse_fail 'k3s version is not the required production version'; return 1; } + raw_status="$(_kse_read_status)" || { kse_fail 'read-only encryption status failed'; return 1; } + canonical_status="$($KSE_JQ -ceS . <<<"$raw_status" 2>/dev/null)" || { kse_fail 'encryption status is invalid'; return 1; } + status_class="$(classify_encryption_status "$canonical_status")" + [[ "$status_class" != invalid ]] || { kse_fail 'encryption status is invalid'; return 1; } + status_hash_line="$(printf '%s' "$canonical_status" | "$KSE_SHA256SUM")" || return 1 + status_sha256="${status_hash_line%% *}" + [[ "$status_sha256" =~ ^[0-9a-f]{64}$ ]] || return 1 + + encryption=Disabled; rotation=unsafe + case "$status_class" in + disabled_no_config) rotation=none ;; + transition_start) rotation=start ;; + enabled_stable) + encryption=Enabled + stage="$($KSE_JQ -er '.stage' <<<"$canonical_status")" || return 1 + [[ "$stage" == start || "$stage" == reencrypt_finished ]] || return 1 + rotation="$stage" + ;; + unsafe_transition|hash_mismatch) encryption=Enabled ;; + *) return 1 ;; + esac + + nodes_json="$(_kse_read_nodes)" || { kse_fail 'read-only node inventory failed'; return 1; } + server_count="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null))] | length' <<<"$nodes_json" 2>/dev/null)" || return 1 + [[ "$server_count" == 1 ]] || { kse_fail 'server count is not exactly one'; return 1; } + node_name="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null))][0].metadata.name' <<<"$nodes_json" 2>/dev/null)" || return 1 + node_uid="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null))][0].metadata.uid | if type == "string" and test("^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$") then . else error("invalid node uid") end' <<<"$nodes_json" 2>/dev/null)" || { kse_fail 'server node UID is invalid'; return 1; } + if [[ "$expectation" != --expect-reencrypted-restore ]]; then + [[ "$node_name" == "$KSE_EXPECTED_NODE" ]] || { kse_fail 'server node identity mismatch'; return 1; } + fi + node_ready="$($KSE_JQ -er '[.items[] | select(.metadata.name == $name)][0].status.conditions | [ .[] | select(.type == "Ready") ] | if length == 1 and .[0].status == "True" then "Ready" else error("not ready") end' --arg name "$node_name" <<<"$nodes_json" 2>/dev/null)" || { kse_fail 'server node is not Ready'; return 1; } + + annotations="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null)) | .metadata.annotations["k3s.io/encryption-config-hash"]?] | if length > 0 and all(.[]; type == "string" and test("^[a-z_]+-[0-9a-f]{64}$")) then .[] else empty end' <<<"$nodes_json" 2>/dev/null)" || annotations='' + if [[ -n "$annotations" ]]; then + first_annotation="${annotations%%$'\n'*}"; annotation="$first_annotation" + while IFS= read -r annotation; do [[ "$annotation" == "$first_annotation" ]] || first_annotation=''; done <<<"$annotations" + [[ -z "$first_annotation" ]] || server_hashes=match + fi + + service_show="$(_kse_read_systemctl_show)" || { kse_fail 'read-only systemd inventory failed'; return 1; } + datastore="$(_kse_detect_datastore "$service_show")" || datastore=ambiguous + case "$datastore" in sqlite|embedded-etcd|external|ambiguous) ;; *) datastore=ambiguous ;; esac + + if [[ "$server_hashes" == match && ( "$status_class" == transition_start || "$status_class" == enabled_stable ) ]]; then + integrity="$(_kse_integrity_from_root_evidence "$first_annotation" "$rotation" 2>/dev/null || true)" + [[ "$integrity" == match ]] || integrity=mismatch + fi + if _kse_api_ready; then api_readyz=pass; else api_readyz=fail; fi + + printf 'K3s version: %s\nServer count: %s\nNode: %s %s\nDatastore: %s\nEncryption status: %s\nRotation stage: %s\nServer annotation hashes: %s\nLocal config/state/annotation integrity: %s\nAPI readyz: %s\n' \ + "$version" "$server_count" "$node_name" "$node_ready" "$datastore" "$encryption" "$rotation" "$server_hashes" "$integrity" "$api_readyz" + + [[ "$api_readyz" == pass ]] || return 1 + case "$expectation" in + '') [[ "$status_class" != unsafe_transition && "$status_class" != hash_mismatch ]] || return 1 ;; + --expect-disabled) [[ "$status_class" == disabled_no_config ]] || return 1 ;; + --expect-transition-start) [[ "$status_class" == transition_start ]] || return 1 ;; + --expect-enabled) [[ "$status_class" == enabled_stable ]] || return 1 ;; + --expect-reencrypted|--expect-reencrypted-restore) + [[ "$status_class" == enabled_stable && "$rotation" == reencrypt_finished && + "$server_hashes" == match && "$integrity" == match ]] || return 1 + ;; + esac + ! "$output_dir_seen" || _kse_write_verified_output "$output_dir" || { kse_fail 'verified output write failed'; return 1; } +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + k3s_secret_encryption_main "$@" +fi diff --git a/scripts/validate/observability-smoke.sh b/scripts/validate/observability-smoke.sh new file mode 100644 index 0000000..1165442 --- /dev/null +++ b/scripts/validate/observability-smoke.sh @@ -0,0 +1,1808 @@ +#!/usr/bin/env bash + +# End-to-end acceptance for the private observability access and alerting +# cutover. No-argument mode is intentionally a no-contact plan printer. +set -Eeuo pipefail +set +x +umask 077 + +readonly EXPECTED_CONTEXT='default' +readonly EXPECTED_API_SERVER='https://127.0.0.1:6443' +readonly EXPECTED_NODE='donghyeon-system-product-name' +readonly NAMESPACE='observability' +readonly GRAFANA_HOST='grafana.learn.hyeonworks.com' +readonly GITEA_HOST='git.learn.hyeonworks.com' +readonly KEYCLOAK_HOST='id.learn.hyeonworks.com' +readonly LAN_IP='192.168.0.107' +readonly TAILSCALE_IP='100.92.240.34' +readonly PROMETHEUS_SERVICE='observability-core-kube-pr-prometheus' +readonly ALERTMANAGER_SERVICE='observability-core-kube-pr-alertmanager' +readonly RUNBOOK_URL='https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' +readonly BUSYBOX_IMAGE='docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0' +readonly KUBECTL_DEFAULT='/usr/local/bin/kubectl' +readonly CURL_DEFAULT='/usr/bin/curl' +readonly DIG_DEFAULT='/usr/bin/dig' +readonly OPENSSL_DEFAULT='/usr/bin/openssl' +readonly NC_DEFAULT='/usr/bin/nc.openbsd' +readonly SUDO_DEFAULT='/usr/bin/sudo' +readonly DATE_DEFAULT='/usr/bin/date' +readonly CERTBOT_BIN='/snap/bin/certbot' +readonly CERTBOT_REAL='/usr/bin/snap' +readonly SYSTEMCTL_BIN='/usr/bin/systemctl' +readonly TIMEOUT_BIN='/usr/bin/timeout' +readonly JQ_BIN='/usr/bin/jq' +readonly PYTHON_BIN='/usr/bin/python3.12' +readonly CAT_BIN='/usr/bin/cat' +readonly CHMOD_BIN='/usr/bin/chmod' +readonly CMP_BIN='/usr/bin/cmp' +readonly MKTEMP_BIN='/usr/bin/mktemp' +readonly MV_BIN='/usr/bin/mv' +readonly READLINK_BIN='/usr/bin/readlink' +readonly RM_BIN='/usr/bin/rm' +readonly STAT_BIN='/usr/bin/stat' +readonly REQUEST_TIMEOUT='10s' +readonly PROCESS_TIMEOUT='20s' +readonly HUMAN_TIMEOUT_SECONDS=300 + +KUBECTL_BIN="$KUBECTL_DEFAULT" +CURL_BIN="$CURL_DEFAULT" +DIG_BIN="$DIG_DEFAULT" +OPENSSL_BIN="$OPENSSL_DEFAULT" +NC_BIN="$NC_DEFAULT" +SUDO_BIN="$SUDO_DEFAULT" +DATE_BIN="$DATE_DEFAULT" +TEST_MODE=false +TEST_CONFIRMATIONS='' +CONFIRMATION_INDEX=0 +declare -a CONFIRMATION_QUEUE=() + +execute_requested=false +WORK='' +API_PROXY_SOCKET='' +API_PROXY_PID='' +API_PROXY_START='' +EXIT_HANDLER_RUNNING=false +MANUAL_RECOVERY=false +SILENCE_ID='' +SILENCE_RUN='' +SILENCE_POLL_ATTEMPTS=30 +SILENCE_POLL_DELAY=1 +MUTATION_CRITICAL_DEPTH=0 +DEFERRED_SIGNAL_RC=0 +EXPECTED_NODE_UID='' +PUBLIC_EDGE_IP='' +KEYCLOAK_TEST_ADDED=false +KEYCLOAK_MEMBERSHIP_PENDING=false +KEYCLOAK_MEMBERSHIP_REMOVED=false +KEYCLOAK_USER_ID='' +KEYCLOAK_VIEWER_GROUP_ID='' +KEYCLOAK_MEMBERSHIP_BEFORE='' +KEYCLOAK_AUTH_CONFIG='' +declare -a OWNED_KIND=() +declare -a OWNED_NAMESPACE=() +declare -a OWNED_NAME=() +declare -a OWNED_UID=() +declare -a OWNED_RUN=() + +fail() { printf 'ERROR: %s\n' "$*" >&2; return 1; } +emit_manual_recovery() { + [[ "$MANUAL_RECOVERY" == true ]] && return 0 + MANUAL_RECOVERY=true + printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2 +} + +usage() { + "$CAT_BIN" <<'USAGE' +Usage: + bash scripts/validate/observability-smoke.sh + bash scripts/validate/observability-smoke.sh --execute + +No arguments prints the exact acceptance plan without external reads or +mutations. Execute requires the visible `APPLY default` gate, creates only +short-lived acceptance objects, restarts only Grafana/Alertmanager Pods, and +requires exact human OIDC, Slack, and external-client evidence tokens. +USAGE +} + +print_dry_run() { + printf '%s\n' \ + 'OBSERVABILITY_SMOKE_DRY_RUN=PASS' \ + 'CHECK_01=Grafana public/private DNS exactness' \ + 'CHECK_02=Grafana certificate chain,hostname,one-SAN,expiry' \ + 'CHECK_03=LAN,Tailscale,denied-source,public-edge HTTPS status' \ + 'CHECK_04=Gitea,Grafana,Keycloak public metrics denial' \ + 'CHECK_05=unknown-SNI and Traefik NodePort boundary' \ + 'CHECK_06=exact Prometheus target allowlist and health' \ + 'CHECK_07=Gitea,Keycloak,CNPG,AIStor,Traefik metric contracts' \ + 'CHECK_08=untrusted-Pod AIStor and blackbox denial' \ + 'CHECK_09=public,private,internal,TLS blackbox series' \ + 'CHECK_10=Grafana datasource health and Loki-to-Tempo trace link' \ + 'CHECK_11=AIStor quota and recording-rule series' \ + 'CHECK_12=Grafana,Alertmanager restart persistence' \ + 'CHECK_13=Certbot dry-run deploy-hook and Nginx health' \ + 'HUMAN_OIDC=admin,viewer,denied,break-glass,session-revoke,relogin-denied' \ + 'HUMAN_SLACK=firing,resolved' \ + 'HUMAN_EXTERNAL_CLIENT=required' \ + 'TEMPORARY_OBJECTS=NetworkPolicy,Pod,AlertmanagerSilence,PrometheusRule' \ + 'PRESERVE=Secret,PVC,preexisting-Keycloak-membership' \ + 'MUTATION=NOT_REQUESTED' +} + +parse_arguments() { + if (( $# == 0 )); then return 0; fi + if (( $# == 1 )) && [[ "$1" == --execute ]]; then execute_requested=true; return 0; fi + usage >&2 + fail 'supported arguments are empty or exactly --execute' +} + +reject_production_overrides() { + local variable + for variable in ${!PLATFORM_OBSERVABILITY_SMOKE_@}; do + fail "production boundary override is forbidden: $variable" + return 1 + done +} + +validate_dns_contract() { + (( $# == 8 )) || return 1 + [[ "$1" == '' && "$2" == '' && "$3" == "$LAN_IP" && "$4" == '' && + "$5" == "$TAILSCALE_IP" && "$6" == '' && "$7" == "$LAN_IP" && "$8" == '' ]] +} + +validate_certificate_contract() { + local sans=$1 verify_rc=$2 expiry_rc=$3 + [[ "$verify_rc" == 0 && "$expiry_rc" == 0 && + "$sans" == "DNS:$GRAFANA_HOST" ]] +} + +validate_certificate_extension_contract() { + local extension=$1 verify_rc=$2 expiry_rc=$3 + [[ "$verify_rc" == 0 && "$expiry_rc" == 0 ]] || return 1 + python_bounded -I -S - "$extension" "$GRAFANA_HOST" <<'PY' +import sys +extension,host=sys.argv[1:] +lines=[line.strip() for line in extension.splitlines() if line.strip()] +if len(lines)<2 or lines[0]!='X509v3 Subject Alternative Name:': + raise SystemExit(1) +names=[name.strip() for name in ' '.join(lines[1:]).split(',')] +if names != ['DNS:'+host]: + raise SystemExit(1) +PY +} + +validate_target_contract() { + local file=$1 + python_bounded -I -S - "$file" <<'PY' +import json,pathlib,re,sys +from urllib.parse import urlsplit +expected={ +"podMonitor/platform-data/platform-postgres/0":1, +"probe/observability/platform-private-edge":3, +"probe/observability/platform-private-internal":3, +"probe/observability/platform-public-edge":2, +"serviceMonitor/gitea/gitea/0":1, +"serviceMonitor/keycloak/keycloak/0":1, +"serviceMonitor/kube-system/traefik/0":1, +"serviceMonitor/object-storage/aistor-bucket-usage/0":1, +"serviceMonitor/observability-agent/alloy/0":1, +"serviceMonitor/observability-agent/node-exporter/0":1, +"serviceMonitor/observability/blackbox-exporter/0":1, +"serviceMonitor/observability/grafana/0":1, +"serviceMonitor/observability/loki/0":1, +"serviceMonitor/observability/observability-core-kube-pr-alertmanager/0":1, +"serviceMonitor/observability/observability-core-kube-pr-alertmanager/1":1, +"serviceMonitor/observability/observability-core-kube-pr-apiserver/0":1, +"serviceMonitor/observability/observability-core-kube-pr-coredns/0":1, +"serviceMonitor/observability/observability-core-kube-pr-kubelet/0":1, +"serviceMonitor/observability/observability-core-kube-pr-kubelet/1":1, +"serviceMonitor/observability/observability-core-kube-pr-kubelet/2":1, +"serviceMonitor/observability/observability-core-kube-pr-operator/0":1, +"serviceMonitor/observability/observability-core-kube-pr-prometheus/0":1, +"serviceMonitor/observability/observability-core-kube-pr-prometheus/1":1, +"serviceMonitor/observability/observability-core-kube-state-metrics/0":1, +"serviceMonitor/observability/tempo/0":1, +} +expected_probe_targets={ +"probe/observability/platform-private-edge":{ + ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fdb-admin.learn.hyeonworks.com%2F","https://db-admin.learn.hyeonworks.com/","blackbox-private-edge","private-edge"), + ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fgrafana.learn.hyeonworks.com%2F","https://grafana.learn.hyeonworks.com/","blackbox-private-edge","private-edge"), + ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fstorage-admin.learn.hyeonworks.com%2F","https://storage-admin.learn.hyeonworks.com/","blackbox-private-edge","private-edge"), +}, +"probe/observability/platform-private-internal":{ + ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=db-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fpgadmin.platform-admin.svc.cluster.local%2Fmisc%2Fping","http://pgadmin.platform-admin.svc.cluster.local/misc/ping","blackbox-private-internal","private-internal"), + ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=grafana.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fgrafana.observability.svc.cluster.local%2Fapi%2Fhealth","http://grafana.observability.svc.cluster.local/api/health","blackbox-private-internal","private-internal"), + ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=storage-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fminio-aistor-console.object-storage.svc.cluster.local%3A9090%2F","http://minio-aistor-console.object-storage.svc.cluster.local:9090/","blackbox-private-internal","private-internal"), +}, +"probe/observability/platform-public-edge":{ + ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fgit.learn.hyeonworks.com%2Fapi%2Fhealthz","https://git.learn.hyeonworks.com/api/healthz","blackbox-public-edge","public-edge"), + ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fid.learn.hyeonworks.com%2Frealms%2Fhyeonworks%2F.well-known%2Fopenid-configuration","https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration","blackbox-public-edge","public-edge"), +}, +} +endpoint_patterns={ +"podMonitor/platform-data/platform-postgres/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9187/metrics$", +"serviceMonitor/gitea/gitea/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3000/metrics$", +"serviceMonitor/keycloak/keycloak/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9000/metrics$", +"serviceMonitor/kube-system/traefik/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9100/metrics$", +"serviceMonitor/object-storage/aistor-bucket-usage/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9000/minio/metrics/v3/cluster/usage/buckets$", +"serviceMonitor/observability-agent/alloy/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:12345/metrics$", +"serviceMonitor/observability-agent/node-exporter/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9100/metrics$", +"serviceMonitor/observability/blackbox-exporter/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9115/metrics$", +"serviceMonitor/observability/grafana/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3000/metrics$", +"serviceMonitor/observability/loki/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3100/metrics$", +"serviceMonitor/observability/observability-core-kube-pr-alertmanager/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9093/metrics$", +"serviceMonitor/observability/observability-core-kube-pr-alertmanager/1":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$", +"serviceMonitor/observability/observability-core-kube-pr-apiserver/0":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:6443/metrics$", +"serviceMonitor/observability/observability-core-kube-pr-coredns/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9153/metrics$", +"serviceMonitor/observability/observability-core-kube-pr-kubelet/0":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:10250/metrics$", +"serviceMonitor/observability/observability-core-kube-pr-kubelet/1":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:10250/metrics/cadvisor$", +"serviceMonitor/observability/observability-core-kube-pr-kubelet/2":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:10250/metrics/probes$", +"serviceMonitor/observability/observability-core-kube-pr-operator/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$", +"serviceMonitor/observability/observability-core-kube-pr-prometheus/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9090/metrics$", +"serviceMonitor/observability/observability-core-kube-pr-prometheus/1":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$", +"serviceMonitor/observability/observability-core-kube-state-metrics/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$", +"serviceMonitor/observability/tempo/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3200/metrics$", +} +target_identities={ +"podMonitor/platform-data/platform-postgres/0":("platform-postgres","platform-data"), +"serviceMonitor/gitea/gitea/0":("gitea","gitea"), +"serviceMonitor/keycloak/keycloak/0":("keycloak-service","keycloak"), +"serviceMonitor/kube-system/traefik/0":("traefik","kube-system"), +"serviceMonitor/object-storage/aistor-bucket-usage/0":("minio-aistor","object-storage"), +"serviceMonitor/observability-agent/alloy/0":("alloy","observability-agent"), +"serviceMonitor/observability-agent/node-exporter/0":("prometheus-node-exporter","observability-agent"), +"serviceMonitor/observability/blackbox-exporter/0":("blackbox-exporter","observability"), +"serviceMonitor/observability/grafana/0":("grafana","observability"), +"serviceMonitor/observability/loki/0":("observability/loki","observability"), +"serviceMonitor/observability/observability-core-kube-pr-alertmanager/0":("observability-core-kube-pr-alertmanager","observability"), +"serviceMonitor/observability/observability-core-kube-pr-alertmanager/1":("observability-core-kube-pr-alertmanager","observability"), +"serviceMonitor/observability/observability-core-kube-pr-apiserver/0":("apiserver","default"), +"serviceMonitor/observability/observability-core-kube-pr-coredns/0":("coredns","kube-system"), +"serviceMonitor/observability/observability-core-kube-pr-kubelet/0":("kubelet","kube-system"), +"serviceMonitor/observability/observability-core-kube-pr-kubelet/1":("kubelet","kube-system"), +"serviceMonitor/observability/observability-core-kube-pr-kubelet/2":("kubelet","kube-system"), +"serviceMonitor/observability/observability-core-kube-pr-operator/0":("observability-core-kube-pr-operator","observability"), +"serviceMonitor/observability/observability-core-kube-pr-prometheus/0":("observability-core-kube-pr-prometheus","observability"), +"serviceMonitor/observability/observability-core-kube-pr-prometheus/1":("observability-core-kube-pr-prometheus","observability"), +"serviceMonitor/observability/observability-core-kube-state-metrics/0":("kube-state-metrics","observability"), +"serviceMonitor/observability/tempo/0":("tempo","observability"), +} +try: item=json.loads(pathlib.Path(sys.argv[1]).read_text()) +except Exception: raise SystemExit(1) +targets=((item.get("data") or {}).get("activeTargets")) +if item.get("status")!="success" or not isinstance(targets,list) or len(targets)!=30: raise SystemExit(1) +actual={} +probe_targets={pool:set() for pool in expected_probe_targets} +for target in targets: + if target.get("health")!="up" or target.get("lastError","") not in ("",None): raise SystemExit(1) + pool=target.get("scrapePool") + url=target.get("scrapeUrl") + if not isinstance(pool,str) or not isinstance(url,str): raise SystemExit(1) + if pool in endpoint_patterns and re.fullmatch(endpoint_patterns[pool],url) is None: raise SystemExit(1) + if pool in target_identities: + labels=target.get("labels") + discovered=target.get("discoveredLabels") + parsed=urlsplit(url) + expected_job,expected_namespace=target_identities[pool] + if (not isinstance(labels,dict) or labels.get("job")!=expected_job or + labels.get("namespace")!=expected_namespace or labels.get("instance")!=parsed.netloc): + raise SystemExit(1) + if (target.get("globalUrl")!=url or not isinstance(discovered,dict) or + discovered.get("__address__")!=parsed.netloc or + discovered.get("__scheme__")!=parsed.scheme or + discovered.get("__metrics_path__")!=parsed.path): + raise SystemExit(1) + if pool in expected_probe_targets: + labels=target.get("labels") + if not isinstance(labels,dict) or set(labels)!={"instance","job","namespace","observability.hyeonworks.com/probe-group"}: raise SystemExit(1) + identity=(url,labels.get("instance"),labels.get("job"),labels.get("observability.hyeonworks.com/probe-group")) + if labels.get("namespace")!="observability" or identity in probe_targets[pool]: raise SystemExit(1) + probe_targets[pool].add(identity) + actual[pool]=actual.get(pool,0)+1 +if actual!=expected or probe_targets!=expected_probe_targets: raise SystemExit(1) +PY +} + +validate_series_contract() { + local file=$1 expected_count=$2 required_csv=$3 expected_values_csv=$4 value_label=$5 + python_bounded -I -S - "$file" "$expected_count" "$required_csv" "$expected_values_csv" "$value_label" <<'PY' +import json,pathlib,sys +path,count,required,values,label=sys.argv[1:] +try: item=json.loads(pathlib.Path(path).read_text()) +except Exception: raise SystemExit(1) +result=((item.get("data") or {}).get("result")) +if item.get("status")!="success" or not isinstance(result,list) or len(result)!=int(count): raise SystemExit(1) +required=set(filter(None,required.split(','))) +expected=sorted(filter(None,values.split(','))) +seen=[] +for series in result: + metric=series.get("metric") + if not isinstance(metric,dict) or not required.issubset(metric): raise SystemExit(1) + value=series.get("value") + if not isinstance(value,list) or len(value)!=2: + raise SystemExit(1) + try: float(value[1]) + except (TypeError,ValueError): raise SystemExit(1) + if label: seen.append(metric.get(label)) +if label and (any(not isinstance(x,str) for x in seen) or sorted(seen)!=expected): raise SystemExit(1) +PY +} + +validate_series_identity_contract() { + local file=$1 expected_json=$2 instance_pattern=${3:-} + "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$PYTHON_BIN" -I -S - \ + "$file" "$expected_json" "$instance_pattern" <<'PY' +import json,pathlib,re,sys +path,expected_raw,instance_pattern=sys.argv[1:] +try: + item=json.loads(pathlib.Path(path).read_text()) + expected=json.loads(expected_raw) +except Exception: raise SystemExit(1) +result=((item.get('data') or {}).get('result')) +if item.get('status')!='success' or not isinstance(result,list) or not isinstance(expected,list) or len(result)!=len(expected) or not expected: + raise SystemExit(1) +if any(not isinstance(value,dict) or not value for value in expected): raise SystemExit(1) +keys=set(expected[0]) +if any(set(value)!=keys or any(not isinstance(x,str) for x in value.values()) for value in expected): raise SystemExit(1) +def canonical(value): return json.dumps(value,sort_keys=True,separators=(',',':')) +expected_rows=sorted(canonical(value) for value in expected) +if len(set(expected_rows))!=len(expected_rows): raise SystemExit(1) +actual=[] +for series in result: + metric=series.get('metric') + if not isinstance(metric,dict) or any(not isinstance(metric.get(key),str) for key in keys): raise SystemExit(1) + if instance_pattern: + instance=metric.get('instance') + if not isinstance(instance,str) or re.fullmatch(instance_pattern,instance) is None: raise SystemExit(1) + actual.append(canonical({key:metric[key] for key in keys})) +if sorted(actual)!=expected_rows: raise SystemExit(1) +PY +} + +validate_datasource_contract() { + local file=$1 + python_bounded -I -S - "$file" <<'PY' +import base64,binascii,json,pathlib,re,sys +try: item=json.loads(pathlib.Path(sys.argv[1]).read_text()) +except Exception: raise SystemExit(1) +if any((item.get(x) or {}).get("status")!="OK" for x in ("prometheus","loki","tempo")): raise SystemExit(1) +loki=item.get("loki_config") or {}; tempo=item.get("tempo_config") or {} +fields=((loki.get("jsonData") or {}).get("derivedFields")) or [] +if not any(x.get("name")=="trace_id" and x.get("datasourceUid")=="tempo" and x.get("url")=="${__value.raw}" for x in fields): raise SystemExit(1) +link=((tempo.get("jsonData") or {}).get("tracesToLogsV2")) or {} +if link.get("datasourceUid")!="loki" or link.get("filterByTraceID") is not True: raise SystemExit(1) +lines=[] +for stream in (((item.get("loki_query") or {}).get("data") or {}).get("result") or []): + for value in stream.get("values") or []: + if isinstance(value,list) and len(value)==2 and isinstance(value[1],str): lines.append(value[1]) +selected=item.get('selected_trace') +if not isinstance(selected,str) or re.fullmatch(r'[0-9a-f]{32}',selected) is None: raise SystemExit(1) +search=(item.get('tempo_search') or {}).get('traces') +if not isinstance(search,list): raise SystemExit(1) +candidates=[x for x in search if isinstance(x,dict) and x.get('traceID')==selected and x.get('rootServiceName')=='platform-smoke'] +if len(candidates)!=1: raise SystemExit(1) +matches=[] +for line in lines: matches.extend(re.findall(r'"trace_id"\s*:\s*"([0-9a-f]{32})"',line)) +if not matches or set(matches)!={selected}: raise SystemExit(1) +batches=(item.get("tempo_query") or {}).get("batches") +if not isinstance(batches,list) or not batches: raise SystemExit(1) +seen_span=False +for batch in batches: + if not isinstance(batch,dict): raise SystemExit(1) + attrs=((batch.get('resource') or {}).get('attributes')) + if not isinstance(attrs,list): raise SystemExit(1) + service=[x for x in attrs if isinstance(x,dict) and x.get('key')=='service.name'] + if len(service)!=1 or ((service[0].get('value') or {}).get('stringValue'))!='platform-smoke': raise SystemExit(1) + for scope in batch.get('scopeSpans') or []: + for span in scope.get('spans') or []: + encoded=span.get('traceId') + if not isinstance(encoded,str): raise SystemExit(1) + try: decoded=base64.b64decode(encoded,validate=True).hex() + except (binascii.Error,ValueError): raise SystemExit(1) + if decoded!=selected: raise SystemExit(1) + seen_span=True +if not seen_span: raise SystemExit(1) +PY +} + +synthetic_loki_query() { + local trace=$1 + [[ "$trace" =~ ^[0-9a-f]{32}$ ]] || return 1 + printf '{cluster="home",namespace="observability"} | json | trace_id="%s"' "$trace" +} + +validate_human_tokens() { + local -a expected=(OIDC_ADMIN_CONFIRMED OIDC_VIEWER_CONFIRMED OIDC_DENIED_CONFIRMED + BREAK_GLASS_CONFIRMED REMOVAL_VIEWER_CONFIRMED SESSION_REVOKED RELOGIN_DENIED + FIRING RESOLVED EXTERNAL_CLIENT_VERIFIED) + local -a actual=() + mapfile -t actual + (( ${#actual[@]} == ${#expected[@]} )) || return 1 + local index + for ((index=0; index<${#expected[@]}; index++)); do + [[ "${actual[$index]}" == "${expected[$index]}" ]] || return 1 + done +} + +validate_ipv4() { + local value=$1 octet + local -a parts=() + [[ "$value" =~ ^[0-9]{1,3}([.][0-9]{1,3}){3}$ ]] || return 1 + IFS=. read -r -a parts <<<"$value" + (( ${#parts[@]} == 4 )) || return 1 + for octet in "${parts[@]}"; do + (( 10#$octet <= 255 )) || return 1 + done +} + +external_attestation_token() { + local public_ip=$1 + validate_ipv4 "$public_ip" || return 1 + printf 'EXTERNAL_CLIENT_VERIFIED public_ip=%s status=403 source=outside-lan-tailscale proxy=disabled' "$public_ip" +} + +render_external_attestation() { + local public_ip=$1 token + token="$(external_attestation_token "$public_ip")" || return 1 + printf '%s\n' \ + 'Run this from a client outside both LAN and Tailscale, with no HTTP(S) proxy:' \ + "/usr/bin/curl --disable --silent --show-error --output /dev/null --write-out '%{http_code}\\n' --connect-timeout 5 --max-time 15 --noproxy '*' --resolve $GRAFANA_HOST:443:$public_ip https://$GRAFANA_HOST/" \ + 'Expected exact result: HTTP 403' \ + "After verifying the client source and result, type: $token" +} + +new_smoke_run_id() { + local prefix=$1 stamp nonce + [[ "$prefix" =~ ^[a-z][a-z0-9-]{0,14}$ ]] || return 1 + stamp="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%Y%m%dT%H%M%SZ)" || return 1 + nonce="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$PYTHON_BIN" -I -S - <<'PY' +import os +print(os.urandom(16).hex(),end='') +PY +)" || return 1 + [[ "$stamp" =~ ^[0-9]{8}T[0-9]{6}Z$ && "$nonce" =~ ^[0-9a-f]{32}$ ]] || return 1 + printf '%s-%s-%s' "$prefix" "${stamp,,}" "$nonce" +} + +kubernetes_smoke_name() { + local run=$1 nonce + [[ "$run" =~ ^[a-z][a-z0-9-]{0,14}-[0-9]{8}t[0-9]{6}z-([0-9a-f]{32})$ ]] || return 1 + nonce=${BASH_REMATCH[1]} + printf 'observability-smoke-%s' "$nonce" +} + +handle_termination_signal() { + local rc=$1 + [[ "$rc" =~ ^(129|130|143)$ ]] || rc=143 + if (( MUTATION_CRITICAL_DEPTH > 0 )); then + (( DEFERRED_SIGNAL_RC == 0 )) && DEFERRED_SIGNAL_RC=$rc + return 0 + fi + exit "$rc" +} + +begin_mutation_critical() { + MUTATION_CRITICAL_DEPTH=$((MUTATION_CRITICAL_DEPTH + 1)) +} + +end_mutation_critical() { + (( MUTATION_CRITICAL_DEPTH > 0 )) || return 1 + MUTATION_CRITICAL_DEPTH=$((MUTATION_CRITICAL_DEPTH - 1)) + if (( MUTATION_CRITICAL_DEPTH == 0 && DEFERRED_SIGNAL_RC != 0 )); then + local rc=$DEFERRED_SIGNAL_RC + DEFERRED_SIGNAL_RC=0 + exit "$rc" + fi +} + +require_confirmation() { + local expected=$1 answer='' + if [[ "$TEST_MODE" == true ]]; then + (( CONFIRMATION_INDEX < ${#CONFIRMATION_QUEUE[@]} )) || return 1 + answer=${CONFIRMATION_QUEUE[$CONFIRMATION_INDEX]} + CONFIRMATION_INDEX=$((CONFIRMATION_INDEX + 1)) + else + [[ -t 0 ]] || { fail '--execute requires an interactive terminal'; return 1; } + printf 'Type %s: ' "$expected" >&2 + IFS= read -r -t "$HUMAN_TIMEOUT_SECONDS" answer || return 1 + fi + [[ "$answer" == "$expected" ]] +} + +confirmation_remaining_seconds() { + local deadline=$1 now=$2 remaining + [[ "$deadline" =~ ^[0-9]+$ && "$now" =~ ^[0-9]+$ ]] || return 1 + remaining=$((deadline - now)) + (( remaining >= 1 && remaining <= HUMAN_TIMEOUT_SECONDS )) || return 1 + printf '%s' "$remaining" +} + +require_confirmation_before() { + local expected=$1 deadline=$2 now remaining answer='' + if [[ "$TEST_MODE" == true ]]; then require_confirmation "$expected"; return; fi + now="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%s)" || return 1 + remaining="$(confirmation_remaining_seconds "$deadline" "$now")" || return 1 + [[ -t 0 ]] || return 1 + printf 'Type %s within the shared deadline: ' "$expected" >&2 + IFS= read -r -t "$remaining" answer || return 1 + [[ "$answer" == "$expected" ]] +} + +read_identity() { + local purpose=$1 answer='' + [[ "$TEST_MODE" == false ]] || return 1 + [[ -t 0 ]] || return 1 + printf 'Enter the existing %s username (not recorded): ' "$purpose" >&2 + IFS= read -r -t "$HUMAN_TIMEOUT_SECONDS" answer || return 1 + [[ "$answer" =~ ^[A-Za-z0-9][A-Za-z0-9._@+-]{0,127}$ ]] || return 1 + printf '%s' "$answer" +} + +process_start_time() { + local pid=$1 + python_bounded -I -S - "$pid" <<'PY' +import pathlib,sys +raw=pathlib.Path('/proc',sys.argv[1],'stat').read_text() +end=raw.rfind(') ') +if end<0: raise SystemExit(1) +fields=raw[end+2:].split() +if len(fields)<20 or not fields[19].isdigit(): raise SystemExit(1) +print(fields[19],end='') +PY +} + +stop_owned_process() { + local pid=$1 expected_start=$2 current attempt + [[ "$pid" =~ ^[0-9]+$ && "$expected_start" =~ ^[0-9]+$ ]] || return 0 + current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; } + [[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; } + /bin/kill -TERM "$pid" 2>/dev/null || true + for ((attempt=0; attempt<40; attempt++)); do + /bin/kill -0 "$pid" 2>/dev/null || { wait "$pid" 2>/dev/null || true; return 0; } + /usr/bin/sleep 0.05 + done + current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; } + [[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; } + /bin/kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true +} + +kubectl_bounded() { + "$TIMEOUT_BIN" --signal=TERM --kill-after=2s "$PROCESS_TIMEOUT" \ + "$KUBECTL_BIN" --request-timeout="$REQUEST_TIMEOUT" "$@" +} + +kubectl_wait_90_bounded() { + "$TIMEOUT_BIN" --signal=TERM --kill-after=2s 100s \ + "$KUBECTL_BIN" --request-timeout=95s "$@" +} + +kubectl_wait_180_bounded() { + "$TIMEOUT_BIN" --signal=TERM --kill-after=2s 190s \ + "$KUBECTL_BIN" --request-timeout=185s "$@" +} + +jq_bounded() { + "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$JQ_BIN" "$@" +} + +python_bounded() { + "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$PYTHON_BIN" "$@" +} + +safe_work_file() { + [[ -n "$WORK" && "$WORK" == /tmp/platform-observability-smoke.???????? && + "$1" == "$WORK"/* && -f "$1" && ! -L "$1" && -O "$1" && + "$("$STAT_BIN" -c '%a:%h' -- "$1")" == 600:1 ]] +} + +make_work() { + WORK="$("$MKTEMP_BIN" -d /tmp/platform-observability-smoke.XXXXXXXX)" || return 1 + [[ "$WORK" == /tmp/platform-observability-smoke.???????? && -d "$WORK" && ! -L "$WORK" ]] || return 1 + "$CHMOD_BIN" 0700 -- "$WORK" +} + +start_api_proxy() { + [[ -n "$WORK" ]] || return 1 + API_PROXY_SOCKET="$WORK/kube-api.sock" + "$KUBECTL_BIN" --request-timeout="$REQUEST_TIMEOUT" proxy \ + --unix-socket="$API_PROXY_SOCKET" --api-prefix=/ \ + --accept-paths='^/(api|apis)/.*$' >"$WORK/kube-api-proxy.log" 2>&1 & + API_PROXY_PID=$! + API_PROXY_START="$(process_start_time "$API_PROXY_PID")" || return 1 + local attempt + for ((attempt=0; attempt<100; attempt++)); do + [[ -S "$API_PROXY_SOCKET" ]] && { "$CHMOD_BIN" 0600 -- "$API_PROXY_SOCKET"; return 0; } + /bin/kill -0 "$API_PROXY_PID" 2>/dev/null || return 1 + /usr/bin/sleep 0.05 + done + return 1 +} + +proxy_request() { + local method=$1 path=$2 output=$3 body=${4:-} extra=${5:-} + local config status + [[ "$method" =~ ^(GET|POST|PUT|DELETE)$ && "$path" == /api* && + "$path" != *$'\n'* && "$path" != *'"'* && "$path" != *'\\'* ]] || return 1 + config="$("$MKTEMP_BIN" "$WORK/curl.XXXXXXXX")" || return 1 + "$CHMOD_BIN" 0600 -- "$config" + : >"$output"; "$CHMOD_BIN" 0600 -- "$output" + { + printf 'silent\nshow-error\nconnect-timeout = 2\nmax-time = 12\n' + printf 'unix-socket = "%s"\nrequest = "%s"\n' "$API_PROXY_SOCKET" "$method" + printf 'url = "http://localhost%s"\noutput = "%s"\nwrite-out = "%%{http_code}"\n' "$path" "$output" + [[ -z "$body" ]] || printf 'header = "Content-Type: application/json"\ndata-binary = "@%s"\n' "$body" + [[ -z "$extra" ]] || /bin/cat -- "$extra" + } >"$config" + safe_work_file "$config" && safe_work_file "$output" || return 1 + status="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$CURL_BIN" --disable --config "$config")" || return 1 + [[ "$status" =~ ^[0-9]{3}$ ]] || return 1 + printf '%s' "$status" +} + +prometheus_path() { + printf '/api/v1/namespaces/observability/services/http:%s:9090/proxy%s' "$PROMETHEUS_SERVICE" "$1" +} + +alertmanager_path() { + printf '/api/v1/namespaces/observability/services/http:%s:9093/proxy%s' "$ALERTMANAGER_SERVICE" "$1" +} + +grafana_path() { printf '/api/v1/namespaces/observability/services/http:grafana:80/proxy%s' "$1"; } +keycloak_path() { printf '/api/v1/namespaces/keycloak/services/http:keycloak-service:8080/proxy%s' "$1"; } + +proxy_get_200() { + local path=$1 output=$2 extra=${3:-} status + status="$(proxy_request GET "$path" "$output" '' "$extra")" || return 1 + [[ "$status" == 200 && -s "$output" ]] +} + +prom_query() { + local expression=$1 output=$2 encoded + encoded="$(jq_bounded -nr --arg value "$expression" '$value|@uri')" || return 1 + proxy_get_200 "$(prometheus_path "/api/v1/query?query=$encoded")" "$output" +} + +api_path_for_owned() { + local kind=$1 namespace=$2 name=$3 + [[ "$namespace" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ && + "$name" =~ ^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$ ]] || return 1 + case "$kind" in + Pod) printf '/api/v1/namespaces/%s/pods/%s' "$namespace" "$name" ;; + NetworkPolicy) printf '/apis/networking.k8s.io/v1/namespaces/%s/networkpolicies/%s' "$namespace" "$name" ;; + PrometheusRule) printf '/apis/monitoring.coreos.com/v1/namespaces/%s/prometheusrules/%s' "$namespace" "$name" ;; + *) return 1 ;; + esac +} + +kind_resource() { + case "$1" in + Pod) printf pod ;; + NetworkPolicy) printf networkpolicy ;; + PrometheusRule) printf prometheusrule ;; + *) return 1 ;; + esac +} + +owned_object_state() { + local kind=$1 namespace=$2 name=$3 resource file uid run + resource="$(kind_resource "$kind")" || return 4 + file="$("$MKTEMP_BIN" "$WORK/state.XXXXXXXX")" || return 4 + "$CHMOD_BIN" 0600 -- "$file" + kubectl_bounded -n "$namespace" get "$resource" "$name" --ignore-not-found -o json >"$file" || return 4 + [[ -s "$file" ]] || return 1 + uid="$(jq_bounded -er '.metadata.uid | select(type=="string" and length>0)' "$file")" || return 4 + run="$(jq_bounded -er '.metadata.labels["platform.hyeonworks.com/smoke-run"] | select(type=="string" and length>0)' "$file")" || return 4 + printf '%s|%s\n' "$uid" "$run" +} + +delete_owned_uid() { + local kind=$1 namespace=$2 name=$3 uid=$4 path body response status state attempt + [[ "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1 + path="$(api_path_for_owned "$kind" "$namespace" "$name")" || return 1 + body="$("$MKTEMP_BIN" "$WORK/delete.XXXXXXXX")" || return 1 + response="$("$MKTEMP_BIN" "$WORK/delete-response.XXXXXXXX")" || return 1 + "$CHMOD_BIN" 0600 -- "$body" "$response" + jq_bounded -nc --arg uid "$uid" '{apiVersion:"meta.k8s.io/v1",kind:"DeleteOptions",propagationPolicy:"Background",preconditions:{uid:$uid}}' >"$body" + status="$(proxy_request DELETE "$path" "$response" "$body")" || status=000 + for ((attempt=0; attempt<30; attempt++)); do + state="$(owned_object_state "$kind" "$namespace" "$name" 2>/dev/null)" && { + [[ "${state%%|*}" == "$uid" ]] || return 1 + } || { + [[ $? == 1 ]] && return 0 + return 1 + } + /usr/bin/sleep 1 + done + return 1 +} + +register_owned() { + OWNED_KIND+=("$1"); OWNED_NAMESPACE+=("$2"); OWNED_NAME+=("$3"); + OWNED_UID+=("$4"); OWNED_RUN+=("$5") +} + +cleanup_owned_objects() { + local index state uid run rc=0 + for ((index=${#OWNED_KIND[@]}-1; index>=0; index--)); do + state="$(owned_object_state "${OWNED_KIND[$index]}" "${OWNED_NAMESPACE[$index]}" "${OWNED_NAME[$index]}" 2>/dev/null)" || { + [[ $? == 1 ]] && continue + emit_manual_recovery; rc=1; continue + } + IFS='|' read -r uid run <<<"$state" + if [[ "$uid" != "${OWNED_UID[$index]}" || "$run" != "${OWNED_RUN[$index]}" ]]; then + emit_manual_recovery; rc=1; continue + fi + delete_owned_uid "${OWNED_KIND[$index]}" "${OWNED_NAMESPACE[$index]}" "${OWNED_NAME[$index]}" "$uid" || { + emit_manual_recovery; rc=1; + } + done + OWNED_KIND=(); OWNED_NAMESPACE=(); OWNED_NAME=(); OWNED_UID=(); OWNED_RUN=() + return "$rc" +} + +validate_commit_environment() { + local permission result context api node_uid + validate_command_boundaries || return 1 + context="$(kubectl_bounded config current-context)" || return 1 + [[ "$context" == "$EXPECTED_CONTEXT" ]] || { fail 'Kubernetes context is not default'; return 1; } + api="$(kubectl_bounded config view --minify -o jsonpath='{.clusters[0].cluster.server}')" || return 1 + [[ "$api" == "$EXPECTED_API_SERVER" ]] || { fail 'Kubernetes API server is not the exact local endpoint'; return 1; } + kubectl_bounded get --raw=/readyz >/dev/null || { fail 'Kubernetes API is not ready'; return 1; } + node_uid="$(kubectl_bounded get node "$EXPECTED_NODE" -o json | jq_bounded -er --arg node "$EXPECTED_NODE" ' + select(.metadata.name==$node and (.metadata.uid|type=="string" and length>0) and + any(.status.conditions[]?; .type=="Ready" and .status=="True")) | .metadata.uid + ')" || { fail 'target node identity/readiness changed'; return 1; } + if [[ -z "$EXPECTED_NODE_UID" ]]; then EXPECTED_NODE_UID=$node_uid; fi + [[ "$node_uid" == "$EXPECTED_NODE_UID" ]] || { fail 'target node UID changed'; return 1; } + for permission in \ + 'get pods observability' 'list pods observability' 'watch pods observability' \ + 'create pods observability' 'delete pods observability' 'get pods/log observability' \ + 'get services observability' 'get services object-storage' 'get services kube-system' \ + 'get services/proxy observability' 'create services/proxy observability' \ + 'delete services/proxy observability' \ + 'get services/proxy keycloak' 'create services/proxy keycloak' \ + 'update services/proxy keycloak' 'delete services/proxy keycloak' \ + 'get persistentvolumeclaims observability' 'list persistentvolumeclaims observability' \ + 'get configmaps observability' 'list configmaps observability' \ + 'get deployments.apps observability' 'watch deployments.apps observability' \ + 'get statefulsets.apps observability' 'watch statefulsets.apps observability' \ + 'get networkpolicies.networking.k8s.io observability' \ + 'create networkpolicies.networking.k8s.io observability' \ + 'delete networkpolicies.networking.k8s.io observability' \ + 'get prometheusrules.monitoring.coreos.com observability' \ + 'create prometheusrules.monitoring.coreos.com observability' \ + 'delete prometheusrules.monitoring.coreos.com observability' \ + 'get secrets observability' 'get secrets keycloak'; do + read -r -a _permission_parts <<<"$permission" + result="$(kubectl_bounded auth can-i "${_permission_parts[0]}" "${_permission_parts[1]}" -n "${_permission_parts[2]}")" || return 1 + [[ "$result" == yes ]] || { fail "missing Kubernetes authority: $permission"; return 1; } + done +} + +require_execute_environment() { + validate_commit_environment || return 1 + make_work || { fail 'cannot create private work directory'; return 1; } + start_api_proxy || { fail 'cannot start bounded Kubernetes API proxy'; return 1; } +} + +validate_command_boundaries() { + local path certbot_real metadata + for path in "$KUBECTL_BIN" "$CURL_BIN" "$DIG_BIN" "$OPENSSL_BIN" "$NC_BIN" \ + "$SUDO_BIN" "$DATE_BIN" "$TIMEOUT_BIN" "$JQ_BIN" "$PYTHON_BIN" \ + "$CAT_BIN" "$CHMOD_BIN" "$CMP_BIN" "$MKTEMP_BIN" "$MV_BIN" \ + "$READLINK_BIN" "$RM_BIN" "$STAT_BIN"; do + [[ -x "$path" && ! -L "$path" ]] || { fail "required command boundary is unavailable: $path"; return 1; } + done + [[ -L "$CERTBOT_BIN" && -x "$CERTBOT_BIN" ]] || { + fail "Certbot command boundary is unavailable: $CERTBOT_BIN"; return 1; + } + certbot_real="$("$READLINK_BIN" -f -- "$CERTBOT_BIN")" || return 1 + [[ "$certbot_real" == "$CERTBOT_REAL" && -x "$CERTBOT_REAL" && ! -L "$CERTBOT_REAL" ]] || { + fail 'Certbot command boundary does not resolve to the fixed snap executable'; return 1; + } + metadata="$("$STAT_BIN" -c '%u:%g:%a:%F' -- "$CERTBOT_REAL")" || return 1 + [[ "$metadata" == '0:0:755:regular file' ]] || { + fail 'Certbot real executable metadata is unsafe'; return 1; + } + [[ -x "$SYSTEMCTL_BIN" && ! -L "$SYSTEMCTL_BIN" && + "$("$STAT_BIN" -c '%u:%g:%a:%F' -- "$SYSTEMCTL_BIN")" == '0:0:755:regular file' ]] || { + fail 'systemctl command boundary is unsafe'; return 1; + } +} + +dig_values() { + local resolver=$1 record=$2 host=$3 + "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 8s "$DIG_BIN" +time=2 +tries=1 +short \ + "@$resolver" "$record" "$host" | /usr/bin/sed '/^[[:space:]]*$/d' | LC_ALL=C /usr/bin/sort -u +} + +http_status() { + local -a args=(--disable --silent --show-error --output /dev/null --write-out '%{http_code}' + --connect-timeout 3 --max-time 12 --noproxy '*') + "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$CURL_BIN" "${args[@]}" "$@" +} + +check_dns_host() { + local public_a public_aaaa lan_a lan_aaaa tail_a tail_aaaa + public_a="$(dig_values 1.1.1.1 A "$GRAFANA_HOST")" || return 1 + public_aaaa="$(dig_values 1.1.1.1 AAAA "$GRAFANA_HOST")" || return 1 + lan_a="$(dig_values "$LAN_IP" A "$GRAFANA_HOST")" || return 1 + lan_aaaa="$(dig_values "$LAN_IP" AAAA "$GRAFANA_HOST")" || return 1 + tail_a="$(dig_values "$TAILSCALE_IP" A "$GRAFANA_HOST")" || return 1 + tail_aaaa="$(dig_values "$TAILSCALE_IP" AAAA "$GRAFANA_HOST")" || return 1 + validate_dns_contract "$public_a" "$public_aaaa" "$lan_a" "$lan_aaaa" \ + "$tail_a" "$tail_aaaa" "$LAN_IP" '' || return 1 + printf 'OBSERVABILITY_MACHINE_DNS_HOST=PASS\n' +} + +check_certificate() { + local raw cert extension verify_rc=0 expiry_rc=0 + raw="$WORK/grafana-sclient.txt"; cert="$WORK/grafana-leaf.pem" + : >"$raw"; : >"$cert"; "$CHMOD_BIN" 0600 -- "$raw" "$cert" + "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$OPENSSL_BIN" s_client \ + -connect "$LAN_IP:443" -servername "$GRAFANA_HOST" -showcerts "$raw" 2>/dev/null || return 1 + /usr/bin/awk '/-----BEGIN CERTIFICATE-----/{copy=1} copy{print} /-----END CERTIFICATE-----/{exit}' "$raw" >"$cert" + [[ -s "$cert" ]] || return 1 + "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$OPENSSL_BIN" s_client -connect "$LAN_IP:443" -servername "$GRAFANA_HOST" \ + -verify_hostname "$GRAFANA_HOST" -verify_return_error /dev/null 2>&1 || verify_rc=$? + "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$OPENSSL_BIN" x509 -in "$cert" -noout -checkend 604800 >/dev/null 2>&1 || expiry_rc=$? + extension="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$OPENSSL_BIN" x509 \ + -in "$cert" -noout -ext subjectAltName 2>/dev/null)" || return 1 + validate_certificate_extension_contract "$extension" "$verify_rc" "$expiry_rc" || return 1 + printf 'OBSERVABILITY_MACHINE_CERTIFICATE=PASS\n' +} + +check_nodeport_identity() { + local service="$WORK/traefik-nodeport-service.json" + kubectl_bounded -n kube-system get service traefik -o json >"$service" || return 1 + "$CHMOD_BIN" 0600 -- "$service" + jq_bounded -e ' + .apiVersion=="v1" and .kind=="Service" and + .metadata.name=="traefik" and .metadata.namespace=="kube-system" and + (.metadata.uid|type=="string" and length>0) and + .spec.type=="NodePort" and .spec.externalTrafficPolicy=="Cluster" and + .spec.selector=={ + "app.kubernetes.io/name":"traefik", + "app.kubernetes.io/instance":"traefik-kube-system"} and + ([.spec.ports[]|{name,port,protocol,targetPort,nodePort}]|sort_by(.name))==([ + {name:"web",port:80,protocol:"TCP",targetPort:"web",nodePort:30080}, + {name:"websecure",port:443,protocol:"TCP",targetPort:"websecure",nodePort:30443} + ]|sort_by(.name)) + ' "$service" >/dev/null || return 1 + [[ "$(http_status --header 'Host: observability-smoke.invalid' http://127.0.0.1:30080/)" == 404 ]] +} + +check_https_boundaries() { + local public_ip + [[ "$(http_status --interface "$LAN_IP" --resolve "$GRAFANA_HOST:443:$LAN_IP" "https://$GRAFANA_HOST/")" =~ ^(200|302)$ ]] || return 1 + [[ "$(http_status --interface "$TAILSCALE_IP" --resolve "$GRAFANA_HOST:443:$TAILSCALE_IP" "https://$GRAFANA_HOST/")" =~ ^(200|302)$ ]] || return 1 + [[ "$(http_status --resolve "$GRAFANA_HOST:443:127.0.0.1" "https://$GRAFANA_HOST/")" == 403 ]] || return 1 + public_ip="$(dig_values 1.1.1.1 A "$GITEA_HOST" | /usr/bin/head -n1)" || return 1 + validate_ipv4 "$public_ip" || return 1 + if [[ -n "$PUBLIC_EDGE_IP" && "$PUBLIC_EDGE_IP" != "$public_ip" ]]; then return 1; fi + PUBLIC_EDGE_IP=$public_ip + [[ "$(http_status --resolve "$GRAFANA_HOST:443:$public_ip" "https://$GRAFANA_HOST/")" == 403 ]] || return 1 + [[ "$(http_status --resolve "$GRAFANA_HOST:443:$LAN_IP" "https://$GRAFANA_HOST/metrics")" == 404 ]] || return 1 + [[ "$(http_status "https://$GITEA_HOST/metrics")" == 404 ]] || return 1 + [[ "$(http_status "https://$KEYCLOAK_HOST/metrics")" == 404 ]] || return 1 + [[ "$(http_status "$RUNBOOK_URL")" == 200 ]] || return 1 + if "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 8s "$OPENSSL_BIN" s_client -connect "$LAN_IP:443" \ + -servername unknown-sni.invalid /dev/null 2>&1; then return 1; fi + ! "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$NC_BIN" -z -w 2 "$LAN_IP" 30080 || return 1 + ! "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$NC_BIN" -z -w 2 "$LAN_IP" 30443 || return 1 + check_nodeport_identity || return 1 + printf 'OBSERVABILITY_MACHINE_EDGE=PASS\n' +} + +check_targets() { + local file="$WORK/prometheus-targets.json" + proxy_get_200 "$(prometheus_path '/api/v1/targets?state=active')" "$file" || return 1 + validate_target_contract "$file" || return 1 + printf 'OBSERVABILITY_MACHINE_TARGETS=PASS\n' +} + +check_one_query() { + local name=$1 expression=$2 count=$3 labels=$4 values=${5:-} value_label=${6:-} + local identity_json=${7:-} instance_pattern=${8:-} + local file="$WORK/query-$name.json" + prom_query "$expression" "$file" || return 1 + validate_series_contract "$file" "$count" "$labels" "$values" "$value_label" || return 1 + [[ -z "$identity_json" ]] || validate_series_identity_contract "$file" "$identity_json" "$instance_pattern" +} + +check_metric_contracts() { + check_one_query gitea 'gitea_build_info{job="gitea",namespace="gitea"}' 1 'instance,job,namespace' '' '' \ + '[{"job":"gitea","namespace":"gitea"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:3000$' || return 1 + check_one_query keycloak 'process_uptime_seconds{job="keycloak-service",namespace="keycloak"}' 1 'instance,job,namespace' '' '' \ + '[{"job":"keycloak-service","namespace":"keycloak"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$' || return 1 + check_one_query cnpg 'cnpg_collector_up{job="platform-postgres",namespace="platform-data",cluster="platform-postgres"}' 1 'cluster,instance,job,namespace' '' '' \ + '[{"cluster":"platform-postgres","job":"platform-postgres","namespace":"platform-data"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9187$' || return 1 + check_one_query traefik 'traefik_config_last_reload_success{job="traefik",namespace="kube-system"}' 1 'instance,job,namespace' '' '' \ + '[{"job":"traefik","namespace":"kube-system"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9100$' || return 1 + check_one_query aistor-usage 'minio_cluster_usage_buckets_total_bytes{bucket=~"loki|tempo"}' 2 \ + 'bucket,instance,job,namespace' 'loki,tempo' bucket \ + '[{"bucket":"loki","job":"minio-aistor","namespace":"object-storage"},{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage"}]' \ + '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$' || return 1 + check_one_query aistor-quota 'minio_cluster_usage_buckets_quota_total_bytes{bucket=~"loki|tempo"} > 0' 2 \ + 'bucket,instance,job,namespace' 'loki,tempo' bucket \ + '[{"bucket":"loki","job":"minio-aistor","namespace":"object-storage"},{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage"}]' \ + '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$' || return 1 + check_one_query quota-rule 'platform:aistor_bucket_quota_usage_percent' 2 'bucket' 'loki,tempo' bucket \ + '[{"bucket":"loki"},{"bucket":"tempo"}]' || return 1 + printf 'OBSERVABILITY_MACHINE_METRICS_QUOTA=PASS\n' +} + +check_blackbox_series() { + local public='[{"instance":"https://git.learn.hyeonworks.com/api/healthz","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"},{"instance":"https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}]' + local private='[{"instance":"https://db-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"},{"instance":"https://grafana.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"},{"instance":"https://storage-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}]' + local internal='[{"instance":"http://grafana.observability.svc.cluster.local/api/health","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"},{"instance":"http://minio-aistor-console.object-storage.svc.cluster.local:9090/","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"},{"instance":"http://pgadmin.platform-admin.svc.cluster.local/misc/ping","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}]' + check_one_query probe-public 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="public-edge"} == 1' 2 \ + 'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$public" || return 1 + check_one_query probe-private 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"} == 1' 3 \ + 'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$private" || return 1 + check_one_query probe-private-status 'probe_http_status_code{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"} == 403' 3 \ + 'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$private" || return 1 + check_one_query probe-internal 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="private-internal"} == 1' 3 \ + 'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$internal" || return 1 + local tls="$WORK/query-probe-tls.json" + prom_query 'probe_ssl_earliest_cert_expiry{job=~"blackbox-public-edge|blackbox-private-edge"} > time()' "$tls" || return 1 + validate_series_identity_contract "$tls" "$(jq_bounded -cn --argjson public "$public" --argjson private "$private" '$public+$private')" || return 1 + printf 'OBSERVABILITY_MACHINE_BLACKBOX=PASS\n' +} + +decode_exact_secret_to_netrc() { + local secret_json=$1 output=$2 + python_bounded -I -S - "$secret_json" "$output" <<'PY' +import base64,binascii,json,os,pathlib,sys +source,destination=sys.argv[1:] +try: item=json.loads(pathlib.Path(source).read_text()) +except Exception: raise SystemExit(1) +meta=item.get('metadata') or {}; data=item.get('data') +if item.get('kind')!='Secret' or item.get('type')!='Opaque' or meta.get('namespace')!='observability' or meta.get('name')!='grafana-admin' or sorted(data or {})!=['admin-password','admin-user']: + raise SystemExit(1) +values=[] +for key in ('admin-user','admin-password'): + try: value=base64.b64decode(data[key],validate=True).decode('utf-8') + except (binascii.Error,UnicodeError,ValueError): raise SystemExit(1) + if not value or any(ch.isspace() for ch in value) or '\x00' in value: raise SystemExit(1) + values.append(value) +fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) +with os.fdopen(fd,'w') as stream: stream.write('machine localhost\nlogin '+values[0]+'\npassword '+values[1]+'\n') +PY +} + +prepare_grafana_auth() { + local secret="$WORK/grafana-admin-secret.json" netrc="$WORK/grafana.netrc" extra="$WORK/grafana-curl.conf" + if safe_work_file "$netrc" && safe_work_file "$extra"; then + printf '%s' "$extra" + return 0 + fi + kubectl_bounded -n observability get secret grafana-admin -o json >"$secret" || return 1 + "$CHMOD_BIN" 0600 -- "$secret"; safe_work_file "$secret" || return 1 + decode_exact_secret_to_netrc "$secret" "$netrc" || return 1 + printf 'netrc\nnetrc-file = "%s"\n' "$netrc" >"$extra"; "$CHMOD_BIN" 0600 -- "$extra" + safe_work_file "$netrc" && safe_work_file "$extra" || return 1 + printf '%s' "$extra" +} + +grafana_get() { + local api_path=$1 output=$2 auth=$3 + proxy_get_200 "$(grafana_path "$api_path")" "$output" "$auth" +} + +check_datasources() { + local auth prometheus loki tempo loki_config tempo_config tempo_search loki_query tempo_query combined trace encoded query + auth="$(prepare_grafana_auth)" || return 1 + prometheus="$WORK/grafana-prometheus-health.json"; loki="$WORK/grafana-loki-health.json"; tempo="$WORK/grafana-tempo-health.json" + loki_config="$WORK/grafana-loki-config.json"; tempo_config="$WORK/grafana-tempo-config.json" + tempo_search="$WORK/grafana-tempo-search.json"; loki_query="$WORK/grafana-loki-query.json" + tempo_query="$WORK/grafana-tempo-query.json"; combined="$WORK/grafana-datasource-contract.json" + grafana_get '/api/datasources/uid/prometheus/health' "$prometheus" "$auth" || return 1 + grafana_get '/api/datasources/uid/loki/health' "$loki" "$auth" || return 1 + grafana_get '/api/datasources/uid/tempo/health' "$tempo" "$auth" || return 1 + grafana_get '/api/datasources/uid/loki' "$loki_config" "$auth" || return 1 + grafana_get '/api/datasources/uid/tempo' "$tempo_config" "$auth" || return 1 + encoded="$(jq_bounded -nr --arg value 'service.name=platform-smoke' '$value|@uri')" || return 1 + grafana_get "/api/datasources/proxy/uid/tempo/api/search?tags=$encoded&limit=20" "$tempo_search" "$auth" || return 1 + trace="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$PYTHON_BIN" -I -S - "$tempo_search" <<'PY' +import json,pathlib,re,sys +try: item=json.loads(pathlib.Path(sys.argv[1]).read_text()) +except Exception: raise SystemExit(1) +traces=item.get('traces') +if not isinstance(traces,list): raise SystemExit(1) +candidates=[] +for trace in traces: + if not isinstance(trace,dict) or trace.get('rootServiceName')!='platform-smoke': continue + identifier=trace.get('traceID') + if isinstance(identifier,str) and re.fullmatch(r'[0-9a-f]{32}',identifier): candidates.append(identifier) +if not candidates: raise SystemExit(1) +print(candidates[0],end='') +PY +)" || return 1 + [[ "$trace" =~ ^[0-9a-f]{32}$ ]] || return 1 + query="$(synthetic_loki_query "$trace")" || return 1 + encoded="$(jq_bounded -nr --arg value "$query" '$value|@uri')" || return 1 + grafana_get "/api/datasources/proxy/uid/loki/loki/api/v1/query_range?query=$encoded&limit=20&direction=backward&since=720h" "$loki_query" "$auth" || return 1 + grafana_get "/api/datasources/proxy/uid/tempo/api/traces/$trace" "$tempo_query" "$auth" || return 1 + jq_bounded -n --slurpfile prometheus "$prometheus" --slurpfile loki "$loki" --slurpfile tempo "$tempo" \ + --slurpfile loki_config "$loki_config" --slurpfile tempo_config "$tempo_config" \ + --slurpfile tempo_search "$tempo_search" --slurpfile loki_query "$loki_query" --slurpfile tempo_query "$tempo_query" \ + --arg selected_trace "$trace" \ + '{prometheus:$prometheus[0],loki:$loki[0],tempo:$tempo[0],loki_config:$loki_config[0],tempo_config:$tempo_config[0],tempo_search:$tempo_search[0],selected_trace:$selected_trace,loki_query:$loki_query[0],tempo_query:$tempo_query[0]}' >"$combined" || return 1 + "$CHMOD_BIN" 0600 -- "$combined" + validate_datasource_contract "$combined" || return 1 + printf 'OBSERVABILITY_MACHINE_DATASOURCES=PASS\n' +} + +run_read_only_machine_checks() { + check_dns_host || { fail 'DNS host acceptance failed'; return 1; } + check_certificate || { fail 'certificate acceptance failed'; return 1; } + check_https_boundaries || { fail 'HTTPS/NodePort boundary acceptance failed'; return 1; } + check_targets || { fail 'Prometheus target acceptance failed'; return 1; } + check_metric_contracts || { fail 'metric/quota acceptance failed'; return 1; } + check_blackbox_series || { fail 'blackbox acceptance failed'; return 1; } + check_datasources || { fail 'Grafana datasource/link acceptance failed'; return 1; } +} + +repeat_commit_gate() { + validate_commit_environment +} + +validate_owned_object_json() { + local file=$1 kind=$2 namespace=$3 name=$4 run=$5 require_uid=${6:-false} + jq_bounded -e --arg kind "$kind" --arg namespace "$namespace" --arg name "$name" --arg run "$run" --argjson require_uid "$require_uid" ' + .kind==$kind and .metadata.namespace==$namespace and .metadata.name==$name and + .metadata.labels["app.kubernetes.io/managed-by"]=="platform-observability-smoke" and + .metadata.labels["platform.hyeonworks.com/smoke-run"]==$run and + (($require_uid|not) or (.metadata.uid|type=="string" and length>0)) + ' "$file" >/dev/null +} + +validate_live_matches_desired() { + local desired=$1 live=$2 + python_bounded -I -S - "$desired" "$live" <<'PY' +import json,pathlib,sys +try: + desired=json.loads(pathlib.Path(sys.argv[1]).read_text()) + live=json.loads(pathlib.Path(sys.argv[2]).read_text()) +except Exception: + raise SystemExit(1) + +def subset(want,got): + if isinstance(want,dict): + return isinstance(got,dict) and all(key in got and subset(value,got[key]) for key,value in want.items()) + if isinstance(want,list): + return isinstance(got,list) and len(want)==len(got) and all(subset(a,b) for a,b in zip(want,got)) + return want==got + +if desired.get('apiVersion')!=live.get('apiVersion') or desired.get('kind')!=live.get('kind'): + raise SystemExit(1) +dmeta=desired.get('metadata') or {}; lmeta=live.get('metadata') or {} +for field in ('name','namespace'): + if dmeta.get(field)!=lmeta.get(field): raise SystemExit(1) +if dmeta.get('labels')!=lmeta.get('labels'): raise SystemExit(1) +if not isinstance(lmeta.get('uid'),str) or not lmeta['uid'] or not isinstance(lmeta.get('resourceVersion'),str) or not lmeta['resourceVersion']: + raise SystemExit(1) +if lmeta.get('deletionTimestamp') or lmeta.get('ownerReferences') or lmeta.get('finalizers'): + raise SystemExit(1) +if not subset(desired.get('spec'),live.get('spec')): raise SystemExit(1) +if desired.get('kind')=='Pod': + dspec=desired.get('spec') or {}; lspec=live.get('spec') or {} + if len(lspec.get('containers') or [])!=len(dspec.get('containers') or []): raise SystemExit(1) + if lspec.get('initContainers') or lspec.get('ephemeralContainers'): raise SystemExit(1) +PY +} + +create_owned_from_file() { + local file=$1 kind=$2 namespace=$3 name=$4 run=$5 resource live rc=0 uid state outcome=0 + resource="$(kind_resource "$kind")" || return 1 + if state="$(owned_object_state "$kind" "$namespace" "$name" 2>/dev/null)"; then + fail "temporary object already exists: $kind/$namespace/$name"; return 1 + else [[ $? == 1 ]] || return 1; fi + validate_owned_object_json "$file" "$kind" "$namespace" "$name" "$run" false || return 1 + kubectl_bounded create --dry-run=server -f "$file" -o json >/dev/null || return 1 + live="$WORK/created-${kind}-${name}.json" + begin_mutation_critical + if kubectl_bounded create -f "$file" -o json >"$live"; then rc=0; else rc=$?; fi + "$CHMOD_BIN" 0600 -- "$live" + if (( rc != 0 )); then + if ! kubectl_bounded -n "$namespace" get "$resource" "$name" --ignore-not-found -o json >"$live"; then + emit_manual_recovery; outcome=1 + fi + fi + if (( outcome == 0 )) && [[ ! -s "$live" ]]; then emit_manual_recovery; outcome=1; fi + if (( outcome == 0 )) && ! validate_owned_object_json "$live" "$kind" "$namespace" "$name" "$run" true; then + emit_manual_recovery; outcome=1 + fi + if (( outcome == 0 )); then + uid="$(jq_bounded -r '.metadata.uid' "$live")" || { emit_manual_recovery; outcome=1; } + fi + if (( outcome == 0 && rc == 0 )); then + # A successful create response is authoritative ownership even if an + # admission/defaulting drift makes the object unsafe; register its UID so + # the exit transaction can remove only that exact object. + register_owned "$kind" "$namespace" "$name" "$uid" "$run" + validate_live_matches_desired "$file" "$live" || outcome=1 + elif (( outcome == 0 )); then + # A nonzero create is response-ambiguous. Copied labels are insufficient: + # only an exact desired-spec match can be classified as ours and cleaned. + if validate_live_matches_desired "$file" "$live"; then + register_owned "$kind" "$namespace" "$name" "$uid" "$run" + else + emit_manual_recovery + fi + outcome=1 + fi + end_mutation_critical + return "$outcome" +} + +validate_deployed_ingress_policy_contracts() { + local aistor=$1 blackbox=$2 + jq_bounded -e ' + .apiVersion=="networking.k8s.io/v1" and .kind=="NetworkPolicy" and + .metadata.name=="object-storage-allow-prometheus-metrics" and + .metadata.namespace=="object-storage" and + (.metadata.uid|type=="string" and length>0) and + (.metadata.resourceVersion|type=="string" and length>0) and + (.metadata.deletionTimestamp==null) and + .spec=={ + podSelector:{matchLabels:{"aistor.min.io/objectStore":"minio-aistor"}}, + policyTypes:["Ingress"], + ingress:[{from:[{ + namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"observability"}}, + podSelector:{matchLabels:{ + "app.kubernetes.io/name":"prometheus", + "app.kubernetes.io/instance":"observability-core-kube-pr-prometheus"}} + }],ports:[{protocol:"TCP",port:9000}]}] + } + ' "$aistor" >/dev/null || return 1 + jq_bounded -e ' + .apiVersion=="networking.k8s.io/v1" and .kind=="NetworkPolicy" and + .metadata.name=="observability-allow-prometheus-to-blackbox" and + .metadata.namespace=="observability" and + (.metadata.uid|type=="string" and length>0) and + (.metadata.resourceVersion|type=="string" and length>0) and + (.metadata.deletionTimestamp==null) and + .spec=={ + podSelector:{matchLabels:{ + "app.kubernetes.io/name":"prometheus-blackbox-exporter", + "app.kubernetes.io/instance":"blackbox-exporter"}}, + policyTypes:["Ingress"], + ingress:[{from:[{podSelector:{matchLabels:{ + "app.kubernetes.io/name":"prometheus", + "app.kubernetes.io/instance":"observability-core-kube-pr-prometheus"}} + }],ports:[{protocol:"TCP",port:9115}]}] + } + ' "$blackbox" >/dev/null +} + +run_network_boundary_transaction() { + local run name policy pod minio_ip blackbox_ip pod_result aistor_ingress blackbox_ingress + run="$(new_smoke_run_id network)" || return 1 + name="$(kubernetes_smoke_name "$run")" || return 1 + policy="$WORK/network-policy.json"; pod="$WORK/network-pod.json" + aistor_ingress="$WORK/aistor-ingress-policy.json" + blackbox_ingress="$WORK/blackbox-ingress-policy.json" + kubectl_bounded -n object-storage get networkpolicy object-storage-allow-prometheus-metrics -o json >"$aistor_ingress" || return 1 + kubectl_bounded -n observability get networkpolicy observability-allow-prometheus-to-blackbox -o json >"$blackbox_ingress" || return 1 + "$CHMOD_BIN" 0600 -- "$aistor_ingress" "$blackbox_ingress" + validate_deployed_ingress_policy_contracts "$aistor_ingress" "$blackbox_ingress" || return 1 + minio_ip="$(kubectl_bounded -n object-storage get service minio -o jsonpath='{.spec.clusterIP}')" || return 1 + blackbox_ip="$(kubectl_bounded -n observability get service blackbox-exporter -o jsonpath='{.spec.clusterIP}')" || return 1 + [[ "$minio_ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ && "$blackbox_ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || return 1 + jq_bounded -n --arg name "$name" --arg run "$run" '{ + apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy", + metadata:{name:$name,namespace:"observability",labels:{ + "app.kubernetes.io/managed-by":"platform-observability-smoke", + "platform.hyeonworks.com/smoke-run":$run}}, + spec:{podSelector:{matchLabels:{"platform.hyeonworks.com/smoke-run":$run}}, + policyTypes:["Egress"],egress:[{to:[{ + namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"kube-system"}}, + podSelector:{matchLabels:{"k8s-app":"kube-dns"}}}], + ports:[{protocol:"UDP",port:53},{protocol:"TCP",port:53}]},{to:[{ + namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"object-storage"}}, + podSelector:{matchLabels:{"aistor.min.io/objectStore":"minio-aistor"}}}], + ports:[{protocol:"TCP",port:9000}]},{to:[{ + namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"observability"}}, + podSelector:{matchLabels:{ + "app.kubernetes.io/name":"prometheus-blackbox-exporter", + "app.kubernetes.io/instance":"blackbox-exporter"}}}], + ports:[{protocol:"TCP",port:9115}]}]}} + ' >"$policy" || return 1 + "$CHMOD_BIN" 0600 -- "$policy" + local command + command="set -eu +resolved=\$(nslookup $GRAFANA_HOST | awk '/^Address: /{print \$2}' | tail -n1) +[ \"\$resolved\" = $LAN_IP ] +if nc -z -w 4 $minio_ip 9000; then exit 41; fi +if nc -z -w 4 $blackbox_ip 9115; then exit 42; fi +printf '%s\\n' OBSERVABILITY_NETWORK_BOUNDARY_PASS" + jq_bounded -n --arg name "$name" --arg run "$run" --arg image "$BUSYBOX_IMAGE" --arg command "$command" '{ + apiVersion:"v1",kind:"Pod",metadata:{name:$name,namespace:"observability",labels:{ + "app.kubernetes.io/managed-by":"platform-observability-smoke", + "platform.hyeonworks.com/smoke-run":$run}}, + spec:{automountServiceAccountToken:false,restartPolicy:"Never",terminationGracePeriodSeconds:1, + securityContext:{runAsNonRoot:true,runAsUser:65534,runAsGroup:65534,seccompProfile:{type:"RuntimeDefault"}}, + containers:[{name:"network-smoke",image:$image,imagePullPolicy:"IfNotPresent", + command:["sh","-ec",$command],securityContext:{allowPrivilegeEscalation:false, + readOnlyRootFilesystem:true,capabilities:{drop:["ALL"]}}}]}} + ' >"$pod" || return 1 + "$CHMOD_BIN" 0600 -- "$pod" + create_owned_from_file "$policy" NetworkPolicy observability "$name" "$run" || return 1 + create_owned_from_file "$pod" Pod observability "$name" "$run" || return 1 + kubectl_wait_90_bounded -n observability wait --for=jsonpath='{.status.phase}'=Succeeded \ + --timeout=90s "pod/$name" >/dev/null || return 1 + pod_result="$(kubectl_bounded -n observability logs "$name" -c network-smoke)" || return 1 + [[ "$pod_result" == OBSERVABILITY_NETWORK_BOUNDARY_PASS ]] || return 1 + cleanup_owned_objects || return 1 + printf 'OBSERVABILITY_MACHINE_NETWORK_POLICY=PASS\n' +} + +canonical_object_snapshot() { + local file=$1 + jq_bounded -S '{metadata:{name:.metadata.name,namespace:.metadata.namespace,uid:.metadata.uid},data:(.data//{}),status:(.status//{})}' "$file" +} + +list_one_ready_pod() { + local selector=$1 output=$2 + kubectl_bounded -n observability get pods -l "$selector" -o json >"$output" || return 1 + "$CHMOD_BIN" 0600 -- "$output" + jq_bounded -e '.items|length==1 and .[0].metadata.uid and any(.[0].status.conditions[]?; .type=="Ready" and .status=="True")' "$output" >/dev/null +} + +delete_pod_uid_for_restart() { + local name=$1 uid=$2 body response path status attempt current + body="$("$MKTEMP_BIN" "$WORK/restart-delete.XXXXXXXX")" || return 1 + response="$("$MKTEMP_BIN" "$WORK/restart-response.XXXXXXXX")" || return 1 + "$CHMOD_BIN" 0600 -- "$body" "$response" + jq_bounded -nc --arg uid "$uid" '{apiVersion:"meta.k8s.io/v1",kind:"DeleteOptions",propagationPolicy:"Background",preconditions:{uid:$uid}}' >"$body" + path="/api/v1/namespaces/observability/pods/$name" + status="$(proxy_request DELETE "$path" "$response" "$body")" || status=000 + for ((attempt=0; attempt<60; attempt++)); do + current="$(kubectl_bounded -n observability get pod "$name" --ignore-not-found -o json 2>/dev/null | jq_bounded -r '.metadata.uid//empty')" || return 1 + [[ -z "$current" || "$current" != "$uid" ]] && return 0 + /usr/bin/sleep 1 + done + return 1 +} + +fetch_silences() { + local output=$1 status + status="$(proxy_request GET "$(alertmanager_path '/api/v2/silences')" "$output")" || return 1 + [[ "$status" == 200 ]] || return 1 + jq_bounded -e 'type=="array"' "$output" >/dev/null +} + +silence_inventory_state() { + local source=$1 run=$2 + python_bounded -I -S - "$source" "$run" <<'PY' +import json,pathlib,re,sys +try: items=json.loads(pathlib.Path(sys.argv[1]).read_text()) +except Exception: raise SystemExit(1) +run=sys.argv[2] +if not isinstance(items,list) or re.fullmatch(r'persistence-[0-9]{8}t[0-9]{6}z-[0-9a-f]{32}',run) is None: raise SystemExit(1) +related=[]; active=[]; expired=[]; structural=[] +expected={ + ('alertname','PlatformObservabilityPersistenceSmoke',False,True), + ('instance',run,False,True), +} +for item in items: + if not isinstance(item,dict): raise SystemExit(1) + matchers=item.get('matchers') + if not isinstance(matchers,list): continue + tuples=[] + malformed=False + for matcher in matchers: + if not isinstance(matcher,dict): malformed=True; break + value=(matcher.get('name'),matcher.get('value'),matcher.get('isRegex'),matcher.get('isEqual')) + tuples.append(value) + if malformed: continue + if any(value[0]=='instance' and value[1]==run for value in tuples): related.append(item) + state=((item.get('status') or {}).get('state')) + if set(tuples)==expected and len(tuples)==2 and item.get('createdBy')=='platform-observability-smoke' and item.get('comment')=='Temporary restart-persistence acceptance '+run: + identifier=item.get('id') + if not isinstance(identifier,str) or re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*',identifier) is None: raise SystemExit(1) + structural.append(identifier) + if state in ('active','pending'): active.append(identifier) + elif state=='expired': expired.append(identifier) +print(f'{len(related)}|{len(active)}|{len(expired)}|{structural[0] if len(structural)==1 else ""}') +PY +} + +read_silence_state() { + local run=$1 output + output="$("$MKTEMP_BIN" "$WORK/silence-list.XXXXXXXX")" || return 1 + "$CHMOD_BIN" 0600 -- "$output" + fetch_silences "$output" || return 1 + silence_inventory_state "$output" "$run" +} + +create_persistence_silence() { + local body="$WORK/silence-create.json" response="$WORK/silence-create-response.json" + local status='' start end state related active expired observed_id response_id='' request_rc=0 attempt outcome=1 classified=false + SILENCE_RUN="$(new_smoke_run_id persistence)" || return 1 + state="$(read_silence_state "$SILENCE_RUN")" || return 1 + IFS='|' read -r related active expired observed_id <<<"$state" + if [[ "$related" != 0 || "$active" != 0 || "$expired" != 0 || -n "$observed_id" ]]; then + emit_manual_recovery; return 1 + fi + start="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%Y-%m-%dT%H:%M:%SZ)" || return 1 + end="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u -d '+20 minutes' +%Y-%m-%dT%H:%M:%SZ)" || return 1 + jq_bounded -n --arg start "$start" --arg end "$end" --arg run "$SILENCE_RUN" '{ + matchers:[ + {name:"alertname",value:"PlatformObservabilityPersistenceSmoke",isRegex:false,isEqual:true}, + {name:"instance",value:$run,isRegex:false,isEqual:true} + ],startsAt:$start,endsAt:$end,createdBy:"platform-observability-smoke", + comment:("Temporary restart-persistence acceptance "+$run) + }' >"$body" || return 1 + "$CHMOD_BIN" 0600 -- "$body" + begin_mutation_critical + status="$(proxy_request POST "$(alertmanager_path '/api/v2/silences')" "$response" "$body")" || request_rc=$? + if (( request_rc == 0 )) && [[ "$status" == 200 ]]; then + response_id="$(jq_bounded -er '.silenceID|select(type=="string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$"))' "$response" 2>/dev/null)" || response_id='' + fi + for ((attempt=0; attempt= 2 )); then SILENCE_ID=''; SILENCE_RUN=''; return 0; fi + else + terminal=0 + if [[ "$related" != 1 || "$active" != 1 || "$expired" != 0 || "$observed_id" != "$SILENCE_ID" ]]; then + emit_manual_recovery; return 1 + fi + fi + (( SILENCE_POLL_DELAY == 0 )) || /usr/bin/sleep "$SILENCE_POLL_DELAY" + done + emit_manual_recovery + return 1 +} + +run_restart_persistence_transaction() { + local grafana_before="$WORK/grafana-pod-before.json" grafana_after="$WORK/grafana-pod-after.json" + local alert_before="$WORK/alertmanager-pod-before.json" alert_after="$WORK/alertmanager-pod-after.json" + local pvc_before="$WORK/pvc-before.json" pvc_after="$WORK/pvc-after.json" + local dashboards_before="$WORK/dashboards-before.json" dashboards_after="$WORK/dashboards-after.json" + local api_before="$WORK/dashboard-api-before.json" api_after="$WORK/dashboard-api-after.json" + local silence="$WORK/silence-after-restart.json" auth name uid state related active expired observed_id + auth="$WORK/grafana-curl.conf"; safe_work_file "$auth" || return 1 + list_one_ready_pod 'app.kubernetes.io/name=grafana,app.kubernetes.io/instance=grafana' "$grafana_before" || return 1 + list_one_ready_pod 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' "$alert_before" || return 1 + { + kubectl_bounded -n observability get pvc grafana -o json + kubectl_bounded -n observability get pvc -l 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' -o json + } | jq_bounded -s '.[0] as $grafana | .[1] as $am | {grafana:{name:$grafana.metadata.name,uid:$grafana.metadata.uid,phase:$grafana.status.phase},alertmanager:[$am.items[]|{name:.metadata.name,uid:.metadata.uid,phase:.status.phase}]}' >"$pvc_before" || return 1 + kubectl_bounded -n observability get configmaps -l grafana_dashboard=1 -o json | jq_bounded -S \ + '[.items[]|{name:.metadata.name,uid:.metadata.uid,data:.data}]|sort_by(.name)' >"$dashboards_before" || return 1 + grafana_get '/api/search?type=dash-db' "$api_before" "$auth" || return 1 + jq_bounded -S '[.[]|{uid,title,type}]|sort_by(.uid)' "$api_before" >"$api_before.canonical" || return 1 + "$MV_BIN" -- "$api_before.canonical" "$api_before" + "$CHMOD_BIN" 0600 -- "$pvc_before" "$dashboards_before" "$api_before" + jq_bounded -e '.grafana.phase=="Bound" and (.alertmanager|length==1) and all(.alertmanager[];.phase=="Bound")' "$pvc_before" >/dev/null || return 1 + jq_bounded -e 'length==5' "$dashboards_before" >/dev/null || return 1 + jq_bounded -e 'length==5 and all(.[];.uid|type=="string" and length>0)' "$api_before" >/dev/null || return 1 + create_persistence_silence || return 1 + + name="$(jq_bounded -r '.items[0].metadata.name' "$grafana_before")"; uid="$(jq_bounded -r '.items[0].metadata.uid' "$grafana_before")" + delete_pod_uid_for_restart "$name" "$uid" || return 1 + kubectl_wait_180_bounded -n observability rollout status deployment/grafana --timeout=180s >/dev/null || return 1 + name="$(jq_bounded -r '.items[0].metadata.name' "$alert_before")"; uid="$(jq_bounded -r '.items[0].metadata.uid' "$alert_before")" + delete_pod_uid_for_restart "$name" "$uid" || return 1 + kubectl_wait_180_bounded -n observability rollout status statefulset/alertmanager-observability-core-kube-pr-alertmanager --timeout=180s >/dev/null || return 1 + + list_one_ready_pod 'app.kubernetes.io/name=grafana,app.kubernetes.io/instance=grafana' "$grafana_after" || return 1 + list_one_ready_pod 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' "$alert_after" || return 1 + [[ "$(jq_bounded -r '.items[0].metadata.uid' "$grafana_before")" != "$(jq_bounded -r '.items[0].metadata.uid' "$grafana_after")" ]] || return 1 + [[ "$(jq_bounded -r '.items[0].metadata.uid' "$alert_before")" != "$(jq_bounded -r '.items[0].metadata.uid' "$alert_after")" ]] || return 1 + { + kubectl_bounded -n observability get pvc grafana -o json + kubectl_bounded -n observability get pvc -l 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' -o json + } | jq_bounded -s '.[0] as $grafana | .[1] as $am | {grafana:{name:$grafana.metadata.name,uid:$grafana.metadata.uid,phase:$grafana.status.phase},alertmanager:[$am.items[]|{name:.metadata.name,uid:.metadata.uid,phase:.status.phase}]}' >"$pvc_after" || return 1 + kubectl_bounded -n observability get configmaps -l grafana_dashboard=1 -o json | jq_bounded -S \ + '[.items[]|{name:.metadata.name,uid:.metadata.uid,data:.data}]|sort_by(.name)' >"$dashboards_after" || return 1 + grafana_get '/api/search?type=dash-db' "$api_after" "$auth" || return 1 + jq_bounded -S '[.[]|{uid,title,type}]|sort_by(.uid)' "$api_after" >"$api_after.canonical" || return 1 + "$MV_BIN" -- "$api_after.canonical" "$api_after" + "$CMP_BIN" --silent -- "$pvc_before" "$pvc_after" || return 1 + "$CMP_BIN" --silent -- "$dashboards_before" "$dashboards_after" || return 1 + "$CMP_BIN" --silent -- "$api_before" "$api_after" || return 1 + fetch_silences "$silence" || return 1 + state="$(silence_inventory_state "$silence" "$SILENCE_RUN")" || return 1 + IFS='|' read -r related active expired observed_id <<<"$state" + [[ "$related" == 1 && "$active" == 1 && "$expired" == 0 && "$observed_id" == "$SILENCE_ID" ]] || return 1 + delete_persistence_silence || return 1 + printf 'OBSERVABILITY_MACHINE_RESTART_PERSISTENCE=PASS\n' +} + +run_certbot_acceptance() { + "$TIMEOUT_BIN" --signal=TERM --kill-after=2s 30s "$SUDO_BIN" -v || return 1 + "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$SUDO_BIN" -n /usr/bin/true || return 1 + "$TIMEOUT_BIN" --signal=TERM --kill-after=5s 610s "$SUDO_BIN" -n "$TIMEOUT_BIN" --signal=TERM --kill-after=5s 600s \ + /snap/bin/certbot renew --dry-run --run-deploy-hooks >/dev/null || return 1 + [[ "$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$SUDO_BIN" -n /usr/bin/systemctl is-active nginx)" == active ]] || return 1 + printf 'OBSERVABILITY_MACHINE_CERTBOT=PASS\n' +} + +decode_keycloak_admin_secret() { + local secret_json=$1 username_file=$2 password_file=$3 + python_bounded -I -S - "$secret_json" "$username_file" "$password_file" <<'PY' +import base64,binascii,json,os,pathlib,sys +source,user_path,password_path=sys.argv[1:] +try: item=json.loads(pathlib.Path(source).read_text()) +except Exception: raise SystemExit(1) +meta=item.get('metadata') or {}; data=item.get('data') +if item.get('kind')!='Secret' or item.get('type')!='Opaque' or meta.get('namespace')!='keycloak' or meta.get('name')!='keycloak-initial-admin' or sorted(data or {})!=['password','username']: + raise SystemExit(1) +for key,destination in [('username',user_path),('password',password_path)]: + try: value=base64.b64decode(data[key],validate=True) + except (binascii.Error,ValueError): raise SystemExit(1) + if not value or b'\x00' in value or b'\r' in value or b'\n' in value: raise SystemExit(1) + fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) + with os.fdopen(fd,'wb') as stream: stream.write(value) +PY +} + +prepare_keycloak_auth() { + local secret="$WORK/keycloak-admin-secret.json" username="$WORK/keycloak-admin-user" password="$WORK/keycloak-admin-password" + local form="$WORK/keycloak-token-form" config="$WORK/keycloak-token-curl.conf" response="$WORK/keycloak-token.json" status token + kubectl_bounded -n keycloak get secret keycloak-initial-admin -o json >"$secret" || return 1 + "$CHMOD_BIN" 0600 -- "$secret"; decode_keycloak_admin_secret "$secret" "$username" "$password" || return 1 + python_bounded -I -S - "$username" "$password" "$form" <<'PY' +import os,pathlib,sys,urllib.parse +user,password,destination=sys.argv[1:] +values={'grant_type':'password','client_id':'admin-cli','username':pathlib.Path(user).read_text(),'password':pathlib.Path(password).read_text()} +fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) +with os.fdopen(fd,'w') as stream: stream.write(urllib.parse.urlencode(values)) +PY + printf 'header = "Content-Type: application/x-www-form-urlencoded"\ndata-binary = "@%s"\n' "$form" >"$config"; "$CHMOD_BIN" 0600 -- "$config" + status="$(proxy_request POST "$(keycloak_path '/realms/master/protocol/openid-connect/token')" "$response" '' "$config")" || return 1 + [[ "$status" == 200 ]] || return 1 + token="$(jq_bounded -er '.access_token|select(type=="string" and test("^[A-Za-z0-9._~-]+$"))' "$response")" || return 1 + KEYCLOAK_AUTH_CONFIG="$WORK/keycloak-auth.conf" + printf 'header = "Authorization: Bearer %s"\n' "$token" >"$KEYCLOAK_AUTH_CONFIG"; "$CHMOD_BIN" 0600 -- "$KEYCLOAK_AUTH_CONFIG" + safe_work_file "$KEYCLOAK_AUTH_CONFIG" +} + +keycloak_request() { + local method=$1 api_path=$2 output=$3 body=${4:-} status + safe_work_file "$KEYCLOAK_AUTH_CONFIG" || return 1 + status="$(proxy_request "$method" "$(keycloak_path "$api_path")" "$output" "$body" "$KEYCLOAK_AUTH_CONFIG")" || return 1 + printf '%s' "$status" +} + +keycloak_lookup_group() { + local name=$1 expected_path=$2 output=$3 encoded status + encoded="$(jq_bounded -nr --arg value "$name" '$value|@uri')" || return 1 + status="$(keycloak_request GET "/admin/realms/hyeonworks/groups?search=$encoded&exact=true&max=2" "$output")" || return 1 + [[ "$status" == 200 ]] || return 1 + jq_bounded -er --arg name "$name" --arg path "$expected_path" ' + [ .[] | select(.name==$name and .path==$path) ] as $items | + select(($items|length)==1) | $items[0].id | select(type=="string" and test("^[A-Za-z0-9._-]+$")) + ' "$output" +} + +keycloak_lookup_user() { + local username=$1 output=$2 encoded status + encoded="$(jq_bounded -nr --arg value "$username" '$value|@uri')" || return 1 + status="$(keycloak_request GET "/admin/realms/hyeonworks/users?username=$encoded&exact=true&max=2" "$output")" || return 1 + [[ "$status" == 200 ]] || return 1 + jq_bounded -er --arg username "$username" ' + [ .[] | select(.username==$username and .enabled==true) ] as $items | + select(($items|length)==1) | $items[0].id | select(type=="string" and test("^[A-Za-z0-9._-]+$")) + ' "$output" +} + +keycloak_memberships() { + local output=$1 status page aggregate next count first=0 pages=0 + aggregate="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-all.XXXXXXXX")" || return 1 + printf '[]\n' >"$aggregate"; "$CHMOD_BIN" 0600 -- "$aggregate" + while (( pages < 100 )); do + page="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-page.XXXXXXXX")" || return 1 + "$CHMOD_BIN" 0600 -- "$page" + status="$(keycloak_request GET "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/groups?briefRepresentation=true&first=$first&max=100" "$page")" || return 1 + [[ "$status" == 200 ]] || return 1 + count="$(jq_bounded -er 'select(type=="array" and length<=100)|length' "$page")" || return 1 + next="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-next.XXXXXXXX")" || return 1 + "$CHMOD_BIN" 0600 -- "$next" + jq_bounded -s '.[0]+.[1]' "$aggregate" "$page" >"$next" || return 1 + "$MV_BIN" -- "$next" "$aggregate" + pages=$((pages + 1)) + (( count == 100 )) || break + first=$((first + 100)) + done + (( pages < 100 || count < 100 )) || return 1 + jq_bounded -S ' + select(type=="array" and all(.[]; + (.id|type=="string" and test("^[A-Za-z0-9._-]+$")) and + (.name|type=="string" and length>0) and (.path|type=="string" and startswith("/")))) | + select(([.[].id]|length)==([.[].id]|unique|length)) | + sort_by(.id) + ' "$aggregate" >"$output" || return 1 + "$CHMOD_BIN" 0600 -- "$output" +} + +keycloak_membership_state() { + local memberships=$1 group_id=$2 + jq_bounded -er --arg id "$group_id" ' + [ .[] | select(.id==$id) ] as $items | + if ($items|length)==0 then "absent" + elif ($items|length)==1 and $items[0].name=="platform-observability-viewers" and + $items[0].path=="/platform-observability-viewers" then "present" + else error("ambiguous viewer membership") end + ' "$memberships" +} + +capture_keycloak_absent_baseline() { + local admin_group_id=$1 first second state + [[ "$admin_group_id" =~ ^[A-Za-z0-9._-]+$ && "$KEYCLOAK_VIEWER_GROUP_ID" =~ ^[A-Za-z0-9._-]+$ ]] || return 1 + first="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-before-a.XXXXXXXX")" || return 1 + second="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-before-b.XXXXXXXX")" || return 1 + keycloak_memberships "$first" || return 1 + keycloak_memberships "$second" || return 1 + "$CMP_BIN" --silent -- "$first" "$second" || return 1 + state="$(keycloak_membership_state "$first" "$KEYCLOAK_VIEWER_GROUP_ID")" || return 1 + [[ "$state" == absent ]] || return 1 + jq_bounded -e --arg admin "$admin_group_id" 'all(.[]?; .id!=$admin)' "$first" >/dev/null || return 1 + [[ -n "$KEYCLOAK_MEMBERSHIP_BEFORE" && "$KEYCLOAK_MEMBERSHIP_BEFORE" == "$WORK"/* ]] || return 1 + "$MV_BIN" -- "$first" "$KEYCLOAK_MEMBERSHIP_BEFORE" + "$CHMOD_BIN" 0600 -- "$KEYCLOAK_MEMBERSHIP_BEFORE" +} + +add_keycloak_viewer_membership() { + local response memberships status=000 state outcome=1 + [[ -f "$KEYCLOAK_MEMBERSHIP_BEFORE" && ! -L "$KEYCLOAK_MEMBERSHIP_BEFORE" ]] || return 1 + [[ "$(keycloak_membership_state "$KEYCLOAK_MEMBERSHIP_BEFORE" "$KEYCLOAK_VIEWER_GROUP_ID")" == absent ]] || return 1 + response="$WORK/keycloak-membership-add-response.json" + memberships="$WORK/keycloak-memberships-added.json" + begin_mutation_critical + KEYCLOAK_MEMBERSHIP_PENDING=true + status="$(keycloak_request PUT "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/groups/$KEYCLOAK_VIEWER_GROUP_ID" "$response")" || status=000 + if keycloak_memberships "$memberships"; then + state="$(keycloak_membership_state "$memberships" "$KEYCLOAK_VIEWER_GROUP_ID")" || state=ambiguous + if [[ "$state" == present ]]; then + KEYCLOAK_TEST_ADDED=true + KEYCLOAK_MEMBERSHIP_PENDING=false + outcome=0 + elif [[ "$state" == absent ]]; then + KEYCLOAK_MEMBERSHIP_PENDING=false + fi + fi + end_mutation_critical + (( outcome == 0 )) || return 1 + [[ "$status" == 204 || "$status" == 000 ]] +} + +cleanup_keycloak_membership() { + local response status memberships verify state + [[ ( "$KEYCLOAK_TEST_ADDED" == true || "$KEYCLOAK_MEMBERSHIP_PENDING" == true ) && + "$KEYCLOAK_MEMBERSHIP_REMOVED" == false ]] || return 0 + [[ "$KEYCLOAK_USER_ID" =~ ^[A-Za-z0-9._-]+$ && "$KEYCLOAK_VIEWER_GROUP_ID" =~ ^[A-Za-z0-9._-]+$ ]] || { + emit_manual_recovery; return 1; + } + [[ -f "$KEYCLOAK_MEMBERSHIP_BEFORE" && ! -L "$KEYCLOAK_MEMBERSHIP_BEFORE" && + "$(keycloak_membership_state "$KEYCLOAK_MEMBERSHIP_BEFORE" "$KEYCLOAK_VIEWER_GROUP_ID" 2>/dev/null)" == absent ]] || { + emit_manual_recovery; return 1; + } + memberships="$WORK/keycloak-cleanup-memberships-before.json" + keycloak_memberships "$memberships" || { emit_manual_recovery; return 1; } + state="$(keycloak_membership_state "$memberships" "$KEYCLOAK_VIEWER_GROUP_ID")" || { + emit_manual_recovery; return 1; + } + if [[ "$state" == absent ]]; then + KEYCLOAK_MEMBERSHIP_PENDING=false; KEYCLOAK_MEMBERSHIP_REMOVED=true + return 0 + fi + response="$WORK/keycloak-cleanup-response.json" + status="$(keycloak_request DELETE "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/groups/$KEYCLOAK_VIEWER_GROUP_ID" "$response")" || status=000 + verify="$WORK/keycloak-cleanup-memberships-after.json" + keycloak_memberships "$verify" || { emit_manual_recovery; return 1; } + [[ "$(keycloak_membership_state "$verify" "$KEYCLOAK_VIEWER_GROUP_ID")" == absent ]] || { + emit_manual_recovery; return 1; + } + KEYCLOAK_MEMBERSHIP_PENDING=false + KEYCLOAK_MEMBERSHIP_REMOVED=true +} + +run_oidc_acceptance() { + local identity admin_identity viewer_identity denied_identity removal_identity + local admin_group viewer_group user memberships response status encoded grafana_user grafana_auth deadline now + printf '%s\n' 'Browser check: organization Admin and server-admin=false.' >&2 + admin_identity="$(read_identity 'OIDC organization Admin')" || return 1 + require_confirmation OIDC_ADMIN_CONFIRMED || return 1 + printf '%s\n' 'Browser check: Viewer and edit/admin actions denied.' >&2 + viewer_identity="$(read_identity 'OIDC Viewer')" || return 1 + require_confirmation OIDC_VIEWER_CONFIRMED || return 1 + printf '%s\n' 'Browser check: no observability groups and login denied.' >&2 + denied_identity="$(read_identity 'OIDC denied-user')" || return 1 + require_confirmation OIDC_DENIED_CONFIRMED || return 1 + printf '%s\n' 'Browser check: local break-glass login and server-admin=true.' >&2 + require_confirmation BREAK_GLASS_CONFIRMED || return 1 + [[ "$admin_identity" != "$viewer_identity" && "$admin_identity" != "$denied_identity" && "$viewer_identity" != "$denied_identity" ]] || return 1 + + removal_identity="$(read_identity 'group-removal test')" || return 1 + [[ "$removal_identity" != "$admin_identity" && "$removal_identity" != "$viewer_identity" && "$removal_identity" != "$denied_identity" ]] || return 1 + prepare_keycloak_auth || return 1 + admin_group="$WORK/keycloak-admin-group.json"; viewer_group="$WORK/keycloak-viewer-group.json"; user="$WORK/keycloak-removal-user.json" + local admin_group_id + admin_group_id="$(keycloak_lookup_group platform-observability-admins /platform-observability-admins "$admin_group")" || return 1 + KEYCLOAK_VIEWER_GROUP_ID="$(keycloak_lookup_group platform-observability-viewers /platform-observability-viewers "$viewer_group")" || return 1 + KEYCLOAK_USER_ID="$(keycloak_lookup_user "$removal_identity" "$user")" || return 1 + KEYCLOAK_MEMBERSHIP_BEFORE="$WORK/keycloak-memberships-before.json" + capture_keycloak_absent_baseline "$admin_group_id" || return 1 + add_keycloak_viewer_membership || return 1 + printf '%s\n' 'Browser check: removal-test user is Viewer before membership removal.' >&2 + require_confirmation REMOVAL_VIEWER_CONFIRMED || return 1 + cleanup_keycloak_membership || return 1 + response="$WORK/keycloak-session-revoke.json" + status="$(keycloak_request POST "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/logout" "$response")" || return 1 + [[ "$status" == 204 ]] || return 1 + + grafana_auth="$WORK/grafana-curl.conf"; safe_work_file "$grafana_auth" || return 1 + encoded="$(jq_bounded -nr --arg value "$removal_identity" '$value|@uri')" || return 1 + grafana_user="$WORK/grafana-removal-user.json" + grafana_get "/api/users/lookup?loginOrEmail=$encoded" "$grafana_user" "$grafana_auth" || return 1 + local grafana_id + grafana_id="$(jq_bounded -er '.id|select(type=="number" and .>0 and floor==.)' "$grafana_user")" || return 1 + response="$WORK/grafana-session-revoke.json" + status="$(proxy_request POST "$(grafana_path "/api/admin/users/$grafana_id/logout")" "$response" '' "$grafana_auth")" || return 1 + [[ "$status" == 200 || "$status" == 204 ]] || return 1 + now="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%s)" || return 1 + [[ "$now" =~ ^[0-9]+$ ]] || return 1 + deadline=$((now + HUMAN_TIMEOUT_SECONDS)) + printf '%s\n' 'Within five minutes, verify old session rejection and OIDC relogin denial.' >&2 + require_confirmation_before SESSION_REVOKED "$deadline" || return 1 + require_confirmation_before RELOGIN_DENIED "$deadline" || return 1 + printf 'OBSERVABILITY_OIDC_ACCEPTANCE=PASS\n' +} + +wait_for_smoke_alert() { + local run=$1 expected=$2 attempt query alerts + for ((attempt=0; attempt<60; attempt++)); do + query="$WORK/slack-query-$attempt.json"; alerts="$WORK/slack-alerts-$attempt.json" + prom_query "ALERTS{alertname=\"PlatformObservabilitySmoke\",instance=\"$run\",alertstate=\"firing\"}" "$query" || return 1 + proxy_get_200 "$(alertmanager_path '/api/v2/alerts')" "$alerts" || return 1 + if [[ "$expected" == present ]]; then + if jq_bounded -e --arg run "$run" --arg runbook "$RUNBOOK_URL" ' + (.status=="success" and (.data.result|length)==1 and + .data.result[0].metric.alertname=="PlatformObservabilitySmoke" and + .data.result[0].metric.instance==$run) + ' "$query" >/dev/null && + jq_bounded -e --arg run "$run" --arg runbook "$RUNBOOK_URL" ' + [.[]|select(.labels.alertname=="PlatformObservabilitySmoke" and .labels.instance==$run)] as $items | + ($items|length)==1 and $items[0].labels.cluster=="home" and + $items[0].labels.namespace=="observability" and $items[0].labels.severity=="info" and + $items[0].annotations.runbook_url==$runbook + ' "$alerts" >/dev/null; then return 0; fi + else + if jq_bounded -e '.status=="success" and (.data.result|length)==0' "$query" >/dev/null && + jq_bounded -e --arg run "$run" '[.[]|select(.labels.alertname=="PlatformObservabilitySmoke" and .labels.instance==$run)]|length==0' "$alerts" >/dev/null; then return 0; fi + fi + /usr/bin/sleep 5 + done + return 1 +} + +run_slack_acceptance() { + local run name rule + run="$(new_smoke_run_id alert)" || return 1 + name="$(kubernetes_smoke_name "$run")" || return 1 + rule="$WORK/slack-smoke-rule.json" + jq_bounded -n --arg name "$name" --arg run "$run" --arg runbook "$RUNBOOK_URL" '{ + apiVersion:"monitoring.coreos.com/v1",kind:"PrometheusRule", + metadata:{name:$name,namespace:"observability",labels:{ + "app.kubernetes.io/managed-by":"platform-observability-smoke", + "platform.hyeonworks.com/smoke-run":$run, + "observability.hyeonworks.com/instance":"home"}}, + spec:{groups:[{name:("platform.observability-smoke."+$run),rules:[{ + alert:"PlatformObservabilitySmoke",expr:"vector(1)",for:"0m", + labels:{cluster:"home",namespace:"observability",instance:$run,severity:"info"}, + annotations:{summary:"Observability acceptance smoke",runbook_url:$runbook} + }]}]}} + ' >"$rule" || return 1 + "$CHMOD_BIN" 0600 -- "$rule" + create_owned_from_file "$rule" PrometheusRule observability "$name" "$run" || return 1 + wait_for_smoke_alert "$run" present || return 1 + printf '%s\n' 'Confirm the Slack message groups cluster, namespace, alertname and includes severity/runbook.' >&2 + require_confirmation FIRING || return 1 + cleanup_owned_objects || return 1 + printf '%s\n' 'Confirm the matching Slack resolved message.' >&2 + require_confirmation RESOLVED || return 1 + wait_for_smoke_alert "$run" absent || return 1 + printf 'OBSERVABILITY_SLACK_ACCEPTANCE=PASS\n' +} + +cleanup_all() { + local rc=0 + cleanup_owned_objects || rc=1 + cleanup_keycloak_membership || rc=1 + delete_persistence_silence || rc=1 + if [[ -n "$API_PROXY_PID" ]]; then + stop_owned_process "$API_PROXY_PID" "$API_PROXY_START" || rc=1 + API_PROXY_PID=''; API_PROXY_START='' + fi + if [[ -n "$WORK" ]]; then + if [[ "$WORK" == /tmp/platform-observability-smoke.???????? && -d "$WORK" && ! -L "$WORK" ]]; then + "$RM_BIN" -rf -- "$WORK" + WORK='' + else + emit_manual_recovery; rc=1 + fi + fi + return "$rc" +} + +on_exit() { + local rc=$? + [[ "$EXIT_HANDLER_RUNNING" == false ]] || exit "$rc" + EXIT_HANDLER_RUNNING=true + trap - EXIT HUP INT TERM + cleanup_all || { (( rc != 0 )) || rc=1; } + exit "$rc" +} + +main() { + parse_arguments "$@" || return 1 + if [[ "$TEST_MODE" == false ]]; then reject_production_overrides || return 1; fi + if [[ "$execute_requested" == false ]]; then print_dry_run; return 0; fi + + trap on_exit EXIT + trap 'handle_termination_signal 129' HUP + trap 'handle_termination_signal 130' INT + trap 'handle_termination_signal 143' TERM + require_execute_environment || { fail 'execute preflight failed'; return 1; } + run_read_only_machine_checks || return 1 + require_confirmation 'APPLY default' || { fail 'confirmation did not match APPLY default'; return 1; } + repeat_commit_gate || { fail 'post-confirmation commit gate changed'; return 1; } + run_network_boundary_transaction || { fail 'network boundary transaction failed'; return 1; } + run_restart_persistence_transaction || { fail 'restart persistence transaction failed'; return 1; } + run_certbot_acceptance || { fail 'Certbot/Nginx acceptance failed'; return 1; } + repeat_commit_gate || { fail 'post-mutation context/API gate changed'; return 1; } + run_read_only_machine_checks || { fail 'post-mutation machine acceptance failed'; return 1; } + printf 'OBSERVABILITY_MACHINE_ACCEPTANCE=PASS\n' + run_oidc_acceptance || { fail 'OIDC human/session acceptance failed'; return 1; } + run_slack_acceptance || { fail 'Slack firing/resolved acceptance failed'; return 1; } + local external_token + external_token="$(external_attestation_token "$PUBLIC_EDGE_IP")" || return 1 + render_external_attestation "$PUBLIC_EDGE_IP" >&2 || return 1 + if ! require_confirmation "$external_token"; then + printf 'OBSERVABILITY_EXTERNAL_BOUNDARY=PARTIAL\n' + return 2 + fi + printf '%s\n' 'OBSERVABILITY_EXTERNAL_BOUNDARY=PASS' 'OBSERVABILITY_SMOKE=PASS' +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/scripts/validate/render-admin-services.sh b/scripts/validate/render-admin-services.sh new file mode 100755 index 0000000..ade10b4 --- /dev/null +++ b/scripts/validate/render-admin-services.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly EXPECTED_HELM_VERSION="v3.19.4" +readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1" +readonly KUBE_VERSION="1.36.2" +readonly PGADMIN_SHA="0f409becdc6a811e852f384cb53b3f11cfb652a19d77cb952a8a176071694f4c" +readonly AISTOR_SHA="50ffa6a4e014cdc48566b237593baab41d039f103cdc50c1b1ac173b8d8bf71e" +readonly IMAGE_SHA="sha256:66a300a7ecdcc1f325af0c430315329bca46cd4a7067227d6899802238167c6e" +readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)" +readonly PG_CACHE="$ROOT/services/pgadmin/.helm/charts/pgadmin4-helm-9.16.0" +readonly AI_CACHE="$ROOT/services/minio-aistor/base/deployment/.helm/charts/aistor-objectstore-1.0.16" +readonly NAMES="admin-namespace pgadmin-local-pv coredns-custom aistor-admin-oidc pgadmin" + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'USAGE' +사용법: + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/validate/render-admin-services.sh + +적용 스크립트 내부 handoff: + bash scripts/validate/render-admin-services.sh \ + --verified-output-dir /tmp/platform-admin-apply.XXXXXX + +고정한 공식 차트의 SHA-256과 렌더 결과를 검사합니다. 변경은 수행하지 않습니다. +USAGE +} + +out="" +if (( $# == 0 )); then + : +elif (( $# == 2 )) && [[ "$1" == "--verified-output-dir" ]]; then + out="$2" +else + usage >&2 + exit 2 +fi + +for cmd in awk find helm install kubectl rg sed sha256sum stat tar wc tr; do + if [[ "$cmd" == helm ]] && [[ -v PLATFORM_HELM_BIN ]]; then + continue + fi + command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다" +done + +if [[ -v PLATFORM_HELM_BIN ]]; then + [[ "$PLATFORM_HELM_BIN" == /* && -x "$PLATFORM_HELM_BIN" ]] || \ + fail "PLATFORM_HELM_BIN은 실행 가능한 절대 경로여야 합니다" + readonly HELM="$PLATFORM_HELM_BIN" +else + readonly HELM="$(command -v helm)" +fi +[[ "$("$HELM" version --template '{{.Version}}')" == "$EXPECTED_HELM_VERSION" ]] || \ + fail "Helm은 정확히 $EXPECTED_HELM_VERSION이어야 합니다" +kv="$(kubectl version --client --output=yaml | sed -n 's/^kustomizeVersion: //p')" +[[ "$kv" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \ + fail "Kustomize는 정확히 $EXPECTED_KUSTOMIZE_VERSION이어야 합니다" + +if [[ -n "$out" ]]; then + [[ "$out" == /tmp/platform-admin-apply.* ]] || fail "handoff 경로 형식이 잘못됐습니다" + [[ -d "$out" && ! -L "$out" && -O "$out" ]] || fail "handoff 경로가 안전하지 않습니다" + [[ "$(stat --format='%a' "$out")" == 700 ]] || fail "handoff 권한은 0700이어야 합니다" + [[ -z "$(find "$out" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail "handoff 경로가 비어 있지 않습니다" +fi + +umask 077 +work="$(mktemp -d /tmp/platform-admin-render.XXXXXX)" +pg_created=false +ai_created=false + +cleanup() { + rc=$? + set +e + [[ "$pg_created" == true ]] && rm -rf -- "$PG_CACHE" + [[ "$ai_created" == true ]] && rm -rf -- "$AI_CACHE" + rmdir -- "$ROOT/services/pgadmin/.helm/charts" "$ROOT/services/pgadmin/.helm" 2>/dev/null || true + rmdir -- "$ROOT/services/minio-aistor/base/deployment/.helm/charts" \ + "$ROOT/services/minio-aistor/base/deployment/.helm" 2>/dev/null || true + case "$work" in + /tmp/platform-admin-render.*) rm -rf -- "$work" ;; + esac + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +prepare() { + label="$1" + ref="$2" + name="$3" + version="$4" + expected="$5" + cache="$6" + archive="$work/$name-$version.tgz" + + [[ ! -e "$cache" && ! -L "$cache" ]] || fail "생성 캐시가 이미 있습니다: $cache" + if [[ "$ref" == oci://* ]]; then + "$HELM" pull "$ref" --version "$version" --destination "$work" + else + "$HELM" pull "$name" --repo "$ref" --version "$version" --destination "$work" + fi + [[ -f "$archive" ]] || fail "$label archive가 없습니다" + actual="$(sha256sum "$archive" | awk '{print $1}')" + [[ "$actual" == "$expected" ]] || fail "$label SHA-256 불일치: $actual" + mkdir -p -- "$cache" + [[ "$cache" == "$PG_CACHE" ]] && pg_created=true + [[ "$cache" == "$AI_CACHE" ]] && ai_created=true + tar -xzf "$archive" -C "$cache" + [[ -f "$cache/$name/Chart.yaml" ]] || fail "$label Chart.yaml이 없습니다" + printf '검증 %-18s SHA-256 %s\n' "$label" "$actual" +} + +plain() { + kubectl kustomize "$ROOT/$2" >"$work/$1.yaml" + [[ -s "$work/$1.yaml" ]] || fail "$1 렌더 결과가 비었습니다" +} + +with_helm() { + kubectl kustomize --enable-helm --helm-command "$HELM" \ + --helm-kube-version "$KUBE_VERSION" "$ROOT/$2" >"$work/$1.yaml" + [[ -s "$work/$1.yaml" ]] || fail "$1 렌더 결과가 비었습니다" +} + +cd -- "$ROOT" +if rg -n --glob '*.yaml' --glob '*.yml' --glob '!**/.helm/**' \ + '^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' infrastructure services bootstrap; then + fail "Git 관리 대상에서 Secret manifest를 발견했습니다" +fi +if rg -n '^[[:space:]]*hostAliases:' services/gitea/profiles/oidc; then + fail "Gitea hostAliases가 남아 있습니다" +fi + +prepare pgadmin oci://docker.io/dpage/pgadmin4-helm pgadmin4-helm 9.16.0 "$PGADMIN_SHA" "$PG_CACHE" +prepare aistor-objectstore https://helm.min.io/ aistor-objectstore 1.0.16 "$AISTOR_SHA" "$AI_CACHE" + +plain admin-namespace infrastructure/namespaces/admin-tools +plain pgadmin-local-pv infrastructure/storage/pgadmin-local-pv +plain coredns-custom infrastructure/networking/private-dns/kubernetes +with_helm aistor-admin-oidc services/minio-aistor/profiles/admin-oidc +with_helm pgadmin services/pgadmin + +if rg -n '^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' "$work"/*.yaml; then + fail "렌더 결과에 Secret 리소스가 있습니다" +fi +[[ "$(rg -c -F "image: docker.io/dpage/pgadmin4@$IMAGE_SHA" "$work/pgadmin.yaml")" == 3 ]] || \ + fail "pgAdmin main/init 이미지 세 개의 digest 고정이 깨졌습니다" +rg -q -F 'type: Recreate' "$work/pgadmin.yaml" || fail "pgAdmin Recreate 전략이 없습니다" +rg -q -F 'storageClassName: ssd-local-pgadmin-retain' "$work/pgadmin.yaml" || fail "pgAdmin StorageClass가 다릅니다" +[[ "$(rg -c -F 'value: db-admin.learn.hyeonworks.com' "$work/pgadmin.yaml")" == 3 ]] || \ + fail "pgAdmin 세 probe의 Host header가 고정되지 않았습니다" +rg -q -F 'host: db-admin.learn.hyeonworks.com' "$work/pgadmin.yaml" || fail "pgAdmin Ingress가 없습니다" +[[ "$(rg -c -F 'ingressClassName: traefik' "$work/pgadmin.yaml")" == 1 ]] || \ + fail "pgAdmin IngressClass가 traefik으로 고정되지 않았습니다" +if rg -q -F 'kubernetes.io/ingress.class' "$work/pgadmin.yaml"; then + fail "pgAdmin Ingress에 레거시 ingress.class annotation이 남아 있습니다" +fi +rg -q -F 'namespace: platform-data' "$work/pgadmin.yaml" || fail "PostgreSQL 정책 namespace가 다릅니다" +bad_pgadmin_namespaces="$( + awk ' + function verify() { + if (pgadmin_instance && namespace != "platform-admin") { + printf "%s/%s=%s\n", kind, name, namespace == "" ? "missing" : namespace + } + } + function reset() { + kind="" + name="" + namespace="" + in_metadata=0 + in_labels=0 + pgadmin_instance=0 + } + /^---$/ { + verify() + reset() + next + } + /^kind:/ { + kind=$2 + next + } + /^metadata:$/ { + in_metadata=1 + in_labels=0 + next + } + in_metadata && /^ name:/ && name == "" { + name=$2 + next + } + in_metadata && /^ namespace:/ && namespace == "" { + namespace=$2 + next + } + in_metadata && /^ labels:$/ { + in_labels=1 + next + } + in_labels && /^ app.kubernetes.io\/instance: pgadmin$/ { + pgadmin_instance=1 + next + } + in_metadata && /^[^ ]/ { + in_metadata=0 + in_labels=0 + } + END { + verify() + } + ' "$work/pgadmin.yaml" +)" +[[ -z "$bad_pgadmin_namespaces" ]] || \ + fail "pgAdmin instance 리소스 namespace 오류: $bad_pgadmin_namespaces" +rg -q -F 'host: storage-admin.learn.hyeonworks.com' "$work/aistor-admin-oidc.yaml" || fail "AIStor Console Ingress가 없습니다" +rg -q -F 'name: MINIO_IDENTITY_OPENID_CLIENT_SECRET' "$work/aistor-admin-oidc.yaml" || fail "AIStor OIDC Secret 참조가 없습니다" +if rg -q -F 'name: MINIO_IDENTITY_OPENID_VENDOR' "$work/aistor-admin-oidc.yaml"; then + fail "claim-only OIDC profile에 Keycloak Admin API vendor 설정이 있습니다" +fi +if rg -q -F 'host: s3.' "$work/aistor-admin-oidc.yaml"; then + fail "S3 API Ingress가 생성됐습니다" +fi + +for name in $NAMES; do + bytes="$(wc -c <"$work/$name.yaml" | tr -d '[:space:]')" + sha="$(sha256sum "$work/$name.yaml" | awk '{print $1}')" + printf '렌더 %-20s %8s bytes SHA-256 %s\n' "$name" "$bytes" "$sha" + if [[ -n "$out" ]]; then + install -m 0600 "$work/$name.yaml" "$out/$name.yaml" + fi +done + +if [[ -n "$out" ]]; then + count="$(find "$out" -mindepth 1 -maxdepth 1 -type f | wc -l | tr -d '[:space:]')" + [[ "$count" == 5 ]] || fail "handoff 파일 수가 다릅니다" +fi +printf 'ADMIN SERVICES STATIC RENDER PASS\n' diff --git a/scripts/validate/render-observability-access.sh b/scripts/validate/render-observability-access.sh new file mode 100644 index 0000000..8fa2b1c --- /dev/null +++ b/scripts/validate/render-observability-access.sh @@ -0,0 +1,825 @@ +#!/usr/bin/env bash + +# Static renderer and sourceable assertion library for the observability access +# layer. It intentionally reuses the pinned core renderer for Helm/cache and +# image/credential boundaries instead of maintaining a second downloader. + +set -o pipefail + +readonly ACCESS_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +# shellcheck source=render-observability-core.sh +source "$ACCESS_ROOT/scripts/validate/render-observability-core.sh" + +access_output_names() { + case "$1" in + grafana) printf '%s\n' grafana.yaml ;; + blackbox) printf '%s\n' blackbox.yaml ;; + targets) printf '%s\n' targets.yaml ;; + rules-alerts) printf '%s\n' dashboards.yaml rules.yaml alertmanager.yaml ;; + complete) printf '%s\n' grafana.yaml blackbox.yaml targets.yaml dashboards.yaml rules.yaml alertmanager.yaml private-dns.yaml ;; + *) return 1 ;; + esac +} + +access_component_requires_inventory() { + [[ "$1" == rules-alerts || "$1" == complete ]] +} + +access_secure_render_context() { + umask 077 +} + +_access_assert() { + local mode=$1 + shift + python3 - "$mode" "$@" <<'PY' +import ipaddress +import json +import pathlib +import re +import sys +import yaml + +class AccessSafeLoader(yaml.SafeLoader): + pass + + +AccessSafeLoader.add_constructor( + "tag:yaml.org,2002:value", + lambda loader, node: loader.construct_scalar(node), +) + +mode = sys.argv[1] +paths = [pathlib.Path(value) for value in sys.argv[2:]] + +def reject(message): + print(f"REJECT: {message}", file=sys.stderr) + raise SystemExit(1) + +def load(path): + with path.open(encoding="utf-8") as stream: + return [item for item in yaml.load_all(stream, Loader=AccessSafeLoader) if item is not None] + +def one(items, kind, name, namespace=None): + matches = [ + item for item in items + if item.get("kind") == kind + and (item.get("metadata") or {}).get("name") == name + and (namespace is None or (item.get("metadata") or {}).get("namespace") == namespace) + ] + if len(matches) != 1: + reject(f"expected one {kind}/{name}, found {len(matches)}") + return matches[0] + +def labels(item): + return (item.get("metadata") or {}).get("labels") or {} + +def exact_policy_specs(items, expected): + policies = { + (item.get("metadata") or {}).get("name"): item.get("spec") or {} + for item in items if item.get("kind") == "NetworkPolicy" + } + for name, spec in expected.items(): + if policies.get(name) != spec: + reject(f"NetworkPolicy spec is not exact: {name}") + +def selector_matches(selector, pod_labels): + selector = selector or {} + for key, value in (selector.get("matchLabels") or {}).items(): + if pod_labels.get(key) != value: + return False + for expression in selector.get("matchExpressions") or []: + key = expression.get("key") + operator = expression.get("operator") + values = expression.get("values") or [] + value = pod_labels.get(key) + if operator == "In" and value not in values: + return False + if operator == "NotIn" and value in values: + return False + if operator == "Exists" and value is None: + return False + if operator == "DoesNotExist" and value is not None: + return False + if operator not in {"In", "NotIn", "Exists", "DoesNotExist"}: + reject("NetworkPolicy selector operator is unsupported") + return True + +def exact_selecting_policy_names(items, namespace, pod_labels, expected): + selected = { + (item.get("metadata") or {}).get("name") + for item in items + if item.get("kind") == "NetworkPolicy" + and (item.get("metadata") or {}).get("namespace") == namespace + and selector_matches((item.get("spec") or {}).get("podSelector") or {}, pod_labels) + } + if selected != set(expected): + reject("NetworkPolicy selecting set is not exact") + +if mode == "grafana": + items, policies = load(paths[0]), load(paths[1]) + service = one(items, "Service", "grafana", "observability") + if (service.get("spec") or {}).get("type") != "ClusterIP": + reject("Grafana Service is not ClusterIP") + ingress = one(items, "Ingress", "grafana", "observability") + spec = ingress.get("spec") or {} + rules = spec.get("rules") or [] + if spec.get("ingressClassName") != "traefik" or spec.get("tls") not in (None, []): + reject("Grafana Ingress boundary is not exact") + if len(rules) != 1 or rules[0].get("host") != "grafana.learn.hyeonworks.com": + reject("Grafana host is not exact") + deployment = one(items, "Deployment", "grafana", "observability") + pod = (((deployment.get("spec") or {}).get("template") or {}).get("spec") or {}) + if pod.get("automountServiceAccountToken") is not False: + reject("Grafana Pod token automount is not disabled") + containers = {container.get("name"): container for container in pod.get("containers") or []} + if set(containers) != {"grafana", "grafana-sc-dashboard"}: + reject("Grafana container set is not exact") + token_volume = "dashboard-sidecar-api-access" + main_mounts = {mount.get("name") for mount in containers["grafana"].get("volumeMounts") or []} + side_mounts = {mount.get("name") for mount in containers["grafana-sc-dashboard"].get("volumeMounts") or []} + if token_volume in main_mounts or token_volume not in side_mounts: + reject("dashboard API token mount is not sidecar-only") + volume = next((volume for volume in pod.get("volumes") or [] if volume.get("name") == token_volume), None) + sources = (((volume or {}).get("projected") or {}).get("sources") or []) + if len(sources) != 3 or not any("serviceAccountToken" in source for source in sources) \ + or not any("configMap" in source for source in sources) \ + or not any("downwardAPI" in source for source in sources): + reject("sidecar projected API volume is not exact") + one(items, "Role", "grafana-dashboard-sidecar", "observability") + if any(item.get("kind") == "ClusterRole" for item in items): + reject("cluster-wide Grafana RBAC is forbidden") + config = one(items, "ConfigMap", "grafana", "observability") + ini = ((config.get("data") or {}).get("grafana.ini") or "") + required = ( + "allowed_groups = /platform-observability-admins /platform-observability-viewers", + "role_attribute_strict = true", + "allow_assign_grafana_admin = false", + "&& 'Admin' || contains(groups[*], '/platform-observability-viewers') && 'Viewer' || null", + ) + if not all(token in ini for token in required) or "&& 'Editor'" in ini or "|| 'Viewer'" in ini: + reject("Grafana OIDC role mapping is not fail-closed") + exact_policy_specs(policies, { + "observability-default-deny": { + "podSelector": {}, "policyTypes": ["Ingress", "Egress"], + }, + "observability-allow-grafana-ingress": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana", + }}, + "policyTypes": ["Ingress"], + "ingress": [ + {"from": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}, + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "traefik-kube-system", + "app.kubernetes.io/name": "traefik", + }}, + }], "ports": [{"protocol": "TCP", "port": 3000}]}, + {"from": [{"podSelector": {"matchLabels": { + "app.kubernetes.io/name": "prometheus", + }}}], "ports": [{"protocol": "TCP", "port": 3000}]}, + {"from": [{"podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "blackbox-exporter", + "app.kubernetes.io/name": "prometheus-blackbox-exporter", + }}}], "ports": [{"protocol": "TCP", "port": 3000}]}, + ], + }, + "observability-allow-grafana-datasources": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana", + }}, + "policyTypes": ["Egress"], + "egress": [ + {"to": [{"podSelector": {"matchLabels": {"app.kubernetes.io/name": "prometheus"}}}], + "ports": [{"protocol": "TCP", "port": 9090}]}, + {"to": [{"podSelector": {"matchLabels": {"app.kubernetes.io/name": "loki"}}}], + "ports": [{"protocol": "TCP", "port": 3100}]}, + {"to": [{"podSelector": {"matchLabels": {"app.kubernetes.io/name": "tempo"}}}], + "ports": [{"protocol": "TCP", "port": 3200}]}, + {"to": [{"ipBlock": {"cidr": "192.168.0.107/32"}}], + "ports": [{"protocol": "TCP", "port": 443}]}, + ], + }, + "observability-allow-grafana-dashboard-api": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana", + }}, + "policyTypes": ["Egress"], + "egress": [ + {"to": [{"ipBlock": {"cidr": "10.43.0.1/32"}}], + "ports": [{"protocol": "TCP", "port": 443}]}, + {"to": [{"ipBlock": {"cidr": "192.168.0.107/32"}}], + "ports": [{"protocol": "TCP", "port": 6443}]}, + ], + }, + }) + exact_selecting_policy_names(policies, "observability", { + "app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana", + }, { + "observability-default-deny", "observability-allow-dns", + "observability-allow-grafana-ingress", "observability-allow-grafana-datasources", + "observability-allow-grafana-dashboard-api", + }) + +elif mode == "blackbox": + items = load(paths[0]) + service = one(items, "Service", "blackbox-exporter", "observability") + if (service.get("spec") or {}).get("type") != "ClusterIP": + reject("Blackbox Service is public") + deployment = one(items, "Deployment", "blackbox-exporter", "observability") + pod = (((deployment.get("spec") or {}).get("template") or {}).get("spec") or {}) + if pod.get("automountServiceAccountToken") is not False or len(pod.get("containers") or []) != 1: + reject("Blackbox Pod attack surface is not exact") + expected = { + "platform-public-edge": { + "https://git.learn.hyeonworks.com/api/healthz", + "https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration", + }, + "platform-private-edge": { + "https://grafana.learn.hyeonworks.com/", "https://storage-admin.learn.hyeonworks.com/", + "https://db-admin.learn.hyeonworks.com/", + }, + "platform-private-internal": { + "http://grafana.observability.svc.cluster.local/api/health", + "http://pgadmin.platform-admin.svc.cluster.local/misc/ping", + "http://minio-aistor-console.object-storage.svc.cluster.local:9090/", + }, + } + for name, targets in expected.items(): + probe = one(items, "Probe", name, "observability") + actual = set((((probe.get("spec") or {}).get("targets") or {}).get("staticConfig") or {}).get("static") or []) + if actual != targets: + reject(f"Blackbox target set drifted: {name}") + exact_policy_specs(items, { + "observability-allow-prometheus-to-blackbox": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "blackbox-exporter", + "app.kubernetes.io/name": "prometheus-blackbox-exporter", + }}, + "policyTypes": ["Ingress"], + "ingress": [{ + "from": [{"podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "observability-core-kube-pr-prometheus", + "app.kubernetes.io/name": "prometheus", + }}}], + "ports": [{"protocol": "TCP", "port": 9115}], + }], + }, + "observability-allow-blackbox-egress": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "blackbox-exporter", + "app.kubernetes.io/name": "prometheus-blackbox-exporter", + }}, + "policyTypes": ["Egress"], + "egress": [ + {"to": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}, + "podSelector": {"matchLabels": {"k8s-app": "kube-dns"}}, + }], "ports": [{"protocol": "UDP", "port": 53}, {"protocol": "TCP", "port": 53}]}, + {"to": [{"ipBlock": {"cidr": "192.168.0.107/32"}}], + "ports": [{"protocol": "TCP", "port": 443}]}, + {"to": [{"podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana", + }}}], "ports": [{"protocol": "TCP", "port": 3000}]}, + {"to": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "platform-admin"}}, + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "pgadmin", "app.kubernetes.io/name": "pgadmin4", + }}, + }], "ports": [{"protocol": "TCP", "port": 5050}]}, + {"to": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "object-storage"}}, + "podSelector": {"matchLabels": {"aistor.min.io/objectStore": "minio-aistor"}}, + }], "ports": [{"protocol": "TCP", "port": 9090}]}, + ], + }, + }) + if { + (item.get("metadata") or {}).get("name") + for item in items if item.get("kind") == "NetworkPolicy" + } != {"observability-allow-prometheus-to-blackbox", "observability-allow-blackbox-egress"}: + reject("Blackbox NetworkPolicy name set is not exact") + +elif mode == "targets": + items = load(paths[0]) + monitors = [(item.get("kind"), (item.get("metadata") or {}).get("namespace"), (item.get("metadata") or {}).get("name")) + for item in items if item.get("kind") in {"ServiceMonitor", "PodMonitor"}] + if sorted(monitors) != sorted([ + ("PodMonitor", "platform-data", "platform-postgres"), + ("ServiceMonitor", "object-storage", "aistor-bucket-usage"), + ]): + reject("manual target monitor set is not exact") + exact_policy_specs(items, { + "traefik-preserve-ingress-and-allow-prometheus-metrics": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "traefik-kube-system", "app.kubernetes.io/name": "traefik", + }}, + "policyTypes": ["Ingress"], + "ingress": [ + {"ports": [ + {"protocol": "TCP", "port": 8000}, {"protocol": "TCP", "port": 8443}, + {"protocol": "TCP", "port": 8080}, + ]}, + {"from": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "observability"}}, + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "observability-core-kube-pr-prometheus", + "app.kubernetes.io/name": "prometheus", + }}, + }], "ports": [{"protocol": "TCP", "port": 9100}]}, + ], + }, + "observability-allow-prometheus-platform-targets": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "observability-core-kube-pr-prometheus", + "app.kubernetes.io/name": "prometheus", + }}, + "policyTypes": ["Egress"], + "egress": [ + {"to": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "gitea"}}, + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "gitea", "app.kubernetes.io/name": "gitea", + }}, + }], "ports": [{"protocol": "TCP", "port": 3000}]}, + {"to": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "keycloak"}}, + "podSelector": {"matchLabels": { + "app": "keycloak", "app.kubernetes.io/instance": "keycloak", + "app.kubernetes.io/managed-by": "keycloak-operator", + }}, + }], "ports": [{"protocol": "TCP", "port": 9000}]}, + {"to": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "platform-data"}}, + "podSelector": {"matchLabels": {"cnpg.io/cluster": "platform-postgres"}}, + }], "ports": [{"protocol": "TCP", "port": 9187}]}, + {"to": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "object-storage"}}, + "podSelector": {"matchLabels": {"aistor.min.io/objectStore": "minio-aistor"}}, + }], "ports": [{"protocol": "TCP", "port": 9000}]}, + {"to": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}, + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "traefik-kube-system", "app.kubernetes.io/name": "traefik", + }}, + }], "ports": [{"protocol": "TCP", "port": 9100}]}, + ], + }, + }) + if { + (item.get("metadata") or {}).get("name") + for item in items if item.get("kind") == "NetworkPolicy" + } != { + "traefik-preserve-ingress-and-allow-prometheus-metrics", + "observability-allow-prometheus-platform-targets", + }: + reject("target NetworkPolicy name set is not exact") + +elif mode == "rules-alerts": + dashboards, rules, alertmanager = map(load, paths) + dashboard_names = sorted((item.get("metadata") or {}).get("name") for item in dashboards) + expected_dashboards = sorted([ + "grafana-dashboard-kubernetes-node", "grafana-dashboard-workload-health", + "grafana-dashboard-platform-services", "grafana-dashboard-observability-backends", + "grafana-dashboard-https-endpoints", + ]) + if dashboard_names != expected_dashboards or any(labels(item).get("grafana_dashboard") != "1" for item in dashboards): + reject("dashboard ConfigMap set is not exact") + rule_names = sorted((item.get("metadata") or {}).get("name") for item in rules) + if rule_names != sorted(["platform-aistor-storage-quota", "platform-certificate-probes", "platform-observability-core", "platform-verified-services"]): + reject("PrometheusRule set is not exact") + am = one(alertmanager, "Alertmanager", "observability-core-kube-pr-alertmanager", "observability") + if (((am.get("spec") or {}).get("alertmanagerConfiguration") or {}).get("name")) != "platform-alertmanager": + reject("global AlertmanagerConfig reference is absent") + config = one(alertmanager, "AlertmanagerConfig", "platform-alertmanager", "observability") + receivers = (config.get("spec") or {}).get("receivers") or [] + slack_receiver = next((receiver for receiver in receivers if receiver.get("name") == "platform-slack"), {}) + slack = ((slack_receiver.get("slackConfigs") or [{}])[0]) + api_url = slack.get("apiURL") or {} + if api_url != {"name": "alertmanager-slack-webhook", "key": "url"} or slack.get("sendResolved") is not True: + reject("Slack Secret selector or resolved route is not exact") + one(alertmanager, "NetworkPolicy", "observability-allow-alertmanager-public-https", "observability") + exact_policy_specs(alertmanager, { + "observability-allow-alertmanager-public-https": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/instance": "observability-core-kube-pr-alertmanager", + "app.kubernetes.io/name": "alertmanager", + }}, + "policyTypes": ["Egress"], + "egress": [{ + "to": [{"ipBlock": { + "cidr": "0.0.0.0/0", + "except": ["10.0.0.0/8", "100.64.0.0/10", "172.16.0.0/12", "192.168.0.0/16"], + }}], + "ports": [{"protocol": "TCP", "port": 443}], + }], + }, + }) + if { + (item.get("metadata") or {}).get("name") + for item in alertmanager if item.get("kind") == "NetworkPolicy" + } != {"observability-allow-alertmanager-public-https"}: + reject("Alertmanager NetworkPolicy name set is not exact") + +elif mode == "scope": + items = [] + for path in paths: + items.extend(load(path)) + forbidden = re.compile(r"spring|jvm|kafka|batch|backup", re.I) + for item in items: + kind = item.get("kind") + name = (item.get("metadata") or {}).get("name") or "" + if kind in {"Probe", "ServiceMonitor", "PodMonitor", "PrometheusRule"}: + if labels(item).get("observability.hyeonworks.com/instance") != "home": + reject(f"missing monitor/rule instance label: {kind}/{name}") + semantic_names = [name] + if kind == "PrometheusRule": + for group in (item.get("spec") or {}).get("groups") or []: + semantic_names.append(group.get("name") or "") + for rule in group.get("rules") or []: + semantic_names.append(rule.get("alert") or rule.get("record") or "") + if any(forbidden.search(value) for value in semantic_names): + reject(f"forbidden product scope: {kind}/{name}") + if kind in {"ServiceMonitor", "PodMonitor"}: + namespace = (item.get("metadata") or {}).get("namespace") + allowed = { + ("observability", "blackbox-exporter"), + ("platform-data", "platform-postgres"), + ("object-storage", "aistor-bucket-usage"), + } + if (namespace, name) not in allowed: + reject(f"out-of-scope access target: {namespace}/{name}") + +elif mode == "private-dns": + items = load(paths[0]) + config = one(items, "ConfigMap", "coredns-custom", "kube-system") + content = "\n".join((config.get("data") or {}).values()) + lan = paths[1].read_text() + tail = paths[2].read_text() + hosts = ("git", "id", "storage-admin", "db-admin", "grafana") + for short in hosts: + host = f"{short}.learn.hyeonworks.com" + if f"192.168.0.107 {host}" not in content \ + or f"address=/{host}/192.168.0.107" not in lan \ + or f"address=/{host}/100.92.240.34" not in tail: + reject(f"private DNS mapping is not exact: {host}") + if re.search(r"(?:^|\s)(?!192\.168\.0\.107\b)\d+\.\d+\.\d+\.\d+\s+grafana\.learn", content): + reject("Grafana DNS assumes a non-private address") + +elif mode == "traefik": + items = load(paths[0]) + hcc = one(items, "HelmChartConfig", "traefik", "kube-system") + values = yaml.safe_load((hcc.get("spec") or {}).get("valuesContent") or "") or {} + web = ((((values.get("ports") or {}).get("web") or {}).get("forwardedHeaders") or {})) + if web.get("trustedIPs") != ["10.42.0.1/32"] or web.get("insecure") is True: + reject("Traefik trust overlay is not exact") + +else: + reject(f"unknown assertion mode: {mode}") +PY +} + +assert_access_grafana_contract() { _access_assert grafana "$@"; } +assert_access_blackbox_contract() { _access_assert blackbox "$@"; } +assert_access_targets_contract() { _access_assert targets "$@"; } +assert_access_rules_alerts_contract() { + (( $# == 3 )) || return 1 + _access_assert rules-alerts "$@" || return 1 + assert_alertmanager_routing_contract "$3" +} +assert_access_scope_contract() { _access_assert scope "$@"; } +assert_access_private_dns_contract() { _access_assert private-dns "$@"; } +assert_access_traefik_trust_contract() { _access_assert traefik "$@"; } + +validate_access_inventory_root() { + local root=$1 phase directory expected actual filename extra entries current_uid + current_uid="$(id -u)" || return 1 + [[ "$root" =~ ^/tmp/platform-observability-metrics\.[A-Za-z0-9]{6}$ ]] || return 1 + [[ -d /tmp && ! -L /tmp && "$(readlink -f -- /tmp)" == /tmp && + "$(stat -c %u:%a -- /tmp)" == 0:1777 ]] || return 1 + [[ -d "$root" && ! -L "$root" && "$(readlink -f -- "$root")" == "$root" && + "$(stat -c %u:%a -- "$root")" == "$current_uid:700" ]] || return 1 + entries="$(find "$root" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" || return 1 + [[ "$entries" == $'post-substrate\ntarget-initial' ]] || return 1 + for phase in target-initial post-substrate; do + directory="$root/$phase" + [[ -d "$directory" && ! -L "$directory" && -f "$directory/inventory.json" && + -f "$directory/inventory.sha256" && ! -L "$directory/inventory.json" && + ! -L "$directory/inventory.sha256" ]] || return 1 + [[ "$(readlink -f -- "$directory")" == "$directory" && + "$(stat -c %u:%a -- "$directory")" == "$current_uid:700" ]] || return 1 + entries="$(find "$directory" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" || return 1 + [[ "$entries" == $'inventory.json\ninventory.sha256' ]] || return 1 + for filename in inventory.json inventory.sha256; do + [[ -f "$directory/$filename" && ! -L "$directory/$filename" && + "$(stat -c %u:%a:%h -- "$directory/$filename")" == "$current_uid:600:1" ]] || return 1 + done + read -r expected filename extra <"$directory/inventory.sha256" + [[ "$expected" =~ ^[0-9a-f]{64}$ && "$filename" == inventory.json && -z "${extra:-}" ]] || return 1 + actual="$(sha256sum "$directory/inventory.json" | awk '{print $1}')" + [[ "$actual" == "$expected" ]] || return 1 + jq -e --arg phase "$phase" ' + .schema == "platform-observability-metric-inventory/v1" and + .phase == $phase and (.targets | type == "array" and length > 0) and + all(.targets[]; .health == "up" and .last_error == "" and (.metrics | type == "array" and length > 0)) + ' "$directory/inventory.json" >/dev/null || return 1 + done +} + +publish_access_outputs() { + local root=$1 work=$2 component=$3 + local -a names=() + mapfile -t names < <(access_output_names "$component") || return 2 + (( ${#names[@]} > 0 )) || return 2 + python3 - "$root" "$work" "${names[@]}" <<'PY' +import hashlib +import os +import re +import stat +import sys + + +class Rejected(Exception): + pass + + +def reject(reason): + raise Rejected(reason) + + +def same_identity(metadata, expected): + return (metadata.st_dev, metadata.st_ino) == expected + + +def sha256_descriptor(descriptor): + digest = hashlib.sha256() + os.lseek(descriptor, 0, os.SEEK_SET) + while True: + chunk = os.read(descriptor, 1024 * 1024) + if not chunk: + break + digest.update(chunk) + os.lseek(descriptor, 0, os.SEEK_SET) + return digest.hexdigest() + + +root, work, *names = sys.argv[1:] +root_descriptor = None +work_descriptor = None +source_descriptors = {} +temporary_names = [] +created = {} +try: + if os.path.dirname(root) != "/tmp" or not re.fullmatch( + r"platform-observability-metrics\.[A-Za-z0-9]{6}", os.path.basename(root) + ): + reject("output root is not the expected direct /tmp handoff") + tmp_metadata = os.lstat("/tmp") + root_metadata = os.lstat(root) + if not stat.S_ISDIR(tmp_metadata.st_mode) or stat.S_ISLNK(tmp_metadata.st_mode): + reject("/tmp is not a physical directory") + if tmp_metadata.st_uid != 0 or stat.S_IMODE(tmp_metadata.st_mode) != 0o1777: + reject("/tmp owner or mode differs") + if os.path.realpath("/tmp") != "/tmp" or os.path.realpath(root) != root: + reject("output root lineage is not physical") + if not stat.S_ISDIR(root_metadata.st_mode) or stat.S_ISLNK(root_metadata.st_mode): + reject("output root is not a physical directory") + if root_metadata.st_uid != os.getuid() or stat.S_IMODE(root_metadata.st_mode) != 0o700: + reject("output root owner or mode differs") + root_identity = (root_metadata.st_dev, root_metadata.st_ino) + root_descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + if not same_identity(os.fstat(root_descriptor), root_identity): + reject("opened output root identity differs") + + base_entries = {"target-initial", "post-substrate"} + if set(os.listdir(root_descriptor)) != base_entries: + reject("output root is not pristine before publication") + for phase in base_entries: + phase_metadata = os.stat(phase, dir_fd=root_descriptor, follow_symlinks=False) + if not stat.S_ISDIR(phase_metadata.st_mode) or phase_metadata.st_uid != os.getuid(): + reject("inventory phase type or owner differs") + if stat.S_IMODE(phase_metadata.st_mode) != 0o700: + reject("inventory phase mode differs") + phase_descriptor = os.open( + phase, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root_descriptor + ) + try: + if set(os.listdir(phase_descriptor)) != {"inventory.json", "inventory.sha256"}: + reject("inventory phase entry set differs") + for filename in ("inventory.json", "inventory.sha256"): + metadata = os.stat(filename, dir_fd=phase_descriptor, follow_symlinks=False) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.getuid(): + reject("inventory file type or owner differs") + if stat.S_IMODE(metadata.st_mode) != 0o600 or metadata.st_nlink != 1: + reject("inventory file mode or link count differs") + finally: + os.close(phase_descriptor) + + if len(set(names)) != len(names) or any("/" in name or name in {".", ".."} for name in names): + reject("publication names are malformed") + if any(name in base_entries for name in names): + reject("publication name collides with inventory") + for name in names: + try: + os.stat(name, dir_fd=root_descriptor, follow_symlinks=False) + except FileNotFoundError: + continue + reject("output destination already exists") + + work_metadata = os.lstat(work) + if not stat.S_ISDIR(work_metadata.st_mode) or stat.S_ISLNK(work_metadata.st_mode): + reject("verified render directory is not physical") + if work_metadata.st_uid != os.getuid() or stat.S_IMODE(work_metadata.st_mode) != 0o700: + reject("verified render directory owner or mode differs") + work_identity = (work_metadata.st_dev, work_metadata.st_ino) + work_descriptor = os.open(work, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + if not same_identity(os.fstat(work_descriptor), work_identity): + reject("opened render directory identity differs") + source_hashes = {} + for name in names: + descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=work_descriptor) + source_descriptors[name] = descriptor + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.getuid(): + reject("verified output type or owner differs") + if stat.S_IMODE(metadata.st_mode) != 0o600 or metadata.st_nlink != 1: + reject("verified output mode or link count differs") + source_hashes[name] = sha256_descriptor(descriptor) + + for position, name in enumerate(names): + temporary = f".access-handoff-{os.getpid()}-{position}" + target_descriptor = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=root_descriptor, + ) + temporary_names.append(temporary) + try: + source_descriptor = source_descriptors[name] + os.lseek(source_descriptor, 0, os.SEEK_SET) + while True: + chunk = os.read(source_descriptor, 1024 * 1024) + if not chunk: + break + view = memoryview(chunk) + while view: + view = view[os.write(target_descriptor, view):] + os.fchmod(target_descriptor, 0o600) + os.fsync(target_descriptor) + staged_identity = (os.fstat(target_descriptor).st_dev, os.fstat(target_descriptor).st_ino) + finally: + os.close(target_descriptor) + os.link( + temporary, + name, + src_dir_fd=root_descriptor, + dst_dir_fd=root_descriptor, + follow_symlinks=False, + ) + created[name] = staged_identity + os.unlink(temporary, dir_fd=root_descriptor) + temporary_names.remove(temporary) + + os.fsync(root_descriptor) + if set(os.listdir(root_descriptor)) != base_entries | set(names): + reject("published output entry set differs") + for name in names: + metadata = os.stat(name, dir_fd=root_descriptor, follow_symlinks=False) + if not same_identity(metadata, created[name]) or not stat.S_ISREG(metadata.st_mode): + reject("published output identity or type differs") + if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o600: + reject("published output owner or mode differs") + descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=root_descriptor) + try: + if sha256_descriptor(descriptor) != source_hashes[name]: + reject("published output content differs") + finally: + os.close(descriptor) + if not same_identity(os.lstat(root), root_identity): + reject("output root identity changed during publication") +except BaseException as error: + if root_descriptor is not None: + for name, identity in reversed(tuple(created.items())): + try: + metadata = os.stat(name, dir_fd=root_descriptor, follow_symlinks=False) + if same_identity(metadata, identity): + os.unlink(name, dir_fd=root_descriptor) + except OSError: + pass + for name in reversed(temporary_names): + try: + os.unlink(name, dir_fd=root_descriptor) + except OSError: + pass + try: + os.fsync(root_descriptor) + except OSError: + pass + if isinstance(error, Rejected): + print(f"REJECT: access output publication failed: {error}", file=sys.stderr) + else: + print("REJECT: access output publication failed: filesystem operation failed", file=sys.stderr) + raise SystemExit(23) +finally: + for descriptor in source_descriptors.values(): + try: + os.close(descriptor) + except OSError: + pass + if work_descriptor is not None: + os.close(work_descriptor) + if root_descriptor is not None: + os.close(root_descriptor) +PY +} + +_access_usage() { + cat <<'USAGE' +Usage: + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/validate/render-observability-access.sh \ + --component grafana|blackbox|targets|rules-alerts|complete \ + [--verified-output-dir /tmp/platform-observability-metrics.XXXXXX] + +Static rendering only; no Kubernetes resource is applied. +USAGE +} + +access_render_main() ( + local component=complete output_root='' argument core_handoff work + access_secure_render_context + while (( $# > 0 )); do + argument=$1 + case "$argument" in + --component) (( $# >= 2 )) || return 2; component=$2; shift 2 ;; + --verified-output-dir) (( $# >= 2 )) || return 2; output_root=$2; shift 2 ;; + -h|--help) _access_usage; return 0 ;; + *) _access_usage >&2; return 2 ;; + esac + done + access_output_names "$component" >/dev/null || return 2 + if access_component_requires_inventory "$component"; then + [[ -n "$output_root" ]] || { printf 'REJECT: rules-alerts/complete requires --verified-output-dir\n' >&2; return 1; } + validate_access_inventory_root "$output_root" || { printf 'REJECT: verified inventory root is invalid\n' >&2; return 1; } + fi + work="$(mktemp -d /tmp/platform-observability-access-render.XXXXXX)" + chmod 0700 "$work" + core_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)" + chmod 0700 "$core_handoff" + trap '[[ -n ${work:-} && $work == /tmp/platform-observability-access-render.* ]] && rm -rf -- "$work"; [[ -n ${core_handoff:-} && $core_handoff == /tmp/platform-observability-core-apply.* ]] && rm -rf -- "$core_handoff"' EXIT INT TERM + /usr/bin/bash "$ACCESS_ROOT/scripts/validate/render-observability-core.sh" \ + --verified-output-dir "$core_handoff" + + cp -- "$core_handoff/grafana.yaml" "$work/grafana.yaml" + cp -- "$core_handoff/blackbox.yaml" "$work/blackbox.yaml" + cp -- "$core_handoff/targets.yaml" "$work/targets.yaml" + cp -- "$core_handoff/dashboards.yaml" "$work/dashboards.yaml" + cp -- "$core_handoff/core-rules.yaml" "$work/rules.yaml" + kubectl kustomize "$ACCESS_ROOT/infrastructure/networking/private-dns/kubernetes" >"$work/private-dns.yaml" + python3 - "$core_handoff/kps.yaml" "$core_handoff/alerting.yaml" "$work/alertmanager.yaml" <<'PY' +import sys, yaml + +class AccessExtractionSafeLoader(yaml.SafeLoader): + pass + +AccessExtractionSafeLoader.add_constructor( + "tag:yaml.org,2002:value", + lambda loader, node: loader.construct_scalar(node), +) + +items=[] +for path in sys.argv[1:3]: + with open(path, encoding="utf-8") as stream: + items.extend( + item + for item in yaml.load_all(stream, Loader=AccessExtractionSafeLoader) + if item is not None + ) +selected=[item for item in items if item.get("kind") in {"Alertmanager", "AlertmanagerConfig", "NetworkPolicy"}] +with open(sys.argv[3], "w", encoding="utf-8") as stream: + yaml.safe_dump_all(selected, stream, explicit_start=True, sort_keys=False) +PY + assert_access_grafana_contract "$work/grafana.yaml" "$core_handoff/core-policies.yaml" + assert_pinned_images "$work/grafana.yaml" + assert_access_blackbox_contract "$work/blackbox.yaml" + assert_pinned_images "$work/blackbox.yaml" + assert_access_targets_contract "$work/targets.yaml" + assert_access_rules_alerts_contract "$work/dashboards.yaml" "$work/rules.yaml" "$work/alertmanager.yaml" + assert_access_scope_contract "$work/blackbox.yaml" "$work/targets.yaml" "$work/rules.yaml" + assert_access_private_dns_contract "$work/private-dns.yaml" \ + "$ACCESS_ROOT/infrastructure/networking/private-dns/host/dnsmasq-lan.conf" \ + "$ACCESS_ROOT/infrastructure/networking/private-dns/host/dnsmasq-tailscale.conf" + assert_no_credentials "$work"/*.yaml + + if [[ -n "$output_root" ]]; then + publish_access_outputs "$output_root" "$work" "$component" + fi + printf 'OBSERVABILITY ACCESS STATIC RENDER PASS\n' +) + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + set -Eeuo pipefail + access_render_main "$@" +fi diff --git a/scripts/validate/render-observability-core.sh b/scripts/validate/render-observability-core.sh new file mode 100755 index 0000000..ea93f07 --- /dev/null +++ b/scripts/validate/render-observability-core.sh @@ -0,0 +1,1650 @@ +#!/usr/bin/env bash + +# This file is intentionally sourceable. Assertions have no cluster or network side +# effects; executable rendering is guarded at the bottom of the file. + +observability_expected_helm_version() { printf '%s\n' 'v3.19.4'; } +observability_expected_kustomize_version() { printf '%s\n' 'v5.8.1'; } +observability_helm_kube_version() { printf '%s\n' '1.36.2'; } + +chart_pin_rows() { + cat <<'ROWS' +prometheus-operator-crds https://prometheus-community.github.io/helm-charts prometheus-operator-crds 31.0.0 v0.93.0 5d11cac50321bc7ae56b35d2701e56f21fe079ca3aea872abb28a3e2119d5cfd infrastructure/controllers/prometheus-operator-crds +kube-prometheus-stack https://prometheus-community.github.io/helm-charts kube-prometheus-stack 88.0.1 v0.93.0 0998b3fae8660cd9ea224caa1c7c4b5d2040d5a02c64e3cdfd3f48d184b50117 services/observability/kube-prometheus-stack +loki https://grafana-community.github.io/helm-charts loki 18.7.1 3.7.4 de9ab59827f2ed24634e94db120ea2467f5d08b02e8d7c250e6bf88c17bfb48d services/observability/loki +tempo https://grafana-community.github.io/helm-charts tempo 2.2.3 2.10.7 14c52efe5d0cad5456ffa5a8be1e5107be47d184be3e209f748be51a7b0316fe services/observability/tempo +alloy https://grafana.github.io/helm-charts alloy 1.11.0 v1.18.0 11d253b62e47beeacd89eb4283fc056962ecbf143984863c1998be13da0772dd services/observability/alloy +prometheus-node-exporter https://prometheus-community.github.io/helm-charts prometheus-node-exporter 4.56.1 1.12.1 70fab291e00a34c58fd943bb075fecc2e2e73fa95db1bcbbe7cdfe008795354a services/observability/node-exporter +grafana https://grafana-community.github.io/helm-charts grafana 12.10.1 13.1.1 ff865a8b9d1d7563901b8e6aa653b4f345bd4d88344c149065bd23958e2beeb8 services/observability/grafana +prometheus-blackbox-exporter https://prometheus-community.github.io/helm-charts prometheus-blackbox-exporter 11.16.0 v0.28.0 932aa65df0538d9dc003c46bb663ebe44a06078af7a6a54577af5339668ac65b services/observability/blackbox-exporter +ROWS +} + +image_pin_rows() { + cat <<'ROWS' +quay.io/prometheus-operator/prometheus-operator:v0.93.0 sha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2 +quay.io/prometheus-operator/prometheus-config-reloader:v0.93.0 sha256:65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905 +quay.io/prometheus/prometheus:v3.13.2-distroless sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b +quay.io/prometheus/alertmanager:v0.33.1 sha256:a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b +registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1 sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4 +quay.io/prometheus/node-exporter:v1.12.1 sha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798 +docker.io/grafana/loki:3.7.4 sha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e +docker.io/grafana/tempo:2.10.7 sha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c +docker.io/grafana/alloy:v1.18.0 sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17 +ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.153.0 sha256:9708a1d33398c8725549104d1f6009be62569e47e094430543da0f66758cc522 +docker.io/library/busybox:1.37.0 sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0 +quay.io/minio/aistor/mc sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b +docker.io/grafana/grafana:13.1.1 sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b +quay.io/kiwigrid/k8s-sidecar:2.10.0 sha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3 +quay.io/prometheus/blackbox-exporter:v0.28.0 sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68 +ROWS +} + +render_root_rows() { + cat <<'ROWS' +infrastructure/namespaces/observability +infrastructure/controllers/prometheus-operator-crds +infrastructure/storage/observability-local-pv +services/observability/kube-prometheus-stack +services/observability/loki +services/observability/tempo +services/observability/alloy +services/observability/node-exporter +services/observability/grafana +services/observability/blackbox-exporter +services/observability/network-policies +services/observability/targets +services/observability/rules/platform +services/observability/alerting +services/observability/dashboards/platform +services/observability +ROWS +} + +_observability_image_allowlist() { + image_pin_rows | awk -F '\t' ' + { + tagged=$1 + base=$1 + sub(/:[^/:]+$/, "", base) + print base "@" $2 + if (tagged != base) { + print tagged "@" $2 + } + } + ' +} + +_observability_yaml_assert() { + local mode="$1" + shift + python3 - "$mode" "$@" <<'PY' +import base64 +import binascii +import json +import os +import re +import sys + +import yaml + + +class ObservabilitySafeLoader(yaml.SafeLoader): + pass + + +# The pinned Prometheus Operator CRDs contain the valid scalar enum value "=". +# PyYAML resolves that token to its YAML value tag but does not provide a safe +# constructor for it, so accept only that scalar tag without enabling unsafe +# object construction. +ObservabilitySafeLoader.add_constructor( + "tag:yaml.org,2002:value", + lambda loader, node: loader.construct_scalar(node), +) + + +def reject(reason): + print(f"REJECT: observability render assertion failed: {reason}", file=sys.stderr) + raise SystemExit(23) + + +def documents(paths): + result = [] + try: + for path in paths: + with open(path, "r", encoding="utf-8") as stream: + for value in yaml.load_all(stream, Loader=ObservabilitySafeLoader): + if value is not None: + if not isinstance(value, dict): + reject("YAML document is not an object") + result.append(value) + except (OSError, yaml.YAMLError): + reject("manifest is unreadable or invalid YAML") + return result + + +def pod_specs(items): + def inherited_metadata(parent, template): + result = dict(template or {}) + for key in ("name", "namespace"): + if not result.get(key) and (parent or {}).get(key): + result[key] = parent[key] + return result + + for item in items: + kind = str(item.get("kind", "")) + metadata = item.get("metadata") or {} + spec = item.get("spec") or {} + if kind == "Pod": + yield metadata, spec + elif kind in {"Deployment", "DaemonSet", "StatefulSet", "Job", "ReplicaSet", "ReplicationController"}: + template = spec.get("template") or {} + yield inherited_metadata(metadata, template.get("metadata")), template.get("spec") or {} + elif kind == "CronJob": + template = (((spec.get("jobTemplate") or {}).get("spec") or {}).get("template") or {}) + yield inherited_metadata(metadata, template.get("metadata")), template.get("spec") or {} + + +def walk(value): + yield value + if isinstance(value, dict): + for child in value.values(): + yield from walk(child) + elif isinstance(value, list): + for child in value: + yield from walk(child) + + +mode = sys.argv[1] +paths = sys.argv[2:] +items = documents(paths) + +if mode == "pinned_images": + allowed = set(filter(None, os.environ.get("OBSERVABILITY_ALLOWED_IMAGES", "").splitlines())) + if not allowed: + reject("image pin table is empty") + images = [] + for _, spec in pod_specs(items): + for key in ("initContainers", "containers", "ephemeralContainers"): + for container in spec.get(key) or []: + image = container.get("image") + if not isinstance(image, str) or not image: + reject("container image is missing") + images.append(image) + for image in images: + if not re.fullmatch(r"[^\s@]+@sha256:[0-9a-f]{64}", image): + reject("an image is not pinned by amd64 digest") + if image not in allowed: + reject("an image is absent from the fixed image table") + +elif mode == "no_credentials": + sensitive_name = re.compile(r"(?:password|passwd|secret|token|access[_-]?key|secret[_-]?key|private[_-]?key)", re.I) + safe_reference = re.compile(r"^(?:\$\{[^}]+\}|\$\([^)]*\)|<[^>]+>)$") + for item in items: + if item.get("kind") == "Secret": + metadata = item.get("metadata") or {} + data = item.get("data") or {} + string_data = item.get("stringData") or {} + allowed_alertmanager_config = ( + metadata.get("namespace") == "observability" + and metadata.get("name") == "alertmanager-observability-core-kube-pr-alertmanager" + and item.get("type") in (None, "Opaque") + and set(data) == {"alertmanager.yaml"} + and not string_data + ) + if not allowed_alertmanager_config: + reject("Secret resources are forbidden") + try: + raw_config = base64.b64decode(data["alertmanager.yaml"], validate=True) + parsed_config = yaml.load(raw_config, Loader=ObservabilitySafeLoader) + except (binascii.Error, UnicodeError, yaml.YAMLError): + reject("allowed Alertmanager config Secret is malformed") + if not isinstance(parsed_config, dict): + reject("allowed Alertmanager config Secret is not an object") + receivers = parsed_config.get("receivers") + route = parsed_config.get("route") or {} + if receivers != [{"name": "null"}] or route.get("receiver") != "null" or route.get("routes") != []: + reject("allowed Alertmanager config Secret is not credential-free") + if re.search( + rb"(?i)(?:password|passwd|secret|token|api[_-]?url|webhook|access[_-]?key|private[_-]?key)", + raw_config, + ): + reject("allowed Alertmanager config Secret contains a credential field") + for _, spec in pod_specs([item]): + for key in ("initContainers", "containers", "ephemeralContainers"): + for container in spec.get(key) or []: + for env in container.get("env") or []: + if sensitive_name.search(str(env.get("name", ""))) and "value" in env: + reject("literal credential environment value is forbidden") + for node in walk(item): + if not isinstance(node, dict): + continue + for key, value in node.items(): + if not sensitive_name.fullmatch(str(key)) or not isinstance(value, str) or not value: + continue + if not safe_reference.fullmatch(value): + reject("literal credential payload is forbidden") + +elif mode == "namespace_security": + for metadata, spec in pod_specs(items): + namespace = str(metadata.get("namespace", "")) + if namespace not in {"observability", "observability-agent"}: + reject("Pod namespace must be observability or observability-agent") + automount = spec.get("automountServiceAccountToken") + api_reader_allowlist = { + ("observability", "observability-core-kube-pr-operator"), + ("observability", "observability-core-kube-state-metrics"), + } + identity = (namespace, str(metadata.get("name", ""))) + if identity in api_reader_allowlist: + if automount is not True: + reject("allowlisted API reader must explicitly mount its service-account token") + elif automount is not False: + reject("Pod service-account automount must be explicit false") + for field in ("hostNetwork", "hostPID", "hostIPC"): + if spec.get(field) is True: + reject(f"{field} is forbidden") + containers = [] + for key in ("initContainers", "containers", "ephemeralContainers"): + containers.extend(spec.get(key) or []) + for container in containers: + if (container.get("securityContext") or {}).get("privileged") is True: + reject("privileged containers are forbidden") + for port in container.get("ports") or []: + if port.get("hostPort") not in (None, 0): + reject("hostPort is forbidden") + host_volumes = {} + for volume in spec.get("volumes") or []: + host_path = volume.get("hostPath") + if isinstance(host_path, dict): + host_volumes[str(volume.get("name", ""))] = str(host_path.get("path", "")) + if namespace == "observability" and host_volumes: + reject("hostPath is forbidden in observability") + if namespace == "observability-agent": + identity_parts = [str(metadata.get("name", ""))] + identity_parts.extend(str(v) for v in (metadata.get("labels") or {}).values()) + identity = " ".join(identity_parts).lower() + if "alloy" in identity: + allowed_paths = {"/var/log/pods"} + elif "node-exporter" in identity or "node_exporter" in identity: + allowed_paths = {"/proc", "/sys", "/"} + else: + allowed_paths = set() + for volume_name, host_path in host_volumes.items(): + if host_path not in allowed_paths: + reject("agent hostPath is outside the workload allowlist") + mounts = [ + mount + for container in containers + for mount in (container.get("volumeMounts") or []) + if mount.get("name") == volume_name + ] + if not mounts or any(mount.get("readOnly") is not True for mount in mounts): + reject("agent hostPath mounts must be read-only") + +elif mode == "no_public_exposure": + for item in items: + kind = str(item.get("kind", "")) + if kind == "Ingress": + metadata = item.get("metadata") or {} + spec = item.get("spec") or {} + expected_spec = { + "ingressClassName": "traefik", + "rules": [ + { + "host": "grafana.learn.hyeonworks.com", + "http": { + "paths": [ + { + "backend": { + "service": { + "name": "grafana", + "port": {"number": 80}, + } + }, + "path": "/", + "pathType": "Prefix", + } + ] + }, + } + ], + } + if ( + metadata.get("namespace") != "observability" + or metadata.get("name") != "grafana" + or spec != expected_spec + ): + reject("only the exact Host-Nginx-fronted Grafana Ingress is allowed") + if kind == "Service": + service_type = str((item.get("spec") or {}).get("type", "ClusterIP")) + if service_type in {"NodePort", "LoadBalancer", "ExternalName"}: + reject("non-ClusterIP Service is forbidden") + spec = item.get("spec") or {} + if spec.get("externalIPs") or spec.get("externalName"): + reject("external Service routing is forbidden") + +elif mode == "no_loki_auxiliary": + forbidden = re.compile(r"(?:^|[-_.])(gateway|canary|chunks-cache|results-cache|test|minio)(?:$|[-_.])", re.I) + for item in items: + metadata = item.get("metadata") or {} + identity = " ".join([str(metadata.get("name", ""))] + [str(v) for v in (metadata.get("labels") or {}).values()]) + if forbidden.search(identity) or str(item.get("kind", "")).lower() == "minio": + reject("forbidden Loki auxiliary resource is present") + +elif mode == "tempo_topology": + forbidden = re.compile(r"(?:metrics-generator|tempo-query|query-frontend)", re.I) + for item in items: + metadata = item.get("metadata") or {} + identity = " ".join([str(metadata.get("name", ""))] + [str(v) for v in (metadata.get("labels") or {}).values()]) + if forbidden.search(identity): + reject("forbidden Tempo auxiliary resource is present") + + def inspect_config(value): + if isinstance(value, dict): + receivers = value.get("receivers") + if isinstance(receivers, dict): + for receiver_name, receiver in receivers.items(): + if not str(receiver_name).lower().startswith("otlp") or not isinstance(receiver, dict): + continue + protocols = receiver.get("protocols") or {} + if "http" in protocols: + reject("Tempo OTLP HTTP receiver is forbidden") + if set(protocols) - {"grpc"}: + reject("Tempo has an unexpected OTLP protocol") + for child in value.values(): + inspect_config(child) + elif isinstance(value, list): + for child in value: + inspect_config(child) + elif isinstance(value, str) and ("receivers:" in value or "4318" in value): + if re.search(r"(?m)^\s*http\s*:", value) or re.search(r"(?{{ end }}\n\n{{ end }}\n · ","footer":"hyeonworks observability · Alertmanager"}]}]}}''') + if configs != [expected_config]: + reject("platform AlertmanagerConfig is not the exact global Slack contract") + + policies = [ + item + for item in items + if item.get("kind") == "NetworkPolicy" + and (item.get("metadata") or {}).get("name") + == "observability-allow-alertmanager-public-https" + ] + expected_policy = { + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": "observability-allow-alertmanager-public-https", + "namespace": "observability", + }, + "spec": { + "podSelector": { + "matchLabels": { + "app.kubernetes.io/name": "alertmanager", + "app.kubernetes.io/instance": "observability-core-kube-pr-alertmanager", + } + }, + "policyTypes": ["Egress"], + "egress": [ + { + "to": [ + { + "ipBlock": { + "cidr": "0.0.0.0/0", + "except": [ + "10.0.0.0/8", + "100.64.0.0/10", + "172.16.0.0/12", + "192.168.0.0/16", + ], + } + } + ], + "ports": [{"protocol": "TCP", "port": 443}], + } + ], + }, + } + if policies != [expected_policy]: + reject("Alertmanager public HTTPS policy is not exact") + +elif mode == "no_crds": + if any(item.get("kind") == "CustomResourceDefinition" for item in items): + reject("CRD is present in a workload render") + +elif mode == "unique_local_pv_storage_classes": + seen = {} + for item in items: + if item.get("kind") != "PersistentVolume": + continue + spec = item.get("spec") or {} + if "local" not in spec: + continue + storage_class = str(spec.get("storageClassName", "")) + if not storage_class: + reject("Local PV StorageClass is missing") + seen[storage_class] = seen.get(storage_class, 0) + 1 + if any(count > 1 for count in seen.values()): + reject("a Local PV StorageClass is shared") + +elif mode == "no_unscoped_monitors_rules": + monitored_kinds = {"ServiceMonitor", "PodMonitor", "Probe", "PrometheusRule"} + forbidden = re.compile(r"(?:spring|kafka|batch|backup)", re.I) + for item in items: + if item.get("kind") not in monitored_kinds: + continue + metadata = item.get("metadata") or {} + if forbidden.search(str(metadata.get("name", ""))): + reject("out-of-scope monitor or rule is present") + if item.get("kind") == "PrometheusRule": + for node in walk((item.get("spec") or {}).get("groups") or []): + if isinstance(node, dict): + for key in ("alert", "record"): + if forbidden.search(str(node.get(key, ""))): + reject("out-of-scope monitor or rule is present") + +else: + reject("unknown assertion mode") +PY +} + +assert_pinned_images() { + local allowed + allowed="$(_observability_image_allowlist)" || return + OBSERVABILITY_ALLOWED_IMAGES="$allowed" _observability_yaml_assert pinned_images "$@" +} + +assert_no_credentials() { _observability_yaml_assert no_credentials "$@"; } +assert_namespace_pod_security() { _observability_yaml_assert namespace_security "$@"; } +assert_no_public_exposure() { _observability_yaml_assert no_public_exposure "$@"; } +assert_no_loki_auxiliary() { _observability_yaml_assert no_loki_auxiliary "$@"; } +assert_tempo_topology() { _observability_yaml_assert tempo_topology "$@"; } +assert_selector_values() { _observability_yaml_assert selector_values "$@"; } +assert_prometheus_selectors() { _observability_yaml_assert prometheus_selectors "$@"; } +assert_alertmanager_routing_contract() { _observability_yaml_assert alertmanager_routing "$@"; } +assert_no_crds() { _observability_yaml_assert no_crds "$@"; } +assert_unique_local_pv_storage_classes() { _observability_yaml_assert unique_local_pv_storage_classes "$@"; } +assert_no_unscoped_monitors_rules() { _observability_yaml_assert no_unscoped_monitors_rules "$@"; } + +validate_chart_metadata() { + local chart_yaml="$1" + local expected_name="$2" + local expected_version="$3" + local expected_app_version="$4" + python3 - "$chart_yaml" "$expected_name" "$expected_version" "$expected_app_version" <<'PY' +import sys +import yaml + +try: + with open(sys.argv[1], "r", encoding="utf-8") as stream: + chart = yaml.safe_load(stream) +except (OSError, yaml.YAMLError): + print("REJECT: Chart.yaml is unreadable or invalid", file=sys.stderr) + raise SystemExit(23) + +expected = {"name": sys.argv[2], "version": sys.argv[3], "appVersion": sys.argv[4]} +if not isinstance(chart, dict) or any(str(chart.get(key, "")) != value for key, value in expected.items()): + print("REJECT: chart name/version/appVersion does not match the fixed table", file=sys.stderr) + raise SystemExit(23) +PY +} + +validate_render_roots() { + local root="$1" + shift + python3 - "$root" "$@" <<'PY' +import os +import stat +import sys + + +def reject(reason): + print(f"REJECT: render root preflight failed: {reason}", file=sys.stderr) + raise SystemExit(23) + + +def physical_directory(path): + current = os.sep + for component in path.split(os.sep)[1:]: + if not component: + reject("path is not normalized") + current = os.path.join(current, component) + try: + metadata = os.lstat(current) + except OSError: + reject("required directory is missing") + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + reject("directory lineage is not physical") + + +root = sys.argv[1] +if not os.path.isabs(root) or os.path.normpath(root) != root: + reject("workspace root must be a normalized absolute path") +physical_directory(root) +if os.path.realpath(root) != root: + reject("workspace root lineage is not physical") + +roots = sys.argv[2:] +if not roots: + reject("render root list is empty") +for relative in roots: + parts = relative.split("/") + if not relative or relative.startswith("/") or any(part in {"", ".", ".."} for part in parts): + reject("relative render root is unsafe") + target = os.path.join(root, *parts) + physical_directory(target) + if os.path.realpath(target) != target: + reject("render root lineage is not physical") +PY +} + +_observability_handoff_operation() { + local action="$1" + local directory="$2" + local identity="${3:-}" + local work="${4:-}" + python3 - "$action" "$directory" "$identity" "$work" <<'PY' +import hashlib +import os +import stat +import sys + + +EXPECTED_NAMES = ( + "namespaces.yaml", "crds.yaml", "storage.yaml", "kps.yaml", "loki.yaml", + "tempo.yaml", "alloy.yaml", "node-exporter.yaml", "grafana.yaml", "blackbox.yaml", "core-policies.yaml", + "targets.yaml", "core-rules.yaml", "alerting.yaml", "dashboards.yaml", + "aggregate.yaml", "resource-index.tsv", +) +PREFIX = "platform-observability-core-apply." + + +class Rejected(Exception): + pass + + +def reject(reason): + raise Rejected(reason) + + +def path_metadata(directory, require_empty): + if os.path.dirname(directory) != "/tmp": + reject("handoff must be a direct /tmp child") + name = os.path.basename(directory) + if not name.startswith(PREFIX) or len(name) == len(PREFIX): + reject("handoff suffix is missing") + try: + tmp_metadata = os.lstat("/tmp") + metadata = os.lstat(directory) + except OSError: + reject("handoff directory is missing") + if stat.S_ISLNK(tmp_metadata.st_mode) or not stat.S_ISDIR(tmp_metadata.st_mode): + reject("/tmp is not a physical directory") + if os.path.realpath("/tmp") != "/tmp": + reject("/tmp lineage is not physical") + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + reject("handoff is not a physical directory") + if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o700: + reject("handoff owner or mode differs") + if require_empty and os.listdir(directory): + reject("handoff directory is not empty") + return metadata + + +def parse_identity(value): + fields = value.split(":") + if len(fields) != 2 or any(not field.isdigit() for field in fields): + reject("handoff identity is malformed") + return tuple(int(field) for field in fields) + + +def same_identity(metadata, expected): + return (metadata.st_dev, metadata.st_ino) == expected + + +def open_pinned_directory(directory, identity): + expected = parse_identity(identity) + metadata = path_metadata(directory, False) + if not same_identity(metadata, expected): + reject("handoff directory identity changed") + try: + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + except OSError: + reject("handoff directory cannot be opened safely") + opened = os.fstat(descriptor) + if not same_identity(opened, expected): + os.close(descriptor) + reject("opened handoff identity differs") + return descriptor, expected + + +def sha256_descriptor(descriptor): + digest = hashlib.sha256() + while True: + chunk = os.read(descriptor, 1024 * 1024) + if not chunk: + return digest.hexdigest() + digest.update(chunk) + + +def validate_entries(directory, identity): + descriptor, expected = open_pinned_directory(directory, identity) + try: + names = os.listdir(descriptor) + if set(names) != set(EXPECTED_NAMES) or len(names) != len(EXPECTED_NAMES): + reject("handoff entry set differs") + for name in EXPECTED_NAMES: + metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.getuid(): + reject("handoff entry type or owner differs") + if stat.S_IMODE(metadata.st_mode) != 0o600: + reject("handoff entry mode differs") + current = os.lstat(directory) + if not same_identity(current, expected): + reject("handoff path identity changed") + finally: + os.close(descriptor) + + +def publish(directory, identity, work): + descriptor, expected = open_pinned_directory(directory, identity) + created = [] + temporary_created = [] + sources = {} + try: + if os.listdir(descriptor): + reject("handoff directory is not empty immediately before publication") + try: + work_metadata = os.lstat(work) + except OSError: + reject("verified render directory is missing") + if stat.S_ISLNK(work_metadata.st_mode) or not stat.S_ISDIR(work_metadata.st_mode): + reject("verified render directory is not physical") + for name in EXPECTED_NAMES: + source = os.path.join(work, name) + try: + source_metadata = os.lstat(source) + except OSError: + reject("verified output is missing") + if stat.S_ISLNK(source_metadata.st_mode) or not stat.S_ISREG(source_metadata.st_mode): + reject("verified output type differs") + source_descriptor = os.open(source, os.O_RDONLY | os.O_NOFOLLOW) + try: + sources[name] = sha256_descriptor(source_descriptor) + finally: + os.close(source_descriptor) + + for position, name in enumerate(EXPECTED_NAMES): + source = os.path.join(work, name) + temporary_name = f".handoff-{os.getpid()}-{position}" + try: + target_descriptor = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=descriptor, + ) + except OSError: + reject("handoff staging no-clobber creation failed") + temporary_created.append(temporary_name) + try: + source_descriptor = os.open(source, os.O_RDONLY | os.O_NOFOLLOW) + try: + while True: + chunk = os.read(source_descriptor, 1024 * 1024) + if not chunk: + break + view = memoryview(chunk) + while view: + written = os.write(target_descriptor, view) + view = view[written:] + finally: + os.close(source_descriptor) + os.fchmod(target_descriptor, 0o600) + os.fsync(target_descriptor) + finally: + os.close(target_descriptor) + try: + os.link( + temporary_name, + name, + src_dir_fd=descriptor, + dst_dir_fd=descriptor, + follow_symlinks=False, + ) + except OSError: + reject("handoff output atomic no-clobber publication failed") + created.append(name) + os.unlink(temporary_name, dir_fd=descriptor) + temporary_created.remove(temporary_name) + + os.fsync(descriptor) + if set(os.listdir(descriptor)) != set(EXPECTED_NAMES): + reject("handoff entry set changed during publication") + for name in EXPECTED_NAMES: + metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.getuid(): + reject("published handoff entry is unsafe") + if stat.S_IMODE(metadata.st_mode) != 0o600: + reject("published handoff entry mode differs") + target_descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=descriptor) + try: + if sha256_descriptor(target_descriptor) != sources[name]: + reject("published handoff content differs") + finally: + os.close(target_descriptor) + current = os.lstat(directory) + if not same_identity(current, expected): + reject("handoff path identity changed during publication") + except BaseException: + for name in reversed(created): + try: + os.unlink(name, dir_fd=descriptor) + except OSError: + pass + for name in reversed(temporary_created): + try: + os.unlink(name, dir_fd=descriptor) + except OSError: + pass + raise + finally: + os.close(descriptor) + + +action, directory, identity, work = sys.argv[1:5] +try: + if action == "validate": + path_metadata(directory, True) + elif action == "pin": + metadata = path_metadata(directory, True) + print(f"{metadata.st_dev}:{metadata.st_ino}") + elif action == "entries": + validate_entries(directory, identity) + elif action == "publish": + publish(directory, identity, work) + else: + reject("unknown handoff operation") +except Rejected as error: + print(f"REJECT: handoff validation failed: {error}", file=sys.stderr) + raise SystemExit(23) +except OSError: + print("REJECT: handoff validation failed: filesystem operation failed", file=sys.stderr) + raise SystemExit(23) +PY +} + +validate_handoff_directory() { _observability_handoff_operation validate "$1"; } +pin_handoff_directory() { _observability_handoff_operation pin "$1"; } +validate_handoff_entry_set() { _observability_handoff_operation entries "$1" "$2"; } +publish_verified_handoff() { _observability_handoff_operation publish "$1" "$2" "$3"; } + +build_resource_index() { + local output="$1" + shift + python3 - "$output" "$@" <<'PY' +import hashlib +import json +import os +import sys +import tempfile + +import yaml + + +class ResourceIndexSafeLoader(yaml.SafeLoader): + pass + + +ResourceIndexSafeLoader.add_constructor( + "tag:yaml.org,2002:value", + lambda loader, node: loader.construct_scalar(node), +) + + +def fail(reason): + print(f"REJECT: resource index failed: {reason}", file=sys.stderr) + raise SystemExit(23) + + +def read_objects(source_specs): + objects = [] + tuples = set() + try: + for source_spec in source_specs: + if ":" not in source_spec: + fail("source specification is malformed") + source, path = source_spec.split(":", 1) + if not source or not path: + fail("source specification is malformed") + with open(path, "r", encoding="utf-8") as stream: + for document in yaml.load_all(stream, Loader=ResourceIndexSafeLoader): + if document is None: + continue + if not isinstance(document, dict): + fail("YAML document is not an object") + metadata = document.get("metadata") or {} + identity = ( + str(document.get("apiVersion", "")), + str(document.get("kind", "")), + str(metadata.get("namespace", "")), + str(metadata.get("name", "")), + ) + if not all((identity[0], identity[1], identity[3])): + fail("resource identity is incomplete") + if identity in tuples: + fail("duplicate resource tuple") + tuples.add(identity) + canonical = json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + objects.append((source, identity, digest)) + except (OSError, yaml.YAMLError): + fail("manifest is unreadable or invalid YAML") + return objects + + +output = sys.argv[1] +objects = read_objects(sys.argv[2:]) +directory = os.path.dirname(output) or "." +fd, temporary = tempfile.mkstemp(prefix=".resource-index.", dir=directory, text=True) +try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write("order\tartifact\tapiVersion\tkind\tnamespace\tname\tcanonicalSha256\n") + for order, (source, identity, digest) in enumerate(objects, 1): + stream.write("\t".join((str(order), source, *identity, digest)) + "\n") + os.chmod(temporary, 0o600) + os.replace(temporary, output) +except BaseException: + try: + os.unlink(temporary) + except OSError: + pass + raise +PY +} + +assert_child_aggregate_equivalence() { + local aggregate="$1" + shift + python3 - "$aggregate" "$@" <<'PY' +import hashlib +import json +import sys + +import yaml + + +class AggregateSafeLoader(yaml.SafeLoader): + pass + + +AggregateSafeLoader.add_constructor( + "tag:yaml.org,2002:value", + lambda loader, node: loader.construct_scalar(node), +) + + +def fail(reason): + print(f"REJECT: aggregate equivalence failed: {reason}", file=sys.stderr) + raise SystemExit(23) + + +def object_map(paths): + result = {} + try: + for path in paths: + with open(path, "r", encoding="utf-8") as stream: + for document in yaml.load_all(stream, Loader=AggregateSafeLoader): + if document is None: + continue + if not isinstance(document, dict): + fail("YAML document is not an object") + metadata = document.get("metadata") or {} + identity = ( + str(document.get("apiVersion", "")), + str(document.get("kind", "")), + str(metadata.get("namespace", "")), + str(metadata.get("name", "")), + ) + if not all((identity[0], identity[1], identity[3])): + fail("resource identity is incomplete") + if identity in result: + fail("duplicate resource tuple") + canonical = json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + result[identity] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + except (OSError, yaml.YAMLError): + fail("manifest is unreadable or invalid YAML") + return result + + +aggregate_path = sys.argv[1] +child_paths = [] +for source_spec in sys.argv[2:]: + if ":" not in source_spec: + fail("source specification is malformed") + source, path = source_spec.split(":", 1) + if not source or not path: + fail("source specification is malformed") + child_paths.append(path) + +children = object_map(child_paths) +aggregate = object_map([aggregate_path]) +if children.keys() != aggregate.keys(): + fail("child and aggregate resource sets differ") +if children != aggregate: + fail("child and aggregate canonical object hashes differ") +PY +} + +_observability_fail() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +_observability_reject() { + printf 'REJECT: %s\n' "$*" >&2 + return 23 +} + +_observability_preflight_chart_cache() { + local relative_root="$1" + local chart="$2" + local version="$3" + local relative_parent cache + validate_render_roots "$_OBS_ROOT" "$relative_root" || return + for relative_parent in "$relative_root/.helm" "$relative_root/.helm/charts"; do + cache="$_OBS_ROOT/$relative_parent" + if [[ -e "$cache" || -L "$cache" ]]; then + validate_render_roots "$_OBS_ROOT" "$relative_parent" || return + fi + done + cache="$_OBS_ROOT/$relative_root/.helm/charts/$chart-$version" + [[ ! -e "$cache" && ! -L "$cache" ]] || \ + _observability_reject 'exact chart cache already exists' +} + +_observability_record_cache_directory() { + local directory="$1" + local cleanup_type="$2" + local identity + identity="$(stat --format='%d:%i' "$directory")" || return + _OBS_CREATED_CACHE_DIRS+=("$directory") + _OBS_CREATED_CACHE_IDENTITIES+=("$identity") + _OBS_CREATED_CACHE_TYPES+=("$cleanup_type") +} + +_observability_create_cache_directory() { + local relative_root="$1" + local chart="$2" + local version="$3" + local created_rows directory identity cleanup_type + [[ -v _OBS_CREATED_CACHE_DIRS ]] || _OBS_CREATED_CACHE_DIRS=() + [[ -v _OBS_CREATED_CACHE_IDENTITIES ]] || _OBS_CREATED_CACHE_IDENTITIES=() + [[ -v _OBS_CREATED_CACHE_TYPES ]] || _OBS_CREATED_CACHE_TYPES=() + + _observability_preflight_chart_cache "$relative_root" "$chart" "$version" || return + created_rows="$(python3 - "$_OBS_ROOT" "$relative_root" "$chart-$version" <<'PY' +import errno +import os +import stat +import sys + + +class Rejected(Exception): + pass + + +def reject(reason): + raise Rejected(reason) + + +def open_directory(name, parent=None): + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + try: + if parent is None: + return os.open(name, flags) + return os.open(name, flags, dir_fd=parent) + except OSError: + reject("cache directory lineage is not physical") + + +root, relative_root, exact_name = sys.argv[1:4] +parts = relative_root.split("/") +if not os.path.isabs(root) or os.path.normpath(root) != root: + reject("workspace root is unsafe") +if not relative_root or any(part in {"", ".", ".."} for part in parts): + reject("cache root is unsafe") +if not exact_name or "/" in exact_name or exact_name in {".", ".."}: + reject("exact cache name is unsafe") + +descriptors = [] +created = [] +cache_parts = [] +try: + current = open_directory("/") + descriptors.append(current) + for component in root.split("/")[1:]: + next_descriptor = open_directory(component, current) + descriptors.append(next_descriptor) + current = next_descriptor + for component in parts: + next_descriptor = open_directory(component, current) + descriptors.append(next_descriptor) + current = next_descriptor + + for name, cleanup_type in ((".helm", "empty"), ("charts", "empty")): + try: + next_descriptor = os.open( + name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=current + ) + except FileNotFoundError: + try: + os.mkdir(name, 0o700, dir_fd=current) + except OSError: + reject("exclusive cache parent creation failed") + next_descriptor = open_directory(name, current) + metadata = os.fstat(next_descriptor) + full_path = os.path.join(root, relative_root, *cache_parts, name) + created.append((current, name, full_path, metadata.st_dev, metadata.st_ino, cleanup_type)) + except OSError: + reject("cache parent is not a physical directory") + descriptors.append(next_descriptor) + current = next_descriptor + cache_parts.append(name) + + try: + os.mkdir(exact_name, 0o700, dir_fd=current) + except OSError: + reject("exclusive exact cache creation failed") + exact_descriptor = open_directory(exact_name, current) + descriptors.append(exact_descriptor) + metadata = os.fstat(exact_descriptor) + exact_path = os.path.join(root, relative_root, ".helm", "charts", exact_name) + created.append((current, exact_name, exact_path, metadata.st_dev, metadata.st_ino, "tree")) + + for _, _, path, device, inode, cleanup_type in created: + print(f"{path}\t{device}:{inode}\t{cleanup_type}") +except Rejected as error: + for parent, name, _, _, _, _ in reversed(created): + try: + os.rmdir(name, dir_fd=parent) + except OSError: + pass + print(f"REJECT: cache creation failed: {error}", file=sys.stderr) + raise SystemExit(23) +finally: + for descriptor in reversed(descriptors): + try: + os.close(descriptor) + except OSError: + pass +PY + )" || return + + while IFS=$'\t' read -r directory identity cleanup_type; do + [[ -n "$directory" && -n "$identity" && -n "$cleanup_type" ]] || \ + _observability_reject 'cache creation result is incomplete' || return + _OBS_CREATED_CACHE_DIRS+=("$directory") + _OBS_CREATED_CACHE_IDENTITIES+=("$identity") + _OBS_CREATED_CACHE_TYPES+=("$cleanup_type") + _OBS_LAST_CACHE_PATH="$directory" + _OBS_LAST_CACHE_IDENTITY="$identity" + done <<<"$created_rows" +} + +_observability_cache_identity_matches() { + local directory="$1" + local identity="$2" + [[ -d "$directory" && ! -L "$directory" ]] || return 1 + [[ "$(stat --format='%d:%i' "$directory")" == "$identity" ]] +} + +_observability_cleanup_owned_cache_directory() { + local directory="$1" + local identity="$2" + local cleanup_type="$3" + local race_hook="${4:-}" + python3 - "$_OBS_ROOT" "$directory" "$identity" "$cleanup_type" "$race_hook" <<'PY' +import ctypes +import errno +import os +import secrets +import stat +import subprocess +import sys + + +RENAME_NOREPLACE = 1 + + +class Rejected(Exception): + pass + + +def reject(reason): + raise Rejected(reason) + + +def parse_identity(value): + fields = value.split(":") + if len(fields) != 2 or any(not field.isdigit() for field in fields): + reject("tracked identity is malformed") + return tuple(int(field) for field in fields) + + +def identity(metadata): + return metadata.st_dev, metadata.st_ino + + +libc = ctypes.CDLL(None, use_errno=True) +try: + renameat2 = libc.renameat2 +except AttributeError: + reject("renameat2 is unavailable") +renameat2.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] +renameat2.restype = ctypes.c_int + + +def rename_no_replace(parent_descriptor, source, target): + result = renameat2( + parent_descriptor, + os.fsencode(source), + parent_descriptor, + os.fsencode(target), + RENAME_NOREPLACE, + ) + if result != 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + + +def random_name(): + return f".observability-cleanup-{os.getpid()}-{secrets.token_hex(16)}" + + +def restore_if_possible(parent_descriptor, quarantine, original): + try: + rename_no_replace(parent_descriptor, quarantine, original) + return True + except OSError: + return False + + +def quarantine_entry(parent_descriptor, name): + try: + original_metadata = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except OSError: + reject("tracked entry disappeared") + quarantine = random_name() + try: + rename_no_replace(parent_descriptor, name, quarantine) + except OSError: + reject("tracked entry quarantine failed") + try: + moved_metadata = os.stat(quarantine, dir_fd=parent_descriptor, follow_symlinks=False) + except OSError: + reject("quarantined entry disappeared") + if identity(moved_metadata) != identity(original_metadata): + restore_if_possible(parent_descriptor, quarantine, name) + reject("entry identity changed during quarantine") + return quarantine, moved_metadata + + +def remove_owned_entry(parent_descriptor, name): + quarantine, metadata = quarantine_entry(parent_descriptor, name) + try: + if stat.S_ISDIR(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode): + child_descriptor = os.open( + quarantine, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=parent_descriptor, + ) + try: + if identity(os.fstat(child_descriptor)) != identity(metadata): + reject("opened child identity differs") + clear_owned_directory(child_descriptor) + finally: + os.close(child_descriptor) + current = os.stat(quarantine, dir_fd=parent_descriptor, follow_symlinks=False) + if identity(current) != identity(metadata): + reject("child identity changed before removal") + os.rmdir(quarantine, dir_fd=parent_descriptor) + else: + current = os.stat(quarantine, dir_fd=parent_descriptor, follow_symlinks=False) + if identity(current) != identity(metadata): + reject("entry identity changed before removal") + os.unlink(quarantine, dir_fd=parent_descriptor) + except BaseException: + restore_if_possible(parent_descriptor, quarantine, name) + raise + + +def clear_owned_directory(descriptor): + for name in os.listdir(descriptor): + remove_owned_entry(descriptor, name) + os.fsync(descriptor) + + +def open_physical_parent(path): + parent = os.path.dirname(path) + descriptor = os.open("/", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + for component in parent.split("/")[1:]: + if not component: + reject("cleanup path is not normalized") + next_descriptor = os.open( + component, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=descriptor, + ) + os.close(descriptor) + descriptor = next_descriptor + return descriptor + except BaseException: + os.close(descriptor) + raise + + +root, directory, expected_value, cleanup_type, race_hook = sys.argv[1:6] +parent_descriptor = None +owned_descriptor = None +top_quarantine = None +top_name = os.path.basename(directory) +try: + if not os.path.isabs(root) or os.path.normpath(root) != root: + reject("workspace root is unsafe") + if not os.path.isabs(directory) or os.path.normpath(directory) != directory: + reject("cleanup path is unsafe") + if os.path.commonpath((root, directory)) != root or directory == root: + reject("cleanup path escapes workspace root") + if cleanup_type == "tree": + if "/.helm/charts/" not in directory: + reject("recursive cleanup target is not an exact chart cache") + elif cleanup_type == "empty": + if not (directory.endswith("/.helm") or directory.endswith("/.helm/charts")): + reject("empty cleanup target is not a cache parent") + else: + reject("cleanup type is invalid") + + expected = parse_identity(expected_value) + parent_descriptor = open_physical_parent(directory) + metadata = os.stat(top_name, dir_fd=parent_descriptor, follow_symlinks=False) + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + reject("tracked cache is not a physical directory") + if identity(metadata) != expected: + reject("tracked cache identity differs") + + top_quarantine, quarantined_metadata = quarantine_entry(parent_descriptor, top_name) + if identity(quarantined_metadata) != expected: + restore_if_possible(parent_descriptor, top_quarantine, top_name) + reject("tracked cache identity changed during quarantine") + owned_descriptor = os.open( + top_quarantine, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=parent_descriptor, + ) + if identity(os.fstat(owned_descriptor)) != expected: + reject("opened tracked cache identity differs") + + if race_hook: + hook_metadata = os.lstat(race_hook) + if ( + stat.S_ISLNK(hook_metadata.st_mode) + or not stat.S_ISREG(hook_metadata.st_mode) + or hook_metadata.st_uid != os.getuid() + or not os.access(race_hook, os.X_OK) + ): + reject("cleanup race hook is unsafe") + result = subprocess.run( + [race_hook, directory], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode != 0: + reject("cleanup race hook failed") + + if cleanup_type == "tree": + clear_owned_directory(owned_descriptor) + elif os.listdir(owned_descriptor): + reject("tracked cache parent is not empty") + os.close(owned_descriptor) + owned_descriptor = None + + current = os.stat(top_quarantine, dir_fd=parent_descriptor, follow_symlinks=False) + if identity(current) != expected: + reject("quarantined cache identity changed before removal") + os.rmdir(top_quarantine, dir_fd=parent_descriptor) + top_quarantine = None +except (Rejected, OSError) as error: + if owned_descriptor is not None: + try: + os.close(owned_descriptor) + except OSError: + pass + if top_quarantine is not None and parent_descriptor is not None: + restore_if_possible(parent_descriptor, top_quarantine, top_name) + if isinstance(error, Rejected): + reason = str(error) + else: + reason = "descriptor-relative filesystem operation failed" + print(f"REJECT: cache cleanup failed: {reason}", file=sys.stderr) + raise SystemExit(23) +finally: + if parent_descriptor is not None: + try: + os.close(parent_descriptor) + except OSError: + pass +PY +} + +_observability_cleanup_cache_dirs() { + local index directory identity cleanup_type cleanup_rc=0 + [[ -v _OBS_CREATED_CACHE_DIRS ]] || return 0 + for ((index=${#_OBS_CREATED_CACHE_DIRS[@]} - 1; index >= 0; index--)); do + directory="${_OBS_CREATED_CACHE_DIRS[$index]}" + identity="${_OBS_CREATED_CACHE_IDENTITIES[$index]}" + cleanup_type="${_OBS_CREATED_CACHE_TYPES[$index]}" + _observability_cleanup_owned_cache_directory \ + "$directory" "$identity" "$cleanup_type" || cleanup_rc=23 + done + _OBS_CREATED_CACHE_DIRS=() + _OBS_CREATED_CACHE_IDENTITIES=() + _OBS_CREATED_CACHE_TYPES=() + return "$cleanup_rc" +} + +_observability_prepare_chart() { + local label="$1" + local repo="$2" + local chart="$3" + local version="$4" + local app_version="$5" + local expected_sha="$6" + local relative_root="$7" + local cache="$_OBS_ROOT/$relative_root/.helm/charts/$chart-$version" + local archive="$_OBS_WORK/$chart-$version.tgz" + local actual_sha chart_yaml cache_identity cache_fd cache_fd_identity + + _observability_preflight_chart_cache "$relative_root" "$chart" "$version" || return + + "$_OBS_HELM" pull "$chart" --repo "$repo" --version "$version" --destination "$_OBS_WORK" + [[ -f "$archive" && ! -L "$archive" ]] || _observability_fail "$label archive is missing" || return + actual_sha="$(sha256sum "$archive" | awk '{print $1}')" + [[ "$actual_sha" == "$expected_sha" ]] || _observability_fail "$label archive SHA-256 mismatch" || return + if ! tar -tzf "$archive" | awk -v prefix="$chart/" ' + $0 == "" || $0 ~ /^\// || $0 ~ /(^|\/)\.\.($|\/)/ || index($0, prefix) != 1 { bad=1 } + END { exit bad ? 1 : 0 } + '; then + _observability_fail "$label archive layout is unsafe" + return + fi + _observability_create_cache_directory "$relative_root" "$chart" "$version" || return + cache="$_OBS_LAST_CACHE_PATH" + cache_identity="$_OBS_LAST_CACHE_IDENTITY" + _observability_cache_identity_matches "$cache" "$cache_identity" || \ + _observability_reject 'exact cache identity changed before extraction' || return + exec {cache_fd}<"$cache" || _observability_reject 'exact cache cannot be pinned for extraction' || return + cache_fd_identity="$(stat --dereference --format='%d:%i' "/proc/$$/fd/$cache_fd")" + if [[ "$cache_fd_identity" != "$cache_identity" ]]; then + exec {cache_fd}<&- + _observability_reject 'opened cache identity differs before extraction' + return + fi + if ! tar --extract --gzip --file "$archive" --directory "/proc/$$/fd/$cache_fd" --keep-old-files; then + exec {cache_fd}<&- + _observability_reject 'chart extraction failed' + return + fi + cache_fd_identity="$(stat --dereference --format='%d:%i' "/proc/$$/fd/$cache_fd")" + exec {cache_fd}<&- + [[ "$cache_fd_identity" == "$cache_identity" ]] || \ + _observability_reject 'opened cache identity changed during extraction' || return + _observability_cache_identity_matches "$cache" "$cache_identity" || \ + _observability_reject 'exact cache identity changed during extraction' || return + chart_yaml="$cache/$chart/Chart.yaml" + [[ -f "$chart_yaml" && ! -L "$chart_yaml" ]] || _observability_fail "$label Chart.yaml is missing" || return + validate_chart_metadata "$chart_yaml" "$chart" "$version" "$app_version" + printf 'Verified %-28s SHA-256 %s\n' "$label" "$actual_sha" +} + +_observability_render_plain() { + local output_name="$1" + local relative_root="$2" + kubectl kustomize "$_OBS_ROOT/$relative_root" >"$_OBS_WORK/$output_name.yaml" + [[ -s "$_OBS_WORK/$output_name.yaml" ]] || _observability_fail "$output_name render is empty" +} + +_observability_render_with_helm() { + local output_name="$1" + local relative_root="$2" + kubectl kustomize --enable-helm --helm-command "$_OBS_HELM" \ + --helm-kube-version "$(observability_helm_kube_version)" \ + "$_OBS_ROOT/$relative_root" >"$_OBS_WORK/$output_name.yaml" + [[ -s "$_OBS_WORK/$output_name.yaml" ]] || _observability_fail "$output_name render is empty" +} + +_observability_cleanup() { + local rc=$? + local cache_cleanup_rc=0 + set +e + _observability_cleanup_cache_dirs || cache_cleanup_rc=$? + case "${_OBS_WORK:-}" in + /tmp/platform-observability-core-render.*) rm -rf -- "$_OBS_WORK" ;; + esac + if [[ "$rc" == 0 && "$cache_cleanup_rc" != 0 ]]; then + rc="$cache_cleanup_rc" + fi + exit "$rc" +} + +_observability_usage() { + cat <<'USAGE' +Usage: + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/validate/render-observability-core.sh + +Verified apply handoff (internal interface): + bash scripts/validate/render-observability-core.sh \ + --verified-output-dir /tmp/platform-observability-core-apply.XXXXXX + +This command performs static rendering only. It never applies resources. +USAGE +} + +observability_render_main() { + local command_name kustomize_version label repo chart version app_version expected_sha relative_root + local output_name bytes digest handoff_identity + local -a all_manifests workload_manifests child_specs output_names render_roots + if (( $# == 0 )); then + _OBS_OUT="" + elif (( $# == 2 )) && [[ "$1" == "--verified-output-dir" ]]; then + _OBS_OUT="$2" + else + _observability_usage >&2 + return 2 + fi + _OBS_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" + + for command_name in awk cat chmod cmp dirname find install kubectl mktemp python3 rm rmdir sed sha256sum stat tar tr wc; do + command -v "$command_name" >/dev/null 2>&1 || _observability_fail "$command_name command is required" || return + done + python3 -c 'import yaml' >/dev/null 2>&1 || _observability_fail 'Python PyYAML is required' || return + + if [[ -v PLATFORM_HELM_BIN ]]; then + [[ "$PLATFORM_HELM_BIN" == /* && -f "$PLATFORM_HELM_BIN" && ! -L "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \ + _observability_fail 'PLATFORM_HELM_BIN must be an executable, non-symlink regular file at an absolute path' || return + _OBS_HELM="$PLATFORM_HELM_BIN" + else + command -v helm >/dev/null 2>&1 || _observability_fail 'helm command is required' || return + _OBS_HELM="$(command -v helm)" + fi + [[ "$("$_OBS_HELM" version --template '{{.Version}}')" == "$(observability_expected_helm_version)" ]] || \ + _observability_fail "Helm must be exactly $(observability_expected_helm_version)" || return + kustomize_version="$(kubectl version --client --output=yaml | sed -n 's/^kustomizeVersion: //p')" + [[ "$kustomize_version" == "$(observability_expected_kustomize_version)" ]] || \ + _observability_fail "Kustomize must be exactly $(observability_expected_kustomize_version)" || return + + if [[ -n "$_OBS_OUT" ]]; then + handoff_identity="$(pin_handoff_directory "$_OBS_OUT")" + fi + + mapfile -t render_roots < <(render_root_rows) + validate_render_roots "$_OBS_ROOT" "${render_roots[@]}" + umask 077 + _OBS_WORK="$(mktemp -d /tmp/platform-observability-core-render.XXXXXX)" + chmod 0700 "$_OBS_WORK" + _OBS_CREATED_CACHE_DIRS=() + _OBS_CREATED_CACHE_IDENTITIES=() + _OBS_CREATED_CACHE_TYPES=() + trap _observability_cleanup EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + while IFS=$'\t' read -r label repo chart version app_version expected_sha relative_root; do + _observability_prepare_chart "$label" "$repo" "$chart" "$version" "$app_version" \ + "$expected_sha" "$relative_root" + done < <(chart_pin_rows) + + _observability_render_plain namespaces infrastructure/namespaces/observability + _observability_render_with_helm crds infrastructure/controllers/prometheus-operator-crds + _observability_render_plain storage infrastructure/storage/observability-local-pv + _observability_render_with_helm kps services/observability/kube-prometheus-stack + _observability_render_with_helm loki services/observability/loki + _observability_render_with_helm tempo services/observability/tempo + _observability_render_with_helm alloy services/observability/alloy + _observability_render_with_helm node-exporter services/observability/node-exporter + _observability_render_with_helm grafana services/observability/grafana + _observability_render_with_helm blackbox services/observability/blackbox-exporter + _observability_render_plain core-policies services/observability/network-policies + _observability_render_plain targets services/observability/targets + _observability_render_plain core-rules services/observability/rules/platform + _observability_render_plain alerting services/observability/alerting + _observability_render_plain dashboards services/observability/dashboards/platform + _observability_render_with_helm aggregate services/observability + + all_manifests=( + "$_OBS_WORK/namespaces.yaml" "$_OBS_WORK/crds.yaml" "$_OBS_WORK/storage.yaml" + "$_OBS_WORK/kps.yaml" "$_OBS_WORK/loki.yaml" "$_OBS_WORK/tempo.yaml" + "$_OBS_WORK/alloy.yaml" "$_OBS_WORK/node-exporter.yaml" + "$_OBS_WORK/grafana.yaml" "$_OBS_WORK/blackbox.yaml" + "$_OBS_WORK/core-policies.yaml" "$_OBS_WORK/targets.yaml" + "$_OBS_WORK/core-rules.yaml" "$_OBS_WORK/alerting.yaml" + "$_OBS_WORK/dashboards.yaml" + ) + workload_manifests=( + "$_OBS_WORK/kps.yaml" "$_OBS_WORK/loki.yaml" "$_OBS_WORK/tempo.yaml" + "$_OBS_WORK/alloy.yaml" "$_OBS_WORK/node-exporter.yaml" + "$_OBS_WORK/grafana.yaml" "$_OBS_WORK/blackbox.yaml" + "$_OBS_WORK/core-policies.yaml" "$_OBS_WORK/targets.yaml" + "$_OBS_WORK/core-rules.yaml" "$_OBS_WORK/alerting.yaml" + "$_OBS_WORK/dashboards.yaml" + ) + assert_pinned_images "${workload_manifests[@]}" + assert_no_credentials "${all_manifests[@]}" "$_OBS_WORK/aggregate.yaml" + assert_namespace_pod_security "${workload_manifests[@]}" + assert_no_public_exposure "${workload_manifests[@]}" + assert_no_loki_auxiliary "$_OBS_WORK/loki.yaml" + assert_tempo_topology "$_OBS_WORK/tempo.yaml" + assert_selector_values "$_OBS_ROOT/services/observability/kube-prometheus-stack/values/home.yaml" + assert_prometheus_selectors "$_OBS_WORK/kps.yaml" + assert_alertmanager_routing_contract "$_OBS_WORK/kps.yaml" "$_OBS_WORK/alerting.yaml" + assert_no_crds "$_OBS_WORK/kps.yaml" "$_OBS_WORK/aggregate.yaml" + assert_unique_local_pv_storage_classes "$_OBS_WORK/storage.yaml" + assert_no_unscoped_monitors_rules "${workload_manifests[@]}" + + child_specs=( + "kps:$_OBS_WORK/kps.yaml" + "loki:$_OBS_WORK/loki.yaml" + "tempo:$_OBS_WORK/tempo.yaml" + "alloy:$_OBS_WORK/alloy.yaml" + "node-exporter:$_OBS_WORK/node-exporter.yaml" + "grafana:$_OBS_WORK/grafana.yaml" + "blackbox:$_OBS_WORK/blackbox.yaml" + "core-policies:$_OBS_WORK/core-policies.yaml" + "targets:$_OBS_WORK/targets.yaml" + "core-rules:$_OBS_WORK/core-rules.yaml" + "alerting:$_OBS_WORK/alerting.yaml" + "dashboards:$_OBS_WORK/dashboards.yaml" + ) + assert_child_aggregate_equivalence "$_OBS_WORK/aggregate.yaml" "${child_specs[@]}" + build_resource_index "$_OBS_WORK/resource-index.tsv" \ + "namespaces:$_OBS_WORK/namespaces.yaml" \ + "crds:$_OBS_WORK/crds.yaml" \ + "storage:$_OBS_WORK/storage.yaml" \ + "${child_specs[@]}" + + output_names=(namespaces crds storage kps loki tempo alloy node-exporter grafana blackbox core-policies targets core-rules alerting dashboards aggregate) + for output_name in "${output_names[@]}"; do + bytes="$(wc -c <"$_OBS_WORK/$output_name.yaml" | tr -d '[:space:]')" + digest="$(sha256sum "$_OBS_WORK/$output_name.yaml" | awk '{print $1}')" + printf 'Rendered %-18s %8s bytes SHA-256 %s\n' "$output_name" "$bytes" "$digest" + done + + if [[ -n "$_OBS_OUT" ]]; then + publish_verified_handoff "$_OBS_OUT" "$handoff_identity" "$_OBS_WORK" + validate_handoff_entry_set "$_OBS_OUT" "$handoff_identity" + fi + + printf 'OBSERVABILITY CORE STATIC RENDER PASS\n' +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + set -Eeuo pipefail + observability_render_main "$@" +fi diff --git a/scripts/validate/render-phase1.sh b/scripts/validate/render-phase1.sh new file mode 100755 index 0000000..1292a45 --- /dev/null +++ b/scripts/validate/render-phase1.sh @@ -0,0 +1,671 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1" +readonly EXPECTED_HELM_VERSION="v3.19.4" +readonly TARGET_KUBERNETES_VERSION="1.36.2" +readonly CNPG_CHART_NAME="cloudnative-pg" +readonly CNPG_CHART_VERSION="0.29.0" +readonly CNPG_CHART_REPOSITORY="https://cloudnative-pg.github.io/charts" +readonly EXPECTED_CNPG_CHART_SHA256="668e065ff53508d58238788fd35b355a925060843629a951df0e6a9362e6d32f" +readonly GITEA_CHART_NAME="gitea" +readonly GITEA_CHART_VERSION="12.7.0" +readonly GITEA_CHART_REPOSITORY="https://dl.gitea.com/charts/" +readonly EXPECTED_GITEA_CHART_SHA256="5881ef9c59400bee2d5547e77c4cd0efb925143c2f5d93fb4f38446db76b0167" +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly -a RENDERED_MANIFEST_NAMES=( + namespaces + ssd-local-pv + cnpg-operator + platform-postgres + gitea + gitea-oidc +) + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +assert_exact_source_text() { + local file="$1" + local expected_text="$2" + local description="$3" + local count + + count="$(rg --count-matches --fixed-strings -- "$expected_text" "$file" || true)" + [[ "$count" == "1" ]] || \ + fail "${description} must appear exactly once in ${file#${REPOSITORY_ROOT}/}" +} + +usage() { + cat <<'USAGE' +Usage: + bash scripts/validate/render-phase1.sh + bash scripts/validate/render-phase1.sh \ + --verified-output-dir /tmp/platform-phase1-apply.XXXXXX + +The output option is an internal handoff used only by the Phase 1 apply +script. The destination must be an existing, empty, non-symlink directory +created directly below /tmp with the platform-phase1-apply.* prefix. +USAGE +} + +verified_output_dir="" +case "$#" in + 0) + ;; + 2) + [[ "$1" == "--verified-output-dir" ]] || { + usage >&2 + exit 2 + } + verified_output_dir="$2" + ;; + *) + usage >&2 + exit 2 + ;; +esac + +if [[ -n "$verified_output_dir" ]]; then + [[ "$verified_output_dir" == /tmp/platform-phase1-apply.* ]] || \ + fail "verified output directory must match /tmp/platform-phase1-apply.*" + [[ -d "$verified_output_dir" && ! -L "$verified_output_dir" ]] || \ + fail "verified output directory must be an existing non-symlink directory" + [[ "$(cd -- "$verified_output_dir" && pwd -P)" == "$verified_output_dir" ]] || \ + fail "verified output directory must be an absolute canonical path" + [[ -O "$verified_output_dir" ]] || \ + fail "verified output directory must be owned by the current user" + [[ "$(stat --format='%a' -- "$verified_output_dir")" == "700" ]] || \ + fail "verified output directory must have mode 0700" + [[ -z "$(find "$verified_output_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || \ + fail "verified output directory must be empty" +fi + +command -v kubectl >/dev/null 2>&1 || fail "kubectl is required" +command -v jq >/dev/null 2>&1 || fail "jq is required" +command -v rg >/dev/null 2>&1 || fail "ripgrep (rg) is required" +command -v sha256sum >/dev/null 2>&1 || fail "sha256sum is required" +command -v tar >/dev/null 2>&1 || fail "tar is required" +command -v cmp >/dev/null 2>&1 || fail "cmp is required" +command -v find >/dev/null 2>&1 || fail "find is required" +command -v install >/dev/null 2>&1 || fail "install is required" +command -v stat >/dev/null 2>&1 || fail "stat is required" + +if [[ -n "${PLATFORM_HELM_BIN:-}" ]]; then + [[ "$PLATFORM_HELM_BIN" == /* ]] || \ + fail "PLATFORM_HELM_BIN must be an absolute path" + [[ -f "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \ + fail "PLATFORM_HELM_BIN is not an executable file: ${PLATFORM_HELM_BIN}" + readonly HELM_BIN="$PLATFORM_HELM_BIN" +else + HELM_BIN="$(command -v helm 2>/dev/null)" || \ + fail "Helm ${EXPECTED_HELM_VERSION} is required" + readonly HELM_BIN +fi + +kustomize_version="$(kubectl version --client --output=yaml | sed -n 's/^kustomizeVersion: //p')" +helm_version="$("$HELM_BIN" version --template '{{.Version}}')" + +[[ "$kustomize_version" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \ + fail "expected Kustomize ${EXPECTED_KUSTOMIZE_VERSION}, found ${kustomize_version:-unknown}" +[[ "$helm_version" == "$EXPECTED_HELM_VERSION" ]] || \ + fail "expected Helm ${EXPECTED_HELM_VERSION}, found ${helm_version:-unknown}" + +render_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-phase1-render.XXXXXX")" +declare -a generated_chart_cache_dirs=() +declare -a generated_chart_cache_parent_dirs=() +cleanup() { + local cache_dir + local parent_dir + + for cache_dir in "${generated_chart_cache_dirs[@]}"; do + case "$cache_dir" in + "${REPOSITORY_ROOT}/infrastructure/controllers/cloudnative-pg/.helm/charts/cloudnative-pg-0.29.0"|\ + "${REPOSITORY_ROOT}/services/gitea/profiles/oidc/.helm/charts/gitea-12.7.0") + rm -rf -- "$cache_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected chart cache: %s\n' \ + "$cache_dir" >&2 + ;; + esac + done + for parent_dir in "${generated_chart_cache_parent_dirs[@]}"; do + case "$parent_dir" in + "${REPOSITORY_ROOT}/infrastructure/controllers/cloudnative-pg/.helm/charts"|\ + "${REPOSITORY_ROOT}/infrastructure/controllers/cloudnative-pg/.helm"|\ + "${REPOSITORY_ROOT}/services/gitea/profiles/oidc/.helm/charts"|\ + "${REPOSITORY_ROOT}/services/gitea/profiles/oidc/.helm") + rmdir -- "$parent_dir" 2>/dev/null || true + ;; + *) + printf 'WARNING: refusing to remove unexpected chart cache parent: %s\n' \ + "$parent_dir" >&2 + ;; + esac + done + + + case "$render_dir" in + /tmp/platform-phase1-render.*|"${TMPDIR:-/tmp}"/platform-phase1-render.*) + rm -rf -- "$render_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected render directory: %s\n' "$render_dir" >&2 + ;; + esac +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +render_plain() { + local label="$1" + local relative_path="$2" + local output="${render_dir}/${label}.yaml" + + kubectl kustomize "${REPOSITORY_ROOT}/${relative_path}" >"$output" + [[ -s "$output" ]] || fail "${label} rendered an empty manifest" + printf 'Rendered %-20s %8s bytes\n' "$label" "$(wc -c <"$output" | tr -d '[:space:]')" +} + +render_helm() { + local label="$1" + local relative_path="$2" + local output="${render_dir}/${label}.yaml" + + kubectl kustomize \ + --enable-helm \ + --helm-command "$HELM_BIN" \ + --helm-kube-version "$TARGET_KUBERNETES_VERSION" \ + "${REPOSITORY_ROOT}/${relative_path}" >"$output" + [[ -s "$output" ]] || fail "${label} rendered an empty manifest" + printf 'Rendered %-20s %8s bytes\n' "$label" "$(wc -c <"$output" | tr -d '[:space:]')" +} + +extract_rendered_document() { + local manifest="$1" + local wanted_kind="$2" + local wanted_name="$3" + local output="$4" + + awk \ + -v wanted_kind="$wanted_kind" \ + -v wanted_name="$wanted_name" \ + ' + function reset_document() { + document = "" + document_kind = "" + document_name = "" + in_metadata = 0 + } + + function flush_document() { + if (document_kind == wanted_kind && document_name == wanted_name) { + matches++ + printf "%s", document + } + } + + BEGIN { + reset_document() + } + + /^---[[:space:]]*$/ { + flush_document() + reset_document() + next + } + + { + document = document $0 ORS + + if ($0 ~ /^kind:[[:space:]]*/) { + document_kind = $0 + sub(/^kind:[[:space:]]*/, "", document_kind) + sub(/[[:space:]]*$/, "", document_kind) + } + + if ($0 == "metadata:") { + in_metadata = 1 + next + } + + if (in_metadata && $0 ~ /^ name:[[:space:]]*/) { + document_name = $0 + sub(/^ name:[[:space:]]*/, "", document_name) + sub(/[[:space:]]*$/, "", document_name) + in_metadata = 0 + } else if (in_metadata && $0 ~ /^[^[:space:]]/) { + in_metadata = 0 + } + } + + END { + flush_document() + if (matches != 1) { + exit 42 + } + } + ' \ + "$manifest" >"$output" +} + +prepare_verified_chart_cache() { + local label="$1" + local chart_name="$2" + local chart_repository="$3" + local chart_version="$4" + local build_root_relative_path="$5" + local expected_digest="$6" + local package_path="${render_dir}/${chart_name}-${chart_version}.tgz" + local cache_version_dir="${REPOSITORY_ROOT}/${build_root_relative_path}/.helm/charts/${chart_name}-${chart_version}" + local checksum_output + local actual_digest + local chart_cache_dir="${cache_version_dir%/*}" + local helm_cache_dir="${chart_cache_dir%/*}" + local cache_parent + + if [[ ! -f "$package_path" ]]; then + "$HELM_BIN" pull "$chart_name" \ + --repo "$chart_repository" \ + --version "$chart_version" \ + --destination "$render_dir" + fi + + [[ -f "$package_path" ]] || fail "${label} chart package was not downloaded: ${package_path}" + checksum_output="$(sha256sum -- "$package_path")" + actual_digest="${checksum_output%% *}" + [[ "$actual_digest" == "$expected_digest" ]] || \ + fail "${label} chart package SHA-256 mismatch: expected ${expected_digest}, found ${actual_digest}" + + case "$cache_version_dir" in + "${REPOSITORY_ROOT}/infrastructure/controllers/cloudnative-pg/.helm/charts/cloudnative-pg-0.29.0"|\ + "${REPOSITORY_ROOT}/services/gitea/profiles/oidc/.helm/charts/gitea-12.7.0") + ;; + *) + fail "refusing to create unexpected chart cache directory: ${cache_version_dir}" + ;; + esac + for cache_parent in "$helm_cache_dir" "$chart_cache_dir"; do + [[ ! -L "$cache_parent" ]] || \ + fail "refusing symlinked chart cache parent: ${cache_parent}" + [[ ! -e "$cache_parent" || -d "$cache_parent" ]] || \ + fail "chart cache parent is not a directory: ${cache_parent}" + done + [[ ! -e "$cache_version_dir" && ! -L "$cache_version_dir" ]] || \ + fail "generated chart cache already exists; remove it only after confirming it is disposable: ${cache_version_dir}" + [[ -e "$chart_cache_dir" ]] || generated_chart_cache_parent_dirs+=("$chart_cache_dir") + [[ -e "$helm_cache_dir" ]] || generated_chart_cache_parent_dirs+=("$helm_cache_dir") + + mkdir -p -- "$cache_version_dir" + generated_chart_cache_dirs+=("$cache_version_dir") + tar -xzf "$package_path" -C "$cache_version_dir" + [[ -f "${cache_version_dir}/${chart_name}/Chart.yaml" ]] || \ + fail "${label} extracted chart is missing Chart.yaml" + + printf 'Verified %-20s SHA-256 %s\n' "$label" "$actual_digest" +} + +cd -- "$REPOSITORY_ROOT" + +if rg --line-number --glob '*.yaml' --glob '*.yml' \ + --glob '!**/.helm/**' --glob '!**/charts/**' \ + '^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' \ + infrastructure services bootstrap clusters components; then + fail "a source-controlled Kubernetes Secret manifest was found" +fi + +if rg --line-number --glob 'kustomization.yaml' 'LoadRestrictionsNone|load-restrictor' .; then + fail "the repository must keep Kustomize LoadRestrictionsRootOnly" +fi + +readonly GITEA_BASELINE_KUSTOMIZATION="${REPOSITORY_ROOT}/services/gitea/kustomization.yaml" +readonly GITEA_OIDC_KUSTOMIZATION="${REPOSITORY_ROOT}/services/gitea/profiles/oidc/kustomization.yaml" +for gitea_kustomization in \ + "$GITEA_BASELINE_KUSTOMIZATION" \ + "$GITEA_OIDC_KUSTOMIZATION"; do + assert_exact_source_text "$gitea_kustomization" \ + 'repo: https://dl.gitea.com/charts/' \ + "the pinned Gitea Chart repository" + assert_exact_source_text "$gitea_kustomization" \ + 'version: 12.7.0' \ + "the pinned Gitea Chart version" + assert_exact_source_text "$gitea_kustomization" \ + 'includeCRDs: false' \ + "the Gitea includeCRDs policy" + assert_exact_source_text "$gitea_kustomization" \ + 'skipTests: true' \ + "the Gitea Helm test policy" +done +assert_exact_source_text "$GITEA_BASELINE_KUSTOMIZATION" \ + 'chartHome: profiles/oidc/.helm/charts' \ + "the baseline Gitea verified Chart cache" +assert_exact_source_text "$GITEA_BASELINE_KUSTOMIZATION" \ + 'valuesFile: profiles/oidc/values/baseline.yaml' \ + "the baseline Gitea values path" +assert_exact_source_text "$GITEA_OIDC_KUSTOMIZATION" \ + 'chartHome: .helm/charts' \ + "the OIDC Gitea verified Chart cache" +assert_exact_source_text "$GITEA_OIDC_KUSTOMIZATION" \ + 'valuesFile: values/baseline.yaml' \ + "the OIDC Gitea baseline values path" +assert_exact_source_text "$GITEA_OIDC_KUSTOMIZATION" \ + 'additionalValuesFiles:' \ + "the OIDC Gitea values merge" +assert_exact_source_text "$GITEA_OIDC_KUSTOMIZATION" \ + 'values/oidc.yaml' \ + "the OIDC Gitea override path" + +render_plain namespaces infrastructure/namespaces/overlays/home +render_plain ssd-local-pv infrastructure/storage/ssd-local-pv +prepare_verified_chart_cache \ + cloudnative-pg-chart \ + "$CNPG_CHART_NAME" \ + "$CNPG_CHART_REPOSITORY" \ + "$CNPG_CHART_VERSION" \ + infrastructure/controllers/cloudnative-pg \ + "$EXPECTED_CNPG_CHART_SHA256" +render_helm cnpg-operator infrastructure/controllers/cloudnative-pg +render_plain platform-postgres services/platform-postgres +prepare_verified_chart_cache \ + gitea-chart \ + "$GITEA_CHART_NAME" \ + "$GITEA_CHART_REPOSITORY" \ + "$GITEA_CHART_VERSION" \ + services/gitea/profiles/oidc \ + "$EXPECTED_GITEA_CHART_SHA256" +render_helm gitea services/gitea + +render_helm gitea-oidc services/gitea/profiles/oidc +[[ "$(awk '$0 == "kind: Cluster" { count++ } END { print count + 0 }' "${render_dir}/platform-postgres.yaml")" == "1" ]] || \ + fail "the Phase 1 PostgreSQL root must contain exactly one Cluster" +[[ "$(awk '$0 == "kind: DatabaseRole" { count++ } END { print count + 0 }' "${render_dir}/platform-postgres.yaml")" == "1" ]] || \ + fail "the Phase 1 PostgreSQL root must contain exactly one Gitea DatabaseRole" +[[ "$(awk '$0 == "kind: Database" { count++ } END { print count + 0 }' "${render_dir}/platform-postgres.yaml")" == "1" ]] || \ + fail "the Phase 1 PostgreSQL root must contain exactly one Gitea Database" +[[ "$(awk '$0 == "kind: NetworkPolicy" { count++ } END { print count + 0 }' "${render_dir}/platform-postgres.yaml")" == "1" ]] || \ + fail "the Phase 1 PostgreSQL root must contain exactly one base NetworkPolicy" +[[ "$(rg --count -- '^[[:space:]]*name:[[:space:]]platform-postgres-gitea[[:space:]]*$' "${render_dir}/platform-postgres.yaml" || true)" == "2" ]] || \ + fail "the Phase 1 PostgreSQL root must contain the Gitea DatabaseRole and Database" +if rg --quiet '^[[:space:]]*name:[[:space:]]platform-postgres-keycloak[[:space:]]*$' "${render_dir}/platform-postgres.yaml"; then + fail "the Phase 1 PostgreSQL root must not contain a Keycloak DatabaseRole or Database" +fi +[[ "$(rg --count -- '^[[:space:]]*-[[:space:]]host gitea gitea all scram-sha-256[[:space:]]*$' "${render_dir}/platform-postgres.yaml" || true)" == "1" ]] || \ + fail "the Gitea role must authenticate only to the Gitea database" +[[ "$(rg --count -- '^[[:space:]]*-[[:space:]]host all gitea all reject[[:space:]]*$' "${render_dir}/platform-postgres.yaml" || true)" == "1" ]] || \ + fail "the Gitea role must be rejected from every other database" +[[ "$(rg --count -- '^[[:space:]]*-[[:space:]]host keycloak keycloak all scram-sha-256[[:space:]]*$' "${render_dir}/platform-postgres.yaml" || true)" == "1" ]] || \ + fail "the Keycloak role must authenticate only to the Keycloak database" +[[ "$(rg --count -- '^[[:space:]]*-[[:space:]]host all keycloak all reject[[:space:]]*$' "${render_dir}/platform-postgres.yaml" || true)" == "1" ]] || \ + fail "the Keycloak role must be rejected from every other database" + + +rg --quiet '^kind: CustomResourceDefinition$' "${render_dir}/cnpg-operator.yaml" || \ + fail "CloudNativePG CRDs are missing from the operator render" +rg --quiet '^kind: Cluster$' "${render_dir}/platform-postgres.yaml" || \ + fail "the platform PostgreSQL Cluster is missing" +rg --quiet '^kind: DatabaseRole$' "${render_dir}/platform-postgres.yaml" || \ + fail "the Gitea DatabaseRole is missing" +rg --quiet '^kind: Database$' "${render_dir}/platform-postgres.yaml" || \ + fail "the Gitea Database is missing" +for gitea_profile in gitea gitea-oidc; do + gitea_profile_manifest="${render_dir}/${gitea_profile}.yaml" + gitea_profile_deployment_document="${render_dir}/${gitea_profile}-deployment.yaml" + gitea_profile_ingress_document="${render_dir}/${gitea_profile}-ingress.yaml" + gitea_profile_servicemonitor_document="${render_dir}/${gitea_profile}-servicemonitor.yaml" + + rg --quiet '^kind: PersistentVolumeClaim$' "$gitea_profile_manifest" || \ + fail "${gitea_profile} is missing the Gitea PVC" + rg --quiet '^kind: Ingress$' "$gitea_profile_manifest" || \ + fail "${gitea_profile} is missing the Gitea Ingress" + rg --quiet \ + '^[[:space:]]*-[[:space:]]*host:[[:space:]]*git\.learn\.hyeonworks\.com[[:space:]]*$' \ + "$gitea_profile_manifest" || \ + fail "${gitea_profile} Ingress host is not git.learn.hyeonworks.com" + [[ "$(rg --count-matches \ + '^[[:space:]]*ROOT_URL=https://git\.learn\.hyeonworks\.com/[[:space:]]*$' \ + "$gitea_profile_manifest" || true)" == "1" ]] || \ + fail "${gitea_profile} must render the external HTTPS ROOT_URL exactly once" + + extract_rendered_document \ + "$gitea_profile_manifest" \ + Deployment \ + gitea \ + "$gitea_profile_deployment_document" || \ + fail "${gitea_profile} must contain exactly one gitea Deployment" + extract_rendered_document \ + "$gitea_profile_manifest" \ + Ingress \ + gitea-http \ + "$gitea_profile_ingress_document" || \ + fail "${gitea_profile} must contain exactly one gitea-http Ingress" + extract_rendered_document \ + "$gitea_profile_manifest" \ + ServiceMonitor \ + gitea \ + "$gitea_profile_servicemonitor_document" || \ + fail "${gitea_profile} must contain exactly one gitea ServiceMonitor" + if rg --quiet '^[[:space:]]{2}tls:[[:space:]]*' "$gitea_profile_ingress_document"; then + fail "${gitea_profile} must not render an in-cluster TLS section" + fi + + if ! kubectl create --dry-run=client \ + -f "$gitea_profile_servicemonitor_document" \ + -o json | jq -e ' + .apiVersion == "monitoring.coreos.com/v1" and + .kind == "ServiceMonitor" and + .metadata.name == "gitea" and + .metadata.namespace == "gitea" and + .metadata.labels["observability.hyeonworks.com/instance"] == "home" and + .spec.jobLabel == "app.kubernetes.io/name" and + .spec.selector.matchLabels == { + "app.kubernetes.io/instance": "gitea", + "app.kubernetes.io/name": "gitea" + } and + .spec.endpoints == [{ + "interval": "30s", + "port": "http", + "scrapeTimeout": "10s" + }] + ' >/dev/null; then + fail "${gitea_profile} ServiceMonitor contract is not exact" + fi + + for restricted_setting in \ + 'allowPrivilegeEscalation: false' \ + 'runAsNonRoot: true' \ + 'type: RuntimeDefault' \ + '- ALL'; do + [[ "$(rg --count-matches --fixed-strings -- "$restricted_setting" \ + "$gitea_profile_deployment_document" || true)" == "4" ]] || \ + fail "${gitea_profile} must apply ${restricted_setting} to all four containers" + done +done + +for forbidden_baseline_marker in \ + 'gitea-keycloak-oidc' \ + 'id.learn.hyeonworks.com' \ + 'gitea-allow-host-nginx-keycloak' \ + 'gitea-branding-assets' \ + 'gitea-branding-templates' \ + 'ALLOW_ONLY_EXTERNAL_REGISTRATION=true'; do + if rg --quiet --fixed-strings -- "$forbidden_baseline_marker" "${render_dir}/gitea.yaml"; then + fail "the baseline Gitea render contains OIDC-only marker ${forbidden_baseline_marker}" + fi +done +for baseline_setting in \ + 'DISABLE_REGISTRATION=true' \ + 'ALLOW_ONLY_EXTERNAL_REGISTRATION=false' \ + 'SHOW_REGISTRATION_BUTTON=false' \ + 'ENABLE_PASSWORD_SIGNIN_FORM=true'; do + [[ "$(rg --count-matches --fixed-strings -- "$baseline_setting" \ + "${render_dir}/gitea.yaml" || true)" == "1" ]] || \ + fail "the baseline Gitea render must contain exactly one ${baseline_setting} setting" +done + +gitea_deployment_document="${render_dir}/gitea-oidc-deployment.yaml" +gitea_keycloak_policy_document="${render_dir}/gitea-keycloak-egress-policy.yaml" +gitea_branding_assets_document="${render_dir}/gitea-branding-assets-configmap.yaml" +gitea_branding_templates_document="${render_dir}/gitea-branding-templates-configmap.yaml" + +extract_rendered_document \ + "${render_dir}/gitea-oidc.yaml" \ + Deployment \ + gitea \ + "$gitea_deployment_document" || \ + fail "the Gitea render must contain exactly one gitea Deployment" +extract_rendered_document \ + "${render_dir}/gitea-oidc.yaml" \ + NetworkPolicy \ + gitea-allow-host-nginx-keycloak \ + "$gitea_keycloak_policy_document" || \ + fail "the Gitea render must contain exactly one dedicated Keycloak egress NetworkPolicy" + +# Assert only references to the externally-created OIDC credential Secret. +# No credential payload is rendered, decoded, read, or printed by these checks. +rg --quiet --multiline \ + '(?s)- name: GITEA_OAUTH_KEY_0\n[[:space:]]+valueFrom:\n[[:space:]]+secretKeyRef:\n[[:space:]]+key: key\n[[:space:]]+name: gitea-keycloak-oidc' \ + "$gitea_deployment_document" || \ + fail "the Gitea Deployment does not reference gitea-keycloak-oidc key=key" +rg --quiet --multiline \ + '(?s)- name: GITEA_OAUTH_SECRET_0\n[[:space:]]+valueFrom:\n[[:space:]]+secretKeyRef:\n[[:space:]]+key: secret\n[[:space:]]+name: gitea-keycloak-oidc' \ + "$gitea_deployment_document" || \ + fail "the Gitea Deployment does not reference gitea-keycloak-oidc key=secret" +[[ "$(rg --count-matches --fixed-strings \ + 'https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration' \ + "${render_dir}/gitea-oidc.yaml" || true)" == "2" ]] || \ + fail "the Gitea OAuth add/update script must use the exact Keycloak discovery URL" + +# These are non-sensitive app.ini policy values rendered by the pinned Chart. +for expected_setting in \ + 'ALLOW_ONLY_EXTERNAL_REGISTRATION=true' \ + 'DISABLE_REGISTRATION=false' \ + 'SHOW_REGISTRATION_BUTTON=false' \ + 'ENABLE_PASSWORD_SIGNIN_FORM=true' \ + 'ENABLE_AUTO_REGISTRATION=true' \ + 'USERNAME=preferred_username' \ + 'ACCOUNT_LINKING=login' \ + 'OPENID_CONNECT_SCOPES=profile email'; do + [[ "$(rg --count-matches --fixed-strings "$expected_setting" \ + "${render_dir}/gitea-oidc.yaml" || true)" == "1" ]] || \ + fail "the Gitea render must contain exactly one ${expected_setting} setting" +done + +if rg --quiet '^[[:space:]]*hostAliases:' "$gitea_deployment_document"; then + fail "the Gitea Deployment must rely on CoreDNS and must not contain hostAliases" +fi + +[[ "$(rg --count-matches \ + '^[[:space:]]*cidr:[[:space:]]*192\.168\.0\.107/32[[:space:]]*$' \ + "$gitea_keycloak_policy_document" || true)" == "1" ]] || \ + fail "the dedicated Keycloak egress policy must allow exactly 192.168.0.107/32" +[[ "$(rg --count-matches \ + '^[[:space:]]*-[[:space:]]*port:[[:space:]]*443[[:space:]]*$' \ + "$gitea_keycloak_policy_document" || true)" == "1" ]] || \ + fail "the dedicated Keycloak egress policy must allow exactly TCP port 443" +[[ "$(rg --count-matches \ + '^[[:space:]]*protocol:[[:space:]]*TCP[[:space:]]*$' \ + "$gitea_keycloak_policy_document" || true)" == "1" ]] || \ + fail "the dedicated Keycloak egress policy must use TCP" +rg --quiet \ + '^[[:space:]]*-[[:space:]]*Egress[[:space:]]*$' \ + "$gitea_keycloak_policy_document" || \ + fail "the dedicated Keycloak NetworkPolicy must select egress traffic" + +gitea_branding_assets_configmap_name="$( + (rg --only-matching --no-filename \ + 'gitea-branding-assets-[a-z0-9]+' \ + "${render_dir}/gitea-oidc.yaml" || true) | + LC_ALL=C sort --unique +)" +gitea_branding_templates_configmap_name="$( + (rg --only-matching --no-filename \ + 'gitea-branding-templates-[a-z0-9]+' \ + "${render_dir}/gitea-oidc.yaml" || true) | + LC_ALL=C sort --unique +)" +[[ "$gitea_branding_assets_configmap_name" =~ ^gitea-branding-assets-[a-z0-9]{10}$ ]] || \ + fail "the Gitea branding assets ConfigMap must have one Kustomize content hash" +[[ "$gitea_branding_templates_configmap_name" =~ ^gitea-branding-templates-[a-z0-9]{10}$ ]] || \ + fail "the Gitea branding templates ConfigMap must have one Kustomize content hash" + +extract_rendered_document \ + "${render_dir}/gitea-oidc.yaml" \ + ConfigMap \ + "$gitea_branding_assets_configmap_name" \ + "$gitea_branding_assets_document" || \ + fail "the Gitea render must contain exactly one branding assets ConfigMap" +extract_rendered_document \ + "${render_dir}/gitea-oidc.yaml" \ + ConfigMap \ + "$gitea_branding_templates_configmap_name" \ + "$gitea_branding_templates_document" || \ + fail "the Gitea render must contain exactly one branding templates ConfigMap" + +for asset_key in hyeonworks.css logo.svg favicon.svg; do + [[ "$(rg --count-matches \ + "^[[:space:]]{2}${asset_key//./\\.}:[[:space:]]*\\|[+-]?[[:space:]]*$" \ + "$gitea_branding_assets_document" || true)" == "1" ]] || \ + fail "the branding assets ConfigMap must contain exactly one ${asset_key}" +done +for template_key in header.tmpl extra_links.tmpl; do + [[ "$(rg --count-matches \ + "^[[:space:]]{2}${template_key//./\\.}:[[:space:]]*\\|[+-]?[[:space:]]*$" \ + "$gitea_branding_templates_document" || true)" == "1" ]] || \ + fail "the branding templates ConfigMap must contain exactly one ${template_key}" +done + +[[ "$(rg --count-matches --fixed-strings \ + "$gitea_branding_assets_configmap_name" \ + "$gitea_deployment_document" || true)" == "1" ]] || \ + fail "the Gitea Deployment must reference the hashed branding assets ConfigMap once" +[[ "$(rg --count-matches --fixed-strings \ + "$gitea_branding_templates_configmap_name" \ + "$gitea_deployment_document" || true)" == "1" ]] || \ + fail "the Gitea Deployment must reference the hashed branding templates ConfigMap once" +rg --quiet --multiline \ + '(?s)- mountPath: /data/gitea/public/assets\n[[:space:]]+name: branding-assets\n[[:space:]]+readOnly: true' \ + "$gitea_deployment_document" || \ + fail "the Gitea branding assets must be mounted read-only at the official custom path" +rg --quiet --multiline \ + '(?s)- mountPath: /data/gitea/templates/custom\n[[:space:]]+name: branding-templates\n[[:space:]]+readOnly: true' \ + "$gitea_deployment_document" || \ + fail "the Gitea branding templates must be mounted read-only at the official custom path" + +if rg --quiet '^[[:space:]]*type:[[:space:]]*(NodePort|LoadBalancer)[[:space:]]*$' \ + "${render_dir}/platform-postgres.yaml" \ + "${render_dir}/gitea.yaml" \ + "${render_dir}/gitea-oidc.yaml"; then + fail "an application service is exposed as NodePort or LoadBalancer" +fi + +if rg --quiet '^[[:space:]]*name:[[:space:]]*gitea-ssh[[:space:]]*$' \ + "${render_dir}/gitea.yaml" "${render_dir}/gitea-oidc.yaml"; then + fail "the disabled Gitea SSH Service is still rendered" +fi + +if [[ -n "$verified_output_dir" ]]; then + for manifest_name in "${RENDERED_MANIFEST_NAMES[@]}"; do + source_manifest="${render_dir}/${manifest_name}.yaml" + output_manifest="${verified_output_dir}/${manifest_name}.yaml" + + [[ -f "$source_manifest" && ! -L "$source_manifest" && -s "$source_manifest" ]] || \ + fail "validated manifest is missing or unsafe: ${source_manifest}" + install -m 0600 -- "$source_manifest" "$output_manifest" + [[ -f "$output_manifest" && ! -L "$output_manifest" && -s "$output_manifest" ]] || \ + fail "verified manifest handoff failed: ${output_manifest}" + cmp --silent -- "$source_manifest" "$output_manifest" || \ + fail "verified manifest changed during handoff: ${manifest_name}.yaml" + done + verified_entry_count="$(find "$verified_output_dir" -mindepth 1 -maxdepth 1 | wc -l | tr -d '[:space:]')" + [[ "$verified_entry_count" == "${#RENDERED_MANIFEST_NAMES[@]}" ]] || \ + fail "verified output directory does not contain exactly six manifest files" + printf 'Preserved six verified manifests for the apply handoff.\n' +fi + +printf 'Phase 1 baseline and Gitea OIDC desired rendering invariants passed.\n' +printf 'Temporary rendered manifests and generated chart caches will be removed on exit.\n' diff --git a/scripts/validate/render-phase2.sh b/scripts/validate/render-phase2.sh new file mode 100755 index 0000000..a214de2 --- /dev/null +++ b/scripts/validate/render-phase2.sh @@ -0,0 +1,439 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1" +readonly EXPECTED_HELM_VERSION="v3.19.4" +readonly TARGET_KUBERNETES_VERSION="1.36.2" +readonly AISTOR_OPERATOR_CHART_NAME="aistor-operator" +readonly AISTOR_OPERATOR_CHART_VERSION="5.10.0" +readonly EXPECTED_AISTOR_OPERATOR_SHA256="e5534f5ae4f6f12a3528a8cda5954d73280cba1a8e979e1628fbf3aac76babd1" +readonly AISTOR_OBJECTSTORE_CHART_NAME="aistor-objectstore" +readonly AISTOR_OBJECTSTORE_CHART_VERSION="1.0.16" +readonly EXPECTED_AISTOR_OBJECTSTORE_SHA256="50ffa6a4e014cdc48566b237593baab41d039f103cdc50c1b1ac173b8d8bf71e" +readonly AISTOR_CHART_REPOSITORY="https://helm.min.io/" +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly -a RENDERED_MANIFEST_NAMES=( + phase2-namespaces + aistor-local-pv + keycloak-operator + platform-postgres-keycloak + keycloak + aistor-operator + minio-aistor + aistor-network-policies +) + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +assert_regex_count() { + local file="$1" + local pattern="$2" + local expected="$3" + local description="$4" + local actual + + [[ -f "$file" ]] || fail "missing validation input: ${file}" + actual="$(rg --count --no-filename -- "$pattern" "$file" || true)" + actual="${actual:-0}" + [[ "$actual" == "$expected" ]] || \ + fail "${description}: expected ${expected}, found ${actual}" +} + +usage() { + cat <<'USAGE' +Usage: + bash scripts/validate/render-phase2.sh + bash scripts/validate/render-phase2.sh \ + --verified-output-dir /tmp/platform-phase2-apply.XXXXXX + +The output option is an internal handoff used only by the AIStor apply script. +The destination must be an existing, empty, non-symlink directory created +below /tmp with the platform-phase2-apply.* prefix. +USAGE +} + +verified_output_dir="" +case "$#" in + 0) + ;; + 2) + [[ "$1" == "--verified-output-dir" ]] || { + usage >&2 + exit 2 + } + verified_output_dir="$2" + ;; + *) + usage >&2 + exit 2 + ;; +esac + +if [[ -n "$verified_output_dir" ]]; then + [[ "$verified_output_dir" == /tmp/platform-phase2-apply.* ]] || \ + fail "verified output directory must match /tmp/platform-phase2-apply.*" + [[ -d "$verified_output_dir" && ! -L "$verified_output_dir" ]] || \ + fail "verified output directory must be an existing non-symlink directory" + [[ "$(cd -- "$verified_output_dir" && pwd -P)" == "$verified_output_dir" ]] || \ + fail "verified output directory must be an absolute canonical path" + [[ -O "$verified_output_dir" ]] || \ + fail "verified output directory must be owned by the current user" + [[ "$(stat --format='%a' -- "$verified_output_dir")" == "700" ]] || \ + fail "verified output directory must have mode 0700" + [[ -z "$(find "$verified_output_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || \ + fail "verified output directory must be empty" +fi + +for command_name in cmp find install kubectl rg sha256sum stat tar; do + command -v "$command_name" >/dev/null 2>&1 || \ + fail "${command_name} is required" +done + +if [[ -n "${PLATFORM_HELM_BIN:-}" ]]; then + [[ "$PLATFORM_HELM_BIN" == /* ]] || \ + fail "PLATFORM_HELM_BIN must be an absolute path" + [[ -f "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \ + fail "PLATFORM_HELM_BIN is not an executable file: ${PLATFORM_HELM_BIN}" + readonly HELM_BIN="$PLATFORM_HELM_BIN" +else + HELM_BIN="$(command -v helm 2>/dev/null)" || \ + fail "Helm ${EXPECTED_HELM_VERSION} is required" + readonly HELM_BIN +fi + +kustomize_version="$(kubectl version --client --output=yaml | sed -n 's/^kustomizeVersion: //p')" +helm_version="$("$HELM_BIN" version --template '{{.Version}}')" +[[ "$kustomize_version" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \ + fail "expected Kustomize ${EXPECTED_KUSTOMIZE_VERSION}, found ${kustomize_version:-unknown}" +[[ "$helm_version" == "$EXPECTED_HELM_VERSION" ]] || \ + fail "expected Helm ${EXPECTED_HELM_VERSION}, found ${helm_version:-unknown}" + +printf 'Validating the Phase 1 baseline and Gitea OIDC desired profile first.\n' +PLATFORM_HELM_BIN="$HELM_BIN" \ + bash "${REPOSITORY_ROOT}/scripts/validate/render-phase1.sh" + +render_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-phase2-render.XXXXXX")" +declare -a generated_chart_cache_dirs=() +declare -a generated_chart_cache_parent_dirs=() + +cleanup() { + local cache_dir + local parent_dir + + for cache_dir in "${generated_chart_cache_dirs[@]}"; do + case "$cache_dir" in + "${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/.helm/charts/aistor-operator-5.10.0"|\ + "${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/.helm/charts/aistor-objectstore-1.0.16") + rm -rf -- "$cache_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected chart cache: %s\n' \ + "$cache_dir" >&2 + ;; + esac + done + for parent_dir in "${generated_chart_cache_parent_dirs[@]}"; do + case "$parent_dir" in + "${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/.helm/charts"|\ + "${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/.helm"|\ + "${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/.helm/charts"|\ + "${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/.helm") + rmdir -- "$parent_dir" 2>/dev/null || true + ;; + *) + printf 'WARNING: refusing to remove unexpected chart cache parent: %s\n' \ + "$parent_dir" >&2 + ;; + esac + done + + + case "$render_dir" in + /tmp/platform-phase2-render.*|"${TMPDIR:-/tmp}"/platform-phase2-render.*) + rm -rf -- "$render_dir" + ;; + *) + printf 'WARNING: refusing to remove unexpected render directory: %s\n' \ + "$render_dir" >&2 + ;; + esac +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +render_plain() { + local label="$1" + local relative_path="$2" + local output="${render_dir}/${label}.yaml" + + kubectl kustomize "${REPOSITORY_ROOT}/${relative_path}" >"$output" + [[ -s "$output" ]] || fail "${label} rendered an empty manifest" + printf 'Rendered %-24s %8s bytes\n' \ + "$label" "$(wc -c <"$output" | tr -d '[:space:]')" +} + +render_helm() { + local label="$1" + local relative_path="$2" + local output="${render_dir}/${label}.yaml" + + kubectl kustomize \ + --enable-helm \ + --helm-command "$HELM_BIN" \ + --helm-kube-version "$TARGET_KUBERNETES_VERSION" \ + "${REPOSITORY_ROOT}/${relative_path}" >"$output" + [[ -s "$output" ]] || fail "${label} rendered an empty manifest" + printf 'Rendered %-24s %8s bytes\n' \ + "$label" "$(wc -c <"$output" | tr -d '[:space:]')" +} + +prepare_verified_chart_cache() { + local label="$1" + local chart_name="$2" + local chart_version="$3" + local build_root_relative_path="$4" + local expected_digest="$5" + local package_path="${render_dir}/${chart_name}-${chart_version}.tgz" + local cache_version_dir="${REPOSITORY_ROOT}/${build_root_relative_path}/.helm/charts/${chart_name}-${chart_version}" + local checksum_output + local actual_digest + local chart_cache_dir="${cache_version_dir%/*}" + local helm_cache_dir="${chart_cache_dir%/*}" + local cache_parent + + case "$cache_version_dir" in + "${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/.helm/charts/aistor-operator-5.10.0"|\ + "${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/.helm/charts/aistor-objectstore-1.0.16") + ;; + *) + fail "refusing to create unexpected chart cache: ${cache_version_dir}" + ;; + esac + for cache_parent in "$helm_cache_dir" "$chart_cache_dir"; do + [[ ! -L "$cache_parent" ]] || \ + fail "refusing symlinked chart cache parent: ${cache_parent}" + [[ ! -e "$cache_parent" || -d "$cache_parent" ]] || \ + fail "chart cache parent is not a directory: ${cache_parent}" + done + + [[ ! -e "$cache_version_dir" && ! -L "$cache_version_dir" ]] || \ + fail "generated chart cache already exists; remove it after confirming it is disposable: ${cache_version_dir}" + + "$HELM_BIN" pull "$chart_name" \ + --repo "$AISTOR_CHART_REPOSITORY" \ + --version "$chart_version" \ + --destination "$render_dir" + + [[ -f "$package_path" ]] || \ + fail "${label} chart package was not downloaded: ${package_path}" + checksum_output="$(sha256sum -- "$package_path")" + actual_digest="${checksum_output%% *}" + [[ "$actual_digest" == "$expected_digest" ]] || \ + fail "${label} chart SHA-256 mismatch: expected ${expected_digest}, found ${actual_digest}" + + [[ -e "$chart_cache_dir" ]] || generated_chart_cache_parent_dirs+=("$chart_cache_dir") + [[ -e "$helm_cache_dir" ]] || generated_chart_cache_parent_dirs+=("$helm_cache_dir") + mkdir -p -- "$cache_version_dir" + generated_chart_cache_dirs+=("$cache_version_dir") + tar -xzf "$package_path" -C "$cache_version_dir" + [[ -f "${cache_version_dir}/${chart_name}/Chart.yaml" ]] || \ + fail "${label} extracted chart is missing Chart.yaml" + printf 'Verified %-24s SHA-256 %s\n' "$label" "$actual_digest" +} + +cd -- "$REPOSITORY_ROOT" + +if rg --line-number --glob '*.yaml' --glob '*.yml' \ + --glob '!**/.helm/**' --glob '!**/charts/**' \ + '^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' \ + infrastructure services bootstrap clusters components; then + fail "a source-controlled Kubernetes Secret manifest was found" +fi + +if rg --line-number --glob 'kustomization.yaml' \ + 'LoadRestrictionsNone|load-restrictor' .; then + fail "the repository must keep Kustomize LoadRestrictionsRootOnly" +fi + +keycloak_operator_kustomization="${REPOSITORY_ROOT}/infrastructure/controllers/keycloak-operator/kustomization.yaml" +aistor_operator_kustomization="${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/kustomization.yaml" +aistor_objectstore_kustomization="${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/kustomization.yaml" +aistor_operator_values="${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/values/home.yaml" +aistor_objectstore_values="${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/values/home.yaml" + +assert_regex_count "$keycloak_operator_kustomization" '^[[:space:]]*-[[:space:]]github\.com/keycloak/keycloak-k8s-resources/kubernetes\?ref=26\.7\.0[[:space:]]*$' 1 "Keycloak Operator remote ref must be 26.7.0" +assert_regex_count "$keycloak_operator_kustomization" 'github\.com/keycloak/keycloak-k8s-resources/kubernetes' 1 "exactly one Keycloak Operator remote resource is allowed" +assert_regex_count "$keycloak_operator_kustomization" '\?ref=' 1 "exactly one pinned remote ref is allowed in the Keycloak Operator root" + +assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*-[[:space:]]name:' 1 "AIStor Operator root must contain one Helm chart" +assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*-[[:space:]]name:[[:space:]]aistor-operator[[:space:]]*$' 1 "AIStor Operator chart name is pinned" +assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*repo:[[:space:]]https://helm\.min\.io/[[:space:]]*$' 1 "AIStor Operator chart repository is pinned" +assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*version:[[:space:]]5\.10\.0[[:space:]]*$' 1 "AIStor Operator chart version is pinned" +assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*valuesFile:[[:space:]]values/home\.yaml[[:space:]]*$' 1 "AIStor Operator values file is pinned" +assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*includeCRDs:[[:space:]]true[[:space:]]*$' 1 "AIStor Operator CRDs must be included" +assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*chartHome:[[:space:]]\.helm/charts[[:space:]]*$' 1 "AIStor Operator chartHome is pinned" + +assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*-[[:space:]]name:' 1 "AIStor ObjectStore root must contain one Helm chart" +assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*-[[:space:]]name:[[:space:]]aistor-objectstore[[:space:]]*$' 1 "AIStor ObjectStore chart name is pinned" +assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*repo:[[:space:]]https://helm\.min\.io/[[:space:]]*$' 1 "AIStor ObjectStore chart repository is pinned" +assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*version:[[:space:]]1\.0\.16[[:space:]]*$' 1 "AIStor ObjectStore chart version is pinned" +assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*valuesFile:[[:space:]]values/home\.yaml[[:space:]]*$' 1 "AIStor ObjectStore values file is pinned" +assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*includeCRDs:[[:space:]]false[[:space:]]*$' 1 "AIStor ObjectStore root must not duplicate CRDs" +assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*chartHome:[[:space:]]\.helm/charts[[:space:]]*$' 1 "AIStor ObjectStore chartHome is pinned" + +assert_regex_count "$aistor_operator_values" '^license:[[:space:]]*""[[:space:]]*$' 1 "AIStor license payload must not be rendered from values" +assert_regex_count "$aistor_objectstore_values" '^[[:space:]]*existingSecret:[[:space:]]true[[:space:]]*$' 1 "AIStor ObjectStore must use an existing Secret" +assert_regex_count "$aistor_objectstore_values" '^[[:space:]]*name:[[:space:]]aistor-root-configuration[[:space:]]*$' 2 "AIStor configuration Secret references are pinned" +assert_regex_count "$aistor_objectstore_values" '^[[:space:]]*serviceType:[[:space:]]ClusterIP[[:space:]]*$' 2 "AIStor S3 and Console services must be ClusterIP" + + + +render_plain phase2-namespaces infrastructure/namespaces/phase2 +render_plain aistor-local-pv infrastructure/storage/aistor-local-pv +render_plain aistor-network-policies infrastructure/networking/aistor +render_plain keycloak-operator infrastructure/controllers/keycloak-operator +render_plain platform-postgres-keycloak services/platform-postgres-keycloak +render_plain keycloak services/keycloak + +prepare_verified_chart_cache \ + aistor-operator-chart \ + "$AISTOR_OPERATOR_CHART_NAME" \ + "$AISTOR_OPERATOR_CHART_VERSION" \ + infrastructure/controllers/aistor-operator \ + "$EXPECTED_AISTOR_OPERATOR_SHA256" +render_helm aistor-operator infrastructure/controllers/aistor-operator + +prepare_verified_chart_cache \ + aistor-objectstore-chart \ + "$AISTOR_OBJECTSTORE_CHART_NAME" \ + "$AISTOR_OBJECTSTORE_CHART_VERSION" \ + services/minio-aistor/base/deployment \ + "$EXPECTED_AISTOR_OBJECTSTORE_SHA256" +render_helm minio-aistor services/minio-aistor + +if rg --line-number '^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' \ + "${render_dir}/phase2-namespaces.yaml" \ + "${render_dir}/aistor-local-pv.yaml" \ + "${render_dir}/keycloak-operator.yaml" \ + "${render_dir}/platform-postgres-keycloak.yaml" \ + "${render_dir}/keycloak.yaml" \ + "${render_dir}/aistor-operator.yaml" \ + "${render_dir}/minio-aistor.yaml" \ + "${render_dir}/aistor-network-policies.yaml"; then + fail "a Phase 2 render unexpectedly contains a Secret" +fi + + +assert_regex_count "${render_dir}/phase2-namespaces.yaml" '^kind: Namespace$' 3 "Phase 2 namespace count" +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^kind: StorageClass$' 1 "AIStor StorageClass count" +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^kind: PersistentVolume$' 1 "AIStor PersistentVolume count" +assert_regex_count "${render_dir}/aistor-network-policies.yaml" '^kind: NetworkPolicy$' 11 "AIStor NetworkPolicy count" +assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^kind: DatabaseRole$' 1 "Keycloak DatabaseRole count" +assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^kind: Database$' 1 "Keycloak Database count" +assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^kind: NetworkPolicy$' 1 "Keycloak PostgreSQL NetworkPolicy count" +assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^kind: Cluster$' 0 "the Phase 2 PostgreSQL extension must not contain a Cluster" +assert_regex_count "${render_dir}/keycloak.yaml" '^kind: Keycloak$' 1 "Keycloak custom resource count" +assert_regex_count "${render_dir}/keycloak.yaml" '^kind: Ingress$' 1 "Keycloak Ingress count" +assert_regex_count "${render_dir}/minio-aistor.yaml" '^kind: ObjectStore$' 1 "AIStor ObjectStore count" + +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*instances:[[:space:]]1[[:space:]]*$' 1 "Keycloak instance count field" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*hostname:[[:space:]]https://id\.learn\.hyeonworks\.com[[:space:]]*$' 1 "Keycloak external hostname" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*strict:[[:space:]]true[[:space:]]*$' 1 "Keycloak strict hostname mode" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*httpEnabled:[[:space:]]true[[:space:]]*$' 1 "Keycloak internal HTTP mode" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*httpPort:[[:space:]]8080[[:space:]]*$' 1 "Keycloak HTTP port" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*serviceHttpPort:[[:space:]]8080[[:space:]]*$' 1 "Keycloak Service HTTP port" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*headers:[[:space:]]xforwarded[[:space:]]*$' 1 "Keycloak forwarded-header mode" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*enabled:[[:space:]]false[[:space:]]*$' 1 "Keycloak Operator-managed Ingress must be disabled" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*ingressClassName:[[:space:]]traefik[[:space:]]*$' 1 "Keycloak Ingress class" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*-[[:space:]]host:[[:space:]]id\.learn\.hyeonworks\.com[[:space:]]*$' 1 "Keycloak Ingress host" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*name:[[:space:]]keycloak-service[[:space:]]*$' 1 "Keycloak Ingress backend Service" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*number:[[:space:]]8080[[:space:]]*$' 1 "Keycloak Ingress backend port" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*kubernetes\.io/metadata\.name:[[:space:]]kube-system[[:space:]]*$' 2 "Keycloak Traefik namespace selectors" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*app\.kubernetes\.io/name:[[:space:]]traefik[[:space:]]*$' 2 "Keycloak Traefik pod selectors" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*kubernetes\.io/metadata\.name:[[:space:]]keycloak[[:space:]]*$' 1 "Keycloak management namespace selector" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*app\.kubernetes\.io/name:[[:space:]]keycloak-operator[[:space:]]*$' 1 "Keycloak management Operator selector" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*xaEnabled:[[:space:]]false[[:space:]]*$' 1 "Keycloak XA transaction mode" +assert_regex_count "${render_dir}/keycloak.yaml" ':[[:space:]]*9000[[:space:]]*$' 0 "Keycloak management port must not be exposed" + +assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^[[:space:]]*name:[[:space:]]platform-postgres-keycloak[[:space:]]*$' 2 "Keycloak database and role names" +assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^[[:space:]]*name:[[:space:]]keycloak-db-credentials[[:space:]]*$' 1 "Keycloak DatabaseRole Secret reference" +assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^[[:space:]]*kubernetes\.io/metadata\.name:[[:space:]]keycloak[[:space:]]*$' 1 "Keycloak PostgreSQL namespace selector" +assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^[[:space:]]*-[[:space:]]port:[[:space:]]5432[[:space:]]*$' 1 "Keycloak PostgreSQL ingress port" +assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^[[:space:]]*cnpg\.io/cluster:[[:space:]]platform-postgres[[:space:]]*$' 1 "Keycloak PostgreSQL pod selector" + +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^provisioner:[[:space:]]kubernetes\.io/no-provisioner[[:space:]]*$' 1 "AIStor static StorageClass provisioner" +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^reclaimPolicy:[[:space:]]Retain[[:space:]]*$' 1 "AIStor StorageClass reclaim policy" +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^volumeBindingMode:[[:space:]]WaitForFirstConsumer[[:space:]]*$' 1 "AIStor volume binding mode" +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*persistentVolumeReclaimPolicy:[[:space:]]Retain[[:space:]]*$' 1 "AIStor PersistentVolume reclaim policy" +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*storage:[[:space:]]900Gi[[:space:]]*$' 1 "AIStor Local PV capacity" +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*path:[[:space:]]/srv/k3s/aistor[[:space:]]*$' 1 "AIStor Local PV host path" +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*-[[:space:]]donghyeon-system-product-name[[:space:]]*$' 1 "AIStor Local PV node affinity" +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*platform\.hyeonworks\.com/filesystem:[[:space:]]xfs[[:space:]]*$' 1 "AIStor Local PV filesystem label" +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*storageClassName:[[:space:]]aistor-local-xfs-retain[[:space:]]*$' 1 "AIStor Local PV StorageClass binding" +assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*-[[:space:]]ReadWriteOnce[[:space:]]*$' 1 "AIStor Local PV access mode" + +assert_regex_count "${render_dir}/aistor-operator.yaml" '^kind: CustomResourceDefinition$' 3 "AIStor Operator CRD count" +assert_regex_count "${render_dir}/aistor-operator.yaml" '^kind: Deployment$' 3 "AIStor Operator Deployment count" +assert_regex_count "${render_dir}/aistor-operator.yaml" '^[[:space:]]*caBundle:' 0 "AIStor webhook runtime CA must not be declaratively owned" +assert_regex_count "${render_dir}/aistor-network-policies.yaml" '^[[:space:]]*namespace:[[:space:]]aistor$' 5 "AIStor control-plane NetworkPolicy namespace count" +assert_regex_count "${render_dir}/aistor-network-policies.yaml" '^[[:space:]]*namespace:[[:space:]]object-storage$' 6 "AIStor data-plane NetworkPolicy namespace count" +assert_regex_count "${render_dir}/aistor-network-policies.yaml" '^[[:space:]]*-[[:space:]]port:[[:space:]]8443$' 1 "AIStor admission webhook policy port" +assert_regex_count "${render_dir}/aistor-network-policies.yaml" '^[[:space:]]*-[[:space:]]port:[[:space:]]4221$' 2 "AIStor upgrade service policy port" +assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*servers:[[:space:]]1[[:space:]]*$' 1 "AIStor server count" +assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*volumesPerServer:[[:space:]]1[[:space:]]*$' 1 "AIStor volume count" +assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*storage:[[:space:]]900Gi[[:space:]]*$' 1 "AIStor claim size" +assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*storageClassName:[[:space:]]aistor-local-xfs-retain[[:space:]]*$' 1 "AIStor claim StorageClass" +assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*name:[[:space:]]aistor-root-configuration[[:space:]]*$' 1 "AIStor configuration Secret reference" +assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*pvcProtection:[[:space:]]true[[:space:]]*$' 1 "AIStor PVC protection" +assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*serviceType:[[:space:]]ClusterIP[[:space:]]*$' 2 "AIStor S3 and Console Service types" +assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*disableAutoCert:[[:space:]]true[[:space:]]*$' 1 "AIStor in-cluster TLS mode" +assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*kubernetes\.io/hostname:[[:space:]]donghyeon-system-product-name[[:space:]]*$' 1 "AIStor ObjectStore node selector" + +if rg --quiet '^[[:space:]]*kind:[[:space:]]*Ingress[[:space:]]*$|^[[:space:]]*(serviceType|type):[[:space:]]*(NodePort|LoadBalancer)[[:space:]]*$' \ + "${render_dir}/minio-aistor.yaml"; then + fail "AIStor must not render Ingress, NodePort, or LoadBalancer exposure" +fi +if rg --quiet '^[[:space:]]*tls:[[:space:]]*(\[\])?[[:space:]]*$' \ + "${render_dir}/keycloak.yaml"; then + fail "Keycloak must not render in-cluster TLS" +fi +if rg --quiet '^[[:space:]]*(serviceType|type):[[:space:]]*(NodePort|LoadBalancer)[[:space:]]*$' \ + "${render_dir}/keycloak.yaml"; then + fail "Keycloak must not render NodePort or LoadBalancer exposure" +fi + +if rg --quiet 'cidr:[[:space:]]*0\.0\.0\.0/0' \ + "${render_dir}/aistor-network-policies.yaml"; then + fail "AIStor NetworkPolicies must not allow unrestricted Internet egress" +fi + +if [[ -n "$verified_output_dir" ]]; then + for manifest_name in "${RENDERED_MANIFEST_NAMES[@]}"; do + source_manifest="${render_dir}/${manifest_name}.yaml" + output_manifest="${verified_output_dir}/${manifest_name}.yaml" + + [[ -f "$source_manifest" && ! -L "$source_manifest" && -s "$source_manifest" ]] || \ + fail "validated manifest is missing or unsafe: ${source_manifest}" + install -m 0600 -- "$source_manifest" "$output_manifest" + [[ -f "$output_manifest" && ! -L "$output_manifest" && -s "$output_manifest" ]] || \ + fail "verified manifest handoff failed: ${output_manifest}" + cmp --silent -- "$source_manifest" "$output_manifest" || \ + fail "verified manifest changed during handoff: ${manifest_name}.yaml" + done + verified_entry_count="$( + find "$verified_output_dir" -mindepth 1 -maxdepth 1 | + wc -l | tr -d '[:space:]' + )" + [[ "$verified_entry_count" == "${#RENDERED_MANIFEST_NAMES[@]}" ]] || \ + fail "verified output directory does not contain exactly eight manifest files" + printf 'Preserved eight verified manifests for the AIStor apply handoff.\n' +fi + +printf 'Phase 2 rendering and source invariants passed.\n' +printf 'No live ObjectStore CRD or Kubernetes cluster access was required.\n' +printf 'Temporary Phase 2 rendered manifests and generated chart caches will be removed on exit.\n' diff --git a/scripts/validate/scan-platform-sensitive-source.sh b/scripts/validate/scan-platform-sensitive-source.sh new file mode 100644 index 0000000..ca433fd --- /dev/null +++ b/scripts/validate/scan-platform-sensitive-source.sh @@ -0,0 +1,1725 @@ +#!/usr/bin/env bash + +set -o pipefail + +readonly PLATFORM_SENSITIVE_SOURCE_EXACT_PLATFORM_ROOT=/home/donghyeon/workspace/platform +readonly PLATFORM_SENSITIVE_SOURCE_EXACT_DOCS_ROOT=/home/donghyeon/workspace/docs/platform +readonly PLATFORM_SENSITIVE_SOURCE_EXACT_SCANNER="$PLATFORM_SENSITIVE_SOURCE_EXACT_PLATFORM_ROOT/scripts/validate/scan-platform-sensitive-source.sh" + +_sensitive_source_fail() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +_sensitive_source_usage() { + cat <<'USAGE' +Usage: + bash scripts/validate/scan-platform-sensitive-source.sh + +Scans the complete platform source and central platform documentation trees. +Only .git and .helm directory contents, plus the exact platform-root +.superpowers/sdd scratch subtree, are excluded. Diagnostics contain file names +and finding classes only; matched values and lines are never printed. + +The production run invokes all five renderer entrypoints in private temporary +contexts. Four output-producing renderers publish private handoffs. The access +renderer uses its output-free Grafana verifier because it does not require +historical metric inventories; its exact two-line success contract is checked. +It still traverses the pinned core renderer, whose separately published handoff +contains the only allowlisted credential-free Secret and is decoded here. +USAGE +} + +_sensitive_source_is_test_file() { + case "$(basename -- "$1")" in + test-*.sh|test_*.sh|test-*.py) + return 0 + ;; + esac + return 1 +} + +_sensitive_source_report_path() { + local path=$1 platform_root=$2 docs_root=$3 relative + if [[ "$path" == "$platform_root"/* ]]; then + relative="platform/${path#"$platform_root"/}" + elif [[ "$path" == "$docs_root"/* ]]; then + relative="docs/${path#"$docs_root"/}" + else + relative="$(basename -- "$path")" + fi + # Preserve filename boundaries without allowing embedded control characters + # to create fake diagnostic records. + printf '%q' "$relative" +} + +_sensitive_source_rg_matches() { + local pattern=$1 file=$2 rc + rg --quiet --no-messages -P -- "$pattern" "$file" + rc=$? + case "$rc" in + 0) return 0 ;; + 1) return 1 ;; + *) return "$rc" ;; + esac +} + +_sensitive_source_python_scan() { + local file=$1 test_file=$2 scanner_source=$3 + python3 - "$file" "$test_file" "$scanner_source" <<'PY' +import ast +import io +import re +import sys +import tokenize +from pathlib import Path + +path = sys.argv[1] +test_file = sys.argv[2] == "1" +scanner_source = sys.argv[3] +suffix = Path(path).suffix.lower() +structured_source = suffix in (".yaml", ".yml", ".json") +python_source = suffix == ".py" +shell_source = suffix in (".sh", ".bash", ".zsh") or not suffix +prose_source = suffix in (".md", ".markdown", ".txt", ".rst") +embedded_python_source = path in ( + "/home/donghyeon/workspace/platform/scripts/validate/observability-smoke.sh", + "/home/donghyeon/workspace/platform/scripts/validate/test-scan-platform-sensitive-source.sh", + "/home/donghyeon/workspace/platform/scripts/validate/test-configure-keycloak-grafana-oidc.sh", + "/home/donghyeon/workspace/platform/scripts/bootstrap/create-observability-secrets.sh", + "/home/donghyeon/workspace/platform/scripts/bootstrap/configure-keycloak-grafana-oidc.sh", +) + +try: + raw = open(path, "rb").read() +except OSError: + raise SystemExit(2) + +if python_source: + try: + encoding, _ = tokenize.detect_encoding(io.BytesIO(raw).readline) + text = raw.decode(encoding) + except (SyntaxError, LookupError, UnicodeDecodeError): + raise SystemExit(2) +else: + if b"\0" in raw: + raise SystemExit(0) + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + raise SystemExit(0) + +url = "https://hooks." + "slack.com/services/" +slack = re.compile(re.escape(url) + r"[^/\s'\"]+/[^/\s'\"]+/[^/\s'\"]+") +credential = re.compile( + r"(?i)(?:^|[^A-Za-z0-9_])(?Pcloudflare[^\r\n:=]{0,24}token|" + r"client[_-]?secret|password|passwd|access[_-]?key|secret[_-]?key|webhook)" + r"[\"']?[ \t]*[:=][ \t]*(?P[^\r\n#]+)" +) +synthetic = re.compile( + r"(?:fixture|synthetic|do-not-leak|should[-_ ]?never|example|t123|b456|s789|" + r"temp[-_ ]?admin|^grafanapassword-[0-9]+$)", + re.I, +) +status_value = re.compile( + r"^(?:CREATE_CONFIRMED|REUSED_UNCHANGED|PASS|FAIL|ABSENT|PRESENT|REQUIRED|BLOCKED)$" +) +unfinished = ("TO" + "DO", "TB" + "D", "나중에" + "채움", "UNRESOLVED_" + "PLACEHOLDER") +definition_line = ( + 'unfinished = ("TO" + "DO", "TB" + "D", "나중에" + "채움", ' + '"UNRESOLVED_" + "PLACEHOLDER")' +) + +def test_context_allowed(value): + return test_file and synthetic.search(value) is not None + +def first_rhs(raw): + value = raw.lstrip() + if not value: + return "" + if value[0] in "\"'": + quote = value[0] + escaped = False + for index, character in enumerate(value[1:], start=1): + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == quote: + trailing = value[index + 1 :].strip() + if trailing and trailing not in (",", ")", ";", ");", "}", "},"): + return value + return value[1:index].strip() + return value + result = [] + stack = [] + quote = None + escaped = False + pairs = {")": "(", "]": "[", "}": "{"} + for character in value: + if quote is not None: + result.append(character) + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == quote: + quote = None + continue + if character in "\"'": + quote = character + result.append(character) + elif character in "([{": + stack.append(character) + result.append(character) + elif character in ")]}": + if stack and stack[-1] == pairs[character]: + stack.pop() + result.append(character) + else: + break + elif not stack and character == "#": + break + else: + result.append(character) + return "".join(result).strip().rstrip("\\\"'").strip() + +shell_reference = re.compile( + r"(?:\$[A-Za-z_][A-Za-z0-9_]*|\$\{[^{}\r\n]+\}|\$\([^\r\n]+\))" +) +base64_expression = re.compile( + r"base64\.b64encode\(" + r"(?:b?(?:\"[^\"\r\n]*\"|'[^'\r\n]*')|" + r"[A-Za-z_][A-Za-z0-9_]*(?:\.encode\((?:\"(?:ascii|utf-8)\"|'(?:ascii|utf-8)')?\))?)" + r"\)\.decode\((?:\"ascii\"|'ascii')?\)" +) +pathlib_read_expression = re.compile( + r"pathlib\.Path\([A-Za-z_][A-Za-z0-9_]*\)\.read_text\(\)" +) +explicit_placeholder = re.compile( + r"^<(?:(?:redacted)|(?:non-credential-placeholder)|(?:runtime-only))>$", re.I +) + +def exact_runtime_reference(value): + if explicit_placeholder.fullmatch(value): + return True + if shell_reference.fullmatch(value): + return True + match = shell_reference.match(value) + return bool(match and re.fullmatch(r"(?:/[A-Za-z0-9._-]+)+", value[match.end():])) + +def exact_python_runtime_expression(value): + if not (python_source or embedded_python_source): + return False + string_literals = re.findall(r"(?i)(?:[bruf]{0,2})([\"'])(.*?)\1", value) + for _, literal in string_literals: + literal = literal.strip() + if len(literal) >= 12 and not test_context_allowed(literal): + return False + expression_shape = re.fullmatch(r"[A-Za-z0-9_.'\"(), -]+", value) + if expression_shape is None: + return False + return ( + base64_expression.fullmatch(value) is not None + or pathlib_read_expression.fullmatch(value) is not None + ) + +def exact_python_mapping_reference(line, match, value): + if not (python_source or embedded_python_source): + return False + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value) is None: + return False + before = line[:match.start("rhs")] + return ( + "{" in before + and re.search(r"[\"'][^\"']+[\"']\s*:\s*$", before) is not None + and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*\s*[,}]", match.group("rhs").strip()) + is not None + ) + +def exact_python_nonliteral_reference(line, match, value): + if python_source or embedded_python_source: + before_rhs = line[:match.start("rhs")] + typed_parameter = re.fullmatch( + r"[A-Za-z_][A-Za-z0-9_.]*" + r"(?:\s*\|\s*(?:[A-Za-z_][A-Za-z0-9_.]*|None))*" + r"(?:\s*=\s*None)?\s*,?", + value, + ) + if ( + re.fullmatch( + r"\s*[A-Za-z_][A-Za-z0-9_]*[\"']?\s*:\s*", + before_rhs, + ) + and typed_parameter is not None + ): + return True + + if not python_source: + return False + + if re.fullmatch( + r"runtime\s*/\s*[\"']input-webhook[\"']\s*[,;)]?", + match.group("rhs").strip(), + ): + return True + + try: + expression = ast.parse(value, mode="eval").body + except SyntaxError: + return False + + def literal_free_call(node): + if isinstance(node, ast.Name): + return isinstance(node.ctx, ast.Load) + if isinstance(node, ast.Attribute): + return isinstance(node.ctx, ast.Load) and literal_free_call(node.value) + if isinstance(node, ast.Call): + return ( + not node.keywords + and literal_free_call(node.func) + and all( + not isinstance(argument, ast.Starred) + and literal_free_call(argument) + for argument in node.args + ) + ) + return False + + return isinstance(expression, ast.Call) and literal_free_call(expression) + +python_string_verdicts = {} +if python_source: + try: + python_tokens = list(tokenize.generate_tokens(io.StringIO(text).readline)) + python_tree = ast.parse(text) + except (SyntaxError, UnicodeDecodeError, tokenize.TokenError): + raise SystemExit(2) + + python_lines = text.splitlines(keepends=True) + python_parents = {} + for python_parent in ast.walk(python_tree): + for python_child in ast.iter_child_nodes(python_parent): + python_parents[id(python_child)] = python_parent + + def token_contains(line_number, column, token): + return ( + (line_number, column) >= token.start + and (line_number, column) < token.end + ) + + def ast_contains(line_number, byte_column, node): + return ( + (line_number, byte_column) >= (node.lineno, node.col_offset) + and (line_number, byte_column) < (node.end_lineno, node.end_col_offset) + ) + + def token_raw_offset(token, line_number, column): + if line_number == token.start[0]: + return column - token.start[1] + offset = len(python_lines[token.start[0] - 1][token.start[1] :]) + for source_line in range(token.start[0], line_number - 1): + offset += len(python_lines[source_line]) + return offset + column + + def decoded_constant(value): + if isinstance(value, bytes): + return value.decode("utf-8") + if isinstance(value, str): + return value + raise ValueError("non-string constant") + + def python_string_verdict(line_number, line, match): + key_column = match.start("key") + token = next( + ( + candidate + for candidate in python_tokens + if candidate.type == tokenize.STRING + and token_contains(line_number, key_column, candidate) + ), + None, + ) + if token is None: + return None + string_open = re.match(r"(?i)[bruf]{0,3}(\"\"\"|'''|[\"'])", token.string) + if string_open is None or not token.string.endswith(string_open.group(1)): + return "unknown" + raw_key_start = token_raw_offset(token, line_number, key_column) + raw_end = len(token.string) - len(string_open.group(1)) + if not (string_open.end() <= raw_key_start < raw_end): + return "unknown" + key_byte_column = len(line[:key_column].encode("utf-8")) + constants = [ + node + for node in ast.walk(python_tree) + if isinstance(node, ast.Constant) + and isinstance(node.value, (str, bytes)) + and ast_contains(line_number, key_byte_column, node) + ] + if not constants: + return "unknown" + constant = min( + constants, + key=lambda node: ( + node.end_lineno - node.lineno, + node.end_col_offset - node.col_offset, + ), + ) + own_rhs = token.string[raw_key_start:raw_end] + if re.fullmatch( + re.escape(match.group("key")) + r"[\"']?[ \t]*:[ \t]*", + own_rhs, + re.I, + ) is None: + mapping = python_parents.get(id(constant)) + if isinstance(mapping, ast.Dict): + for index, mapping_key in enumerate(mapping.keys): + if mapping_key is not constant: + continue + mapping_value = mapping.values[index] + if not any( + isinstance(leaf, ast.Constant) + and isinstance(leaf.value, (str, bytes)) + and decoded_constant(leaf.value) + for leaf in ast.walk(mapping_value) + ): + return "safe" + return "unsafe" + return "unsafe" + try: + prompt = decoded_constant(constant.value) + except UnicodeDecodeError: + return "unknown" + if re.search( + re.escape(match.group("key")) + r"[\"']?[ \t]*:[ \t]*$", + prompt, + re.I, + ) is None: + return "unsafe" + expression = constant + while isinstance(python_parents.get(id(expression)), ast.expr): + expression = python_parents[id(expression)] + nonempty_constants = 0 + try: + for leaf in ast.walk(expression): + if ( + isinstance(leaf, ast.Constant) + and isinstance(leaf.value, (str, bytes)) + and decoded_constant(leaf.value) + ): + nonempty_constants += 1 + except (UnicodeDecodeError, ValueError): + return "unknown" + return "safe" if nonempty_constants == 1 else "unsafe" + + for python_line_number, python_line in enumerate(text.splitlines(), start=1): + for python_match in credential.finditer(python_line): + verdict = python_string_verdict( + python_line_number, python_line, python_match + ) + if verdict is not None: + python_string_verdicts[ + (python_line_number, python_match.start("key")) + ] = verdict + if verdict == "unknown": + raise SystemExit(2) + +def exact_shell_argument_reference(line, match): + if not shell_source: + return None + prefix = line[:match.start()] + if "--from-file=" not in prefix: + return None + rhs = match.group("rhs").lstrip() + closing_quote = re.search(r"[\"']", rhs) + if closing_quote is None: + return None + token_value = rhs[:closing_quote.start()].strip() + if not exact_runtime_reference(token_value): + return None + return match.start("rhs") + closing_quote.end() + +for line in text.splitlines(): + if line == definition_line: + if path == scanner_source: + continue + print("UNFINISHED_MARKER") + break + hits = [marker for marker in unfinished if marker in line] + if hits: + print("UNFINISHED_MARKER") + break + +for line in text.splitlines(): + matches = list(slack.finditer(line)) + if matches and not all(test_context_allowed(match.group(0)) for match in matches): + print("SLACK_WEBHOOK") + break + +for line_number, line in enumerate(text.splitlines(), start=1): + offset = 0 + while offset < len(line): + match = credential.search(line, offset) + if match is None: + break + next_offset = max(match.start() + 1, match.end()) + python_string_verdict = python_string_verdicts.get( + (line_number, match.start("key")) + ) + if python_string_verdict == "safe": + offset = next_offset + continue + if python_string_verdict == "unsafe": + print("CREDENTIAL_LITERAL") + raise SystemExit(0) + if python_string_verdict == "unknown": + raise SystemExit(2) + prefix = line[:match.start()] + statement_start = max(prefix.rfind(";"), prefix.rfind("&&"), prefix.rfind("||")) + 1 + statement_prefix = prefix[statement_start:] + regex_literal_context = ( + not statement_prefix.strip() + and "[[:space:]]" in match.group(0) + and ("|" in match.group(0) or "`" in match.group(0) or "[^" in match.group(0)) + ) + detector_context = ( + re.search(r"(?i)(?:rg|grep|check_pattern|regex|pattern)", statement_prefix) + and ("[[:" in match.group(0) or "\\" in match.group(0) or "|" in match.group(0)) + ) + quoted_regex_context = ( + re.search(r"[\"'][^\"']*$", statement_prefix) + and re.search(r"[\"'](?:\s*\\)?\s*$", match.group("rhs")) + and ("[[:" in match.group(0) or "\\" in match.group(0) or "|" in match.group(0)) + ) + if regex_literal_context or detector_context or quoted_regex_context: + delimiter = re.search(r"(?:;|&&|\|\|)", match.group("rhs")) + quoted_end = re.search(r"[\"']", match.group("rhs")) + if delimiter is not None: + offset = match.start("rhs") + delimiter.end() + elif quoted_end is not None: + offset = match.start("rhs") + quoted_end.end() + else: + break + continue + value = first_rhs(match.group("rhs")) + value = re.sub(r"\\[nrt]$", "", value) + if not value or exact_runtime_reference(value): + offset = next_offset + continue + if exact_python_runtime_expression(value): + offset = next_offset + continue + if exact_python_nonliteral_reference(line, match, value): + offset = next_offset + continue + if exact_python_mapping_reference(line, match, value): + offset = next_offset + continue + shell_argument_end = exact_shell_argument_reference(line, match) + if shell_argument_end is not None: + offset = shell_argument_end + continue + line_starts_with_assignment = re.match( + r"^[ \t]*(?:cloudflare[^:=]{0,24}token|client[_-]?secret|password|passwd|" + r"access[_-]?key|secret[_-]?key|webhook)[\"']?[ \t]*[:=]", + line, + re.I, + ) is not None + if ( + prose_source + and not line_starts_with_assignment + and ":" in match.group(0) + and re.match(r"^[ \t]*(?:[-*+]\s+|\d+[.)]\s+|[^:=`]*\s+)", line) + ): + offset = next_offset + continue + if len(value) < 12: + offset = next_offset + continue + if status_value.fullmatch(value): + offset = next_offset + continue + if test_context_allowed(value): + offset = next_offset + continue + print("CREDENTIAL_LITERAL") + raise SystemExit(0) +PY +} + +_sensitive_source_yaml_secret_scan() { + local file=$1 + python3 - "$file" <<'PY' +import sys + +try: + import yaml + from yaml.constructor import ConstructorError + from yaml.nodes import MappingNode + from yaml.resolver import BaseResolver +except Exception: + raise SystemExit(2) + + +class StrictSourceLoader(yaml.BaseLoader): + pass + + +def construct_unique_mapping(loader, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, "mapping node required", node.start_mark) + result = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if not isinstance(key, str): + raise ConstructorError(None, None, "scalar mapping key required", key_node.start_mark) + if key in result: + raise ConstructorError(None, None, "duplicate mapping key", key_node.start_mark) + result[key] = loader.construct_object(value_node, deep=deep) + return result + + +StrictSourceLoader.add_constructor(BaseResolver.DEFAULT_MAPPING_TAG, construct_unique_mapping) + +path = sys.argv[1] +try: + with open(path, encoding="utf-8") as stream: + for item in yaml.load_all(stream, Loader=StrictSourceLoader): + if item is None or not isinstance(item, dict) or item.get("kind") != "Secret": + continue + data = item.get("data") or {} + string_data = item.get("stringData") or {} + if not isinstance(data, dict) or not isinstance(string_data, dict): + print("SECRET_SOURCE_PAYLOAD", flush=True) + continue + if data or string_data: + print("SECRET_SOURCE_PAYLOAD", flush=True) +except (OSError, UnicodeError, yaml.YAMLError): + print("SECRET_SOURCE_MALFORMED", flush=True) +PY +} + +_sensitive_source_structured_credential_scan() { + local file=$1 test_file=$2 + python3 - "$file" "$test_file" <<'PY' +import json +import re +import sys +from pathlib import Path + +try: + import yaml + from yaml.constructor import ConstructorError + from yaml.nodes import MappingNode + from yaml.resolver import BaseResolver +except Exception: + raise SystemExit(2) + + +class StrictCredentialLoader(yaml.BaseLoader): + pass + + +def construct_unique_mapping(loader, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, "mapping node required", node.start_mark) + result = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if not isinstance(key, str): + raise ConstructorError(None, None, "scalar mapping key required", key_node.start_mark) + if key in result: + raise ConstructorError(None, None, "duplicate mapping key", key_node.start_mark) + result[key] = loader.construct_object(value_node, deep=deep) + return result + + +StrictCredentialLoader.add_constructor( + BaseResolver.DEFAULT_MAPPING_TAG, construct_unique_mapping +) + +path = Path(sys.argv[1]) +test_file = sys.argv[2] == "1" +credential_segments = ( + ("password",), + ("passwd",), + ("client", "secret"), + ("access", "key"), + ("secret", "key"), + ("webhook",), +) +synthetic = re.compile( + r"(?:fixture|synthetic|do-not-leak|should[-_ ]?never|example|t123|b456|s789|" + r"temp[-_ ]?admin|^grafanapassword-[0-9]+$)", + re.I, +) +status_value = re.compile( + r"^(?:CREATE_CONFIRMED|REUSED_UNCHANGED|PASS|FAIL|ABSENT|PRESENT|REQUIRED|BLOCKED)$" +) +runtime_reference = re.compile( + r"^(?:\$[A-Za-z_][A-Za-z0-9_]*|\$\{[^{}\r\n]+\}|\$\([^\r\n]+\)|" + r"<(?:(?:redacted)|(?:non-credential-placeholder)|(?:runtime-only))>)$", + re.I, +) + + +def unsafe_scalar(value): + if not isinstance(value, str): + return False + value = value.strip() + return ( + len(value) >= 12 + and runtime_reference.fullmatch(value) is None + and status_value.fullmatch(value) is None + and not (test_file and synthetic.search(value)) + ) + + +def normalized_key(value): + separated = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", value) + separated = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", separated) + return tuple( + segment + for segment in separated.lower() + .replace("-", "_") + .split("_") + if segment + ) + + +def is_credential_key(value): + segments = normalized_key(value) + if segments and segments[-1] == "token" and "cloudflare" in segments: + return True + return any( + len(segments) >= len(candidate) + and segments[-len(candidate) :] == candidate + for candidate in credential_segments + ) + + +def permitted_reference_container(key, value): + key_segments = normalized_key(key) + if not ( + len(key_segments) >= 4 + and key_segments[:2] == ("gf", "auth") + and key_segments[-2:] == ("client", "secret") + ): + return False + if not isinstance(value, dict) or set(value) != {"secretKeyRef"}: + return False + reference = value["secretKeyRef"] + return ( + isinstance(reference, dict) + and set(reference) == {"name", "key"} + and all(isinstance(reference[field], str) and reference[field] for field in reference) + ) + + +def permitted_webhook_feature(path, key, value): + return ( + tuple(str(part) for part in path) == ("operators", "object-store") + and key == "webhook" + and isinstance(value, dict) + and set(value) == {"enabled", "replicas"} + and str(value["enabled"]).lower() in ("true", "false") + and str(value["replicas"]).isdigit() + ) + + +def strict_pairs(pairs): + result = {} + exact = set() + normalized = set() + for key, value in pairs: + if not isinstance(key, str): + raise ValueError("non-string JSON key") + key_normalized = normalized_key(key) + if key in exact or key_normalized in normalized: + raise ValueError("duplicate JSON key") + exact.add(key) + normalized.add(key_normalized) + result[key] = value + return result + + +def walk(value, path=()): + if isinstance(value, dict): + for key, child in value.items(): + if isinstance(key, str) and is_credential_key(key): + if permitted_webhook_feature(path, key, child): + pass + elif permitted_reference_container(key, child): + pass + elif not isinstance(child, str) or unsafe_scalar(child): + return True + if walk(child, path + (key,)): + return True + elif isinstance(value, list): + return any(walk(child, path + (index,)) for index, child in enumerate(value)) + return False + + +try: + with path.open(encoding="utf-8") as stream: + if path.suffix.lower() == ".json": + documents = (json.load(stream, object_pairs_hook=strict_pairs),) + else: + documents = yaml.load_all(stream, Loader=StrictCredentialLoader) + for document in documents: + if walk(document): + print("CREDENTIAL_LITERAL", flush=True) + break +except (OSError, UnicodeError, ValueError, json.JSONDecodeError, yaml.YAMLError): + raise SystemExit(2) +PY +} + +_sensitive_source_rendered_secret_scan() { + local manifest=$1 label=${2:-any} artifact=${3:-any} + python3 - "$manifest" "$label" "$artifact" <<'PY' +import base64 +import binascii +import re +import sys + +try: + import yaml + from yaml.constructor import ConstructorError + from yaml.nodes import MappingNode + from yaml.resolver import BaseResolver +except Exception: + raise SystemExit(2) + + +class StrictBaseLoader(yaml.BaseLoader): + pass + + +def construct_unique_mapping(loader, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, "mapping node required", node.start_mark) + result = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if not isinstance(key, str): + raise ConstructorError(None, None, "scalar mapping key required", key_node.start_mark) + if key in result: + raise ConstructorError(None, None, "duplicate mapping key", key_node.start_mark) + result[key] = loader.construct_object(value_node, deep=deep) + return result + + +StrictBaseLoader.add_constructor(BaseResolver.DEFAULT_MAPPING_TAG, construct_unique_mapping) + +path = sys.argv[1] +label = sys.argv[2] +artifact = sys.argv[3] + +gitea_common = { + "gitea": frozenset(("assertions", "config_environment.sh")), + "gitea-init": frozenset( + ( + "configure_gitea.sh", + "configure_gpg_environment.sh", + "init_directory_structure.sh", + ) + ), +} +gitea_inline_baseline = frozenset( + ( + "_generals_", + "actions", + "cache", + "database", + "indexer", + "metrics", + "packages", + "queue", + "repository", + "security", + "server", + "service", + "session", + ) +) +gitea_inline_oidc = gitea_inline_baseline | {"oauth2_client"} + +slack_url = re.compile( + re.escape("https://hooks." + "slack.com/services/") + + r"[^/\s'\"]+/[^/\s'\"]+/[^/\s'\"]+" +) +private_key = re.compile( + "-----BEGIN (?:[A-Z0-9 ]+ )?PRI" + "VATE KEY-----|" + "-----BEGIN OPENSSH PRI" + "VATE KEY-----" +) +credential_assignment = re.compile( + r"(?i)(?:^|[^A-Za-z0-9_])" + r"(?:cloudflare[^\r\n:=]{0,24}token|client[_-]?secret|password|passwd|token|" + r"api[_-]?url|webhook|access[_-]?key|secret[_-]?key|authorization)" + r"[\"']?[ \t]*[:=][ \t]*([^\r\n#]+)" +) +safe_reference = re.compile( + r"^(?:\$[A-Za-z_][A-Za-z0-9_]*|\$\{[^{}\r\n]+\}|\$\([^\r\n]+\)|" + r"<(?:(?:redacted)|(?:non-credential-placeholder)|(?:runtime-only))>)$", + re.I, +) + + +def first_assignment_rhs(raw): + value = raw.strip() + if not value: + return "" + if value[0] in "\"'": + quote = value[0] + escaped = False + result = [] + for index, character in enumerate(value[1:], start=1): + if escaped: + result.append(character) + escaped = False + elif character == "\\": + result.append(character) + escaped = True + elif character == quote: + trailing = value[index + 1 :].strip() + if trailing and trailing not in (",", ")", ";", ");"): + return value + return "".join(result).strip() + else: + result.append(character) + return "".join(result).strip() + return value.rstrip(",;)\\").rstrip().rstrip("\"'").strip() + + +def contains_literal_credential(raw): + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + return True + if slack_url.search(text) or private_key.search(text): + return True + for line in text.splitlines(): + for match in credential_assignment.finditer(line): + value = first_assignment_rhs(match.group(1)) + if not value or safe_reference.fullmatch(value): + continue + if len(value) >= 12: + return True + return False + + +def decoded_values(data, string_data): + result = [] + for value in data.values(): + if not isinstance(value, str): + raise ValueError("non-scalar data value") + try: + result.append(base64.b64decode(value, validate=True)) + except (binascii.Error, ValueError, TypeError): + raise ValueError("invalid base64") + for value in string_data.values(): + if not isinstance(value, str): + raise ValueError("non-scalar stringData value") + result.append(value.encode("utf-8")) + return result + + +def allowlisted_secret(metadata, secret_type, data, string_data): + namespace = metadata.get("namespace") + name = metadata.get("name") + if namespace == "observability" and name == ( + "alertmanager-observability-core-kube-pr-alertmanager" + ): + return ( + secret_type in (None, "Opaque") + and set(data) == {"alertmanager.yaml"} + and not string_data + ) + if namespace != "gitea" or secret_type != "Opaque" or data: + return False + keys = frozenset(string_data) + if name in gitea_common: + return keys == gitea_common[name] + if name == "gitea-inline-config": + if label == "phase1" and artifact == "gitea": + return keys == gitea_inline_baseline + if label == "phase1" and artifact == "gitea-oidc": + return keys == gitea_inline_oidc + return keys in (gitea_inline_baseline, gitea_inline_oidc) + return False + + +def expected_secret_identities(): + if label == "phase1" and artifact == "gitea": + return { + ("gitea", "gitea"), + ("gitea", "gitea-init"), + ("gitea", "gitea-inline-config"), + } + if label == "phase1" and artifact == "gitea-oidc": + return { + ("gitea", "gitea"), + ("gitea", "gitea-init"), + ("gitea", "gitea-inline-config"), + } + if label == "observability-core" and artifact in ("kps", "aggregate"): + return { + ("observability", "alertmanager-observability-core-kube-pr-alertmanager"), + } + return set() + + +try: + identities = [] + with open(path, encoding="utf-8") as stream: + for item in yaml.load_all(stream, Loader=StrictBaseLoader): + if item is None: + continue + if not isinstance(item, dict) or item.get("kind") != "Secret": + continue + metadata = item.get("metadata") or {} + data = item.get("data") or {} + string_data = item.get("stringData") or {} + if not isinstance(metadata, dict) or not isinstance(data, dict) or not isinstance( + string_data, dict + ): + print("RENDERED_SECRET_MALFORMED") + continue + if not allowlisted_secret(metadata, item.get("type"), data, string_data): + print("RENDERED_SECRET_NOT_ALLOWLISTED") + continue + identities.append((metadata.get("namespace"), metadata.get("name"))) + try: + values = decoded_values(data, string_data) + except ValueError: + print("RENDERED_SECRET_MALFORMED") + continue + if any(contains_literal_credential(value) for value in values): + print("RENDERED_SECRET_CREDENTIAL") + continue + print("RENDERED_SECRET_ALLOWLISTED") + if label != "any": + expected = expected_secret_identities() + if len(identities) != len(expected) or set(identities) != expected: + print("RENDERED_REQUIRED_SECRET_MISSING") +except (OSError, UnicodeError, yaml.YAMLError): + raise SystemExit(2) +PY +} + +_sensitive_source_render_results_are_safe() { + local results=$1 line + while IFS= read -r line; do + [[ -n "$line" ]] || continue + [[ "$line" == RENDERED_SECRET_ALLOWLISTED ]] || return 1 + done <<<"$results" +} + +_sensitive_source_access_log_is_safe() { + _sensitive_source_renderer_log_is_safe observability-access "$1" +} + +_sensitive_source_renderer_log_is_safe() { + local label=$1 log=$2 + python3 - "$label" "$log" <<'PY' +import os +import re +import stat +import sys + +label, path = sys.argv[1:] + + +def exact(value): + return re.compile(re.escape(value)) + + +def rendered(name, width=None, digest=False, korean=False): + prefix = "렌더" if korean else "Rendered" + suffix = r" bytes SHA-256 [0-9a-f]{64}" if digest else r" bytes" + return re.compile(re.escape(prefix) + r" +" + re.escape(name) + r" +[0-9]+" + suffix) + + +def verified(name, korean=False): + prefix = "검증" if korean else "Verified" + return re.compile(re.escape(prefix) + r" +" + re.escape(name) + r" +SHA-256 +[0-9a-f]{64}") + + +def phase1(published): + result = [ + rendered("namespaces"), + rendered("ssd-local-pv"), + verified("cloudnative-pg-chart"), + rendered("cnpg-operator"), + rendered("platform-postgres"), + verified("gitea-chart"), + rendered("gitea"), + rendered("gitea-oidc"), + ] + if published: + result.append(exact("Preserved six verified manifests for the apply handoff.")) + result.extend( + ( + exact("Phase 1 baseline and Gitea OIDC desired rendering invariants passed."), + exact("Temporary rendered manifests and generated chart caches will be removed on exit."), + ) + ) + return result + + +patterns = {} +patterns["phase1"] = phase1(True) +patterns["phase2"] = [ + exact("Validating the Phase 1 baseline and Gitea OIDC desired profile first."), + *phase1(False), + *(rendered(name) for name in ( + "phase2-namespaces", + "aistor-local-pv", + "aistor-network-policies", + "keycloak-operator", + "platform-postgres-keycloak", + "keycloak", + )), + verified("aistor-operator-chart"), + rendered("aistor-operator"), + verified("aistor-objectstore-chart"), + rendered("minio-aistor"), + exact("Preserved eight verified manifests for the AIStor apply handoff."), + exact("Phase 2 rendering and source invariants passed."), + exact("No live ObjectStore CRD or Kubernetes cluster access was required."), + exact("Temporary Phase 2 rendered manifests and generated chart caches will be removed on exit."), +] +patterns["admin"] = [ + exact("Pulled: docker.io/dpage/pgadmin4-helm:9.16.0"), + re.compile(r"Digest: sha256:[0-9a-f]{64}"), + verified("pgadmin", korean=True), + verified("aistor-objectstore", korean=True), + *(rendered(name, digest=True, korean=True) for name in ( + "admin-namespace", + "pgadmin-local-pv", + "coredns-custom", + "aistor-admin-oidc", + "pgadmin", + )), + exact("ADMIN SERVICES STATIC RENDER PASS"), +] +patterns["observability-core"] = [ + *(verified(name) for name in ( + "prometheus-operator-crds", + "kube-prometheus-stack", + "loki", + "tempo", + "alloy", + "prometheus-node-exporter", + "grafana", + "prometheus-blackbox-exporter", + )), + *(rendered(name, digest=True) for name in ( + "namespaces", + "crds", + "storage", + "kps", + "loki", + "tempo", + "alloy", + "node-exporter", + "grafana", + "blackbox", + "core-policies", + "targets", + "core-rules", + "alerting", + "dashboards", + "aggregate", + )), + exact("OBSERVABILITY CORE STATIC RENDER PASS"), +] +patterns["observability-access"] = [ + *patterns["observability-core"], + exact("OBSERVABILITY ACCESS STATIC RENDER PASS"), +] + +try: + metadata = os.lstat(path) + if ( + not stat.S_ISREG(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != os.getuid() + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + ): + raise ValueError("unsafe log") + with open(path, "rb") as stream: + raw = stream.read(65537) + if len(raw) > 65536 or not raw or b"\0" in raw or b"\r" in raw or not raw.endswith(b"\n"): + raise ValueError("unsafe log bytes") + lines = raw.decode("utf-8").splitlines() + expected = patterns[label] + if len(lines) != len(expected): + raise ValueError("renderer log line count differs") + if any(pattern.fullmatch(line) is None for pattern, line in zip(expected, lines)): + raise ValueError("renderer log grammar differs") +except (KeyError, OSError, UnicodeError, ValueError): + raise SystemExit(1) +PY +} + +_sensitive_source_validate_handoff_entries() { + local label=$1 directory=$2 + shift 2 + python3 - "$label" "$directory" "$@" <<'PY' +import os +import stat +import sys + +label, directory, *expected = sys.argv[1:] +try: + metadata = os.lstat(directory) + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != os.getuid() + or stat.S_IMODE(metadata.st_mode) != 0o700 + ): + raise ValueError("unsafe handoff directory") + entries = os.listdir(directory) + if len(entries) != len(expected) or set(entries) != set(expected): + raise ValueError("handoff entry set differs") + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + for name in expected: + item = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if ( + not stat.S_ISREG(item.st_mode) + or item.st_uid != os.getuid() + or stat.S_IMODE(item.st_mode) != 0o600 + or item.st_nlink != 1 + ): + raise ValueError("unsafe handoff entry") + finally: + os.close(descriptor) +except (OSError, ValueError): + raise SystemExit(1) +PY +} + +_sensitive_source_validate_core_index() { + local directory=$1 + python3 - "$directory" <<'PY' +import hashlib +import json +import os +import re +import sys + +try: + import yaml + from yaml.constructor import ConstructorError + from yaml.nodes import MappingNode + from yaml.resolver import BaseResolver +except Exception: + raise SystemExit(2) + + +class StrictIndexLoader(yaml.SafeLoader): + pass + + +def construct_unique_mapping(loader, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, "mapping node required", node.start_mark) + result = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if key in result: + raise ConstructorError(None, None, "duplicate mapping key", key_node.start_mark) + result[key] = loader.construct_object(value_node, deep=deep) + return result + + +StrictIndexLoader.add_constructor(BaseResolver.DEFAULT_MAPPING_TAG, construct_unique_mapping) +StrictIndexLoader.add_constructor( + "tag:yaml.org,2002:value", lambda loader, node: loader.construct_scalar(node) +) + +artifacts = ( + "namespaces", + "crds", + "storage", + "kps", + "loki", + "tempo", + "alloy", + "node-exporter", + "grafana", + "blackbox", + "core-policies", + "targets", + "core-rules", + "alerting", + "dashboards", +) +header = ( + "order", + "artifact", + "apiVersion", + "kind", + "namespace", + "name", + "canonicalSha256", +) + + +def expected_rows(directory): + rows = [] + identities = set() + for artifact in artifacts: + path = os.path.join(directory, artifact + ".yaml") + with open(path, encoding="utf-8") as stream: + for document in yaml.load_all(stream, Loader=StrictIndexLoader): + if document is None: + continue + if not isinstance(document, dict): + raise ValueError("non-object manifest") + metadata = document.get("metadata") or {} + if not isinstance(metadata, dict): + raise ValueError("malformed metadata") + identity = ( + str(document.get("apiVersion", "")), + str(document.get("kind", "")), + str(metadata.get("namespace", "")), + str(metadata.get("name", "")), + ) + if not identity[0] or not identity[1] or not identity[3] or identity in identities: + raise ValueError("invalid resource identity") + identities.add(identity) + canonical = json.dumps( + document, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + rows.append((str(len(rows) + 1), artifact, *identity, digest)) + return rows + + +def actual_rows(directory): + path = os.path.join(directory, "resource-index.tsv") + with open(path, "rb") as stream: + raw = stream.read() + if not raw or b"\0" in raw or not raw.endswith(b"\n"): + raise ValueError("unsafe index bytes") + text = raw.decode("utf-8") + lines = text.splitlines() + if not lines or tuple(lines[0].split("\t")) != header: + raise ValueError("index header differs") + rows = [] + identities = set() + for position, line in enumerate(lines[1:], 1): + fields = tuple(line.split("\t")) + if len(fields) != 7 or fields[0] != str(position): + raise ValueError("index shape differs") + if ( + fields[1] not in artifacts + or not fields[2] + or not fields[3] + or not fields[5] + or re.fullmatch(r"[0-9a-f]{64}", fields[6]) is None + ): + raise ValueError("index field differs") + identity = fields[2:6] + if identity in identities: + raise ValueError("duplicate index identity") + identities.add(identity) + rows.append(fields) + return rows + + +try: + directory = sys.argv[1] + if actual_rows(directory) != expected_rows(directory): + raise ValueError("canonical index mismatch") +except (OSError, UnicodeError, ValueError, yaml.YAMLError): + raise SystemExit(1) +PY +} + +scan_platform_sensitive_roots() ( + local platform_root=$1 docs_root=$2 + shift 2 + local argument rendered file test_file finding rc report + local -a rendered_manifests=() + local -a files=() + local failed=false + local list_file='' + + cleanup_sensitive_source_scan() { + case "$list_file" in + /tmp/platform-sensitive-source-files.??????) + rm -f -- "$list_file" + ;; + esac + } + trap cleanup_sensitive_source_scan EXIT HUP INT TERM + + while (( $# > 0 )); do + argument=$1 + case "$argument" in + --rendered-manifest) + (( $# >= 2 )) || return 2 + rendered_manifests+=("$2") + shift 2 + ;; + *) + return 2 + ;; + esac + done + + for argument in rg find mktemp python3 rm stat; do + command -v "$argument" >/dev/null 2>&1 || { + printf 'SCANNER_ERROR: required command unavailable: %s\n' "$argument" >&2 + return 1 + } + done + [[ "$platform_root" == /* && "$docs_root" == /* && "$platform_root" != "$docs_root" ]] || + _sensitive_source_fail 'scan roots must be distinct absolute paths' || return + for argument in "$platform_root" "$docs_root"; do + [[ -d "$argument" && ! -L "$argument" && "$(cd -- "$argument" && pwd -P)" == "$argument" ]] || + _sensitive_source_fail "scan root is not a physical directory: $argument" || return + done + + list_file="$(mktemp /tmp/platform-sensitive-source-files.XXXXXX)" || { + printf 'SCANNER_ERROR: file inventory allocation failed\n' >&2 + return 1 + } + chmod 0600 "$list_file" || { + printf 'SCANNER_ERROR: file inventory hardening failed\n' >&2 + return 1 + } + if ! find "$platform_root" "$docs_root" \ + \( -type d -path "$platform_root/.superpowers/sdd" -prune \) -o \ + \( -type d \( -name .git -o -name .helm \) -prune \) -o \ + \( -type f -o -type l \) -print0 >"$list_file"; then + printf 'SCANNER_ERROR: find failed\n' >&2 + return 1 + fi + while IFS= read -r -d '' file; do + files+=("$file") + done <"$list_file" + + for file in "${files[@]}"; do + report="$(_sensitive_source_report_path "$file" "$platform_root" "$docs_root")" + if [[ -L "$file" ]]; then + printf 'SENSITIVE_SOURCE_FINDING=SYMLINK FILE=%s\n' "$report" >&2 + failed=true + continue + fi + [[ -f "$file" ]] || continue + test_file=0 + _sensitive_source_is_test_file "$file" && test_file=1 + + private_key_pattern='-----BEGIN (?:[A-Z0-9 ]+ )?PRI''VATE KEY-----|-----BEGIN OPENSSH PRI''VATE KEY-----' + if _sensitive_source_rg_matches "$private_key_pattern" "$file"; then + printf 'SENSITIVE_SOURCE_FINDING=PRIVATE_KEY FILE=%s\n' "$report" >&2 + failed=true + else + rc=$? + if (( rc > 1 )); then + printf 'SCANNER_ERROR: rg failed for FILE=%s\n' "$report" >&2 + return 1 + fi + fi + + finding="$(_sensitive_source_python_scan \ + "$file" "$test_file" "$PLATFORM_SENSITIVE_SOURCE_EXACT_SCANNER")" || { + printf 'SCANNER_ERROR: content scan failed for FILE=%s\n' "$report" >&2 + return 1 + } + if [[ -n "$finding" ]]; then + printf 'SENSITIVE_SOURCE_FINDING=%s FILE=%s\n' "${finding%%$'\n'*}" "$report" >&2 + failed=true + fi + + case "${file,,}" in + *.yaml|*.yml|*.json) + finding="$(_sensitive_source_structured_credential_scan "$file" "$test_file")" || { + printf 'SCANNER_ERROR: structured credential scan failed for FILE=%s\n' "$report" >&2 + return 1 + } + if [[ -n "$finding" ]]; then + printf 'SENSITIVE_SOURCE_FINDING=%s FILE=%s\n' "${finding%%$'\n'*}" "$report" >&2 + failed=true + fi + ;; + esac + + case "${file,,}" in + *.yaml|*.yml) + finding="$(_sensitive_source_yaml_secret_scan "$file")" || { + printf 'SCANNER_ERROR: Secret source scan failed for FILE=%s\n' "$report" >&2 + return 1 + } + if [[ -n "$finding" ]] && ! _sensitive_source_render_results_are_safe "$finding"; then + printf 'SENSITIVE_SOURCE_FINDING=%s FILE=%s\n' "${finding%%$'\n'*}" "$report" >&2 + failed=true + fi + ;; + esac + + if [[ "$file" == "$platform_root/services/observability/"* ]]; then + future_target_pattern='(?i)(?:spring[ -]?boot|\bjvm\b|\bkafka\b|consumer[ _-]?lag|(?:target|monitor|dashboard|alert|rule)[^\r\n]{0,40}\b(?:batch|backup)\b|\b(?:batch|backup)\b[^\r\n]{0,40}(?:target|monitor|dashboard|alert|rule))' + if _sensitive_source_rg_matches "$future_target_pattern" "$file"; then + printf 'SENSITIVE_SOURCE_FINDING=FUTURE_OBSERVABILITY_TARGET FILE=%s\n' "$report" >&2 + failed=true + else + rc=$? + if (( rc > 1 )); then + printf 'SCANNER_ERROR: rg failed for FILE=%s\n' "$report" >&2 + return 1 + fi + fi + fi + done + + for file in "${rendered_manifests[@]}"; do + [[ "$file" == /* && -f "$file" && ! -L "$file" ]] || { + printf 'SCANNER_ERROR: rendered manifest is not a regular absolute file\n' >&2 + return 1 + } + report="$(_sensitive_source_report_path "$file" "$platform_root" "$docs_root")" + finding="$(_sensitive_source_rendered_secret_scan "$file")" || { + printf 'SCANNER_ERROR: rendered Secret scan failed for FILE=%s\n' "$report" >&2 + return 1 + } + if [[ -n "$finding" ]] && ! _sensitive_source_render_results_are_safe "$finding"; then + printf 'SENSITIVE_SOURCE_FINDING=%s FILE=%s\n' "${finding%%$'\n'*}" "$report" >&2 + failed=true + fi + done + + [[ "$failed" == false ]] || return 1 + printf 'PLATFORM SENSITIVE SOURCE SCAN PASS\n' +) + +scan_platform_renderer_secrets() ( + set -o pipefail + local platform_root=$1 renderer label output_prefix publishes log file results line rc entry artifact + local helm_bin list_file='' + local failed=false + local -a renderers=( + 'phase1|render-phase1.sh|platform-phase1-apply|yes' + 'phase2|render-phase2.sh|platform-phase2-apply|yes' + 'admin|render-admin-services.sh|platform-admin-apply|yes' + 'observability-core|render-observability-core.sh|platform-observability-core-apply|yes' + 'observability-access|render-observability-access.sh|-|no' + ) + local -a outputs=() + local -a logs=() + local -a expected_entries=() + local -a manifest_files=() + local -a manifest_labels=() + local -a manifest_artifacts=() + + cleanup_renderer_scan() { + local path + for path in "${outputs[@]}"; do + case "$path" in + /tmp/platform-phase1-apply.??????|\ + /tmp/platform-phase2-apply.??????|\ + /tmp/platform-admin-apply.??????|\ + /tmp/platform-observability-core-apply.??????|\ + /tmp/platform-observability-metrics.??????) + rm -rf -- "$path" + ;; + esac + done + for path in "${logs[@]}"; do + case "$path" in + /tmp/platform-sensitive-source-render-log.??????) + rm -f -- "$path" + ;; + esac + done + case "$list_file" in + /tmp/platform-sensitive-source-render-files.??????) + rm -f -- "$list_file" + ;; + esac + } + trap cleanup_renderer_scan EXIT HUP INT TERM + + (( $# == 1 )) || return 2 + for label in bash chmod find mktemp python3 rm timeout; do + command -v "$label" >/dev/null 2>&1 || { + printf 'SCANNER_ERROR: renderer command unavailable: %s\n' "$label" >&2 + return 1 + } + done + [[ "$platform_root" == /* && -d "$platform_root" && ! -L "$platform_root" && + "$(cd -- "$platform_root" && pwd -P)" == "$platform_root" ]] || { + printf 'SCANNER_ERROR: renderer root is not a physical absolute directory\n' >&2 + return 1 + } + helm_bin="$(command -v helm 2>/dev/null)" || { + printf 'SCANNER_ERROR: Helm is unavailable for renderer Secret validation\n' >&2 + return 1 + } + [[ "$helm_bin" == /* && -f "$helm_bin" && -x "$helm_bin" && ! -L "$helm_bin" ]] || { + printf 'SCANNER_ERROR: Helm path is unsafe\n' >&2 + return 1 + } + + for renderer in "${renderers[@]}"; do + IFS='|' read -r label renderer output_prefix publishes <<<"$renderer" + renderer="$platform_root/scripts/validate/$renderer" + [[ -f "$renderer" && ! -L "$renderer" ]] || { + printf 'SCANNER_ERROR: required renderer is unsafe: %s\n' "$label" >&2 + return 1 + } + output='' + if [[ "$publishes" == yes ]]; then + output="$(mktemp -d "/tmp/$output_prefix.XXXXXX")" || { + printf 'SCANNER_ERROR: renderer output allocation failed: %s\n' "$label" >&2 + return 1 + } + outputs+=("$output") + chmod 0700 "$output" || { + printf 'SCANNER_ERROR: renderer output hardening failed: %s\n' "$label" >&2 + return 1 + } + elif [[ "$publishes" != no || "$label" != observability-access ]]; then + printf 'SCANNER_ERROR: invalid renderer publication contract: %s\n' "$label" >&2 + return 1 + fi + log="$(mktemp /tmp/platform-sensitive-source-render-log.XXXXXX)" || { + printf 'SCANNER_ERROR: renderer log allocation failed: %s\n' "$label" >&2 + return 1 + } + logs+=("$log") + chmod 0600 "$log" || return 1 + + if [[ "$label" == observability-access ]]; then + PLATFORM_HELM_BIN="$helm_bin" timeout --signal=TERM --kill-after=5s 300s \ + bash "$renderer" --component grafana >"$log" 2>&1 || rc=$? + else + PLATFORM_HELM_BIN="$helm_bin" timeout --signal=TERM --kill-after=5s 300s \ + bash "$renderer" --verified-output-dir "$output" >"$log" 2>&1 || rc=$? + fi + if (( ${rc:-0} != 0 )); then + printf 'SCANNER_ERROR: renderer failed: %s RC=%s\n' "$label" "$rc" >&2 + return 1 + fi + unset rc + if [[ "$label" == observability-access ]]; then + _sensitive_source_access_log_is_safe "$log" || { + printf 'SCANNER_ERROR: unexpected observability-access output contract\n' >&2 + return 1 + } + else + _sensitive_source_renderer_log_is_safe "$label" "$log" || { + printf 'SCANNER_ERROR: unexpected renderer output contract: %s\n' "$label" >&2 + return 1 + } + fi + if [[ "$publishes" == yes ]]; then + expected_entries=() + case "$label" in + phase1) + expected_entries=( + namespaces.yaml ssd-local-pv.yaml cnpg-operator.yaml + platform-postgres.yaml gitea.yaml gitea-oidc.yaml + ) + ;; + phase2) + expected_entries=( + phase2-namespaces.yaml aistor-local-pv.yaml keycloak-operator.yaml + platform-postgres-keycloak.yaml keycloak.yaml aistor-operator.yaml + minio-aistor.yaml aistor-network-policies.yaml + ) + ;; + admin) + expected_entries=( + admin-namespace.yaml pgadmin-local-pv.yaml coredns-custom.yaml + aistor-admin-oidc.yaml pgadmin.yaml + ) + ;; + observability-core) + expected_entries=( + namespaces.yaml crds.yaml storage.yaml kps.yaml loki.yaml tempo.yaml + alloy.yaml node-exporter.yaml grafana.yaml blackbox.yaml + core-policies.yaml targets.yaml core-rules.yaml alerting.yaml + dashboards.yaml aggregate.yaml resource-index.tsv + ) + ;; + *) + printf 'SCANNER_ERROR: unknown renderer handoff contract: %s\n' "$label" >&2 + return 1 + ;; + esac + _sensitive_source_validate_handoff_entries "$label" "$output" "${expected_entries[@]}" || { + printf 'SCANNER_ERROR: renderer handoff entry contract failed: %s\n' "$label" >&2 + return 1 + } + if [[ "$label" == observability-core ]]; then + _sensitive_source_validate_core_index "$output" || { + printf 'SCANNER_ERROR: observability-core resource index contract failed\n' >&2 + return 1 + } + fi + for entry in "${expected_entries[@]}"; do + [[ "$entry" == *.yaml ]] || continue + manifest_files+=("$output/$entry") + manifest_labels+=("$label") + manifest_artifacts+=("${entry%.yaml}") + done + fi + printf 'SENSITIVE_SOURCE_RENDERER=%s PASS\n' "$label" + done + + for ((rc=0; rc < ${#manifest_files[@]}; rc++)); do + file=${manifest_files[$rc]} + label=${manifest_labels[$rc]} + artifact=${manifest_artifacts[$rc]} + if [[ -L "$file" || ! -f "$file" ]]; then + printf 'SENSITIVE_SOURCE_FINDING=RENDERED_SYMLINK FILE=%q\n' "$(basename -- "$file")" >&2 + failed=true + continue + fi + results="$(_sensitive_source_rendered_secret_scan "$file" "$label" "$artifact")" || { + printf 'SCANNER_ERROR: rendered Secret scan failed: FILE=%q\n' "$(basename -- "$file")" >&2 + return 1 + } + while IFS= read -r line; do + [[ -n "$line" ]] || continue + if [[ "$line" == RENDERED_SECRET_ALLOWLISTED ]]; then + continue + else + printf 'SENSITIVE_SOURCE_FINDING=%s FILE=%q\n' "$line" "$(basename -- "$file")" >&2 + failed=true + fi + done <<<"$results" + done + + [[ "$failed" == false ]] || return 1 + printf 'PLATFORM RENDERED SECRET SCAN PASS\n' +) + +scan_platform_sensitive_source_main() { + (( $# == 0 )) || { + _sensitive_source_usage >&2 + return 2 + } + scan_platform_sensitive_roots \ + "$PLATFORM_SENSITIVE_SOURCE_EXACT_PLATFORM_ROOT" \ + "$PLATFORM_SENSITIVE_SOURCE_EXACT_DOCS_ROOT" >/dev/null || return + scan_platform_renderer_secrets "$PLATFORM_SENSITIVE_SOURCE_EXACT_PLATFORM_ROOT" || return + printf 'PLATFORM SENSITIVE SOURCE SCAN PASS\n' +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + set -Eeuo pipefail + set +x + umask 077 + scan_platform_sensitive_source_main "$@" +fi diff --git a/scripts/validate/test-apply-host-nginx-observability.sh b/scripts/validate/test-apply-host-nginx-observability.sh new file mode 100755 index 0000000..837ea2b --- /dev/null +++ b/scripts/validate/test-apply-host-nginx-observability.sh @@ -0,0 +1,1181 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly APPLY="$ROOT/scripts/bootstrap/apply-host-nginx-observability.sh" +readonly DENY="$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf" +readonly FULL="$ROOT/infrastructure/networking/host-nginx/learn-services-observability.conf" +readonly ROLLBACK_ID=20260812T120000Z +WORK='' +ASSERTIONS=0 + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +pass() { + ASSERTIONS=$((ASSERTIONS + 1)) + printf 'PASS: %s\n' "$1" +} + +cleanup() { + local rc=$? + trap - EXIT HUP INT TERM + case "$WORK" in + /tmp/platform-host-nginx-test.*) rm -rf -- "$WORK" ;; + esac + exit "$rc" +} +trap cleanup EXIT HUP INT TERM + +make_fakes() { + local root=$1 + mkdir -m 0700 -p "$root/bin" + cat >"$root/bin/nginx" <<'SH' +#!/usr/bin/env bash +[[ "${PLATFORM_TEST_VIA_SUDO:-0}" == 1 ]] || { + printf 'nginx-without-sudo %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" + exit 91 +} +printf 'nginx %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" +exit 0 +SH + cat >"$root/bin/systemctl" <<'SH' +#!/usr/bin/env bash +printf 'systemctl %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" +if [[ "${1:-} ${2:-}" == 'reload nginx' ]]; then + count=0 + [[ ! -f "$PLATFORM_TEST_RELOAD_COUNTER" ]] || read -r count <"$PLATFORM_TEST_RELOAD_COUNTER" + count=$((count + 1)) + printf '%d\n' "$count" >"$PLATFORM_TEST_RELOAD_COUNTER" + if [[ "${PLATFORM_TEST_SYSTEMCTL_FAIL_RELOAD_AT:-0}" == "$count" ]]; then + exit 94 + fi +fi +exit 0 +SH + cat >"$root/bin/sudo" <<'SH' +#!/usr/bin/env bash +printf 'sudo %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" +if [[ "${1:-}" == -v ]]; then + : >"$PLATFORM_TEST_SUDO_REFRESHED" + exit 0 +fi +if [[ "${1:-}" == -n ]]; then shift; fi +(( $# == 0 )) && exit 0 +[[ -f "$PLATFORM_TEST_SUDO_REFRESHED" ]] || exit 92 +export PLATFORM_TEST_VIA_SUDO=1 +exec "$@" +SH + cat >"$root/bin/test" <<'SH' +#!/usr/bin/env bash +printf 'test %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" +exec /usr/bin/test "$@" +SH + cat >"$root/bin/install" <<'SH' +#!/usr/bin/env bash +args=() +while (( $# > 0 )); do + case "$1" in + -o|-g) shift 2 ;; + *) args+=("$1"); shift ;; + esac +done +if (( ${#args[@]} > 0 )); then + destination=${args[$((${#args[@]} - 1))]} + if [[ -n "${PLATFORM_TEST_INSTALL_FAIL_MATCH:-}" && + "${args[*]}" == *"$PLATFORM_TEST_INSTALL_FAIL_MATCH"* ]]; then + exit 93 + fi + if [[ -n "${PLATFORM_HOST_NGINX_ACTIVE:-}" && + "$destination" == "$PLATFORM_HOST_NGINX_ACTIVE" ]]; then + count=0 + [[ ! -f "$PLATFORM_TEST_ACTIVE_INSTALL_COUNTER" ]] || + read -r count <"$PLATFORM_TEST_ACTIVE_INSTALL_COUNTER" + count=$((count + 1)) + printf '%d\n' "$count" >"$PLATFORM_TEST_ACTIVE_INSTALL_COUNTER" + if [[ "${PLATFORM_TEST_FAIL_ACTIVE_INSTALL_AT:-0}" == "$count" ]]; then + exit 95 + fi + fi +fi +/usr/bin/install "${args[@]}" +rc=$? +if (( rc == 0 )) && [[ -n "${PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH:-}" && + "${args[*]}" == *"$PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH"* ]]; then + chmod 0644 "$destination" +fi +exit "$rc" +SH + cat >"$root/bin/openssl" <<'SH' +#!/usr/bin/env bash +printf 'openssl %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" +# This boundary deliberately reports a valid exact SAN and successful checkend. +# A missing privkey must therefore be rejected by our validator, not by OpenSSL. +if [[ " $* " == *' -ext subjectAltName '* ]]; then + printf 'X509v3 Subject Alternative Name:\n DNS:grafana.learn.hyeonworks.com\n' +fi +exit 0 +SH + for command in dig curl kubectl; do + cat >"$root/bin/$command" <<'SH' +#!/usr/bin/env bash +printf '%s %s\n' "${0##*/}" "$*" >>"$PLATFORM_TEST_COMMAND_LOG" +exit 0 +SH + done + cat >"$root/bin/preflight" <<'SH' +#!/usr/bin/env bash +printf 'preflight\n' >>"$PLATFORM_TEST_COMMAND_LOG" +exit "${PLATFORM_TEST_PREFLIGHT_RC:-0}" +SH + cat >"$root/bin/postcheck" <<'SH' +#!/usr/bin/env bash +printf 'postcheck\n' >>"$PLATFORM_TEST_COMMAND_LOG" +exit "${PLATFORM_TEST_POSTCHECK_RC:-0}" +SH + cat >"$root/bin/certbot" <<'SH' +#!/usr/bin/env bash +printf 'certbot %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" +if [[ "${1:-}" == plugins ]]; then + printf 'dns-cloudflare\n' +fi +exit 0 +SH + cat >"$root/bin/date" <<'SH' +#!/usr/bin/env bash +if [[ -n "${PLATFORM_TEST_DATE_NOW_EPOCH:-}" && "$*" == '-u +%s' ]]; then + printf '%s\n' "$PLATFORM_TEST_DATE_NOW_EPOCH" + exit 0 +fi +exec /usr/bin/date "$@" +SH + chmod 0755 "$root/bin/"* +} + +write_inventory() { + local root=$1 phase=$2 sha + mkdir -m 0700 -p -- "$root/$phase" + printf '{"phase":"%s"}\n' "$phase" >"$root/$phase/inventory.json" + chmod 0600 "$root/$phase/inventory.json" + sha="$(sha256sum "$root/$phase/inventory.json" | awk '{print $1}')" + printf '%s inventory.json\n' "$sha" >"$root/$phase/inventory.sha256" + chmod 0600 "$root/$phase/inventory.sha256" + printf '%s' "$sha" +} + +write_acceptance_v2() { + local path=$1 initial_sha=$2 post_sha=$3 + local gate=${4:-RISK_ACCEPTED} ref uid + case "$gate" in + RECOVERY) + ref=strict-recovery-evidence-v1 + uid=not-applicable + ;; + RISK_ACCEPTED) + ref=2026-08-14-observability-slack-recovery-risk-acceptance-design + uid=1000 + ;; + *) return 2 ;; + esac + cat >"$path" <"$path" <"$rollback_root/blackbox-source-proof.env" <"$fixture/output" 2>&1; then + rc=0 + else + rc=$? + fi + unset PLATFORM_TEST_NGINX_BIN PLATFORM_TEST_SYSTEMCTL_BIN PLATFORM_TEST_TEST_BIN PLATFORM_TEST_CERTBOT_BIN \ + PLATFORM_TEST_PREFLIGHT_COMMAND PLATFORM_TEST_POSTCHECK_COMMAND + (( rc != 0 )) || fail "test mode accepted an escaped $label command override" +} + +assert_test_command_boundary_mutation_rejected() { + local mutation=$1 fixture rc + fixture="$(new_fixture "test-boundary-$mutation")" + case "$mutation" in + missing) rm -f -- "$fixture/bin/test" ;; + unsafe-mode) chmod 0775 "$fixture/bin/test" ;; + *) fail "unknown test command boundary mutation: $mutation" ;; + esac + if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ + >"$fixture/output" 2>&1; then + rc=0 + else + rc=$? + fi + (( rc != 0 )) || fail "test mode accepted $mutation test command boundary" + grep -Fq 'test command boundary is unsafe: test' "$fixture/output" || + fail "$mutation test command boundary did not identify the test boundary" + cmp -s "$fixture/active/learn-services" "$DENY" || + fail "$mutation test command boundary changed active bytes" + [[ ! -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" && + ! -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]] || + fail "$mutation test command boundary created a Host ledger" + [[ ! -s "$fixture/commands.log" ]] || + fail "$mutation test command boundary reached a command boundary" + ! grep -Fq 'Type APPLY' "$fixture/output" || + fail "$mutation test command boundary reached the APPLY prompt" + pass "test mode rejects $mutation test command boundary before contact, ledger, or prompt" +} + +new_fixture() { + local name=$1 fixture deny_sha + fixture="$WORK/$name" + mkdir -m 0700 -p "$fixture" + mkdir -p "$fixture/active" "$fixture/enabled" "$fixture/rollbacks/observability-$ROLLBACK_ID" + cp "$DENY" "$fixture/active/learn-services" + ln -s "$fixture/active/learn-services" "$fixture/enabled/learn-services" + chmod 0644 "$fixture/active/learn-services" + chmod 0700 "$fixture/rollbacks/observability-$ROLLBACK_ID" + : >"$fixture/commands.log" + make_fakes "$fixture" + deny_sha="$(sha256sum "$DENY" | awk '{print $1}')" + write_evidence "$fixture/rollbacks/observability-$ROLLBACK_ID" "$fixture/metrics" "$deny_sha" + printf '%s' "$fixture" +} + +acceptance_inventory_shas() { + local fixture=$1 initial_sha post_sha + initial_sha="$(sha256sum "$fixture/metrics/target-initial/inventory.json" | awk '{print $1}')" + post_sha="$(sha256sum "$fixture/metrics/post-substrate/inventory.json" | awk '{print $1}')" + printf '%s|%s' "$initial_sha" "$post_sha" +} + +evidence_contract_failures=() +task2_contract_failures=() + +record_evidence_contract_failure() { + evidence_contract_failures+=("$1") + printf 'RED: %s\n' "$1" >&2 +} + +record_task2_contract_failure() { + task2_contract_failures+=("$1") + printf 'RED: %s\n' "$1" >&2 +} + +host_tree_fingerprint() { + local root=$1 entry + if [[ ! -e "$root" && ! -L "$root" ]]; then + printf 'ABSENT' + return 0 + fi + { + stat -c '.|%F|%u:%g|%a|%h|%N' -- "$root" + if [[ -d "$root" && ! -L "$root" ]]; then + find -P "$root" -mindepth 1 -printf '%P|%y|%U:%G|%m|%n|%l\n' | sort + while IFS= read -r entry; do + printf '%s|' "${entry#"$root"/}" + sha256sum -- "$entry" | awk '{print $1}' + done < <(find -P "$root" -type f -links 1 -print | sort) + fi + } | sha256sum | awk '{print $1}' +} + +exercise_first_cutover_residue() { + local mutation=$1 fixture host_root payload_root ledger before after rc failed=false + fixture="$(new_fixture "first-cutover-residue-$mutation")" + host_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" + payload_root="$host_root/payloads" + ledger="$host_root/stages.tsv" + case "$mutation" in + empty-root) + mkdir -m 0700 "$host_root" + ;; + normal-ledger) + mkdir -m 0700 "$host_root" "$payload_root" + printf '%s\n' $'stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc' >"$ledger" + chmod 0600 "$ledger" + ;; + normal-payload) + mkdir -m 0700 "$host_root" "$payload_root" + cp "$DENY" "$payload_root/full-prior-0001.conf" + chmod 0600 "$payload_root/full-prior-0001.conf" + ;; + *) fail "unknown first-cutover residue mutation: $mutation" ;; + esac + before="$(host_tree_fingerprint "$host_root")" + if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ + >"$fixture/output" 2>&1; then + rc=0 + else + rc=$? + fi + after="$(host_tree_fingerprint "$host_root")" + if (( rc == 0 )); then + record_task2_contract_failure "$mutation residue returned success" + failed=true + fi + if ! cmp -s "$fixture/active/learn-services" "$DENY"; then + record_task2_contract_failure "$mutation residue changed active deny bytes" + failed=true + fi + if [[ "$before" != "$after" ]]; then + record_task2_contract_failure "$mutation residue changed the pre-existing Host evidence tree" + failed=true + fi + if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then + record_task2_contract_failure "$mutation residue reached preflight or mutation" + failed=true + fi + if grep -Fq 'Type APPLY' "$fixture/output"; then + record_task2_contract_failure "$mutation residue reached the APPLY prompt" + failed=true + fi + if [[ "$failed" == false ]]; then + pass "full first cutover rejects $mutation residue before preflight, mutation, and prompt" + fi +} + +exercise_active_state_gate() { + local state=$1 fixture before rc failed=false expected_prefix attempt + fixture="$(new_fixture "active-state-$state")" + case "$state" in + full) cp "$FULL" "$fixture/active/learn-services" ;; + third) + expected_prefix="$(sha256sum "$DENY" | cut -c1)" + for ((attempt=0; attempt<256; attempt++)); do + printf 'server { listen 127.0.0.1:65534; } # %d\n' "$attempt" \ + >"$fixture/active/learn-services" + [[ "$(sha256sum "$fixture/active/learn-services" | cut -c1)" == "$expected_prefix" ]] && break + done + [[ "$attempt" -lt 256 ]] || fail 'cannot construct deterministic same-prefix third state' + ;; + *) fail "unknown active state: $state" ;; + esac + chmod 0644 "$fixture/active/learn-services" + before="$(sha256sum "$fixture/active/learn-services")" + if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ + >"$fixture/output" 2>&1; then + rc=0 + else + rc=$? + fi + if (( rc == 0 )); then + record_task2_contract_failure "$state active state returned success" + failed=true + fi + if [[ "$(sha256sum "$fixture/active/learn-services")" != "$before" ]]; then + record_task2_contract_failure "$state active state changed active bytes" + failed=true + fi + if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then + record_task2_contract_failure "$state active state reached preflight or mutation" + failed=true + fi + if [[ -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" || + -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]]; then + record_task2_contract_failure "$state active state created a Host ledger" + failed=true + fi + if grep -Fq 'Type APPLY' "$fixture/output"; then + record_task2_contract_failure "$state active state reached the APPLY prompt" + failed=true + fi + if [[ "$state" == full ]] && grep -Fq 'ALREADY_ACTIVE' "$fixture/output"; then + record_task2_contract_failure 'full active state emitted ALREADY_ACTIVE' + failed=true + fi + if [[ "$failed" == false ]]; then + pass "full cutover rejects $state active state before preflight, ledger, and prompt" + fi +} + +exercise_source_boundary_contract() { + local output rc=0 + output="$(env APPLY="$APPLY" /usr/bin/bash <<'SH' 2>&1 +set -Eeuo pipefail +source "$APPLY" +[[ "$PATH" == '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' ]] +[[ "$TEST_BIN" == /usr/bin/test ]] +print_plan_body="$(declare -f print_plan)" +if grep -Eq '(^|[[:space:];(])(dig|curl|kubectl|sudo|nginx|systemctl)([[:space:];|)])' <<<"$print_plan_body"; then + exit 1 +fi +expected=$'awk=/usr/bin/awk\nbash=/usr/bin/bash\ncat=/usr/bin/cat\ncurl=/usr/bin/curl\ndate=/usr/bin/date\ndig=/usr/bin/dig\ndirname=/usr/bin/dirname\ngrep=/usr/bin/grep\nhead=/usr/bin/head\nid=/usr/bin/id\ninstall=/usr/bin/install\njq=/usr/bin/jq\nkubectl=/usr/local/bin/kubectl\nmktemp=/usr/bin/mktemp\nopenssl=/usr/bin/openssl\npython3=/usr/bin/python3\nreadlink=/usr/bin/readlink\nrm=/usr/bin/rm\nsed=/usr/bin/sed\nsha256sum=/usr/bin/sha256sum\nsleep=/usr/bin/sleep\nsort=/usr/bin/sort\nss=/usr/bin/ss\nstat=/usr/bin/stat\ntail=/usr/bin/tail\ntest=/usr/bin/test\ntr=/usr/bin/tr' +actual="$(for name in "${!HOST_NGINX_PRODUCTION_COMMANDS[@]}"; do printf '%s=%s\n' "$name" "${HOST_NGINX_PRODUCTION_COMMANDS[$name]}"; done | sort)" +if (( ${#HOST_NGINX_PRODUCTION_COMMANDS[@]} != 27 )); then + printf 'production command allowlist cardinality is not 27\n' >&2 + exit 1 +fi +if [[ "$actual" != "$expected" ]]; then + printf 'production command allowlist lacks exact test=/usr/bin/test boundary\n' >&2 + exit 1 +fi +declare -F production_executable_metadata_is_safe >/dev/null +reject_production_metadata() { + if production_executable_metadata_is_safe "$@"; then + return 1 + fi +} +production_executable_metadata_is_safe sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4755 1 sudo +reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 0755 1 sudo +reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 1 0 4755 1 sudo +reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 1 4755 1 sudo +reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4755 2 sudo +reject_production_metadata sudo /usr/bin/sudo /usr/local/bin/sudo 'regular file' 0 0 4755 1 sudo +reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 6755 1 sudo +reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4775 1 sudo +reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4757 1 sudo +production_executable_metadata_is_safe awk /usr/bin/awk /usr/bin/awk 'regular file' 0 0 0755 1 ordinary +reject_production_metadata awk /usr/bin/awk /usr/bin/awk 'regular file' 0 0 4755 1 ordinary +reject_production_metadata awk /usr/bin/awk /usr/bin/awk 'regular file' 0 0 2755 1 ordinary +validate_production_command_boundaries +SH +)" || rc=$? + if (( rc != 0 )); then + record_task2_contract_failure "fixed production command boundary contract failed: ${output//$'\n'/; }" + else + pass 'fixed production PATH, exact inventory, sudo exception, and ordinary metadata policy are enforced' + fi +} + +assert_privileged_test_source_boundary() { + local source=$1 + /usr/bin/python3 - "$source" <<'PY' +import pathlib +import re +import sys + +text = pathlib.Path(sys.argv[1]).read_text() +exact = '"$SUDO_BIN" -n "$TEST_BIN"' +expected_count = 27 +commands = re.findall(r'"\$SUDO_BIN"\s+-n\s+([^\s;&|()]+)', text) +test_commands = [ + command for command in commands + if command == '"$TEST_BIN"' + or command.strip('"\'') == 'test' + or command.strip('"\'').endswith('/test') +] +if text.count(exact) != expected_count: + raise SystemExit( + f'privileged test boundary exact-call cardinality is {text.count(exact)}, expected {expected_count}' + ) +if len(test_commands) != expected_count: + raise SystemExit( + f'privileged test boundary candidate cardinality is {len(test_commands)}, expected {expected_count}' + ) +if any(command != '"$TEST_BIN"' for command in test_commands): + raise SystemExit('privileged test boundary includes a non-fixed test command') +for operand in ( + '-s "$CERT_DIR/fullchain.pem"', + '-s "$CERT_DIR/privkey.pem"', + '-f "$CREDENTIALS"', +): + if text.count(f'{exact} {operand}') != 1: + raise SystemExit(f'privileged test boundary misses exact certificate/credential probe: {operand}') +PY +} + +exercise_privileged_test_source_mutation_matrix() { + local mutation ordinal rc + assert_privileged_test_source_boundary "$APPLY" || + fail 'current Host source violates the closed privileged test boundary' + for (( ordinal=0; ordinal<27; ordinal++ )); do + mutation="$WORK/privileged-test-bare-$ordinal.sh" + /usr/bin/python3 - "$APPLY" "$mutation" "$ordinal" <<'PY' +import pathlib +import sys + +source = pathlib.Path(sys.argv[1]) +destination = pathlib.Path(sys.argv[2]) +ordinal = int(sys.argv[3]) +text = source.read_text() +exact = '"$SUDO_BIN" -n "$TEST_BIN"' +bare = '"$SUDO_BIN" -n test' +parts = text.split(exact) +if len(parts) - 1 != 27: + raise SystemExit('unexpected privileged test boundary cardinality while mutating') +if ordinal < 0 or ordinal >= len(parts) - 1: + raise SystemExit('privileged test mutation ordinal is out of range') +mutated = exact.join(parts[:ordinal + 1]) + bare + exact.join(parts[ordinal + 1:]) +destination.write_text(mutated) +destination.chmod(0o600) +PY + if assert_privileged_test_source_boundary "$mutation" >"$mutation.output" 2>&1; then + fail "privileged test boundary accepted bare-test mutation $ordinal" + fi + grep -Eq 'privileged test boundary (exact-call|candidate) cardinality is|privileged test boundary includes a non-fixed test command' \ + "$mutation.output" || + fail "privileged test boundary mutation $ordinal failed outside the boundary assertion" + done + pass 'closed source matrix rejects every privileged bare-test mutation including certificate and credentials probes' +} + +exercise_active_site_symlink_diagnostic() { + local fixture rc expected + fixture="$(new_fixture active-site-symlink-diagnostic)" + rm -f -- "$fixture/active/learn-services" + ln -s "$DENY" "$fixture/active/learn-services" + if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ + >"$fixture/output" 2>&1; then + rc=0 + else + rc=$? + fi + (( rc != 0 )) || fail 'active-site symlink was accepted' + expected="ERROR: active site is a symlink: $fixture/active/learn-services" + grep -Fqx "$expected" "$fixture/output" || + fail 'active-site symlink diagnostic drifted from its exact contract' + [[ ! -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" && + ! -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]] || + fail 'active-site symlink diagnostic created a Host ledger' + ! grep -Fq 'Type APPLY' "$fixture/output" || + fail 'active-site symlink diagnostic reached the APPLY prompt' + pass 'active-site symlink retains its exact diagnostic before ledger or prompt' +} + +exercise_precommit_staging_failure() { + local stage=$1 fixture host_root rc failed=false reloads=0 + fixture="$(new_fixture "staging-failure-$stage")" + host_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" + unset PLATFORM_TEST_INSTALL_FAIL_MATCH PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH + case "$stage" in + directory) PLATFORM_TEST_INSTALL_FAIL_MATCH="$host_root" ;; + ledger) PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH="$host_root/stages.tsv" ;; + payload) PLATFORM_TEST_INSTALL_FAIL_MATCH="$host_root/payloads/full-prior-0001.conf" ;; + timestamp) PLATFORM_TEST_INSTALL_FAIL_MATCH='.before-observability-full-' ;; + *) fail "unknown precommit staging failure: $stage" ;; + esac + if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ + >"$fixture/output" 2>&1; then + rc=0 + else + rc=$? + fi + unset PLATFORM_TEST_INSTALL_FAIL_MATCH PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH + [[ ! -f "$fixture/reload-counter" ]] || read -r reloads <"$fixture/reload-counter" + if (( rc == 0 )); then + record_task2_contract_failure "$stage staging failure returned success" + failed=true + fi + if ! cmp -s "$fixture/active/learn-services" "$DENY"; then + record_task2_contract_failure "$stage staging failure changed active deny bytes" + failed=true + fi + if [[ "$reloads" != 0 ]]; then + record_task2_contract_failure "$stage staging failure reloaded Nginx" + failed=true + fi + if grep -Fq 'HOST_NGINX_OBSERVABILITY_ROLLBACK=' "$fixture/output"; then + record_task2_contract_failure "$stage pre-transaction staging failure emitted a rollback marker" + failed=true + fi + if ! grep -Fq 'fresh rollback ID + source proof + complete Task 6 required' "$fixture/output"; then + record_task2_contract_failure "$stage staging failure omitted the frozen-ID terminal classification" + failed=true + fi + case "$stage" in + directory) + if [[ -e "$host_root" || -L "$host_root" ]]; then + record_task2_contract_failure 'directory staging failure unexpectedly created the Host root' + failed=true + fi + ;; + ledger) + if [[ ! -d "$host_root/payloads" || ! -f "$host_root/stages.tsv" || + "$(stat -c %a "$host_root/stages.tsv")" != 644 ]]; then + record_task2_contract_failure 'ledger staging failure did not preserve its unsafe staged evidence for review' + failed=true + fi + ;; + payload) + if [[ ! -f "$host_root/stages.tsv" || -e "$host_root/payloads/full-prior-0001.conf" ]]; then + record_task2_contract_failure 'payload staging failure evidence topology is not exact' + failed=true + fi + ;; + timestamp) + if [[ ! -f "$host_root/stages.tsv" || ! -f "$host_root/payloads/full-prior-0001.conf" ]] || + compgen -G "$fixture/active/learn-services.before-observability-full-*" >/dev/null; then + record_task2_contract_failure 'timestamp staging failure evidence topology is not exact' + failed=true + fi + ;; + esac + if [[ "$failed" == false ]]; then + pass "$stage pre-transaction staging failure freezes the ID without a rollback claim" + fi +} + +exercise_valid_acceptance_tuple() { + local gate=$1 fixture acceptance inventory_shas initial_sha post_sha rc failed=false + fixture="$(new_fixture "valid-${gate,,}-acceptance")" + acceptance="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" + inventory_shas="$(acceptance_inventory_shas "$fixture")" + initial_sha=${inventory_shas%%|*} + post_sha=${inventory_shas#*|} + write_acceptance_v2 "$acceptance" "$initial_sha" "$post_sha" "$gate" + if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ + >"$fixture/output" 2>&1; then + rc=0 + else + rc=$? + fi + if (( rc != 0 )); then + record_evidence_contract_failure "valid $gate v2 acceptance was rejected with RC $rc" + failed=true + if cmp -s "$fixture/active/learn-services" "$DENY"; then + printf 'RED-PROOF: valid %s v2 rejection preserved exact deny bytes\n' "$gate" >&2 + else + record_evidence_contract_failure "valid $gate v2 rejection changed the active deny bytes" + fi + fi + if ! cmp -s "$fixture/active/learn-services" "$FULL"; then + record_evidence_contract_failure "valid $gate v2 acceptance did not install the full candidate" + failed=true + fi + if [[ ! -f "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/stages.tsv" ]]; then + record_evidence_contract_failure "valid $gate v2 acceptance did not create the Host ledger" + failed=true + fi + if ! grep -Fq 'Type APPLY' "$fixture/output"; then + record_evidence_contract_failure "valid $gate v2 acceptance did not reach the APPLY prompt" + failed=true + fi + if [[ "$failed" == false ]]; then + pass "full evidence accepts the exact $gate v2 tuple" + fi +} + +exercise_invalid_acceptance() { + local mutation=$1 fixture acceptance inventory_shas initial_sha post_sha rc failed=false + fixture="$(new_fixture "invalid-${mutation}")" + acceptance="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" + inventory_shas="$(acceptance_inventory_shas "$fixture")" + initial_sha=${inventory_shas%%|*} + post_sha=${inventory_shas#*|} + case "$mutation" in + v1) + write_acceptance_v1 "$acceptance" "$initial_sha" "$post_sha" + ;; + wrong-schema) + sed -i 's/^schema=.*/schema=wrong-schema/' "$acceptance" + ;; + missing-slack-gate) + sed -i '/^slack_deployment_gate=/d' "$acceptance" + ;; + extra-field) + printf 'unexpected=value\n' >>"$acceptance" + ;; + duplicate-field) + printf 'state=accepted\n' >>"$acceptance" + ;; + recovery-with-risk-ref) + write_acceptance_v2 "$acceptance" "$initial_sha" "$post_sha" RECOVERY + sed -i 's/^slack_gate_approval_ref=.*/slack_gate_approval_ref=2026-08-14-observability-slack-recovery-risk-acceptance-design/' "$acceptance" + ;; + recovery-with-uid-1000) + write_acceptance_v2 "$acceptance" "$initial_sha" "$post_sha" RECOVERY + sed -i 's/^slack_gate_accepted_by_uid=.*/slack_gate_accepted_by_uid=1000/' "$acceptance" + ;; + risk-with-strict-ref) + sed -i 's/^slack_gate_approval_ref=.*/slack_gate_approval_ref=strict-recovery-evidence-v1/' "$acceptance" + ;; + risk-with-not-applicable) + sed -i 's/^slack_gate_accepted_by_uid=.*/slack_gate_accepted_by_uid=not-applicable/' "$acceptance" + ;; + arbitrary-gate) + sed -i 's/^slack_deployment_gate=.*/slack_deployment_gate=ARBITRARY/' "$acceptance" + ;; + arbitrary-ref) + sed -i 's/^slack_gate_approval_ref=.*/slack_gate_approval_ref=arbitrary-ref/' "$acceptance" + ;; + arbitrary-uid) + sed -i 's/^slack_gate_accepted_by_uid=.*/slack_gate_accepted_by_uid=2000/' "$acceptance" + ;; + rollback-id) + sed -i 's/^rollback_id=.*/rollback_id=20260812T120001Z/' "$acceptance" + ;; + target-initial-hash) + sed -i 's/^target_initial_sha256=.*/target_initial_sha256=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/' "$acceptance" + ;; + post-substrate-hash) + sed -i 's/^post_substrate_sha256=.*/post_substrate_sha256=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc/' "$acceptance" + ;; + state) + sed -i 's/^state=.*/state=pending/' "$acceptance" + ;; + malformed-accepted-at) + sed -i 's/^accepted_at_utc=.*/accepted_at_utc=2026-02-30T00:00:00Z/' "$acceptance" + ;; + *) fail "unknown acceptance mutation: $mutation" ;; + esac + if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ + >"$fixture/output" 2>&1; then + rc=0 + else + rc=$? + fi + if (( rc == 0 )); then + record_evidence_contract_failure "$mutation acceptance returned success" + failed=true + fi + if ! cmp -s "$fixture/active/learn-services" "$DENY"; then + record_evidence_contract_failure "$mutation acceptance changed the active deny bytes" + failed=true + fi + if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then + record_evidence_contract_failure "$mutation acceptance reached preflight or mutation" + failed=true + fi + if [[ -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" || + -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]]; then + record_evidence_contract_failure "$mutation acceptance created a Host ledger" + failed=true + fi + if grep -Fq 'Type APPLY' "$fixture/output"; then + record_evidence_contract_failure "$mutation acceptance reached the APPLY prompt" + failed=true + fi + if [[ "$failed" == false ]]; then + pass "full evidence rejects $mutation before preflight, mutation, ledger, and prompt" + fi +} + +exercise_invalid_source_proof() { + local mutation=$1 fixture proof rc failed=false mutated_time fixed_now='' + fixture="$(new_fixture "invalid-source-${mutation}")" + proof="$fixture/rollbacks/observability-$ROLLBACK_ID/blackbox-source-proof.env" + case "$mutation" in + deny-sha) + sed -i 's/^nginx_sha256=.*/nginx_sha256=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/' "$proof" + ;; + status) + sed -i 's/^grafana_status=403$/grafana_status=200/' "$proof" + ;; + stale) + mutated_time="$(date -u -d 'now - 86401 seconds' +%Y-%m-%dT%H:%M:%SZ)" + sed -i "s/^tested_at_utc=.*/tested_at_utc=$mutated_time/" "$proof" + ;; + future) + fixed_now="$(date -u +%s)" + mutated_time="$(date -u -d "@$((fixed_now + 301))" +%Y-%m-%dT%H:%M:%SZ)" + sed -i "s/^tested_at_utc=.*/tested_at_utc=$mutated_time/" "$proof" + ;; + invalid-calendar) + sed -i 's/^tested_at_utc=.*/tested_at_utc=2026-02-30T00:00:00Z/' "$proof" + ;; + *) fail "unknown source-proof mutation: $mutation" ;; + esac + if PLATFORM_TEST_DATE_NOW_EPOCH="$fixed_now" run_apply "$fixture" \ + --execute --verified-output-dir "$fixture/metrics" \ + >"$fixture/output" 2>&1; then + rc=0 + else + rc=$? + fi + if (( rc == 0 )); then + record_evidence_contract_failure "source-proof $mutation returned success" + failed=true + fi + if ! cmp -s "$fixture/active/learn-services" "$DENY"; then + record_evidence_contract_failure "source-proof $mutation changed the active deny bytes" + failed=true + fi + if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then + record_evidence_contract_failure "source-proof $mutation reached preflight or mutation" + failed=true + fi + if [[ -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" || + -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]]; then + record_evidence_contract_failure "source-proof $mutation created a Host ledger" + failed=true + fi + if grep -Fq 'Type APPLY' "$fixture/output"; then + record_evidence_contract_failure "source-proof $mutation reached the APPLY prompt" + failed=true + fi + if [[ "$failed" == false ]]; then + pass "full evidence rejects source-proof $mutation before preflight, mutation, ledger, and prompt" + fi +} + +WORK="$(mktemp -d /tmp/platform-host-nginx-test.XXXXXX)" +chmod 0700 "$WORK" + +if env PLATFORM_HOST_NGINX_ACTIVE="$DENY" bash "$APPLY" >"$WORK/production-path-override.out" 2>&1; then + fail 'production dry-run accepted a Host Nginx path override' +fi +pass 'production rejects Host Nginx path overrides' + +if env PLATFORM_HOST_NGINX_NGINX_BIN=/bin/true bash "$APPLY" >"$WORK/production-binary-override.out" 2>&1; then + fail 'production dry-run accepted a Host Nginx binary override' +fi +pass 'production rejects Host Nginx binary overrides' + +fixture="$(new_fixture rejects-real-sudo)" +if PLATFORM_TEST_SUDO_BIN=/usr/bin/sudo run_apply "$fixture" >"$fixture/output" 2>&1; then + fail 'test-mode dry-run accepted the real sudo binary' +fi +grep -Fq 'test command boundary is unsafe' "$fixture/output" || + fail 'test-mode real sudo rejection did not identify the command boundary' +pass 'test mode rejects the real sudo binary before it can execute' + +for command in nginx systemctl test certbot preflight postcheck; do + assert_test_command_override_rejected "$command" /usr/bin/true + fixture="$WORK/rejects-$command-override" + grep -Fq 'test command boundary is unsafe' "$fixture/output" || + fail "test-mode escaped $command rejection did not identify the command boundary" + pass "test mode rejects escaped $command command overrides" +done + +fixture="$(new_fixture rejects-foreign-fixture-bin)" +foreign_fixture="$(new_fixture foreign-fixture-bin)" +if PLATFORM_TEST_SUDO_BIN="$foreign_fixture/bin/sudo" \ + PLATFORM_TEST_NGINX_BIN="$foreign_fixture/bin/nginx" \ + PLATFORM_TEST_SYSTEMCTL_BIN="$foreign_fixture/bin/systemctl" \ + PLATFORM_TEST_TEST_BIN="$foreign_fixture/bin/test" \ + PLATFORM_TEST_CERTBOT_BIN="$foreign_fixture/bin/certbot" \ + PLATFORM_TEST_PREFLIGHT_COMMAND="$foreign_fixture/bin/preflight" \ + PLATFORM_TEST_POSTCHECK_COMMAND="$foreign_fixture/bin/postcheck" \ + run_apply "$fixture" >"$fixture/output" 2>&1; then + fail 'test mode accepted command boundaries from a different fixture tree' +fi +grep -Fq 'test command boundary is unsafe: sudo escapes the fixture bin' "$fixture/output" || + fail 'foreign fixture command boundary rejection did not identify the fixture mismatch' +pass 'test mode requires command boundaries from the active and rollback fixture tree' + +for mutation in missing unsafe-mode; do + assert_test_command_boundary_mutation_rejected "$mutation" +done + +fixture="$(new_fixture dry-run)" +before="$(sha256sum "$fixture/active/learn-services")" +output="$(run_apply "$fixture")" || fail 'dry-run failed' +[[ "$output" == *'HOST_NGINX_OBSERVABILITY_DRY_RUN=PASS'* ]] || fail 'dry-run marker missing' +[[ "$(sha256sum "$fixture/active/learn-services")" == "$before" ]] || fail 'dry-run mutated active config' +[[ ! -s "$fixture/commands.log" ]] || fail 'dry-run invoked a mutating boundary' +dry_run_contract_ok=true +for marker in \ + 'HOST_NGINX_CERTIFICATE_EXPECTED_SAN=grafana.learn.hyeonworks.com' \ + 'HOST_NGINX_CERTIFICATE_SAN=NOT_CHECKED_DRY_RUN' \ + 'HOST_NGINX_GRAFANA_PUBLIC_DNS=NOT_CHECKED_DRY_RUN'; do + if [[ "$(grep -Fxc "$marker" <<<"$output")" != 1 ]]; then + record_task2_contract_failure "dry-run marker is not exact-once: $marker" + dry_run_contract_ok=false + fi +done +for forbidden in HOST_NGINX_CERTIFICATE_EXACT_SAN= HOST_NGINX_GRAFANA_PUBLIC_DNS=ABSENT; do + if grep -Fq "$forbidden" <<<"$output"; then + record_task2_contract_failure "dry-run retained false live claim: $forbidden" + dry_run_contract_ok=false + fi +done +if [[ "$dry_run_contract_ok" == true ]]; then + pass 'dry-run is no-contact and emits truthful expected/not-checked markers' +fi + +hostile="$WORK/hostile-path" +mkdir -m 0700 "$hostile" +cat >"$hostile/dirname" <<'SH' +#!/usr/bin/env bash +printf 'dirname %s\n' "$*" >>"$PLATFORM_TEST_HOSTILE_LOG" +exec /usr/bin/dirname "$@" +SH +chmod 0700 "$hostile/dirname" +: >"$WORK/hostile-path.log" +if env PATH="$hostile:$PATH" PLATFORM_TEST_HOSTILE_LOG="$WORK/hostile-path.log" \ + PLATFORM_HOST_NGINX_TEST_MODE=invalid /usr/bin/bash "$APPLY" >"$WORK/hostile-path.out" 2>&1; then + record_task2_contract_failure 'invalid test mode unexpectedly returned success under hostile PATH' +elif [[ -s "$WORK/hostile-path.log" ]]; then + record_task2_contract_failure 'ambient dirname executed before hostile test mode rejection' +else + pass 'fixed PATH is installed before ROOT resolution and invalid-mode rejection' +fi + +exercise_source_boundary_contract +exercise_privileged_test_source_mutation_matrix +exercise_active_site_symlink_diagnostic + +fixture="$(new_fixture missing-acceptance)" +rm -f "$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" +if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then + fail 'missing acceptance was accepted' +fi +cmp -s "$fixture/active/learn-services" "$DENY" || fail 'failed preflight changed active config' +if grep -Eq '^(nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then + fail 'missing acceptance reached a mutation boundary' +fi +pass 'full mode refuses missing acceptance with zero mutation' + +fixture="$(new_fixture unsafe-inventory-mode)" +chmod 0755 "$fixture/metrics/target-initial" +if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then + fail 'unsafe inventory mode was hidden by the later inventory hash' +fi +cmp -s "$fixture/active/learn-services" "$DENY" || fail 'unsafe inventory mode changed active config' +if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then + fail 'unsafe inventory mode reached a mutation or preflight boundary' +fi +pass 'unsafe inventory metadata cannot be hidden by a valid hash' + +fixture="$(new_fixture symlinked-inventory-json)" +mv "$fixture/metrics/target-initial/inventory.json" \ + "$fixture/metrics/target-initial/inventory.real.json" +ln -s inventory.real.json "$fixture/metrics/target-initial/inventory.json" +if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then + fail 'symlinked inventory JSON was hidden by its valid target hash' +fi +cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked inventory JSON changed active config' +if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then + fail 'symlinked inventory JSON reached a mutation or preflight boundary' +fi +pass 'symlinked inventory JSON cannot be hidden by a valid target hash' + +fixture="$(new_fixture missing-certificate-key)" +mkdir -p "$fixture/cert" +printf 'test certificate\n' >"$fixture/cert/fullchain.pem" +if run_apply "$fixture" --execute --certificate-only >"$fixture/output" 2>&1; then + fail 'missing certificate private key was hidden by later SAN validation' +fi +if grep -Fq 'HOST_NGINX_GRAFANA_CERTIFICATE=READY' "$fixture/output"; then + fail 'missing certificate private key emitted READY' +fi +pass 'exact certificate validation cannot hide a missing private key' + +for gate in RECOVERY RISK_ACCEPTED; do + exercise_valid_acceptance_tuple "$gate" +done + +for mutation in \ + v1 wrong-schema missing-slack-gate extra-field duplicate-field \ + recovery-with-risk-ref recovery-with-uid-1000 \ + risk-with-strict-ref risk-with-not-applicable \ + arbitrary-gate arbitrary-ref arbitrary-uid \ + rollback-id target-initial-hash post-substrate-hash state malformed-accepted-at; do + exercise_invalid_acceptance "$mutation" +done + +for mutation in deny-sha status stale future invalid-calendar; do + exercise_invalid_source_proof "$mutation" +done + +exercise_active_state_gate full +exercise_active_state_gate third +for mutation in empty-root normal-ledger normal-payload; do + exercise_first_cutover_residue "$mutation" +done + +if (( ${#evidence_contract_failures[@]} > 0 )); then + fail "full evidence contract mismatches: ${evidence_contract_failures[*]}" +fi + +fixture="$(new_fixture symlinked-payload-leaf)" +mkdir -m 0700 "$fixture/escaped-payloads" \ + "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" +ln -s "$fixture/escaped-payloads" \ + "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/payloads" +if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then + fail 'symlinked payloads ledger leaf was followed' +fi +cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked payload leaf changed active config' +[[ ! -e "$fixture/escaped-payloads/full-prior-0001.conf" ]] || + fail 'prior payload escaped through a symlinked leaf' +pass 'ledger payload leaf symlink is rejected before any write' + +fixture="$(new_fixture symlinked-ledger-parent)" +mkdir -m 0700 "$fixture/escaped-host-nginx" +ln -s "$fixture/escaped-host-nginx" \ + "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" +if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then + fail 'symlinked host-nginx ledger parent was followed' +fi +cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked ledger parent changed active config' +[[ ! -e "$fixture/escaped-host-nginx/stages.tsv" ]] || fail 'ledger escaped through a symlinked parent' +pass 'ledger parent symlink is rejected before any write' + +fixture="$(new_fixture symlinked-ledger-leaf)" +ledger_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" +mkdir -m 0700 "$ledger_root" "$ledger_root/payloads" +printf '%s\n' $'stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc' \ + >"$fixture/escaped-stages.tsv" +chmod 0600 "$fixture/escaped-stages.tsv" +ln -s "$fixture/escaped-stages.tsv" "$ledger_root/stages.tsv" +if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then + fail 'symlinked stages.tsv ledger leaf was accepted' +fi +cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked ledger leaf changed active config' +if grep -Fq "sudo -n cat -- $ledger_root/stages.tsv" "$fixture/commands.log"; then + fail 'symlinked stages.tsv was read before no-follow rejection' +fi +pass 'ledger file symlink is rejected before read or write' + +fixture="$(new_fixture hardlinked-ledger-leaf)" +ledger_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" +mkdir -m 0700 "$ledger_root" "$ledger_root/payloads" +printf '%s\n' $'stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc' \ + >"$ledger_root/stages.tsv" +chmod 0600 "$ledger_root/stages.tsv" +ln "$ledger_root/stages.tsv" "$fixture/linked-stages.tsv" +if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then + fail 'multiply-linked stages.tsv ledger leaf was accepted' +fi +cmp -s "$fixture/active/learn-services" "$DENY" || fail 'multiply-linked ledger leaf changed active config' +pass 'ledger file requires exactly one link before read or write' + +fixture="$(new_fixture success)" +run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1 || { + sed -n '1,260p' "$fixture/output" >&2 + fail 'full transition failed' +} +cmp -s "$fixture/active/learn-services" "$FULL" || fail 'full candidate was not installed' +ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/stages.tsv" +[[ -f "$ledger" && ! -L "$ledger" ]] || fail 'authoritative ledger missing' +[[ "$(wc -l <"$ledger" | tr -d ' ')" == 2 ]] || fail 'ledger does not have header plus exact stage' +awk -F '\t' 'NR == 1 { exit($0 != "stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc") } + NR == 2 { exit(NF != 6 || $1 != "full" || $4 != "payloads/full-prior-0001.conf") }' "$ledger" || + fail 'ledger schema or full row is not exact' +payload="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/payloads/full-prior-0001.conf" +cmp -s "$payload" "$DENY" || fail 'ledger payload is not exact prior active config' +[[ "$(stat -c %a "$ledger")" == 600 && "$(stat -c %a "$payload")" == 600 ]] || + fail 'ledger or payload is not mode 0600' +grep -Fqx 'systemctl reload nginx' "$fixture/commands.log" || fail 'successful transition did not reload nginx' +[[ "$(grep -Fxc 'postcheck' "$fixture/commands.log")" == 3 ]] || + fail 'postcheck did not require three consecutive stable results' +first_sudo="$(grep '^sudo ' "$fixture/commands.log" | head -n1)" +[[ "$first_sudo" == 'sudo -v' ]] || fail 'sudo refresh did not precede root metadata reads' +awk -v nginx="$fixture/bin/nginx" ' + $0 == "sudo -n " nginx " -t" { sudo_test=NR } + $0 == "preflight" { exit(!(sudo_test > 0 && sudo_test < NR)) } + END { if (!sudo_test) exit 1 } +' "$fixture/commands.log" || fail 'preflight nginx -t did not route through sudo before the probe hook' +! grep -q '^nginx-without-sudo ' "$fixture/commands.log" || fail 'nginx was invoked outside sudo' +grep -Eq "^sudo -n $fixture/bin/test " "$fixture/commands.log" || + fail 'full transition did not route test probes through the fixed test executable' +! grep -Eq '^sudo -n test( |$)' "$fixture/commands.log" || + fail 'full transition invoked a bare sudo test subcommand' +pass 'full transition routes all sudo test probes through the fixed test executable' +pass 'full transition records exact prior payload and installs candidate' + +fixture="$(new_fixture rollback)" +if PLATFORM_TEST_POSTCHECK_RC=1 run_apply "$fixture" --execute \ + --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then + fail 'failed postcheck returned success' +fi +cmp -s "$fixture/active/learn-services" "$DENY" || fail 'rollback did not restore exact ledger payload' +grep -Fq 'HOST_NGINX_OBSERVABILITY_ROLLBACK=PASS' "$fixture/output" || fail 'rollback PASS marker missing' +[[ "$(grep -Fxc 'systemctl reload nginx' "$fixture/commands.log")" == 2 ]] || + fail 'apply and rollback did not each reload nginx' +pass 'failed postcheck restores the exact prior payload' + +fixture="$(new_fixture rollback-failure)" +if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_SYSTEMCTL_FAIL_RELOAD_AT=2 \ + run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ + >"$fixture/output" 2>&1; then + fail 'failed rollback returned success' +fi +grep -Fq 'HOST_NGINX_OBSERVABILITY_ROLLBACK=FAIL' "$fixture/output" || + fail 'rollback failure marker missing' +grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" || + fail 'manual recovery marker missing after rollback failure' +pass 'rollback failure is truthfully classified for manual recovery' + +for stage in directory ledger payload timestamp; do + exercise_precommit_staging_failure "$stage" +done + +if (( ${#task2_contract_failures[@]} > 0 )); then + fail "Task 2 contract mismatches: ${task2_contract_failures[*]}" +fi + +printf 'Assertions: %d\n' "$ASSERTIONS" +printf 'APPLY HOST NGINX OBSERVABILITY TEST PASS\n' diff --git a/scripts/validate/test-apply-observability-access.sh b/scripts/validate/test-apply-observability-access.sh new file mode 100644 index 0000000..de67401 --- /dev/null +++ b/scripts/validate/test-apply-observability-access.sh @@ -0,0 +1,1651 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly TEST_SCRIPT="$(readlink -f -- "${BASH_SOURCE[0]}")" +readonly APPLY="$ROOT/scripts/bootstrap/apply-observability-access.sh" +readonly ROLLBACK_ID=20260812T120000Z +readonly INITIAL_PRODUCTION_SHA=79688d017d38eec9a6f100f8d0f784a5474e79802046ef1c2c11b30d170b0b0c +readonly POST_PRODUCTION_SHA=b1c3049206a1a88165ee672ae9aceac7945673a3bb9c3cf3670b7f0d56c3f291 +readonly INITIAL_TARGET_COUNT=21 +readonly POST_TARGET_COUNT=30 +readonly SUITE_WALL_BOUND_SECONDS=300 +WORK='' +declare -a METRIC_ROOTS=() +ASSERTIONS=0 + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +pass() { + ASSERTIONS=$((ASSERTIONS + 1)) + printf 'PASS: %s\n' "$1" +} + +case_progress() { + local case_name=$1 marker="CASE_BEGIN=$1" + printf '%s\n' "$marker" + printf '%s\n' "$marker" >>"$PROGRESS_LOG" +} + +run_with_wall_bound() { + local seconds=$1 + shift + [[ "$seconds" =~ ^[1-9][0-9]*$ && $# -gt 0 ]] || return 2 + timeout --signal=TERM --kill-after=2s "${seconds}s" "$@" +} + +cleanup() { + trap - EXIT HUP INT TERM + case "$WORK" in + /tmp/platform-observability-access-test.*) rm -rf -- "$WORK" ;; + esac + local root + for root in "${METRIC_ROOTS[@]}"; do + case "$root" in + /tmp/platform-observability-metrics.??????) rm -rf -- "$root" ;; + esac + done +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +write_executable() { + local path=$1 + shift + printf '%s\n' "$@" >"$path" + chmod 0755 "$path" +} + +make_fakes() { + local fixture=$1 + mkdir -m 0700 -p "$fixture/bin" "$fixture/state" + chmod 0700 "$fixture/bin" "$fixture/state" + : >"$fixture/commands.log" + + write_executable "$fixture/bin/sudo" \ + '#!/usr/bin/env bash' \ + 'printf '\''sudo'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'if [[ "${1:-}" == -v ]]; then : >"$PLATFORM_TEST_SUDO_REFRESHED"; exit 0; fi' \ + '[[ -f "$PLATFORM_TEST_SUDO_REFRESHED" ]] || exit 92' \ + 'if [[ "${1:-}" == -n ]]; then shift; fi' \ + '(( $# > 0 )) || exit 0' \ + 'if [[ "${PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE:-0}" == 1 && " $* " == *" install "* && "$*" == *"acceptance.env"* ]]; then exit 81; fi' \ + 'args=()' \ + 'while (( $# > 0 )); do case "$1" in -o|-g) shift 2 ;; *) args+=("$1"); shift ;; esac; done' \ + 'exec "${args[@]}"' + + write_executable "$fixture/bin/encryption" \ + '#!/usr/bin/env bash' \ + 'printf '\''encryption'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'count=0; [[ -f "$PLATFORM_TEST_ENCRYPTION_COUNT" ]] && read -r count <"$PLATFORM_TEST_ENCRYPTION_COUNT"' \ + 'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_ENCRYPTION_COUNT"' \ + 'if [[ "$count" == 2 && -n "${PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE:-}" ]]; then printf '\''# changed\n'\'' >>"$PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE"; fi' \ + 'action="${PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE:-}"' \ + 'if [[ "$count" == 2 && -n "$action" ]]; then' \ + ' fixture_root="$(dirname -- "$PLATFORM_TEST_COMMAND_LOG")"; expected_root="$fixture_root/metrics"' \ + ' [[ "$action" == raw-json || "$action" == rechecksum-count || "$action" == hardlink ]] || exit 73' \ + ' [[ "$PLATFORM_TEST_INVENTORY_ROOT" == "$expected_root" && -d "$PLATFORM_TEST_INVENTORY_ROOT" && ! -L "$PLATFORM_TEST_INVENTORY_ROOT" && "$(readlink -f -- "$PLATFORM_TEST_INVENTORY_ROOT")" == "$expected_root" ]] || exit 74' \ + ' inventory="$expected_root/target-initial/inventory.json"; checksum="$expected_root/target-initial/inventory.sha256"' \ + ' case "$action" in' \ + ' raw-json) printf '\'' \n'\'' >>"$inventory" ;;' \ + ' rechecksum-count) jq -c '\''.targets |= .[:-1]'\'' "$inventory" >"$inventory.next"; chmod 0600 "$inventory.next"; mv -f -- "$inventory.next" "$inventory"; sha="$(sha256sum -- "$inventory" | awk '\''{print $1}'\'')"; printf '\''%s inventory.json\n'\'' "$sha" >"$checksum"; chmod 0600 "$checksum" ;;' \ + ' hardlink) ln -- "$inventory" "$fixture_root/inventory-hardlink-control" ;;' \ + ' esac' \ + 'fi' \ + 'if [[ "$count" == "${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" ]]; then exit 71; fi' + + write_executable "$fixture/bin/restore" \ + '#!/usr/bin/env bash' \ + 'printf '\''restore'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'count=0; [[ -f "$PLATFORM_TEST_RESTORE_COUNT" ]] && read -r count <"$PLATFORM_TEST_RESTORE_COUNT"' \ + 'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_RESTORE_COUNT"' \ + 'if [[ "$count" == "${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" ]]; then exit 72; fi' + + write_executable "$fixture/bin/recovery" \ + '#!/usr/bin/env bash' \ + 'printf '\''recovery'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'count=0; [[ -f "$PLATFORM_TEST_RECOVERY_COUNT" ]] && read -r count <"$PLATFORM_TEST_RECOVERY_COUNT"' \ + 'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_RECOVERY_COUNT"' \ + 'response_var="PLATFORM_TEST_RECOVERY_OUTPUT_${count}"; response="${!response_var:-SLACK_DEPLOYMENT_GATE=RECOVERY\\n}"' \ + 'rc_var="PLATFORM_TEST_RECOVERY_RC_${count}"; rc="${!rc_var:-${PLATFORM_TEST_RECOVERY_RC:-0}}"' \ + 'printf '\''%b'\'' "$response"' \ + 'exit "$rc"' + + cat >"$fixture/bin/kubectl" <<'PY' +#!/usr/bin/env python3 +import json +import hashlib +import os +import pathlib +import signal +import socketserver +import sys +import time +from http.server import BaseHTTPRequestHandler +import yaml + +raw_args = sys.argv[1:] +log = pathlib.Path(os.environ["PLATFORM_TEST_COMMAND_LOG"]) +with log.open("a", encoding="utf-8") as stream: + stream.write("kubectl " + " ".join(raw_args) + "\n") +if not raw_args or raw_args[0] != "--request-timeout=10s": + raise SystemExit(93) +args = raw_args[1:] +state = pathlib.Path(os.environ["PLATFORM_TEST_STATE"]) + +kind_alias = { + "configmap": "ConfigMap", "configmaps": "ConfigMap", + "prometheusrule": "PrometheusRule", "prometheusrules": "PrometheusRule", + "alertmanagerconfig": "AlertmanagerConfig", "alertmanagerconfigs": "AlertmanagerConfig", + "alertmanager": "Alertmanager", "alertmanagers": "Alertmanager", + "networkpolicy": "NetworkPolicy", "networkpolicies": "NetworkPolicy", +} + +def option(*names, default=None): + for position, value in enumerate(args): + if value in names and position + 1 < len(args): + return args[position + 1] + for name in names: + if value.startswith(name + "="): + return value.split("=", 1)[1] + return default + +def state_path(kind, namespace, name): + return state / f"{kind}__{namespace}__{name}.json" + +def load_documents(filename): + with open(filename, encoding="utf-8") as stream: + return [item for item in yaml.safe_load_all(stream) if item] + +if args == ["config", "current-context"]: + print("default") + raise SystemExit(0) +if args[:2] == ["config", "view"]: + print("https://127.0.0.1:6443", end="") + raise SystemExit(0) +if args[:2] == ["get", "node"]: + print(json.dumps({ + "metadata": {"name": "donghyeon-system-product-name", "uid": "node-uid"}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + })) + raise SystemExit(0) +if args[:3] == ["get", "secret", "alertmanager-slack-webhook"]: + schema_count_path = pathlib.Path(os.environ["PLATFORM_TEST_SLACK_SCHEMA_COUNT"]) + schema_count = int(schema_count_path.read_text()) if schema_count_path.exists() else 0 + schema_count += 1 + schema_count_path.write_text(str(schema_count)) + extra_at = os.environ.get("PLATFORM_TEST_SLACK_EXTRA_KEY_AT", "0") + extra = (os.environ.get("PLATFORM_TEST_SLACK_EXTRA_KEY", "0") == "1" or + (extra_at.isdigit() and schema_count == int(extra_at))) + keys = ["extra", "url"] if extra else ["url"] + print(json.dumps({ + "apiVersion": "v1", "kind": "Secret", "type": "Opaque", + "metadata": {"name": "alertmanager-slack-webhook", "namespace": "observability"}, + "data": {key: "cmVkYWN0ZWQ=" for key in keys}, + })) + raise SystemExit(0) +if args[:2] == ["get", "probe"]: + print(json.dumps({"items": [ + {"metadata": {"name": "platform-private-edge"}}, + {"metadata": {"name": "platform-private-internal"}}, + {"metadata": {"name": "platform-public-edge"}}, + ]})) + raise SystemExit(0) +if args[:3] == ["get", "servicemonitor", "aistor-bucket-usage"]: + print("servicemonitor.monitoring.coreos.com/aistor-bucket-usage") + raise SystemExit(0) +if args[:3] == ["get", "podmonitor", "platform-postgres"]: + print("podmonitor.monitoring.coreos.com/platform-postgres") + raise SystemExit(0) +if args and args[0] == "get" and option("--raw"): + raw_path = option("--raw") + pathlib.Path(os.environ["PLATFORM_TEST_POSTCHECK_REACHED"]).touch() + if os.environ.get("PLATFORM_TEST_POSTCHECK_RC", "0") != "0": + raise SystemExit(74) + if raw_path.endswith("/proxy/-/ready") and "prometheus" in raw_path: + print("Prometheus Server is Ready.") + raise SystemExit(0) + if raw_path.endswith("/proxy/-/ready") and "alertmanager" in raw_path: + print("OK") + raise SystemExit(0) + if raw_path.endswith("/proxy/api/v1/rules"): + groups = [] + for path in sorted(state.glob("PrometheusRule__observability__*.json")): + item = json.loads(path.read_text(encoding="utf-8")) + for group in (item.get("spec") or {}).get("groups") or []: + rules = [] + for rule in group.get("rules") or []: + rules.append({ + "name": rule.get("alert") or rule.get("record"), + "type": "alerting" if rule.get("alert") else "recording", + "health": "ok", "lastError": "", "query": str(rule.get("expr", "")), + }) + groups.append({"name": group.get("name"), "rules": rules}) + if os.environ.get("PLATFORM_TEST_PROMETHEUS_DROP_RULE", "0") == "1" and groups: + groups[0]["rules"] = groups[0]["rules"][1:] + print(json.dumps({"status": "success", "data": {"groups": groups}})) + raise SystemExit(0) + if raw_path.endswith("/proxy/api/v2/receivers"): + mode = os.environ.get("PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE", "qualified") + platform_null = "observability/platform-alertmanager/platform-null" + platform_slack = "observability/platform-alertmanager/platform-slack" + def receiver(name): + return {"labels": {"name": name}, "name": name} + receiver_cases = { + "qualified": [receiver(platform_null), receiver(platform_slack)], + "qualified-reversed": [receiver(platform_slack), receiver(platform_null)], + "raw": [receiver("platform-null"), receiver("platform-slack")], + "empty": [], + "null": [None, receiver(platform_slack)], + "duplicate": [receiver(platform_null), receiver(platform_null)], + "extra": [receiver(platform_null), receiver(platform_slack), receiver("null")], + "wrong-namespace": [receiver("other/platform-alertmanager/platform-null"), receiver(platform_slack)], + "wrong-config": [receiver(platform_null), receiver("observability/other/platform-slack")], + "wrong-local": [receiver(platform_null), receiver("observability/platform-alertmanager/other")], + } + if mode == "malformed": + print('{"name":') + elif mode in receiver_cases: + print(json.dumps(receiver_cases[mode])) + else: + raise SystemExit(76) + raise SystemExit(0) + raise SystemExit(75) +if args and args[0] == "get" and len(args) >= 3: + kind = kind_alias.get(args[1].lower()) + if not kind: + raise SystemExit(44) + name = args[2] + namespace = option("-n", "--namespace", default="default") + path = state_path(kind, namespace, name) + if not path.exists(): + if "--ignore-not-found" in args: + raise SystemExit(0) + raise SystemExit(1) + item = json.loads(path.read_text()) + drift = os.environ.get("PLATFORM_TEST_DRIFT_RESOURCE", "") + reached = pathlib.Path(os.environ["PLATFORM_TEST_POSTCHECK_REACHED"]).exists() + if reached and drift == f"{kind}/{name}": + item["metadata"]["uid"] = "uid-drifted-by-other-owner" + print(json.dumps(item)) + raise SystemExit(0) +if args and args[0] == "apply": + filename = option("-f", "--filename") + if not filename: + raise SystemExit(45) + if option("--dry-run") == "server" or "--dry-run=server" in args: + raise SystemExit(0) + counter = pathlib.Path(os.environ["PLATFORM_TEST_APPLY_COUNT"]) + count = int(counter.read_text()) if counter.exists() else 0 + for item in load_documents(filename): + count += 1 + counter.write_text(str(count)) + if count == int(os.environ.get("PLATFORM_TEST_FAIL_APPLY_AT", "0")): + raise SystemExit(73) + metadata = item.setdefault("metadata", {}) + namespace = metadata.get("namespace", "default") + kind = item["kind"] + path = state_path(kind, namespace, metadata["name"]) + prior = json.loads(path.read_text()) if path.exists() else None + metadata["uid"] = (prior or {}).get("metadata", {}).get("uid", f"uid-new-{kind}-{metadata['name']}") + metadata["resourceVersion"] = str(1000 + count) + metadata["generation"] = 2 + path.write_text(json.dumps(item)) + if count == int(os.environ.get("PLATFORM_TEST_FAIL_AFTER_APPLY_AT", "0")): + raise SystemExit(73) + raise SystemExit(0) +if args and args[0] == "replace": + filename = option("-f", "--filename") + documents = load_documents(filename) if filename else [] + if len(documents) != 1: + raise SystemExit(47) + item = documents[0] + metadata = item.get("metadata") or {} + namespace = metadata.get("namespace", "default") + path = state_path(item["kind"], namespace, metadata["name"]) + if not path.exists(): + raise SystemExit(48) + prior = json.loads(path.read_text()) + prior_metadata = prior.get("metadata") or {} + with log.open("a", encoding="utf-8") as stream: + stream.write( + f"replace-preconditions {item['kind']}/{metadata.get('name')} " + f"uid={metadata.get('uid')} resourceVersion={metadata.get('resourceVersion')}\n" + ) + if (metadata.get("uid") != prior_metadata.get("uid") or + metadata.get("resourceVersion") != prior_metadata.get("resourceVersion")): + raise SystemExit(49) + if os.environ.get("PLATFORM_TEST_REPLACE_CONFLICT", "0") == "1": + raise SystemExit(409) + metadata["resourceVersion"] = "replaced" + path.write_text(json.dumps(item)) + raise SystemExit(0) +if args and args[0] == "exec": + expected = { + "grafana-dashboard-https-endpoints": "https-endpoints.json", + "grafana-dashboard-kubernetes-node": "kubernetes-node.json", + "grafana-dashboard-observability-backends": "observability-backends.json", + "grafana-dashboard-platform-services": "platform-services.json", + "grafana-dashboard-workload-health": "workload-health.json", + } + lines = [] + for name, filename in expected.items(): + item = json.loads(state_path("ConfigMap", "observability", name).read_text(encoding="utf-8")) + data = item.get("data") or {} + if set(data) != {filename}: + raise SystemExit(76) + digest = hashlib.sha256(data[filename].encode()).hexdigest() + lines.append(f"{digest} /tmp/dashboards/{filename}") + if os.environ.get("PLATFORM_TEST_GRAFANA_STALE_DASHBOARD", "0") == "1": + lines[0] = "0" * 64 + " /tmp/dashboards/https-endpoints.json" + print("\n".join(lines)) + raise SystemExit(0) +if args and args[0] == "proxy": + socket_path = option("--unix-socket") + if not socket_path: + raise SystemExit(77) + socket_file = pathlib.Path(socket_path) + if socket_file.exists(): + socket_file.unlink() + + api_prefixes = { + "/api/v1/namespaces/observability/configmaps/": "ConfigMap", + "/apis/monitoring.coreos.com/v1/namespaces/observability/prometheusrules/": "PrometheusRule", + "/apis/monitoring.coreos.com/v1alpha1/namespaces/observability/alertmanagerconfigs/": "AlertmanagerConfig", + "/apis/monitoring.coreos.com/v1/namespaces/observability/alertmanagers/": "Alertmanager", + "/apis/networking.k8s.io/v1/namespaces/observability/networkpolicies/": "NetworkPolicy", + } + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_): + return + + def reply(self, status, payload): + encoded = json.dumps(payload, separators=(",", ":")).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def do_DELETE(self): + length = int(self.headers.get("Content-Length", "0")) + try: + delete_options = json.loads(self.rfile.read(length)) + except Exception: + self.reply(400, {"kind": "Status", "reason": "BadRequest"}) + return + kind = None + name = None + for prefix, candidate in api_prefixes.items(): + if self.path.startswith(prefix): + kind = candidate + name = self.path[len(prefix):] + break + path = state_path(kind, "observability", name) if kind and name else None + item = json.loads(path.read_text(encoding="utf-8")) if path and path.exists() else None + expected_uid = ((item or {}).get("metadata") or {}).get("uid") + supplied_uid = ((delete_options.get("preconditions") or {}).get("uid")) + exact_options = ( + delete_options == { + "apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": supplied_uid}, + "propagationPolicy": "Background", + } + and isinstance(supplied_uid, str) and supplied_uid == expected_uid + ) + with log.open("a", encoding="utf-8") as stream: + stream.write( + f"raw-delete {self.path} uid={supplied_uid} " + f"propagation={delete_options.get('propagationPolicy')} exact={str(exact_options).lower()}\n" + ) + if not item: + self.reply(404, {"kind": "Status", "reason": "NotFound"}) + return + if not exact_options: + self.reply(409, {"kind": "Status", "reason": "Conflict"}) + return + mode = os.environ.get("PLATFORM_TEST_DELETE_MODE", "success") + count_file = pathlib.Path(os.environ["PLATFORM_TEST_DELETE_COUNT"]) + delete_count = int(count_file.read_text()) if count_file.exists() else 0 + delete_count += 1 + count_file.write_text(str(delete_count)) + active_mode = mode if delete_count == 1 else "success" + if active_mode == "conflict": + self.reply(409, {"kind": "Status", "reason": "Conflict"}) + return + if active_mode == "timeout": + time.sleep(5) + return + path.unlink() + if active_mode == "response-loss": + self.close_connection = True + return + self.reply(200, {"kind": "Status", "status": "Success"}) + + with socketserver.UnixStreamServer(str(socket_file), Handler) as server: + server.serve_forever() +if args and args[0] == "delete": + raise SystemExit(96) +if args and args[0] in {"wait", "rollout"}: + if "--timeout=60s" in args and os.environ.get("PLATFORM_TEST_PREFLIGHT_RC", "0") != "0": + raise SystemExit(78) + if "--timeout=10s" in args: + pathlib.Path(os.environ["PLATFORM_TEST_POSTCHECK_REACHED"]).touch() + delay = float(os.environ.get("PLATFORM_TEST_POSTCHECK_DELAY", "0")) + if delay: + time.sleep(delay) + if os.environ.get("PLATFORM_TEST_POSTCHECK_RC", "0") != "0": + raise SystemExit(79) + raise SystemExit(0) +raise SystemExit(46) +PY + chmod 0755 "$fixture/bin/kubectl" +} + +write_inventory() { + local root=$1 phase=$2 captured_at=$3 target_count=$4 sha + [[ "$target_count" =~ ^[1-9][0-9]*$ ]] || fail "fixture target count is not a canonical positive decimal: $phase" + mkdir -m 0700 -p -- "$root/$phase" + jq -c -n \ + --arg phase "$phase" \ + --arg captured_at "$captured_at" \ + --argjson target_count "$target_count" ' + { + schema: "platform-observability-metric-inventory/v1", + phase: $phase, + context: "default", + api_server: "https://127.0.0.1:6443", + captured_at_utc: $captured_at, + targets: [range(0; $target_count) | { + job: "fixture", + health: "up", + last_error: "", + metrics: [{name: "up", label_names: ["job"]}] + }] + } + ' >"$root/$phase/inventory.json" + chmod 0600 "$root/$phase/inventory.json" + jq -e --arg phase "$phase" --arg captured_at "$captured_at" --argjson target_count "$target_count" ' + .schema == "platform-observability-metric-inventory/v1" and + .phase == $phase and .context == "default" and + .api_server == "https://127.0.0.1:6443" and + .captured_at_utc == $captured_at and + (.targets | type == "array" and length == $target_count) and + all(.targets[]; + .health == "up" and .last_error == "" and + (.metrics | type == "array" and length > 0) and + all(.metrics[]; + (.name | type == "string" and length > 0) and + (.label_names | type == "array") and + all(.label_names[]; type == "string"))) + ' "$root/$phase/inventory.json" >/dev/null || fail "fixture inventory generation was invalid: $phase" + sha="$(sha256sum "$root/$phase/inventory.json" | awk '{print $1}')" + printf '%s inventory.json\n' "$sha" >"$root/$phase/inventory.sha256" + chmod 0600 "$root/$phase/inventory.sha256" + printf '%s' "$sha" +} + +refresh_inventory_checksum() { + local root=$1 phase=$2 sha + sha="$(sha256sum -- "$root/$phase/inventory.json" | awk '{print $1}')" + printf '%s inventory.json\n' "$sha" >"$root/$phase/inventory.sha256" + chmod 0600 "$root/$phase/inventory.sha256" + printf '%s' "$sha" +} + +rewrite_inventory_filter() { + local root=$1 phase=$2 filter=$3 next + next="$root/$phase/inventory.json.next" + jq -c "$filter" "$root/$phase/inventory.json" >"$next" || fail "fixture inventory rewrite failed: $phase" + chmod 0600 "$next" + mv -f -- "$next" "$root/$phase/inventory.json" + refresh_inventory_checksum "$root" "$phase" +} + +rewrite_inventory_count() { + local root=$1 phase=$2 target_count=$3 next + next="$root/$phase/inventory.json.next" + [[ "$target_count" =~ ^(0|[1-9][0-9]*)$ ]] || fail "fixture rewrite count is malformed: $phase" + jq -c --argjson target_count "$target_count" ' + .targets[0] as $target | + .targets = [range(0; $target_count) | $target] + ' "$root/$phase/inventory.json" >"$next" || fail "fixture count rewrite failed: $phase" + chmod 0600 "$next" + mv -f -- "$next" "$root/$phase/inventory.json" + refresh_inventory_checksum "$root" "$phase" +} + +write_handoff() { + local root=$1 runbook=${2-https://git.learn.hyeonworks.com/platform/runbooks/observability.md} + cat >"$root/dashboards.yaml" <<'YAML' +apiVersion: v1 +kind: ConfigMap +metadata: {name: grafana-dashboard-https-endpoints, namespace: observability, labels: {grafana_dashboard: "1"}} +data: {https-endpoints.json: '{"uid":"platform-https-endpoints","title":"Platform / HTTPS Endpoints"}'} +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: grafana-dashboard-kubernetes-node, namespace: observability, labels: {grafana_dashboard: "1"}} +data: {kubernetes-node.json: '{"uid":"platform-kubernetes-node","title":"Platform / Kubernetes Node"}'} +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: grafana-dashboard-observability-backends, namespace: observability, labels: {grafana_dashboard: "1"}} +data: {observability-backends.json: '{"uid":"platform-observability-backends","title":"Platform / Observability Backends"}'} +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: grafana-dashboard-platform-services, namespace: observability, labels: {grafana_dashboard: "1"}} +data: {platform-services.json: '{"uid":"platform-services","title":"Platform / Services"}'} +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: grafana-dashboard-workload-health, namespace: observability, labels: {grafana_dashboard: "1"}} +data: {workload-health.json: '{"uid":"platform-workload-health","title":"Platform / Workload Health"}'} +YAML + : >"$root/rules.yaml" + local name alert + while IFS='|' read -r name alert; do + cat >>"$root/rules.yaml" <"$root/alertmanager.yaml" <<'YAML' +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: {name: observability-core-kube-pr-alertmanager, namespace: observability} +spec: {alertmanagerConfiguration: {name: platform-alertmanager}} +--- +apiVersion: monitoring.coreos.com/v1alpha1 +kind: AlertmanagerConfig +metadata: + name: platform-alertmanager + namespace: observability + labels: + observability.hyeonworks.com/instance: home +spec: + route: + receiver: platform-slack + groupBy: + - cluster + - namespace + - alertname + - severity + groupWait: 30s + groupInterval: 5m + repeatInterval: 4h + routes: + - receiver: platform-null + matchers: + - name: alertname + matchType: "=" + value: InfoInhibitor + inhibitRules: + - sourceMatch: + - name: alertname + matchType: "=" + value: InfoInhibitor + targetMatch: + - name: severity + matchType: "=" + value: info + equal: + - namespace + receivers: + - name: platform-null + - name: platform-slack + slackConfigs: + - apiURL: + name: alertmanager-slack-webhook + key: url + sendResolved: true + linkNames: false + mrkdwnIn: + - text + - fields + fallback: >- + {{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }} + title: >- + {{ if eq .Status "firing" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }} + titleLink: https://grafana.learn.hyeonworks.com/ + color: >- + {{ if eq .Status "resolved" }}good{{ else if or (eq .CommonLabels.severity "emergency") (eq .CommonLabels.severity "critical") }}danger{{ else if eq .CommonLabels.severity "warning" }}warning{{ else }}#439FE0{{ end }} + fields: + - title: Status + value: '{{ .Status | toUpper }}' + short: true + - title: Severity + value: '{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}' + short: true + - title: Location + value: '{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}' + short: true + - title: Alert count + value: '{{ len .Alerts }}' + short: true + text: |- + {{ range .Alerts }} + *Alert status:* {{ .Status | toUpper }} + *Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }} + *Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }} + *Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }} + *Started:* {{ .StartsAt.Format "2006-01-02T15:04:05Z07:00" }} + {{ if eq .Status "resolved" }}*Ended:* {{ .EndsAt.Format "2006-01-02T15:04:05Z07:00" }}{{ end }} + {{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }} + + {{ end }} + · + footer: hyeonworks observability · Alertmanager +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: observability-allow-alertmanager-public-https, namespace: observability} +spec: + podSelector: + matchLabels: + app.kubernetes.io/instance: observability-core-kube-pr-alertmanager + app.kubernetes.io/name: alertmanager + policyTypes: [Egress] + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: [10.0.0.0/8, 100.64.0.0/10, 172.16.0.0/12, 192.168.0.0/16] + ports: + - {protocol: TCP, port: 443} +YAML + chmod 0600 "$root/dashboards.yaml" "$root/rules.yaml" "$root/alertmanager.yaml" +} + +new_fixture() { + local name=$1 runbook=${2-https://git.learn.hyeonworks.com/platform/runbooks/observability.md} + local captured_at=${3-} initial_count=${4-$INITIAL_TARGET_COUNT} post_count=${5-$POST_TARGET_COUNT} + local fixture metric initial_sha post_sha + [[ -n "$captured_at" ]] || captured_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + fixture="$WORK/$name" + mkdir -m 0700 -- "$fixture" + mkdir -m 0700 -- "$fixture/rollbacks" + metric="$fixture/metrics" + mkdir -m 0700 -- "$metric" + mkdir -m 0700 -- "$fixture/rollbacks/observability-$ROLLBACK_ID" + make_fakes "$fixture" + initial_sha="$(write_inventory "$metric" target-initial "$captured_at" "$initial_count")" + post_sha="$(write_inventory "$metric" post-substrate "$captured_at" "$post_count")" + write_handoff "$metric" "$runbook" + cat >"$fixture/state/Alertmanager__observability__observability-core-kube-pr-alertmanager.json" <<'JSON' +{"apiVersion":"monitoring.coreos.com/v1","kind":"Alertmanager","metadata":{"name":"observability-core-kube-pr-alertmanager","namespace":"observability","uid":"uid-alertmanager-prior","resourceVersion":"88","generation":1,"creationTimestamp":"2026-08-01T00:00:00Z","managedFields":[{"manager":"operator"}],"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"remove"}},"spec":{"replicas":1},"status":{"availableReplicas":1}} +JSON + chmod 0700 "$fixture/rollbacks/observability-$ROLLBACK_ID" + printf '%s|%s|%s|%s' "$fixture" "$metric" "$initial_sha" "$post_sha" +} + +run_apply() { + local fixture=$1 metric=$2 initial_sha=$3 post_sha=$4 + shift 4 + env \ + PATH="$fixture/bin:$PATH" \ + PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 \ + PLATFORM_OBSERVABILITY_ACCESS_KUBECTL_BIN="$fixture/bin/kubectl" \ + PLATFORM_OBSERVABILITY_ACCESS_SUDO_BIN="${PLATFORM_TEST_SUDO_BIN:-$fixture/bin/sudo}" \ + PLATFORM_OBSERVABILITY_ACCESS_ENCRYPTION_SCRIPT="$fixture/bin/encryption" \ + PLATFORM_OBSERVABILITY_ACCESS_RESTORE_SCRIPT="$fixture/bin/restore" \ + PLATFORM_OBSERVABILITY_ACCESS_RECOVERY_SCRIPT="$fixture/bin/recovery" \ + PLATFORM_OBSERVABILITY_ACCESS_ROLLBACK_BASE="$fixture/rollbacks" \ + PLATFORM_TEST_EXPECTED_INITIAL_SHA="$initial_sha" \ + PLATFORM_TEST_EXPECTED_POST_SHA="$post_sha" \ + PLATFORM_OBSERVABILITY_ACCESS_CONFIRMATION=APPLY \ + PLATFORM_OBSERVABILITY_ROLLBACK_ID="$ROLLBACK_ID" \ + PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \ + PLATFORM_TEST_SUDO_REFRESHED="$fixture/sudo-refreshed" \ + PLATFORM_TEST_ENCRYPTION_COUNT="$fixture/encryption-count" \ + PLATFORM_TEST_RESTORE_COUNT="$fixture/restore-count" \ + PLATFORM_TEST_RECOVERY_COUNT="$fixture/recovery-count" \ + PLATFORM_TEST_SLACK_SCHEMA_COUNT="$fixture/slack-schema-count" \ + PLATFORM_TEST_APPLY_COUNT="$fixture/apply-count" \ + PLATFORM_TEST_DELETE_COUNT="$fixture/delete-count" \ + PLATFORM_TEST_STATE="$fixture/state" \ + PLATFORM_TEST_POSTCHECK_REACHED="$fixture/postcheck-reached" \ + PLATFORM_TEST_ENCRYPTION_FAIL_AT="${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" \ + PLATFORM_TEST_RESTORE_FAIL_AT="${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" \ + PLATFORM_TEST_RECOVERY_RC="${PLATFORM_TEST_RECOVERY_RC:-0}" \ + PLATFORM_TEST_RECOVERY_RC_1="${PLATFORM_TEST_RECOVERY_RC_1:-}" \ + PLATFORM_TEST_RECOVERY_RC_2="${PLATFORM_TEST_RECOVERY_RC_2:-}" \ + PLATFORM_TEST_RECOVERY_OUTPUT_1="${PLATFORM_TEST_RECOVERY_OUTPUT_1:-}" \ + PLATFORM_TEST_RECOVERY_OUTPUT_2="${PLATFORM_TEST_RECOVERY_OUTPUT_2:-}" \ + PLATFORM_TEST_PREFLIGHT_RC="${PLATFORM_TEST_PREFLIGHT_RC:-0}" \ + PLATFORM_TEST_POSTCHECK_RC="${PLATFORM_TEST_POSTCHECK_RC:-0}" \ + PLATFORM_TEST_POSTCHECK_DELAY="${PLATFORM_TEST_POSTCHECK_DELAY:-0}" \ + PLATFORM_TEST_PARENT_PID="$BASHPID" \ + PLATFORM_TEST_FAIL_APPLY_AT="${PLATFORM_TEST_FAIL_APPLY_AT:-0}" \ + PLATFORM_TEST_FAIL_AFTER_APPLY_AT="${PLATFORM_TEST_FAIL_AFTER_APPLY_AT:-0}" \ + PLATFORM_TEST_SIGNAL_AT="${PLATFORM_TEST_SIGNAL_AT:-0}" \ + PLATFORM_TEST_SLACK_EXTRA_KEY="${PLATFORM_TEST_SLACK_EXTRA_KEY:-0}" \ + PLATFORM_TEST_SLACK_EXTRA_KEY_AT="${PLATFORM_TEST_SLACK_EXTRA_KEY_AT:-0}" \ + PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE="${PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE:-0}" \ + PLATFORM_TEST_DRIFT_RESOURCE="${PLATFORM_TEST_DRIFT_RESOURCE:-}" \ + PLATFORM_TEST_DELETE_MODE="${PLATFORM_TEST_DELETE_MODE:-success}" \ + PLATFORM_TEST_REPLACE_CONFLICT="${PLATFORM_TEST_REPLACE_CONFLICT:-0}" \ + PLATFORM_TEST_PROMETHEUS_DROP_RULE="${PLATFORM_TEST_PROMETHEUS_DROP_RULE:-0}" \ + PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE="${PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE-qualified}" \ + PLATFORM_TEST_GRAFANA_STALE_DASHBOARD="${PLATFORM_TEST_GRAFANA_STALE_DASHBOARD:-0}" \ + PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE="${PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE:-}" \ + PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE="${PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE:-}" \ + PLATFORM_TEST_INVENTORY_ROOT="$metric" \ + bash "$APPLY" "$@" +} + +run_inventory_validator() { + local metric=$1 phase=$2 expected_sha=$3 expected_count=$4 + env PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 bash -c ' + source "$1" + verified_output_dir=$2 + validate_inventory_phase "$3" "$4" "$5" + ' _ "$APPLY" "$metric" "$phase" "$expected_sha" "$expected_count" +} + +assert_pre_prompt_rejection() { + local fixture=$1 label=$2 ledger + ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts" + [[ "$(<"$fixture/output")" != *'Type APPLY'* ]] || fail "$label reached confirmation" + ! grep -Fq 'kubectl --request-timeout=10s apply --server-side --dry-run=server -f' "$fixture/commands.log" || + fail "$label reached server dry-run" + ! grep -Eq '^sudo .* install ' "$fixture/commands.log" || fail "$label installed rollback data" + [[ ! -e "$ledger/objects.tsv" ]] || fail "$label created a rollback ledger" + ! grep -Fq 'kubectl --request-timeout=10s apply -f' "$fixture/commands.log" || fail "$label reached Kubernetes apply" + [[ ! -e "$ledger/acceptance.env" ]] || fail "$label wrote acceptance" +} + +assert_post_confirmation_rejection() { + local fixture=$1 label=$2 ledger + ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts" + [[ -f "$ledger/objects.tsv" ]] || fail "$label did not preserve the durable rollback ledger" + [[ ! -e "$fixture/apply-count" ]] || fail "$label reached an apply invocation" + ! grep -Fq 'kubectl --request-timeout=10s apply -f' "$fixture/commands.log" || fail "$label reached Kubernetes apply" + [[ ! -e "$ledger/acceptance.env" ]] || fail "$label wrote acceptance" +} + +assert_reconcile_rollback() { + local fixture=$1 label=$2 ledger prior + ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts" + prior="$fixture/state/Alertmanager__observability__observability-core-kube-pr-alertmanager.json" + [[ -e "$fixture/postcheck-reached" ]] || fail "$label did not reach reconciliation" + grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS' "$fixture/output" || + fail "$label did not roll back" + [[ -f "$ledger/objects.tsv" ]] || fail "$label did not preserve the durable rollback ledger" + [[ ! -e "$ledger/acceptance.env" ]] || fail "$label wrote acceptance" + [[ -f "$prior" ]] || fail "$label did not restore the pre-existing Alertmanager" + [[ "$(find "$fixture/state" -type f | wc -l | tr -d ' ')" == 1 ]] || + fail "$label left newly-created fake state" +} + +run_deployment_gate_helper() { + local fixture=$1 first_output=$2 second_output=$3 expected_value=${4:-} + env \ + PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 \ + PLATFORM_OBSERVABILITY_ACCESS_RECOVERY_SCRIPT="$fixture/bin/recovery" \ + PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \ + PLATFORM_TEST_RECOVERY_COUNT="$fixture/recovery-count" \ + PLATFORM_TEST_RECOVERY_OUTPUT_1="$first_output" \ + PLATFORM_TEST_RECOVERY_OUTPUT_2="$second_output" \ + bash -c ' + source "$1" + trap cleanup EXIT + capture_slack_deployment_gate slack_deployment_gate_first helper-first + capture_slack_deployment_gate slack_deployment_gate helper-second + require_matching_slack_deployment_gates + map_slack_deployment_gate_acceptance + [[ "$slack_deployment_gate_value" == "$2" ]] + cleanup + ' _ "$APPLY" "$expected_value" +} + +split_fixture() { + local value=$1 + IFS='|' read -r FIXTURE METRIC INITIAL_SHA POST_SHA <<<"$value" + METRIC_ROOTS+=("$METRIC") +} + +supervise_focused_suite() { + (( $# == 0 )) || fail 'focused suite accepts no caller arguments' + WORK="$(mktemp -d /tmp/platform-observability-access-test.XXXXXX)" + chmod 0700 "$WORK" + forged_root="$(mktemp -d /tmp/platform-observability-access-test.XXXXXX)" + chmod 0700 "$forged_root" + forged_attestation="$forged_root/worker.attestation" + printf '%s\n' "$(stat -c '%d:%i' -- "$TEST_SCRIPT")" >"$forged_attestation" + chmod 0600 "$forged_attestation" + if run_with_wall_bound 2 bash "$TEST_SCRIPT" --internal-worker "$forged_root" "$forged_attestation" \ + >"$WORK/forged-worker.out" 2>&1; then + forged_rc=0 + else + forged_rc=$? + fi + case "$forged_root" in + /tmp/platform-observability-access-test.??????) rm -rf -- "$forged_root" ;; + esac + (( forged_rc != 124 )) && ! grep -q '^PASS:' "$WORK/forged-worker.out" || + fail 'direct forged internal worker bypassed the suite supervisor' + printf 'SUPERVISOR_FORGED_WORKER_REJECTED=PASS\n' + suite_started=$SECONDS + if run_with_wall_bound "$SUITE_WALL_BOUND_SECONDS" \ + /usr/bin/bash -c 'source "$1"; run_focused_suite_worker "$2"' \ + _ "$TEST_SCRIPT" "$WORK"; then + suite_rc=0 + else + suite_rc=$? + fi + suite_elapsed=$((SECONDS - suite_started)) + mapfile -t suite_orphan_pids < <(pgrep -f -- "$WORK" || true) + suite_orphan_count=${#suite_orphan_pids[@]} + if (( suite_orphan_count > 0 )); then + kill -TERM "${suite_orphan_pids[@]}" 2>/dev/null || true + sleep 0.2 + kill -KILL "${suite_orphan_pids[@]}" 2>/dev/null || true + fi + (( suite_rc != 124 )) || fail "focused suite exceeded its ${SUITE_WALL_BOUND_SECONDS}s hard wall-clock bound" + (( suite_orphan_count == 0 )) || fail 'focused suite left a fixture-owned orphan process' + (( suite_elapsed <= SUITE_WALL_BOUND_SECONDS + 2 )) || fail 'focused suite wall-clock accounting exceeded its hard bound' + printf 'SUITE_WALL_SECONDS=%s\n' "$suite_elapsed" + printf 'SUITE_WALL_BOUND_SECONDS=%s\n' "$SUITE_WALL_BOUND_SECONDS" + printf 'SUITE_ORPHAN_PROCESSES=0\n' + return "$suite_rc" +} + +run_focused_suite_worker() { +(( $# == 1 )) || fail 'invalid focused suite worker arity' +WORK=$1 +[[ "$WORK" =~ ^/tmp/platform-observability-access-test\.[A-Za-z0-9]{6}$ && + -d "$WORK" && ! -L "$WORK" && "$(readlink -f -- "$WORK")" == "$WORK" && + "$(stat -c '%u:%a' -- "$WORK")" == "$(id -u):700" ]] || + fail 'focused suite supervisor work root is not exact, canonical, and private' +PROGRESS_LOG="$WORK/suite-progress.log" +: >"$PROGRESS_LOG" +chmod 0600 "$PROGRESS_LOG" + +deadline_probe="platform-observability-deadline-probe-$BASHPID" +if run_with_wall_bound 1 /usr/bin/bash -c 'exec -a "$1" /usr/bin/sleep 5' _ "$deadline_probe"; then + deadline_probe_rc=0 +else + deadline_probe_rc=$? +fi +[[ "$deadline_probe_rc" == 124 ]] || fail 'suite wall-clock helper did not enforce its exact deadline' +! pgrep -f -- "$deadline_probe" >/dev/null || fail 'suite wall-clock helper left an orphan process' +pass 'suite wall-clock helper terminates its process group without an orphan' + +# Break caught: production dry-run used to return before validating the +# environment boundary, so a caller-controlled binary override silently +# survived until a later execute invocation. +if env PLATFORM_OBSERVABILITY_ACCESS_KUBECTL_BIN=/usr/bin/true \ + bash "$APPLY" >"$WORK/production-override.out" 2>&1; then + fail 'production dry-run accepted a test boundary override' +fi +pass 'production rejects every access boundary override before dry-run' + +# Break caught: TEST_MODE used to be an env-only switch. In particular the +# real sudo binary was accepted because dry-run skipped fixture validation. +split_fixture "$(new_fixture unsafe-real-sudo)" +if PLATFORM_TEST_SUDO_BIN=/usr/bin/sudo \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + >"$FIXTURE/output" 2>&1; then + fail 'test mode accepted the real sudo binary' +fi +grep -Fq 'test command boundary is unsafe' "$FIXTURE/output" || + fail 'real sudo rejection did not identify the unsafe boundary' +pass 'test mode rejects real sudo before dry-run' + +split_fixture "$(new_fixture dry-run)" +output="$(run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA")" || fail 'dry-run failed' +[[ "$output" == *'OBSERVABILITY_ACCESS_DRY_RUN=PASS'* ]] || fail 'dry-run PASS marker missing' +[[ "$output" == *"TARGET_INITIAL_SHA256=$INITIAL_PRODUCTION_SHA"* ]] || fail 'dry-run did not show pinned initial hash' +[[ "$output" == *"POST_SUBSTRATE_SHA256=$POST_PRODUCTION_SHA"* ]] || fail 'dry-run did not show pinned post hash' +[[ ! -s "$FIXTURE/commands.log" ]] || fail 'dry-run invoked a system boundary' +pass 'source-only dry-run prints pinned readiness without system access' + +split_fixture "$(new_fixture alertmanager-missing-title)" +python3 - "$METRIC/alertmanager.yaml" <<'PY' +import sys +import yaml + +path = sys.argv[1] +with open(path, encoding="utf-8") as stream: + items = [item for item in yaml.safe_load_all(stream) if item is not None] +config = next(item for item in items if item.get("kind") == "AlertmanagerConfig") +receiver = next(item for item in config["spec"]["receivers"] if item["name"] == "platform-slack") +receiver["slackConfigs"][0].pop("title") +with open(path, "w", encoding="utf-8") as stream: + yaml.safe_dump_all(items, stream, explicit_start=True, sort_keys=False) +PY +chmod 0600 "$METRIC/alertmanager.yaml" +if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'Alertmanager handoff without Slack title was accepted' +fi +assert_pre_prompt_rejection "$FIXTURE" 'Alertmanager handoff without Slack title' +pass 'Alertmanager handoff without Slack title fails before prompt, ledger, server dry-run, and apply' + +# Break caught: the production receiver predicate treated the operator's +# qualified receiver pair as absent, causing a valid rules-alerts transaction +# to roll back after all resources had been applied. +split_fixture "$(new_fixture qualified-receiver-pair)" +if ! run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + assert_reconcile_rollback "$FIXTURE" 'qualified receiver pair' + fail 'qualified receiver pair was rejected' +fi +grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" || + fail 'qualified receiver pair omitted transaction success' +pass 'qualified Alertmanager receiver pair is accepted exactly once' + +split_fixture "$(new_fixture qualified-receiver-pair-reversed)" +if ! PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE=qualified-reversed \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + assert_reconcile_rollback "$FIXTURE" 'reversed qualified receiver pair' + fail 'reversed qualified receiver pair was rejected' +fi +grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" || + fail 'reversed qualified receiver pair omitted transaction success' +pass 'reversed qualified Alertmanager receiver pair is accepted exactly once' + +# Break caught: exact source-pinned historical evidence older than the former +# 24-hour ceiling was rejected even though age is not an authority signal. +old_capture="$(date -u -d '49 hours ago' +%Y-%m-%dT%H:%M:%SZ)" +split_fixture "$(new_fixture old-authoritative-pair \ + https://git.learn.hyeonworks.com/platform/runbooks/observability.md "$old_capture")" +if ! run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + [[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || + fail 'rejected old authoritative pair unexpectedly reached confirmation' + ! grep -Eq '^sudo .* install ' "$FIXTURE/commands.log" || + fail 'rejected old authoritative pair unexpectedly invoked ledger install' + ! grep -Fq 'kubectl --request-timeout=10s apply -f' "$FIXTURE/commands.log" || + fail 'rejected old authoritative pair unexpectedly invoked Kubernetes apply' + [[ ! -e "$FIXTURE/apply-count" ]] || fail 'rejected old authoritative pair unexpectedly mutated Kubernetes' + [[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] || + fail 'rejected old authoritative pair unexpectedly installed a rollback ledger' + [[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] || + fail 'rejected old authoritative pair unexpectedly wrote acceptance' + fail 'old exact 21/30 authoritative pair was rejected' +fi +grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" || + fail 'old exact 21/30 authoritative pair omitted transaction success' +grep -Fq 'schema=platform-observability-rules-alerts-v2' \ + "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" || + fail 'old exact 21/30 authoritative pair omitted v2 acceptance' +pass 'old exact 21/30 source-pinned inventories remain authoritative' + +split_fixture "$(new_fixture valid-inventory-pair)" +if ! env PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 \ + PLATFORM_OBSERVABILITY_ACCESS_ROLLBACK_BASE="$FIXTURE/rollbacks" \ + bash -c ' + source "$1" + verified_output_dir=$2 + expected_initial_sha=$3 + expected_post_sha=$4 + validate_handoff_root + verify_handoff_unchanged + ' _ "$APPLY" "$METRIC" "$INITIAL_SHA" "$POST_SHA" >"$FIXTURE/output" 2>&1; then + sed -n '1,160p' "$FIXTURE/output" >&2 + fail 'valid exact 21/30 pair failed an initial or unchanged source-only gate' +fi +[[ ! -s "$FIXTURE/commands.log" ]] || fail 'source-only inventory pair control reached a system boundary' +pass 'valid exact 21/30 pair passes both initial and unchanged source-only gates' + +for timestamp_case in malformed-utc invalid-calendar future-skew; do + split_fixture "$(new_fixture "timestamp-$timestamp_case")" + case "$timestamp_case" in + malformed-utc) timestamp_value='not-a-utc-second' ;; + invalid-calendar) timestamp_value='2026-02-30T12:00:00Z' ;; + future-skew) timestamp_value="$(date -u -d '600 seconds' +%Y-%m-%dT%H:%M:%SZ)" ;; + esac + INITIAL_SHA="$(rewrite_inventory_filter "$METRIC" target-initial \ + ".captured_at_utc = \"$timestamp_value\"")" + if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail "invalid inventory timestamp was accepted: $timestamp_case" + fi + assert_pre_prompt_rejection "$FIXTURE" "inventory timestamp $timestamp_case" + pass "inventory timestamp $timestamp_case fails closed before confirmation" +done + +for count_case in initial-20 initial-22 post-29 post-31; do + split_fixture "$(new_fixture "count-$count_case")" + case "$count_case" in + initial-20) phase=target-initial; actual_count=20 ;; + initial-22) phase=target-initial; actual_count=22 ;; + post-29) phase=post-substrate; actual_count=29 ;; + post-31) phase=post-substrate; actual_count=31 ;; + esac + changed_sha="$(rewrite_inventory_count "$METRIC" "$phase" "$actual_count")" + if [[ "$phase" == target-initial ]]; then + INITIAL_SHA=$changed_sha + else + POST_SHA=$changed_sha + fi + run_inventory_validator "$METRIC" "$phase" "$changed_sha" "$actual_count" \ + >"$FIXTURE/source-control.out" 2>&1 || + fail "source-only actual-count control failed: $count_case" + if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail "production expected target count accepted drift: $count_case" + fi + assert_pre_prompt_rejection "$FIXTURE" "inventory count $count_case" + pass "inventory count $count_case is self-consistent but not production-authoritative" +done + +split_fixture "$(new_fixture invalid-expected-count)" +for invalid_expected_count in 0 01 x; do + if run_inventory_validator "$METRIC" target-initial "$INITIAL_SHA" "$invalid_expected_count" \ + >"$FIXTURE/expected-count-$invalid_expected_count.out" 2>&1; then + fail "invalid expected count reached inventory validation: $invalid_expected_count" + fi + grep -Fq 'inventory expected target count is invalid' \ + "$FIXTURE/expected-count-$invalid_expected_count.out" || + fail "invalid expected count lacked its entry-gate error: $invalid_expected_count" +done +[[ ! -s "$FIXTURE/commands.log" ]] || fail 'invalid expected count reached a system boundary' +pass 'expected count rejects 0, 01, and x at the source-only function entry' + +for semantic_case in schema phase context api health last-error empty-metrics metric-name label-type; do + split_fixture "$(new_fixture "semantic-$semantic_case")" + case "$semantic_case" in + schema) filter='.schema = "platform-observability-metric-inventory/v2"' ;; + phase) filter='.phase = "post-substrate"' ;; + context) filter='.context = "other"' ;; + api) filter='.api_server = "https://127.0.0.1:7443"' ;; + health) filter='.targets[0].health = "down"' ;; + last-error) filter='.targets[0].last_error = "failed"' ;; + empty-metrics) filter='.targets[0].metrics = []' ;; + metric-name) filter='.targets[0].metrics[0].name = 7' ;; + label-type) filter='.targets[0].metrics[0].label_names[0] = 7' ;; + esac + INITIAL_SHA="$(rewrite_inventory_filter "$METRIC" target-initial "$filter")" + if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail "self-consistent inventory semantic drift was accepted: $semantic_case" + fi + assert_pre_prompt_rejection "$FIXTURE" "inventory semantic drift $semantic_case" +done +pass 'schema, phase, context, API, health, error, metrics, name, and label-type drift fail closed' + +for authority_case in wrong-production-pin one-good-phase swapped-phases; do + split_fixture "$(new_fixture "authority-$authority_case")" + case "$authority_case" in + wrong-production-pin) expected_initial=$INITIAL_PRODUCTION_SHA; expected_post=$POST_PRODUCTION_SHA ;; + one-good-phase) expected_initial=$INITIAL_SHA; expected_post=$POST_PRODUCTION_SHA ;; + swapped-phases) expected_initial=$POST_SHA; expected_post=$INITIAL_SHA ;; + esac + if run_apply "$FIXTURE" "$METRIC" "$expected_initial" "$expected_post" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail "invalid source pin pair was accepted: $authority_case" + fi + assert_pre_prompt_rejection "$FIXTURE" "source pin $authority_case" +done + +split_fixture "$(new_fixture mixed-generation-a)" +mixed_fixture=$FIXTURE mixed_metric=$METRIC mixed_initial=$INITIAL_SHA mixed_post=$POST_SHA +older_generation="$(date -u -d '2 hours ago' +%Y-%m-%dT%H:%M:%SZ)" +split_fixture "$(new_fixture mixed-generation-b \ + https://git.learn.hyeonworks.com/platform/runbooks/observability.md "$older_generation")" +other_post=$POST_SHA +FIXTURE=$mixed_fixture METRIC=$mixed_metric INITIAL_SHA=$mixed_initial POST_SHA=$mixed_post +if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$other_post" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'mixed-generation source pin pair was accepted' +fi +assert_pre_prompt_rejection "$FIXTURE" 'mixed-generation source pin pair' +pass 'wrong, partial, swapped, and mixed-generation source pins fail closed' + +for integrity_case in stale-checksum wrong-checksum-grammar extra-phase-entry missing-phase-entry json-symlink checksum-symlink json-hardlink checksum-hardlink wrong-file-mode; do + split_fixture "$(new_fixture "integrity-$integrity_case")" + case "$integrity_case" in + stale-checksum) printf ' \n' >>"$METRIC/target-initial/inventory.json" ;; + wrong-checksum-grammar) printf 'not-a-sha inventory.json\n' >"$METRIC/target-initial/inventory.sha256" ;; + extra-phase-entry) printf 'extra\n' >"$METRIC/target-initial/extra"; chmod 0600 "$METRIC/target-initial/extra" ;; + missing-phase-entry) rm -- "$METRIC/target-initial/inventory.sha256" ;; + json-symlink) mv "$METRIC/target-initial/inventory.json" "$FIXTURE/inventory-json-control"; ln -s "$FIXTURE/inventory-json-control" "$METRIC/target-initial/inventory.json" ;; + checksum-symlink) mv "$METRIC/target-initial/inventory.sha256" "$FIXTURE/inventory-checksum-control"; ln -s "$FIXTURE/inventory-checksum-control" "$METRIC/target-initial/inventory.sha256" ;; + json-hardlink) ln "$METRIC/target-initial/inventory.json" "$FIXTURE/inventory-json-control" ;; + checksum-hardlink) ln "$METRIC/target-initial/inventory.sha256" "$FIXTURE/inventory-checksum-control" ;; + wrong-file-mode) chmod 0644 "$METRIC/target-initial/inventory.json" ;; + esac + if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail "inventory integrity drift was accepted: $integrity_case" + fi + assert_pre_prompt_rejection "$FIXTURE" "inventory integrity $integrity_case" +done +pass 'checksum, entry-set, symlink, hardlink, and mode drift fail closed' + +split_fixture "$(new_fixture wrong-owner)" +write_executable "$FIXTURE/bin/id" \ + '#!/usr/bin/env bash' \ + 'if [[ "${1:-}" == -u ]]; then printf '\''999999\n'\''; else exec /usr/bin/id "$@"; fi' +if PATH="$FIXTURE/bin:$PATH" run_inventory_validator \ + "$METRIC" target-initial "$INITIAL_SHA" "$INITIAL_TARGET_COUNT" >"$FIXTURE/output" 2>&1; then + fail 'inventory owner drift was accepted' +fi +grep -Fq 'inventory directory metadata changed' "$FIXTURE/output" || + fail 'owner drift did not reach the ownership metadata gate' +[[ ! -s "$FIXTURE/commands.log" ]] || fail 'owner drift reached a system boundary' +pass 'inventory owner drift fails at the source-only metadata gate' + +argv_case=0 +for args in \ + '--execute' \ + '--execute --rules-alerts' \ + '--execute --verified-output-dir /tmp/example' \ + '--execute --rules-alerts --substrate --verified-output-dir /tmp/example' \ + '--execute --rules-alerts --verified-output-dir /tmp/example --verified-output-dir /tmp/example'; do + argv_case=$((argv_case + 1)) + split_fixture "$(new_fixture "argv-$argv_case")" + read -r -a argv <<<"$args" + if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" "${argv[@]}" >"$FIXTURE/output" 2>&1; then + fail "invalid argv was accepted: $args" + fi + [[ ! -s "$FIXTURE/commands.log" ]] || fail "invalid argv reached a system boundary: $args" +done +pass 'execute accepts only one exact rules-alerts mode and verified handoff argument' + +split_fixture "$(new_fixture unsafe-inventory-mode)" +chmod 0755 "$METRIC/target-initial" +if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'unsafe inventory mode was accepted' +fi +[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'unsafe inventory mode reached prompt' +! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'unsafe inventory mode reached mutation' +pass 'inventory ownership, mode, link count, and exact hash gate precedes prompt' + +split_fixture "$(new_fixture extra-handoff-entry)" +cp "$METRIC/dashboards.yaml" "$METRIC/grafana.yaml" +chmod 0600 "$METRIC/grafana.yaml" +if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'extra renderer handoff entry was accepted' +fi +[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'extra renderer handoff entry reached prompt' +! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'extra renderer handoff entry reached mutation' +pass 'renderer handoff rejects Grafana and every entry outside three exact rules-alerts files' + +split_fixture "$(new_fixture missing-runbook '')" +if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'missing HTTPS runbook URL was accepted' +fi +grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_READINESS=BLOCKED_RUNBOOK_URL' "$FIXTURE/output" || + { sed -n '1,160p' "$FIXTURE/output" >&2; fail 'missing runbook blocker marker absent'; } +! grep -q '^kubectl --request-timeout=10s apply ' "$FIXTURE/commands.log" || fail 'missing runbook reached apply' +! grep -q '^sudo .*install ' "$FIXTURE/commands.log" || fail 'missing runbook wrote ledger' +pass 'missing runbook URL blocks before prompt and mutation' + +split_fixture "$(new_fixture encryption-pre)" +if PLATFORM_TEST_ENCRYPTION_FAIL_AT=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'failed initial encryption gate was accepted' +fi +[[ "$(<"$FIXTURE/commands.log")" == $'sudo -v\nsudo -n true\nencryption --expect-reencrypted' ]] || + fail 'initial encryption failure did not stop at exact first gate' +pass 'initial encryption failure is zero mutation' + +split_fixture "$(new_fixture slack-schema)" +if PLATFORM_TEST_SLACK_EXTRA_KEY=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'Slack Secret with an extra key was accepted' +fi +! grep -q '^kubectl --request-timeout=10s apply ' "$FIXTURE/commands.log" || fail 'Slack schema mismatch reached apply' +[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'Slack schema mismatch reached prompt' +pass 'Slack Secret checks only the exact non-value schema before prompt' + +split_fixture "$(new_fixture recovery)" +if PLATFORM_TEST_RECOVERY_RC=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'missing Slack recovery marker was accepted' +fi +grep -Fqx 'recovery --check-slack-deployment-evidence' "$FIXTURE/commands.log" || + fail 'Slack deployment checker argv is not exact' +! grep -q '^kubectl --request-timeout=10s apply ' "$FIXTURE/commands.log" || fail 'missing recovery marker reached apply' +pass 'Slack deployment evidence is an exact pre-prompt gate' + +split_fixture "$(new_fixture second-gate-failure)" +if PLATFORM_TEST_RECOVERY_RC_2=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'second Slack deployment gate failure was accepted' +fi +[[ "$(<"$FIXTURE/recovery-count")" == 2 ]] || fail 'second gate failure did not invoke both exact deployment gates' +[[ ! -e "$FIXTURE/apply-count" ]] || fail 'second gate failure reached cluster mutation' +[[ -f "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] || + fail 'second gate failure did not preserve the durable rollback ledger' +[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] || + fail 'second gate failure wrote acceptance' +pass 'second deployment gate failure preserves ledger and remains zero mutation' + +for gate_case in recovery-to-risk risk-to-recovery duplicate-output extra-output partial-output; do + split_fixture "$(new_fixture "deployment-gate-$gate_case")" + first='SLACK_DEPLOYMENT_GATE=RECOVERY\n' + second='SLACK_DEPLOYMENT_GATE=RECOVERY\n' + expected=RECOVERY + case "$gate_case" in + recovery-to-risk) second='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n' ;; + risk-to-recovery) first='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n'; expected=RISK_ACCEPTED ;; + duplicate-output) first='SLACK_DEPLOYMENT_GATE=RECOVERY\nSLACK_DEPLOYMENT_GATE=RECOVERY\n' ;; + extra-output) first='SLACK_DEPLOYMENT_GATE=RECOVERY\nextra\n' ;; + partial-output) first='SLACK_DEPLOYMENT_GATE=RECOVERY' ;; + esac + if run_deployment_gate_helper "$FIXTURE" "$first" "$second" "$expected" \ + >"$FIXTURE/output" 2>&1; then + fail "invalid Slack deployment gate was accepted: $gate_case" + fi + [[ ! -e "$FIXTURE/apply-count" && ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] || + fail "invalid Slack deployment gate escaped the helper boundary: $gate_case" +done +pass 'verdict swaps and malformed deployment gate output fail closed before mutation' + +split_fixture "$(new_fixture last-secret-schema)" +if PLATFORM_TEST_SLACK_EXTRA_KEY_AT=2 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'last Slack Secret schema drift was accepted' +fi +[[ ! -e "$FIXTURE/apply-count" ]] || fail 'last Slack Secret schema drift reached mutation' +[[ -f "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] || + fail 'last Slack Secret schema drift did not preserve the durable ledger' +pass 'last Slack Secret schema check precedes the second deployment gate and mutation' + +split_fixture "$(new_fixture production-preflight)" +if PLATFORM_TEST_PREFLIGHT_RC=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'failed production preflight was accepted' +fi +grep -Fq 'kubectl --request-timeout=10s wait --namespace observability --for=condition=Available deployment/grafana deployment/blackbox-exporter --timeout=60s' \ + "$FIXTURE/commands.log" || fail 'focused failure did not execute the production preflight' +[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'failed production preflight reached prompt' +! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || + fail 'failed production preflight reached mutation' +pass 'production preflight is materially exercised and remains zero mutation on failure' + +split_fixture "$(new_fixture last-gate)" +if PLATFORM_TEST_RESTORE_FAIL_AT=2 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'failed last restore gate was accepted' +fi +[[ "$(grep -c '^encryption --expect-reencrypted$' "$FIXTURE/commands.log")" == 2 ]] || + { sed -n '1,220p' "$FIXTURE/commands.log" >&2; sed -n '1,160p' "$FIXTURE/output" >&2; fail 'encryption gate did not run twice'; } +[[ "$(grep -c '^restore --check$' "$FIXTURE/commands.log")" == 2 ]] || + fail 'restore gate did not run twice' +! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'last restore failure reached apply' +[[ -f "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] || + fail 'last gate did not occur after immutable ledger capture' +pass 'last encryption and restore gate failure remains zero cluster mutation' + +split_fixture "$(new_fixture handoff-race)" +if PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE="$METRIC/rules.yaml" \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'handoff mutation after confirmation was accepted' +fi +! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'handoff race reached apply' +pass 'verified renderer handoff hashes are rechecked after the last gate' + +for inventory_drift_case in raw-json rechecksum-count hardlink; do + split_fixture "$(new_fixture "inventory-last-gate-$inventory_drift_case")" + if PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE="$inventory_drift_case" \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail "post-confirmation inventory drift was accepted: $inventory_drift_case" + fi + assert_post_confirmation_rejection "$FIXTURE" "post-confirmation inventory drift $inventory_drift_case" + pass "post-confirmation inventory drift $inventory_drift_case stops before first apply" +done + +split_fixture "$(new_fixture last-gate-count-wiring)" +INITIAL_SHA="$(rewrite_inventory_count "$METRIC" target-initial 20)" +run_inventory_validator "$METRIC" target-initial "$INITIAL_SHA" 20 \ + >"$FIXTURE/source-control.out" 2>&1 || + fail '20-target source-only positive control did not accept its matching expected count' +if env PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 bash -c ' + source "$1" + verified_output_dir=$2 + target_initial_sha=$3 + post_substrate_sha=$4 + verify_handoff_unchanged +' _ "$APPLY" "$METRIC" "$INITIAL_SHA" "$POST_SHA" >"$FIXTURE/output" 2>&1; then + fail 'last inventory gate lost the production initial target count argument' +fi +grep -Fq 'inventory semantic contract changed: target-initial' "$FIXTURE/output" || + fail 'last inventory count gate did not reject the self-consistent 20-target inventory' +[[ ! -s "$FIXTURE/commands.log" ]] || fail 'last-gate count wiring control reached a system boundary' +pass 'last inventory gate carries expected count 21 while direct expected count 20 remains a positive control' + +split_fixture "$(new_fixture symlinked-ledger-root)" +mkdir -m 0700 "$FIXTURE/escaped-ledger" +ln -s "$FIXTURE/escaped-ledger" \ + "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts" +if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'symlinked root ledger was followed' +fi +! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'symlinked ledger root reached cluster mutation' +[[ -z "$(find "$FIXTURE/escaped-ledger" -mindepth 1 -print -quit)" ]] || + fail 'root ledger write escaped through symlink' +pass 'root ledger lineage rejects symlink escape before cluster mutation' + +split_fixture "$(new_fixture success)" +run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1 || { + sed -n '1,260p' "$FIXTURE/output" >&2 + fail 'rules-alerts transition failed' +} +grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" || fail 'success marker missing' +grep -Fq 'Type APPLY default to apply only observability rules-alerts: APPLY' "$FIXTURE/output" || + fail 'interactive prompt contract changed' +expected_gate_trace='secret-schema-1 -> deployment-gate-1 -> preflight -> server-dry-run -> confirmation -> rollback-ledger -> encryption-last -> handoff/live-last -> secret-schema-2 -> deployment-gate-2 -> first-apply' +mapfile -t gate_trace_parts < <(awk ' + /get secret alertmanager-slack-webhook/ { secret++; if (secret == 1) print "secret-schema-1"; else if (secret == 2) print "secret-schema-2" } + /^recovery --check-slack-deployment-evidence$/ { gate++; if (gate == 1) print "deployment-gate-1"; else if (gate == 2) print "deployment-gate-2" } + /wait --namespace observability --for=condition=Available deployment\/grafana/ { print "preflight" } + /apply --dry-run=server/ && !dry++ { print "server-dry-run" } + /^sudo -n mv -T -- .*\/mutations.tsv.next-[0-9]+ .*\/mutations.tsv$/ && !ledger++ { print "rollback-ledger" } + /^encryption --expect-reencrypted$/ && ++encryption == 2 { print "encryption-last" } + /^restore --check$/ && ++restore == 2 { print "handoff/live-last" } + /^kubectl --request-timeout=10s apply -f / && !apply++ { print "first-apply" } +' "$FIXTURE/commands.log") +actual_gate_trace='' +for gate_trace_part in "${gate_trace_parts[@]}"; do + [[ -z "$actual_gate_trace" ]] || actual_gate_trace+=' -> ' + actual_gate_trace+="$gate_trace_part" +done +actual_gate_trace="${actual_gate_trace/server-dry-run -> rollback-ledger/server-dry-run -> confirmation -> rollback-ledger}" +[[ "$actual_gate_trace" == "$expected_gate_trace" ]] || + fail "two Slack gates and first apply did not preserve the exact required order: $actual_gate_trace" +pass 'second deployment gate preserves the required pre-prompt-to-first-apply order' +# Break caught: the former test hook bypassed the production preflight and +# reconcile path, so a focused PASS did not prove Prometheus or Grafana had +# accepted the rendered rules and dashboards. +grep -Fq 'kubectl --request-timeout=10s get probe --namespace observability --output=json' \ + "$FIXTURE/commands.log" || fail 'focused success bypassed the production preflight path' +grep -Fq 'kubectl --request-timeout=10s get --raw=/api/v1/namespaces/observability/services/http:observability-core-kube-pr-prometheus:9090/proxy/api/v1/rules' \ + "$FIXTURE/commands.log" || fail 'focused success did not exercise Prometheus rule acceptance' +grep -Fq 'kubectl --request-timeout=10s get --raw=/api/v1/namespaces/observability/services/http:observability-core-kube-pr-alertmanager:9093/proxy/api/v2/receivers' \ + "$FIXTURE/commands.log" || fail 'focused success did not exercise the exact Alertmanager receiver API' +grep -Fq 'kubectl --request-timeout=10s exec --namespace observability deployment/grafana --container grafana --' \ + "$FIXTURE/commands.log" || fail 'focused success did not prove Grafana consumed the dashboard ConfigMaps' +! grep -Eq '^(preflight|postcheck)$' "$FIXTURE/commands.log" || + fail 'focused success still used a test-only health hook' +ledger="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" +[[ -f "$ledger" && ! -L "$ledger" && "$(stat -c %a "$ledger")" == 600 ]] || fail 'root ledger is unsafe' +[[ "$(wc -l <"$ledger" | tr -d ' ')" == 13 ]] || fail 'ledger is not header plus exact twelve resources' +awk -F '\t' ' + NR == 1 { exit($0 != "phase\towner\tapiVersion\tkind\tnamespace\tname\texisted\trestore-mode\tpayload-file\tpayload-sha256") } + NR > 1 { if (NF != 10 || $1 != "access-rules-alerts" || ($2 != "kube-prometheus-stack" && $2 != "platform-rules")) exit 1 } +' "$ledger" || fail 'ledger header, fields, phase, or owner is not exact' +[[ "$(awk -F '\t' '$4=="ConfigMap"{n++} END{print n+0}' "$ledger")" == 5 ]] || fail 'dashboard ledger set is not exact' +[[ "$(awk -F '\t' '$4=="PrometheusRule"{n++} END{print n+0}' "$ledger")" == 4 ]] || fail 'rule ledger set is not exact' +[[ "$(awk -F '\t' '$4=="AlertmanagerConfig"{n++} END{print n+0}' "$ledger")" == 1 ]] || fail 'AlertmanagerConfig ledger set is not exact' +[[ "$(awk -F '\t' '$4=="Alertmanager"{n++} END{print n+0}' "$ledger")" == 1 ]] || fail 'Alertmanager ledger set is not exact' +[[ "$(awk -F '\t' '$4=="NetworkPolicy"{n++} END{print n+0}' "$ledger")" == 1 ]] || fail 'NetworkPolicy ledger set is not exact' +! grep -Eq $'\t(Secret|Probe|Ingress|Deployment|Service)\t' "$ledger" || fail 'forbidden substrate or Secret resource entered ledger' +payload_rel="$(awk -F '\t' '$4=="Alertmanager"{print $9}' "$ledger")" +payload="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/$payload_rel" +python3 - "$payload" <<'PY' || fail 'prior Alertmanager payload is not sanitized' +import json, sys +item=json.load(open(sys.argv[1])) +meta=item.get("metadata", {}) +for key in ("uid","resourceVersion","generation","creationTimestamp","managedFields"): + assert key not in meta +assert "status" not in item +assert "kubectl.kubernetes.io/last-applied-configuration" not in meta.get("annotations", {}) +PY +acceptance="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" +[[ -f "$acceptance" && "$(stat -c %a "$acceptance")" == 600 ]] || fail 'acceptance marker is unsafe' +grep -Fqx 'schema=platform-observability-rules-alerts-v2' "$acceptance" || fail 'acceptance schema missing' +grep -Fqx "rollback_id=$ROLLBACK_ID" "$acceptance" || fail 'acceptance rollback id missing' +grep -Fqx "target_initial_sha256=$INITIAL_SHA" "$acceptance" || fail 'acceptance initial hash mismatch' +grep -Fqx "post_substrate_sha256=$POST_SHA" "$acceptance" || fail 'acceptance post hash mismatch' +grep -Fqx 'slack_deployment_gate=RECOVERY' "$acceptance" || fail 'acceptance deployment verdict missing' +grep -Fqx 'slack_gate_approval_ref=strict-recovery-evidence-v1' "$acceptance" || fail 'recovery acceptance reference mismatch' +grep -Fqx 'slack_gate_accepted_by_uid=not-applicable' "$acceptance" || fail 'recovery acceptance uid mismatch' +grep -Fqx 'state=accepted' "$acceptance" || fail 'acceptance state missing' +[[ "$(wc -l <"$acceptance" | tr -d ' ')" == 9 ]] || fail 'v2 acceptance field count is not exact' +reconcile_last_line="$(awk ' + /\/proxy\/api\/v1\/rules|\/proxy\/api\/v2\/receivers| exec --namespace observability deployment\/grafana / {line=NR} + END {print line+0} +' "$FIXTURE/commands.log")" +acceptance_install_line="$(awk '/^sudo -n install .*acceptance\.env/ {line=NR} END {print line+0}' "$FIXTURE/commands.log")" +(( reconcile_last_line > 0 && acceptance_install_line > reconcile_last_line )) || + fail 'acceptance marker was installed before all production acceptance evidence' +[[ "$(grep -c '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log")" == 12 ]] || fail 'apply was not one exact resource at a time' +! grep -Eqi '^kubectl --request-timeout=10s apply -f .*/(grafana|blackbox|targets|private-dns)\.yaml|^kubectl --request-timeout=10s delete (probe|ingress|deployment|service) ' "$FIXTURE/commands.log" || + fail 'rules-alerts mode mutated a substrate, Probe, or Nginx resource' +first_sudo="$(awk '/^sudo / {print; exit}' "$FIXTURE/commands.log")" +[[ "$first_sudo" == 'sudo -v' ]] || fail 'sudo refresh did not precede root reads/writes' +awk ' + $0 == "encryption --expect-reencrypted" { encryption[++encryption_count]=NR } + $0 == "restore --check" { restore[++restore_count]=NR } + /^sudo -n mv -T -- .*\/mutations.tsv.next-[0-9]+ .*\/mutations.tsv$/ && !ledger { ledger=NR } + /^kubectl --request-timeout=10s apply -f / && !mutation { mutation=NR } + END { + exit(!(encryption_count==2 && restore_count==2 && + encryption[1] < restore[1] && restore[1] < ledger && + ledger < encryption[2] && encryption[2] < restore[2] && restore[2] < mutation)) + } +' "$FIXTURE/commands.log" || fail 'pre/last gate, ledger, and mutation order changed' +pass 'current exact 21/30 success captures the sanitized ledger and applies only twelve rules-alerts resources' + +split_fixture "$(new_fixture risk-accepted-success)" +PLATFORM_TEST_RECOVERY_OUTPUT_1='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n' \ +PLATFORM_TEST_RECOVERY_OUTPUT_2='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n' \ +run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1 || + fail 'risk-accepted deployment gate success fixture failed' +acceptance="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" +grep -Fqx 'slack_deployment_gate=RISK_ACCEPTED' "$acceptance" || fail 'risk acceptance verdict mismatch' +grep -Fqx 'slack_gate_approval_ref=2026-08-14-observability-slack-recovery-risk-acceptance-design' "$acceptance" || fail 'risk acceptance reference mismatch' +grep -Fqx 'slack_gate_accepted_by_uid=1000' "$acceptance" || fail 'risk acceptance uid mismatch' +pass 'risk-accepted deployment verdict records the exact v2 acceptance mapping' + +split_fixture "$(new_fixture acceptance-write-failure)" +if PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE=1 \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'v2 acceptance write failure returned success' +fi +grep -Fq 'APPLIED_ACCEPTANCE_UNRECORDED' "$FIXTURE/output" || fail 'acceptance write failure did not identify missing acceptance' +[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] || + fail 'acceptance write failure left an acceptance marker' +pass 'v2 acceptance write failure remains unaccepted' + +for reconcile_case in prometheus-rule grafana-sidecar; do + case_progress "reconcile-$reconcile_case" + split_fixture "$(new_fixture "reconcile-$reconcile_case")" + prometheus_drop=0 + grafana_stale=0 + case "$reconcile_case" in + prometheus-rule) + prometheus_drop=1 + ;; + grafana-sidecar) + grafana_stale=1 + ;; + esac + if PLATFORM_TEST_PROMETHEUS_DROP_RULE="$prometheus_drop" \ + PLATFORM_TEST_GRAFANA_STALE_DASHBOARD="$grafana_stale" \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail "incomplete production acceptance was accepted: $reconcile_case" + fi + assert_reconcile_rollback "$FIXTURE" "incomplete production acceptance $reconcile_case" +done +for receiver_case in raw empty null duplicate extra wrong-namespace wrong-config wrong-local malformed; do + case_progress "reconcile-alertmanager-receiver-$receiver_case" + split_fixture "$(new_fixture "reconcile-alertmanager-receiver-$receiver_case")" + if PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE="$receiver_case" \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail "invalid Alertmanager receiver was accepted: $receiver_case" + fi + assert_reconcile_rollback "$FIXTURE" "invalid Alertmanager receiver $receiver_case" +done +expected_reconcile_progress=$'CASE_BEGIN=reconcile-prometheus-rule\nCASE_BEGIN=reconcile-grafana-sidecar\nCASE_BEGIN=reconcile-alertmanager-receiver-raw\nCASE_BEGIN=reconcile-alertmanager-receiver-empty\nCASE_BEGIN=reconcile-alertmanager-receiver-null\nCASE_BEGIN=reconcile-alertmanager-receiver-duplicate\nCASE_BEGIN=reconcile-alertmanager-receiver-extra\nCASE_BEGIN=reconcile-alertmanager-receiver-wrong-namespace\nCASE_BEGIN=reconcile-alertmanager-receiver-wrong-config\nCASE_BEGIN=reconcile-alertmanager-receiver-wrong-local\nCASE_BEGIN=reconcile-alertmanager-receiver-malformed' +[[ "$(<"$PROGRESS_LOG")" == "$expected_reconcile_progress" ]] || + fail 'long reconciliation failure matrix did not emit exact per-case progress' +pass 'Prometheus rules, exact Alertmanager receiver matrix, and Grafana sidecar evidence each fail closed before acceptance' + +split_fixture "$(new_fixture apply-outcome-unknown)" +if PLATFORM_TEST_FAIL_APPLY_AT=5 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'failed apply returned success' +fi +grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" || + fail 'non-zero apply result was incorrectly reported as a certain no-op' +grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" || + fail 'unknown apply outcome did not require manual recovery' +[[ "$(find "$FIXTURE/state" -type f | wc -l | tr -d ' ')" == 1 ]] || + fail 'recorded prefix was not rolled back after an ambiguous apply result' +pass 'non-zero apply response is treated as an unknown outcome while owned prefix rolls back' + +split_fixture "$(new_fixture apply-response-loss)" +if PLATFORM_TEST_FAIL_AFTER_APPLY_AT=5 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'lost apply response returned success' +fi +grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" || + fail 'lost apply response was reported as a certain rollback' +grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" || + fail 'lost apply response omitted manual recovery' +[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] || + fail 'lost apply response wrote acceptance' +pass 'apply response loss preserves truthful ambiguity and no acceptance marker' + +split_fixture "$(new_fixture rollback)" + if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=0 \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'failed reconcile returned success' +fi +grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS' "$FIXTURE/output" || fail 'rollback PASS marker missing' +prior="$FIXTURE/state/Alertmanager__observability__observability-core-kube-pr-alertmanager.json" +jq -e '.metadata.uid == "uid-alertmanager-prior" and .spec.replicas == 1' "$prior" >/dev/null || + fail 'rollback did not restore the prior Alertmanager owner payload' +[[ "$(find "$FIXTURE/state" -type f | wc -l | tr -d ' ')" == 1 ]] || fail 'rollback left newly-created objects' +! grep -Eq '^kubectl --request-timeout=10s (delete|apply).*(Secret|PersistentVolumeClaim|Probe)' "$FIXTURE/commands.log" || + fail 'rollback touched a preserved or non-rules resource' +delete_order="$(awk '$1=="raw-delete" {path=$2; sub(".*/","",path); print path}' "$FIXTURE/commands.log")" +expected_delete_order=$'observability-allow-alertmanager-public-https\nplatform-alertmanager\nplatform-verified-services\nplatform-observability-core\nplatform-certificate-probes\nplatform-aistor-storage-quota\ngrafana-dashboard-workload-health\ngrafana-dashboard-platform-services\ngrafana-dashboard-observability-backends\ngrafana-dashboard-kubernetes-node\ngrafana-dashboard-https-endpoints' +[[ "$delete_order" == "$expected_delete_order" ]] || fail 'rollback did not delete owned creations in exact reverse dependency order' +[[ "$(grep -c '^raw-delete .* exact=true$' "$FIXTURE/commands.log")" == 11 ]] || + fail 'rollback did not send exact UID-preconditioned DeleteOptions through the raw API' +grep -Fq 'replace-preconditions Alertmanager/observability-core-kube-pr-alertmanager uid=uid-alertmanager-prior resourceVersion=' \ + "$FIXTURE/commands.log" || fail 'rollback restore did not bind both UID and live resourceVersion' +! grep -Fq 'kubectl --request-timeout=10s delete ' "$FIXTURE/commands.log" || + fail 'rollback used a plain kubectl delete branch' +pass 'failed reconcile reverses only ledger-owned resources and restores exact prior owner' + +for delete_mode in response-loss conflict timeout; do + split_fixture "$(new_fixture "delete-$delete_mode")" + started_at="$(date +%s)" + if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=0 PLATFORM_TEST_DELETE_MODE="$delete_mode" \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail "raw DELETE $delete_mode returned success" + fi + elapsed=$(( $(date +%s) - started_at )) + (( elapsed <= 30 )) || fail "raw DELETE $delete_mode exceeded its bounded timeout" + grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" || + fail "raw DELETE $delete_mode was not reported truthfully" + grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" || + fail "raw DELETE $delete_mode omitted manual recovery" + grep -Eq '^raw-delete .* exact=true$' "$FIXTURE/commands.log" || + fail "raw DELETE $delete_mode did not carry exact UID preconditions" + [[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] || + fail "raw DELETE $delete_mode wrote acceptance" +done +pass 'raw DELETE response loss, conflict, and timeout stay bounded and truthfully ambiguous' + +split_fixture "$(new_fixture replace-conflict)" +if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=0 PLATFORM_TEST_REPLACE_CONFLICT=1 \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'UID/resourceVersion restore conflict returned success' +fi +grep -Fq 'replace-preconditions Alertmanager/observability-core-kube-pr-alertmanager uid=uid-alertmanager-prior resourceVersion=' \ + "$FIXTURE/commands.log" || fail 'conflicted restore did not carry UID and resourceVersion' +grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" || + fail 'restore conflict did not remain ambiguous' +grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" || + fail 'restore conflict omitted manual recovery' +pass 'UID and resourceVersion restore conflict fails closed with manual recovery' + +split_fixture "$(new_fixture signal)" +( + trap 'exit 143' TERM + PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=5 \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" +) >"$FIXTURE/output" 2>&1 & +signal_pid=$! +for ((signal_attempt=1; signal_attempt<=300; signal_attempt++)); do + [[ -e "$FIXTURE/postcheck-reached" ]] && break + kill -0 "$signal_pid" 2>/dev/null || break + sleep 0.05 +done +[[ -e "$FIXTURE/postcheck-reached" ]] || fail 'signal fixture never reached a mutation-owned reconciliation boundary' +apply_pid="$(pgrep -P "$signal_pid" -f "$APPLY" | head -n1)" +[[ "$apply_pid" =~ ^[0-9]+$ ]] || fail 'signal fixture could not bind the apply process' +kill -TERM "$apply_pid" +if wait "$signal_pid"; then + fail 'signal during mutation returned success' +fi +grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS' "$FIXTURE/output" || + { sed -n '1,200p' "$FIXTURE/output" >&2; fail 'signal during reconciliation did not complete owned rollback'; } +[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] || + fail 'signal during mutation wrote acceptance' +pass 'signal during reconciliation invokes owned rollback and leaves no false acceptance' + +split_fixture "$(new_fixture uid-drift)" +if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_DRIFT_RESOURCE='ConfigMap/grafana-dashboard-workload-health' \ + run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \ + --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then + fail 'UID-drifted rollback returned success' +fi +grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" || + fail 'UID drift did not produce truthful ambiguity' +grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" || fail 'manual recovery marker missing' +[[ -f "$FIXTURE/state/ConfigMap__observability__grafana-dashboard-workload-health.json" ]] || + fail 'UID-drifted object was deleted without ownership' +pass 'rollback refuses UID drift and reports manual recovery truthfully' + +printf 'Assertions: %d\n' "$ASSERTIONS" +printf 'APPLY OBSERVABILITY ACCESS TEST PASS\n' +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + supervise_focused_suite "$@" +fi diff --git a/scripts/validate/test-backup-slack-webhook-recovery.sh b/scripts/validate/test-backup-slack-webhook-recovery.sh new file mode 100755 index 0000000..e31a246 --- /dev/null +++ b/scripts/validate/test-backup-slack-webhook-recovery.sh @@ -0,0 +1,2197 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly PROD="$ROOT/scripts/bootstrap/backup-slack-webhook-recovery.sh" +readonly LIB="$ROOT/scripts/lib/slack-webhook-recovery.sh" +fixture_dir='' +runtime_fixture='' +owned_work_fixture='' +boundary_cases=0 +review_failures=0 +transaction_cases=0 +production_runtime_root_definition='' + +fail() { + printf 'TEST FAILURE: %s\n' "$*" >&2 + exit 1 +} + +assert_eq() { + local expected="$1" actual="$2" description="$3" + [[ "$actual" == "$expected" ]] || fail "$description" +} + +assert_fails() { + if "$@" >/dev/null 2>&1; then + fail "expected failure: $*" + fi +} + +pass_boundary_case() { + boundary_cases=$((boundary_cases + 1)) +} + +pass_transaction_case() { + transaction_cases=$((transaction_cases + 1)) +} + +write_recovery_contract() { + local destination="$1" + builtin printf '%s\n' \ + 'K3SLR_SCHEMA_VERSION=1' \ + 'K3SLR_RECOVERY_DISK_BY_ID=/dev/disk/by-id/wwn-0x500a0751e6aa6254' \ + 'K3SLR_RECOVERY_PARTITION_BY_ID=/dev/disk/by-id/wwn-0x500a0751e6aa6254-part3' \ + 'K3SLR_RECOVERY_FS_UUID=4EA0196C0C5FA27E' \ + 'K3SLR_RECOVERY_PARTUUID=4670aa9f-9045-4bce-930d-9e84dfec9f38' \ + 'K3SLR_RECOVERY_MODEL=CT1000MX500SSD1' \ + 'K3SLR_RECOVERY_SERIAL=2306E6AA6254' \ + 'K3SLR_RECOVERY_WWN=0x500a0751e6aa6254' \ + 'K3SLR_K3S_DISK_BY_ID=/dev/disk/by-id/wwn-0x500a07512df28bd1' \ + 'K3SLR_K3S_PARTITION_BY_ID=/dev/disk/by-id/wwn-0x500a07512df28bd1-part1' \ + 'K3SLR_K3S_FS_UUID=b86086ef-2b3c-4638-abcf-fc7f137dcb97' \ + 'K3SLR_K3S_PARTUUID=b081b955-d6bc-442c-ac0f-db76560a5245' \ + 'K3SLR_K3S_MODEL=CT250MX500SSD1' \ + 'K3SLR_K3S_SERIAL=21132DF28BD1' \ + 'K3SLR_K3S_WWN=0x500a07512df28bd1' \ + 'K3SLR_OWNER_UID=1000' \ + 'K3SLR_OWNER_GID=1000' \ + 'K3SLR_OUTER_MOUNT=/mnt/k3s-recovery-ssd' \ + 'K3SLR_INNER_MOUNT=/srv/recovery/k3s' \ + 'K3SLR_ROOT_RELATIVE=HyeonworksRecovery' \ + 'K3SLR_DATABASE_RELATIVE=HyeonworksRecovery/vault/hyeonworks-recovery.kdbx' \ + 'K3SLR_CONTAINER_RELATIVE=HyeonworksRecovery/containers/k3s-recovery.luks' \ + 'K3SLR_RUNTIME_METADATA_RELATIVE=HyeonworksRecovery/volume.env' \ + 'K3SLR_MAPPING_NAME=k3s-recovery' \ + 'K3SLR_PROOF_MAPPING_NAME=k3s-recovery-proof' \ + 'K3SLR_INNER_LABEL=K3S_RECOVERY' \ + 'K3SLR_CONTAINER_SIZE_BYTES=34359738368' \ + 'K3SLR_MINIMUM_FREE_BYTES=10737418240' \ + 'K3SLR_OUTER_MIN_REMAINING_PERCENT=20' >"$destination" + chmod 0600 -- "$destination" +} + +reset_fake_boundary() { + : >"$fake_log" + builtin printf '%s\n' absent >"$fake_mount_state" + builtin printf '%s\n' 0 >"$fake_probe_status" + builtin printf '%s\n' 0 >"$fake_mount_status" + builtin printf '%s\n' 0 >"$fake_umount_status" + builtin printf '%s\n' absent >"$fake_umount_after" + builtin printf '%s\n' 0 >"$fake_sudo_status" + builtin printf '%s\n' 0 >"$fake_source_query_count" + builtin printf '%s\n' 0 >"$fake_action_count" + builtin printf '%s\n' 0 >"$fake_mount_uncertain" + builtin printf '%s\n' valid >"$fake_case_file" +} + +fake_log_command() { + local separator='' argument + for argument in "$@"; do + builtin printf '%s%s' "$separator" "$argument" >>"$fake_log" + separator=$'\t' + done + builtin printf '\n' >>"$fake_log" +} + +_swr_fake_runtime_root() { + builtin printf '%s\n' "$fake_runtime_root" +} + +_swr_fake_command() { + fake_log_command "$@" + local executable="$1" fake_case mount_state phase options + shift + if [[ "$executable" == /fixture/fake-only-sentinel ]]; then + [[ $# -eq 0 ]] + return + fi + fake_case="$(<"$fake_case_file")" + case "$executable" in + /usr/bin/readlink) + [[ "$1" == -e && "$2" == -- && $# -eq 3 ]] || return 97 + case "$3" in + /run/user/1000) builtin printf '%s\n' /run/user/1000 ;; + /dev/disk/by-id/wwn-0x500a0751e6aa6254) builtin printf '%s\n' /dev/fixture-recovery-disk ;; + /dev/disk/by-id/wwn-0x500a0751e6aa6254-part3) builtin printf '%s\n' /dev/fixture-recovery-partition ;; + /dev/disk/by-id/wwn-0x500a07512df28bd1) builtin printf '%s\n' /dev/fixture-k3s-disk ;; + /dev/disk/by-id/wwn-0x500a07512df28bd1-part1) builtin printf '%s\n' /dev/fixture-k3s-partition ;; + *) return 97 ;; + esac + ;; + /usr/sbin/blkid) + [[ "$1" == --output && "$2" == value && "$3" == --match-tag && "$5" == -- && $# -eq 6 ]] || return 97 + case "$4|$6" in + 'UUID|/dev/fixture-recovery-partition') + [[ "$fake_case" == wrong_uuid ]] && builtin printf '%s\n' WRONG || builtin printf '%s\n' 4EA0196C0C5FA27E + ;; + 'PARTUUID|/dev/fixture-recovery-partition') builtin printf '%s\n' 4670aa9f-9045-4bce-930d-9e84dfec9f38 ;; + 'TYPE|/dev/fixture-recovery-partition') + [[ "$fake_case" == wrong_type ]] && builtin printf '%s\n' ext4 || builtin printf '%s\n' ntfs + ;; + *) return 97 ;; + esac + ;; + /usr/bin/lsblk) + [[ "$1" == --noheadings && "$2" == --raw && "$3" == --paths ]] || return 97 + if [[ "$4" == --output && "$6" == -- && $# -eq 7 ]]; then + case "$5|$7" in + 'PKNAME|/dev/fixture-recovery-partition') + [[ "$fake_case" == wrong_parent ]] && builtin printf '%s\n' /dev/fixture-other-disk || builtin printf '%s\n' /dev/fixture-recovery-disk + ;; + 'PKNAME|/dev/fixture-k3s-partition') + [[ "$fake_case" == same_parent ]] && builtin printf '%s\n' /dev/fixture-recovery-disk || builtin printf '%s\n' /dev/fixture-k3s-disk + ;; + 'MODEL|/dev/fixture-recovery-disk') + [[ "$fake_case" == wrong_model ]] && builtin printf '%s\n' WRONGMODEL '' || builtin printf '%s\n' CT1000MX500SSD1 '' + ;; + 'SERIAL|/dev/fixture-recovery-disk') builtin printf '%s\n' 2306E6AA6254 '' ;; + 'WWN|/dev/fixture-recovery-disk') builtin printf '%s\n' 0x500a0751e6aa6254 '' ;; + *) return 97 ;; + esac + elif [[ "$4" == --nodeps && "$5" == --output && "$7" == -- && $# -eq 8 ]]; then + case "$6|$8" in + 'MODEL|/dev/fixture-recovery-disk') + [[ "$fake_case" == wrong_model ]] && builtin printf '%s\n' WRONGMODEL || builtin printf '%s\n' CT1000MX500SSD1 + ;; + 'SERIAL|/dev/fixture-recovery-disk') builtin printf '%s\n' 2306E6AA6254 ;; + 'WWN|/dev/fixture-recovery-disk') builtin printf '%s\n' 0x500a0751e6aa6254 ;; + *) return 97 ;; + esac + else + return 97 + fi + ;; + /usr/bin/findmnt) + if [[ "$1" == --json && "$2" == --source && "$3" == /dev/fixture-recovery-partition && + "$4" == --output && "$5" == SOURCE,TARGET,FSTYPE,VFS-OPTIONS,FS-OPTIONS && $# -eq 5 ]]; then + local source_query_count + source_query_count="$(( $(<"$fake_source_query_count") + 1 ))" + builtin printf '%s\n' "$source_query_count" >"$fake_source_query_count" + mount_state="$(<"$fake_mount_state")" + [[ "$fake_case" == source_ambiguous ]] && return 2 + if [[ "$fake_case" == source_rc1_nonempty || + "$fake_case" == source_rc1_nonempty_poll1 && "$source_query_count" == 1 || + "$fake_case" == source_rc1_nonempty_poll2 && "$source_query_count" == 2 ]]; then + builtin printf '%s\n' '{"filesystems":[' + return 1 + fi + if [[ "$fake_case" == source_rc1_overflow ]]; then + local overflow + builtin printf -v overflow '%*s' 16385 '' + builtin printf '%s' "${overflow// /x}" + return 1 + fi + case "$fake_case" in + source_rc1_newline_1) builtin printf '\n'; return 1 ;; + source_rc1_newline_2) builtin printf '\n\n'; return 1 ;; + source_rc1_newline_overflow) + local newline_overflow + builtin printf -v newline_overflow '%*s' 16385 '' + builtin printf '%s' "${newline_overflow// /$'\n'}" + return 1 + ;; + esac + [[ "$mount_state" == absent ]] && return 1 + builtin printf '{"filesystems":[{"source":"/dev/fixture-recovery-partition","target":"%s","fstype":"fuseblk","vfs-options":"%s,nosuid,nodev,noexec","fs-options":"%s,allow_other,uid=1000,gid=1000,fmask=0177,dmask=0077"}]}\n' \ + "$fake_mountpoint" "$mount_state" "$mount_state" + return 0 + fi + if [[ "$1" == --json && "$2" == --mountpoint && + "$3" == "$fake_runtime_root/slack-webhook-recovery."* && + "$4" == --output && "$5" == SOURCE,TARGET,FSTYPE,VFS-OPTIONS,FS-OPTIONS && $# -eq 5 ]]; then + fake_mountpoint="$3" + mount_state="$(<"$fake_mount_state")" + [[ "$fake_case" == target_reused && "$mount_state" == absent ]] && { + builtin printf '{"filesystems":[{"source":"/dev/foreign","target":"%s","fstype":"ext4","vfs-options":"rw","fs-options":"rw"}]}\n' "$fake_mountpoint" + return 0 + } + [[ "$mount_state" == absent ]] && return 1 + local source=/dev/fixture-recovery-partition fstype=fuseblk + local vfs_options="nodev,noexec,nosuid,relatime,${mount_state}" + local fs_options="allow_other,blksize=4096,default_permissions,group_id=0,${mount_state},user_id=0" + [[ "$fake_case" == foreign_source ]] && source=/dev/foreign + [[ "$fake_case" == foreign_fstype ]] && fstype=ext4 + [[ "$fake_case" == unsafe_allow_other ]] && fs_options="${mount_state},allow_other" + [[ "$fake_case" == missing_nosuid ]] && vfs_options="nodev,noexec,relatime,${mount_state}" + [[ "$fake_case" == vfs_relatime_missing ]] && vfs_options="nodev,noexec,nosuid,${mount_state}" + [[ "$fake_case" == vfs_relatime_duplicate ]] && vfs_options+=",relatime" + [[ "$fake_case" == vfs_phase_missing ]] && vfs_options="nodev,noexec,nosuid,relatime" + [[ "$fake_case" == executable ]] && vfs_options="nodev,exec,nosuid,relatime,${mount_state}" + [[ "$fake_case" == suid ]] && vfs_options="nodev,noexec,relatime,suid,${mount_state}" + [[ "$fake_case" == dev ]] && vfs_options="dev,noexec,nosuid,relatime,${mount_state}" + [[ "$fake_case" == wrong_phase ]] && vfs_options="nodev,noexec,nosuid,relatime,rw" + [[ "$fake_case" == fs_allow_other_missing ]] && fs_options="${fs_options/allow_other,/}" + [[ "$fake_case" == fs_allow_other_duplicate ]] && fs_options+=",allow_other" + [[ "$fake_case" == fs_phase_missing ]] && fs_options="${fs_options/,${mount_state},/,}" + [[ "$fake_case" == fs_wrong_phase ]] && fs_options="${fs_options/,${mount_state},/,rw,}" + [[ "$fake_case" == legacy_explicit_mapping ]] && { + vfs_options="${mount_state},nosuid,nodev,noexec" + fs_options="${mount_state},allow_other,uid=1000,gid=1000,fmask=0177,dmask=0077" + } + [[ "$fake_case" == fs_default_permissions_missing ]] && fs_options="${fs_options/default_permissions,/}" + [[ "$fake_case" == fs_default_permissions_duplicate ]] && fs_options+=",default_permissions" + [[ "$fake_case" == fs_user_id_missing ]] && fs_options="${fs_options/,user_id=0/}" + [[ "$fake_case" == fs_user_id_wrong ]] && fs_options="${fs_options/user_id=0/user_id=1000}" + [[ "$fake_case" == fs_user_id_duplicate ]] && fs_options+=",user_id=0" + [[ "$fake_case" == fs_group_id_missing ]] && fs_options="${fs_options/group_id=0,/}" + [[ "$fake_case" == fs_group_id_wrong ]] && fs_options="${fs_options/group_id=0/group_id=1000}" + [[ "$fake_case" == fs_group_id_duplicate ]] && fs_options+=",group_id=0" + [[ "$fake_case" == fs_blksize_missing ]] && fs_options="${fs_options/blksize=4096,/}" + [[ "$fake_case" == fs_blksize_wrong ]] && fs_options="${fs_options/blksize=4096/blksize=8192}" + [[ "$fake_case" == fs_blksize_duplicate ]] && fs_options+=",blksize=4096" + [[ "$fake_case" == field_placement_vfs_default_permissions ]] && vfs_options+=",default_permissions" + [[ "$fake_case" == field_placement_fs_nodev ]] && fs_options+=",nodev" + [[ "$fake_case" == hybrid_actual_legacy ]] && fs_options+=",uid=1000,gid=1000,fmask=0177,dmask=0077" + [[ "$fake_case" == observed_permuted ]] && { + vfs_options="${mount_state},relatime,nosuid,noexec,nodev" + fs_options="${mount_state},user_id=0,group_id=0,default_permissions,blksize=4096,allow_other" + } + [[ "$fake_case" == json_fs_trailing_lf ]] && fs_options+=$'\n' + [[ "$fake_case" == json_fs_trailing_lf_garbage ]] && fs_options+=$'\nunknown' + [[ "$fake_case" == json_vfs_cross_field_reframe ]] && { + vfs_options+=$'\n'"$fs_options" + fs_options='' + } + [[ "$fake_case" == fs_uid_conflict ]] && fs_options+=",uid=0" + [[ "$fake_case" == fs_uid_duplicate ]] && fs_options+=",uid=1000" + [[ "$fake_case" == fs_gid_conflict ]] && fs_options+=",gid=0" + [[ "$fake_case" == fs_gid_duplicate ]] && fs_options+=",gid=1000" + [[ "$fake_case" == fs_fmask_conflict ]] && fs_options+=",fmask=0000" + [[ "$fake_case" == fs_fmask_duplicate ]] && fs_options+=",fmask=0177" + [[ "$fake_case" == fs_dmask_conflict ]] && fs_options+=",dmask=0000" + [[ "$fake_case" == fs_dmask_duplicate ]] && fs_options+=",dmask=0077" + [[ "$fake_case" == fs_umask ]] && fs_options+=",umask=0000" + [[ "$fake_case" == fs_phase_conflict ]] && fs_options+=",rw" + [[ "$fake_case" == fs_phase_duplicate ]] && fs_options+=",ro" + [[ "$fake_case" == vfs_phase_duplicate ]] && vfs_options+=",ro" + [[ "$fake_case" == vfs_nosuid_duplicate ]] && vfs_options+=",nosuid" + [[ "$fake_case" == vfs_nodev_duplicate ]] && vfs_options+=",nodev" + [[ "$fake_case" == vfs_noexec_duplicate ]] && vfs_options+=",noexec" + [[ "$fake_case" == vfs_suid_conflict ]] && vfs_options+=",suid" + [[ "$fake_case" == vfs_dev_conflict ]] && vfs_options+=",dev" + [[ "$fake_case" == vfs_exec_conflict ]] && vfs_options+=",exec" + case "$fake_case" in + vfs_permissions) vfs_options+=",permissions" ;; + vfs_acl) vfs_options+=",acl" ;; + vfs_inherit) vfs_options+=",inherit" ;; + vfs_unknown) vfs_options+=",unknown_fixture_token" ;; + fs_permissions) fs_options+=",permissions" ;; + fs_acl) fs_options+=",acl" ;; + fs_inherit) fs_options+=",inherit" ;; + fs_unknown) fs_options+=",unknown_fixture_token" ;; + fs_arbitrary) fs_options+=",arbitrary_fixture_option" ;; + vfs_empty) vfs_options='' ;; + vfs_leading_comma) vfs_options=",${vfs_options}" ;; + vfs_trailing_comma) vfs_options+="," ;; + vfs_double_comma) vfs_options="${vfs_options/nosuid,/nosuid,,}" ;; + fs_empty) fs_options='' ;; + fs_leading_comma) fs_options=",${fs_options}" ;; + fs_trailing_comma) fs_options+="," ;; + fs_double_comma) fs_options="${fs_options/ro,/ro,,}" ;; + esac + builtin printf '{"filesystems":[{"source":"%s","target":"%s","fstype":"%s","vfs-options":"%s","fs-options":"%s"}]}\n' \ + "$source" "$fake_mountpoint" "$fstype" "$vfs_options" "$fs_options" + return 0 + fi + return 97 + ;; + /usr/bin/stat) + if [[ "$1" == --format=%u:%g:%a && "$2" == -- && + "$3" == /run/user/1000/platform-slack-keepass.* && $# -eq 3 ]]; then + command /usr/bin/stat "$@" + return + fi + if [[ "$1" == --file-system && "$2" == --format=%T && "$3" == -- && + "$4" == /run/user/1000/platform-slack-keepass.* && $# -eq 4 ]]; then + command /usr/bin/stat "$@" + return + fi + [[ "$1" == --format=%u:%g:%a && "$2" == -- && + "$3" == "$fake_runtime_root/slack-webhook-recovery."* && $# -eq 3 ]] || return 97 + [[ "$fake_case" == wrong_owner ]] && builtin printf '%s\n' 1001:1000:700 && return 0 + [[ "$fake_case" == wrong_gid ]] && builtin printf '%s\n' 1000:1001:700 && return 0 + [[ "$fake_case" == wrong_mode ]] && builtin printf '%s\n' 1000:1000:755 && return 0 + builtin printf '%s\n' 1000:1000:700 + ;; + /usr/bin/mktemp) + [[ "$1" == -d && "$2" == -- && "$3" == "$fake_runtime_root/slack-webhook-recovery.XXXXXXXX" && $# -eq 3 ]] || return 97 + command /usr/bin/mktemp "$@" + ;; + /usr/bin/chmod) + command /usr/bin/chmod "$@" + ;; + /usr/bin/rmdir) + command /usr/bin/rmdir "$@" + ;; + /usr/bin/jq) + command /usr/bin/jq "$@" + ;; + /usr/bin/base64) + command /usr/bin/base64 "$@" + ;; + /usr/bin/sudo) + if [[ "$1" == -v && $# -eq 1 ]]; then + return "$(<"$fake_sudo_status")" + fi + [[ "$1" == -n && "$2" == /usr/bin/timeout && "$3" == --signal=TERM ]] || return 97 + if [[ "$4" == --kill-after=5s && "$5" == 20s && "$6" == /usr/bin/ntfs-3g.probe && + "$7" == --readwrite && "$8" == /dev/fixture-recovery-partition && $# -eq 8 ]]; then + return "$(<"$fake_probe_status")" + fi + if [[ "$4" == --kill-after=10s && "$5" == 45s && "$6" == /usr/bin/ntfs-3g && + "$7" == -o && "$9" == /dev/fixture-recovery-partition && + "${10}" == "$fake_runtime_root/slack-webhook-recovery."* && $# -eq 10 ]]; then + fake_mountpoint="${10}" + options="$8" + case ",$options," in + *,ro,*) phase=ro ;; + *,rw,*) phase=rw ;; + *) return 97 ;; + esac + if [[ "$(<"$fake_mount_status")" != 0 ]]; then + [[ "$(<"$fake_mount_uncertain")" == 1 ]] && builtin printf '%s\n' "$phase" >"$fake_mount_state" + return "$(<"$fake_mount_status")" + fi + builtin printf '%s\n' "$phase" >"$fake_mount_state" + return 0 + fi + if [[ "$4" == --kill-after=10s && "$5" == 45s && "$6" == /usr/bin/umount && + "$7" == -- && "$8" == "$fake_mountpoint" && $# -eq 8 ]]; then + builtin printf '%s\n' "$(<"$fake_umount_after")" >"$fake_mount_state" + return "$(<"$fake_umount_status")" + fi + return 97 + ;; + *) + return 97 + ;; + esac +} + +_swr_fake_volume_action() { + local result_name="$1" phase="$2" + builtin printf '%s\n' "$(( $(<"$fake_action_count") + 1 ))" >"$fake_action_count" + case "$fake_action" in + noop) builtin printf -v "$result_name" '%s' exact-noop ;; + mutation) + [[ "$phase" == ro ]] && builtin printf -v "$result_name" '%s' mutation || builtin printf -v "$result_name" '%s' complete + ;; + error) return 71 ;; + signal) + kill -TERM "$BASHPID" + builtin printf -v "$result_name" '%s' exact-noop + ;; + *) return 97 ;; + esac +} + +assert_log_line() { + local expected="$1" description="$2" + /usr/bin/grep -Fqx -- "$expected" "$fake_log" || fail "$description" +} + +test_contract_and_device_boundary() { + local contract="$fixture_dir/local-recovery.env" variant="$fixture_dir/variant.env" + local _swr_device='' _swr_no_nodeps='' identity_trace + write_recovery_contract "$contract" + + _swr_load_recovery_contract "$contract" || fail 'strict synthetic recovery contract was rejected' + assert_eq /dev/disk/by-id/wwn-0x500a0751e6aa6254 "$SWR_RECOVERY_DISK_BY_ID" 'recovery disk contract differs' + assert_eq /dev/disk/by-id/wwn-0x500a0751e6aa6254-part3 "$SWR_RECOVERY_PARTITION_BY_ID" 'recovery partition contract differs' + assert_eq 4EA0196C0C5FA27E "$SWR_RECOVERY_FS_UUID" 'recovery UUID contract differs' + pass_boundary_case + + /usr/bin/awk 'NR != 3' "$contract" >"$variant" + assert_fails _swr_load_recovery_contract "$variant" + pass_boundary_case + { command /usr/bin/cp -- "$contract" "$variant"; builtin printf '%s\n' 'K3SLR_SCHEMA_VERSION=1' >>"$variant"; } + assert_fails _swr_load_recovery_contract "$variant" + pass_boundary_case + /usr/bin/awk 'NR == 2 { a=$0; next } NR == 3 { print; print a; next } { print }' "$contract" >"$variant" + assert_fails _swr_load_recovery_contract "$variant" + pass_boundary_case + /usr/bin/sed '2s/^K3SLR_RECOVERY_DISK_BY_ID=/K3SLR_UNKNOWN=/' "$contract" >"$variant" + assert_fails _swr_load_recovery_contract "$variant" + pass_boundary_case + /usr/bin/sed '2s/=/ /' "$contract" >"$variant" + assert_fails _swr_load_recovery_contract "$variant" + pass_boundary_case + + reset_fake_boundary + assert_fails _swr_capture_single_line _swr_no_nodeps /usr/bin/lsblk \ + --noheadings --raw --paths --output MODEL -- /dev/fixture-recovery-disk + pass_boundary_case + + reset_fake_boundary + _swr_load_recovery_contract "$contract" || fail 'valid contract reload failed' + _swr_resolve_device _swr_device || fail 'exact synthetic device was rejected' + assert_eq /dev/fixture-recovery-partition "$_swr_device" 'canonical recovery partition differs' + assert_log_line $'/usr/sbin/blkid\t--output\tvalue\t--match-tag\tUUID\t--\t/dev/fixture-recovery-partition' 'fixed UUID probe argv differs' + assert_log_line $'/usr/bin/lsblk\t--noheadings\t--raw\t--paths\t--output\tPKNAME\t--\t/dev/fixture-recovery-partition' 'fixed parent probe argv differs' + identity_trace="$(/usr/bin/grep -E $'^/usr/bin/lsblk\t.*\t(MODEL|SERIAL|WWN)\t' "$fake_log")" + assert_eq \ + $'/usr/bin/lsblk\t--noheadings\t--raw\t--paths\t--nodeps\t--output\tMODEL\t--\t/dev/fixture-recovery-disk\n/usr/bin/lsblk\t--noheadings\t--raw\t--paths\t--nodeps\t--output\tSERIAL\t--\t/dev/fixture-recovery-disk\n/usr/bin/lsblk\t--noheadings\t--raw\t--paths\t--nodeps\t--output\tWWN\t--\t/dev/fixture-recovery-disk' \ + "$identity_trace" 'fixed disk identity probe argv/order differs' + pass_boundary_case + + local rejection + for rejection in wrong_uuid wrong_type wrong_parent same_parent wrong_model; do + reset_fake_boundary + builtin printf '%s\n' "$rejection" >"$fake_case_file" + assert_fails _swr_resolve_device _swr_device + pass_boundary_case + done + + reset_fake_boundary + _swr_require_unmounted /dev/fixture-recovery-partition || fail 'stable absent source was rejected' + pass_boundary_case + builtin printf '%s\n' ro >"$fake_mount_state" + assert_fails _swr_require_unmounted /dev/fixture-recovery-partition + pass_boundary_case + reset_fake_boundary + builtin printf '%s\n' source_ambiguous >"$fake_case_file" + assert_fails _swr_require_unmounted /dev/fixture-recovery-partition + pass_boundary_case +} + +test_probe_mount_and_validation_boundary() { + local status first_stat_line findmnt_line last_stat_line _swr_created='' + reset_fake_boundary + _swr_probe_rw /dev/fixture-recovery-partition || fail 'successful read-write probe was rejected' + assert_log_line $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=5s\t20s\t/usr/bin/ntfs-3g.probe\t--readwrite\t/dev/fixture-recovery-partition' 'fixed privileged probe argv differs' + pass_boundary_case + for status in 1 124 137 143 255; do + reset_fake_boundary + builtin printf '%s\n' "$status" >"$fake_probe_status" + assert_fails _swr_probe_rw /dev/fixture-recovery-partition + pass_boundary_case + done + + reset_fake_boundary + _swr_create_mountpoint _swr_created || fail 'private fixture mountpoint creation failed' + fake_mountpoint="$_swr_created" + [[ "$fake_mountpoint" == "$fake_runtime_root/slack-webhook-recovery."* ]] || fail 'mountpoint escaped private runtime root' + [[ "$(/usr/bin/stat -c %a -- "$fake_mountpoint")" == 700 ]] || fail 'mountpoint mode differs' + + # Production break caught: installed ntfs-3g does not accept a standalone + # `--` before DEVICE. The strict fake accepts only its documented argv. + _swr_mount_volume /dev/fixture-recovery-partition "$fake_mountpoint" ro || fail 'closed read-only mount argv was rejected' + assert_log_line $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=10s\t45s\t/usr/bin/ntfs-3g\t-o\tnorecover,no_def_opts,allow_other,nodev,nosuid,noexec,uid=1000,gid=1000,fmask=0177,dmask=0077,ro\t/dev/fixture-recovery-partition\t'"$fake_mountpoint" 'fixed read-only mount argv differs' + + # Production break caught: removing the unsupported separator must not + # allow an untrusted device or mountpoint to reach the privileged leaf. + local adversarial_mountpoint + : >"$fake_log" + assert_fails _swr_mount_volume_command -o,allow_other "$fake_mountpoint" ro + assert_fails _swr_mount_volume_command /dev/fixture-recovery-partition -o,allow_other ro + assert_fails _swr_mount_volume_command /tmp/arbitrary-device "$fake_mountpoint" ro + assert_fails _swr_mount_volume_command /dev/fixture-recovery-partition-sibling "$fake_mountpoint" ro + assert_fails _swr_mount_volume_command /dev/fixture-recovery-partition /tmp/arbitrary-mountpoint ro + + adversarial_mountpoint="$fake_runtime_root/slack-webhook-recovery.foreign" + assert_fails _swr_mount_volume_command /dev/fixture-recovery-partition "$adversarial_mountpoint" ro + /usr/bin/ln -s -- "$fixture_dir" "$adversarial_mountpoint" + assert_fails _swr_mount_volume_command /dev/fixture-recovery-partition "$adversarial_mountpoint" ro + /usr/bin/rm -f -- "$adversarial_mountpoint" + + builtin printf '%s\n' wrong_owner >"$fake_case_file" + assert_fails _swr_mount_volume_command /dev/fixture-recovery-partition "$fake_mountpoint" ro + builtin printf '%s\n' wrong_mode >"$fake_case_file" + assert_fails _swr_mount_volume_command /dev/fixture-recovery-partition "$fake_mountpoint" ro + builtin printf '%s\n' valid >"$fake_case_file" + + ! /usr/bin/grep -Fq $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=10s\t45s\t/usr/bin/ntfs-3g\t' "$fake_log" || + fail 'untrusted mount operand reached the privileged command boundary' + pass_boundary_case + # Production break caught: findmnt reports the effective ntfs-3g FUSE + # policy in normalized kernel-facing form rather than echoing uid/gid/masks. + : >"$fake_log" + builtin printf '%s\n' observed_normalized >"$fake_case_file" + _swr_validate_mount /dev/fixture-recovery-partition "$fake_mountpoint" ro || + fail 'observed normalized read-only FUSE evidence was rejected' + assert_eq 2 "$(/usr/bin/grep -Fc $'/usr/bin/stat\t--format=%u:%g:%a\t--\t'"$fake_mountpoint" "$fake_log")" \ + 'read-only mount validation did not couple option evidence to two private-root attestations' + assert_log_line $'/usr/bin/findmnt\t--json\t--mountpoint\t'"$fake_mountpoint"$'\t--output\tSOURCE,TARGET,FSTYPE,VFS-OPTIONS,FS-OPTIONS' \ + 'read-only mount validation findmnt argv differs' + first_stat_line="$(/usr/bin/grep -nF $'/usr/bin/stat\t--format=%u:%g:%a\t--\t'"$fake_mountpoint" "$fake_log" | /usr/bin/head -n 1)" + findmnt_line="$(/usr/bin/grep -nF $'/usr/bin/findmnt\t--json\t--mountpoint\t'"$fake_mountpoint"$'\t--output\tSOURCE,TARGET,FSTYPE,VFS-OPTIONS,FS-OPTIONS' "$fake_log")" + last_stat_line="$(/usr/bin/grep -nF $'/usr/bin/stat\t--format=%u:%g:%a\t--\t'"$fake_mountpoint" "$fake_log" | /usr/bin/tail -n 1)" + (( ${first_stat_line%%:*} < ${findmnt_line%%:*} && ${findmnt_line%%:*} < ${last_stat_line%%:*} )) || + fail 'read-only option observation was not bracketed by private-root attestations' + pass_boundary_case + + : >"$fake_log" + builtin printf '%s\n' rw >"$fake_mount_state" + _swr_validate_mount /dev/fixture-recovery-partition "$fake_mountpoint" rw || + fail 'observed normalized writable FUSE evidence was rejected' + assert_eq 2 "$(/usr/bin/grep -Fc $'/usr/bin/stat\t--format=%u:%g:%a\t--\t'"$fake_mountpoint" "$fake_log")" \ + 'writable mount validation did not couple option evidence to two private-root attestations' + assert_log_line $'/usr/bin/findmnt\t--json\t--mountpoint\t'"$fake_mountpoint"$'\t--output\tSOURCE,TARGET,FSTYPE,VFS-OPTIONS,FS-OPTIONS' \ + 'writable mount validation findmnt argv differs' + first_stat_line="$(/usr/bin/grep -nF $'/usr/bin/stat\t--format=%u:%g:%a\t--\t'"$fake_mountpoint" "$fake_log" | /usr/bin/head -n 1)" + findmnt_line="$(/usr/bin/grep -nF $'/usr/bin/findmnt\t--json\t--mountpoint\t'"$fake_mountpoint"$'\t--output\tSOURCE,TARGET,FSTYPE,VFS-OPTIONS,FS-OPTIONS' "$fake_log")" + last_stat_line="$(/usr/bin/grep -nF $'/usr/bin/stat\t--format=%u:%g:%a\t--\t'"$fake_mountpoint" "$fake_log" | /usr/bin/tail -n 1)" + (( ${first_stat_line%%:*} < ${findmnt_line%%:*} && ${findmnt_line%%:*} < ${last_stat_line%%:*} )) || + fail 'writable option observation was not bracketed by private-root attestations' + pass_boundary_case + + builtin printf '%s\n' observed_permuted >"$fake_case_file" + _swr_validate_mount /dev/fixture-recovery-partition "$fake_mountpoint" rw || + fail 'permuted normalized writable FUSE evidence was rejected' + pass_boundary_case + + builtin printf '%s\n' ro >"$fake_mount_state" + + local rejection + for rejection in \ + foreign_source foreign_fstype wrong_owner wrong_gid wrong_mode unsafe_allow_other \ + missing_nosuid vfs_relatime_missing vfs_relatime_duplicate vfs_phase_missing \ + executable suid dev wrong_phase legacy_explicit_mapping \ + fs_allow_other_missing fs_allow_other_duplicate fs_phase_missing fs_wrong_phase \ + fs_default_permissions_missing fs_default_permissions_duplicate \ + fs_user_id_missing fs_user_id_wrong fs_user_id_duplicate \ + fs_group_id_missing fs_group_id_wrong fs_group_id_duplicate \ + fs_blksize_missing fs_blksize_wrong fs_blksize_duplicate \ + field_placement_vfs_default_permissions field_placement_fs_nodev hybrid_actual_legacy \ + fs_uid_conflict fs_uid_duplicate fs_gid_conflict fs_gid_duplicate \ + fs_fmask_conflict fs_fmask_duplicate fs_dmask_conflict fs_dmask_duplicate \ + fs_permissions fs_acl fs_inherit fs_arbitrary; do + builtin printf '%s\n' "$rejection" >"$fake_case_file" + assert_fails _swr_validate_mount /dev/fixture-recovery-partition "$fake_mountpoint" ro + pass_boundary_case + done + builtin printf '%s\n' valid >"$fake_case_file" + builtin printf '%s\n' absent >"$fake_mount_state" + builtin printf '%s\n' target_reused >"$fake_case_file" + assert_fails _swr_require_mountpoint_unused "$fake_mountpoint" + pass_boundary_case + builtin printf '%s\n' valid >"$fake_case_file" + command /usr/bin/rmdir -- "$fake_mountpoint" +} + +test_unmount_and_cycle_boundary() { + local status mode second_probe_line mount_line post_probe_trace probes mounts _swr_created='' + for status in 0 32 124 143 255; do + reset_fake_boundary + builtin printf '%s\n' ro >"$fake_mount_state" + builtin printf '%s\n' "$status" >"$fake_umount_status" + _swr_unmount_volume /dev/fixture-recovery-partition "$fake_mountpoint" || + fail "stable absence after unmount status $status was rejected" + assert_log_line $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=10s\t45s\t/usr/bin/umount\t--\t'"$fake_mountpoint" 'fixed privileged unmount argv differs' + pass_boundary_case + done + + reset_fake_boundary + builtin printf '%s\n' ro >"$fake_mount_state" + builtin printf '%s\n' ro >"$fake_umount_after" + assert_fails _swr_unmount_volume /dev/fixture-recovery-partition "$fake_mountpoint" + [[ "$(/usr/bin/grep -Fc $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=10s\t45s\t/usr/bin/umount\t--\t'"$fake_mountpoint" "$fake_log")" == 1 ]] || fail 'unmount retried a retained source' + ! /usr/bin/grep -Eq $'\t(-l|-f|--lazy|--force)(\t|$)' "$fake_log" || fail 'lazy or force unmount was attempted' + pass_boundary_case + + reset_fake_boundary + builtin printf '%s\n' ro >"$fake_mount_state" + builtin printf '%s\n' source_ambiguous >"$fake_case_file" + assert_fails _swr_unmount_volume /dev/fixture-recovery-partition "$fake_mountpoint" + pass_boundary_case + + for mode in noop mutation error signal; do + reset_fake_boundary + fake_action="$mode" + _swr_create_mountpoint _swr_created || fail 'cycle mountpoint creation failed' + fake_mountpoint="$_swr_created" + if [[ "$mode" == noop || "$mode" == mutation ]]; then + _swr_run_volume_cycle /dev/fixture-recovery-partition "$fake_mountpoint" || fail "$mode cycle failed" + else + assert_fails _swr_run_volume_cycle /dev/fixture-recovery-partition "$fake_mountpoint" + fi + assert_eq absent "$(<"$fake_mount_state")" "$mode cycle did not unmount" + probes="$(/usr/bin/grep -Fc $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=5s\t20s\t/usr/bin/ntfs-3g.probe' "$fake_log")" + mounts="$(/usr/bin/grep -Fc $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=10s\t45s\t/usr/bin/ntfs-3g\t-o' "$fake_log")" + if [[ "$mode" == mutation ]]; then + assert_eq 2 "$probes" 'mutation cycle probe count differs' + assert_eq 2 "$mounts" 'mutation cycle mount count differs' + second_probe_line="$(/usr/bin/grep -nF $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=5s\t20s\t/usr/bin/ntfs-3g.probe\t--readwrite\t/dev/fixture-recovery-partition' "$fake_log" | /usr/bin/tail -n 1)" + mount_line="$(/usr/bin/grep -nF $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=10s\t45s\t/usr/bin/ntfs-3g\t-o\tnorecover,no_def_opts,allow_other,nodev,nosuid,noexec,uid=1000,gid=1000,fmask=0177,dmask=0077,rw\t/dev/fixture-recovery-partition\t'"$fake_mountpoint" "$fake_log" | /usr/bin/tail -n 1)" + post_probe_trace="$(/usr/bin/sed -n "$(( ${second_probe_line%%:*} + 1 )),$(( ${mount_line%%:*} ))p" "$fake_log")" + assert_eq \ + $'/usr/bin/stat\t--format=%u:%g:%a\t--\t'"$fake_mountpoint"$'\n/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=10s\t45s\t/usr/bin/ntfs-3g\t-o\tnorecover,no_def_opts,allow_other,nodev,nosuid,noexec,uid=1000,gid=1000,fmask=0177,dmask=0077,rw\t/dev/fixture-recovery-partition\t'"$fake_mountpoint" \ + "$post_probe_trace" 'fresh mountpoint attestation was not immediately between second probe and writable mount' + else + assert_eq 1 "$probes" "$mode cycle probe count differs" + assert_eq 1 "$mounts" "$mode cycle mount count differs" + fi + command /usr/bin/rmdir -- "$fake_mountpoint" + pass_boundary_case + done +} + +test_full_fake_boundary_ordering() { + local contract="$fixture_dir/local-recovery.env" first_sudo last_preflight _swr_result='' + reset_fake_boundary + fake_action=noop + write_recovery_contract "$contract" + _swr_open_recovery_boundary _swr_result "$contract" || fail 'full fake recovery boundary failed' + assert_eq exact-noop "$_swr_result" 'full fake no-op result differs' + first_sudo="$(/usr/bin/grep -n '^/usr/bin/sudo' "$fake_log" | /usr/bin/head -n 1)" + last_preflight="$(/usr/bin/grep -n $'^/usr/bin/findmnt\t--json\t--source' "$fake_log" | /usr/bin/head -n 1)" + (( ${first_sudo%%:*} > ${last_preflight%%:*} )) || fail 'sudo validation ran before user-space preflight completed' + assert_log_line $'/usr/bin/sudo\t-v' 'fixed sudo validation argv differs' + [[ ! -e "$fake_mountpoint" ]] || fail 'full boundary left its mountpoint behind' + pass_boundary_case +} + +review_findmnt_initial_rc1_nonempty() { + reset_fake_boundary + builtin printf '%s\n' source_rc1_nonempty >"$fake_case_file" + _swr_require_unmounted /dev/fixture-recovery-partition && + fail 'findmnt RC1 nonempty initial output was accepted as absence' + return 0 +} + +review_findmnt_post_poll1_rc1_nonempty() { + reset_fake_boundary + builtin printf '%s\n' ro >"$fake_mount_state" + builtin printf '%s\n' source_rc1_nonempty_poll1 >"$fake_case_file" + _swr_unmount_volume /dev/fixture-recovery-partition "$fake_mountpoint" >/dev/null 2>&1 && + fail 'findmnt RC1 nonempty first cleanup poll was accepted as absence' + return 0 +} + +review_findmnt_post_poll2_rc1_nonempty() { + reset_fake_boundary + builtin printf '%s\n' ro >"$fake_mount_state" + builtin printf '%s\n' source_rc1_nonempty_poll2 >"$fake_case_file" + _swr_unmount_volume /dev/fixture-recovery-partition "$fake_mountpoint" >/dev/null 2>&1 && + fail 'findmnt RC1 nonempty second cleanup poll was accepted as absence' + return 0 +} + +review_findmnt_rc1_overflow() { + reset_fake_boundary + builtin printf '%s\n' source_rc1_overflow >"$fake_case_file" + _swr_require_unmounted /dev/fixture-recovery-partition && + fail 'findmnt RC1 overflowing output was accepted as absence' + return 0 +} + +review_sudo_after_complete_preflight() { + local contract="$fixture_dir/local-recovery.env" _swr_result='' + local sudo_line mktemp_line chmod_line stat_line unused_line probe_line + local post_sudo_trace + reset_fake_boundary + fake_action=noop + write_recovery_contract "$contract" + _swr_open_recovery_boundary _swr_result "$contract" || fail 'ordered fake boundary unexpectedly failed' + sudo_line="$(/usr/bin/grep -n $'^/usr/bin/sudo\t-v$' "$fake_log" | /usr/bin/head -n 1)" + mktemp_line="$(/usr/bin/grep -n $'^/usr/bin/mktemp\t-d\t--' "$fake_log" | /usr/bin/head -n 1)" + chmod_line="$(/usr/bin/grep -n $'^/usr/bin/chmod\t0700\t--' "$fake_log" | /usr/bin/head -n 1)" + stat_line="$(/usr/bin/grep -n $'^/usr/bin/stat\t--format=%u:%g:%a\t--' "$fake_log" | /usr/bin/head -n 1)" + unused_line="$(/usr/bin/grep -n $'^/usr/bin/findmnt\t--json\t--mountpoint' "$fake_log" | /usr/bin/head -n 1)" + [[ -n "$sudo_line" && -n "$mktemp_line" && -n "$chmod_line" && -n "$stat_line" && -n "$unused_line" ]] || + fail 'complete mountpoint preflight trace was absent' + (( ${mktemp_line%%:*} < ${sudo_line%%:*} && ${chmod_line%%:*} < ${sudo_line%%:*} && + ${stat_line%%:*} < ${sudo_line%%:*} && ${unused_line%%:*} < ${sudo_line%%:*} )) || + fail 'sudo refresh preceded complete mountpoint preflight' + probe_line="$(/usr/bin/grep -n $'^/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=5s' "$fake_log" | /usr/bin/head -n 1)" + post_sudo_trace="$(/usr/bin/sed -n "$(( ${sudo_line%%:*} + 1 )),$(( ${probe_line%%:*} - 1 ))p" "$fake_log")" + [[ "$post_sudo_trace" == *$'/usr/bin/stat\t--format=%u:%g:%a\t--'* && + "$post_sudo_trace" == *$'/usr/bin/findmnt\t--json\t--mountpoint'* ]] || + fail 'post-sudo defensive mountpoint recheck was absent' +} + +review_sudo_failure_cleans_owned_mountpoint() { + local contract="$fixture_dir/local-recovery.env" _swr_result='' cleaned_mountpoint + reset_fake_boundary + fake_action=noop + write_recovery_contract "$contract" + builtin printf '%s\n' 1 >"$fake_sudo_status" + _swr_open_recovery_boundary _swr_result "$contract" >/dev/null 2>&1 && + fail 'sudo refresh failure was accepted' + cleaned_mountpoint="$(/usr/bin/grep $'^/usr/bin/chmod\t0700\t--\t' "$fake_log" | /usr/bin/cut -f4 | /usr/bin/head -n 1)" + [[ "$(/usr/bin/grep -Fc $'/usr/bin/mktemp\t-d\t--' "$fake_log")" == 1 && + "$(/usr/bin/grep -Fc $'/usr/bin/rmdir\t--' "$fake_log")" == 1 && + "$(/usr/bin/grep -Fc $'/usr/bin/sudo\t-n\t' "$fake_log")" == 0 && + -n "$cleaned_mountpoint" && ! -e "$cleaned_mountpoint" ]] || fail 'sudo failure did not clean only its owned unused mountpoint' +} + +review_preflight_failure_has_no_privileged_call() { + local contract="$fixture_dir/local-recovery.env" _swr_result='' + reset_fake_boundary + fake_action=noop + write_recovery_contract "$contract" + builtin printf '%s\n' target_reused >"$fake_case_file" + _swr_open_recovery_boundary _swr_result "$contract" >/dev/null 2>&1 && + fail 'reused mountpoint preflight was accepted' + ! /usr/bin/grep -q '^/usr/bin/sudo' "$fake_log" || + fail 'privileged call occurred before reused-mountpoint rejection' +} + +review_mount_option_conflicts() { + local rejection _swr_created='' + local -a rejections=( + legacy_explicit_mapping + vfs_relatime_missing vfs_relatime_duplicate vfs_phase_missing + fs_allow_other_missing fs_allow_other_duplicate fs_phase_missing fs_wrong_phase + fs_default_permissions_missing fs_default_permissions_duplicate + fs_user_id_missing fs_user_id_wrong fs_user_id_duplicate + fs_group_id_missing fs_group_id_wrong fs_group_id_duplicate + fs_blksize_missing fs_blksize_wrong fs_blksize_duplicate + field_placement_vfs_default_permissions field_placement_fs_nodev hybrid_actual_legacy + fs_uid_conflict fs_uid_duplicate fs_gid_conflict fs_gid_duplicate + fs_fmask_conflict fs_fmask_duplicate fs_dmask_conflict fs_dmask_duplicate + fs_umask fs_phase_conflict fs_phase_duplicate vfs_phase_duplicate + vfs_nosuid_duplicate vfs_nodev_duplicate vfs_noexec_duplicate + vfs_suid_conflict vfs_dev_conflict vfs_exec_conflict + ) + reset_fake_boundary + _swr_create_mountpoint _swr_created || fail 'option-conflict mountpoint creation failed' + fake_mountpoint="$_swr_created" + builtin printf '%s\n' ro >"$fake_mount_state" + for rejection in "${rejections[@]}"; do + builtin printf '%s\n' "$rejection" >"$fake_case_file" + _swr_validate_mount /dev/fixture-recovery-partition "$fake_mountpoint" ro >/dev/null 2>&1 && + fail "mount option conflict was accepted: $rejection" + done + builtin printf '%s\n' observed_normalized >"$fake_case_file" + _swr_validate_mount /dev/fixture-recovery-partition "$fake_mountpoint" ro || + fail 'observed normalized mount options were rejected' + builtin printf '%s\n' observed_permuted >"$fake_case_file" + _swr_validate_mount /dev/fixture-recovery-partition "$fake_mountpoint" ro || + fail 'permuted normalized mount options were rejected' + builtin printf '%s\n' valid >"$fake_case_file" + command /usr/bin/rmdir -- "$fake_mountpoint" +} + +review_normalized_mount_option_grammar_both_phases() { + local phase opposite vfs fs candidate + local -a invalid_vfs=() invalid_fs=() + for phase in ro rw; do + [[ "$phase" == ro ]] && opposite=rw || opposite=ro + vfs="nodev,noexec,nosuid,relatime,${phase}" + fs="allow_other,blksize=4096,default_permissions,group_id=0,${phase},user_id=0" + _swr_mount_options_are_safe "$phase" "$vfs" "$fs" || + fail "normalized $phase mount option set was rejected" + _swr_mount_options_are_safe "$phase" \ + "${phase},relatime,nosuid,noexec,nodev" \ + "${phase},user_id=0,group_id=0,default_permissions,blksize=4096,allow_other" || + fail "permuted normalized $phase mount option set was rejected" + + invalid_vfs=( + "nodev,noexec,nosuid,${phase}" + "nodev,noexec,nosuid,relatime,relatime,${phase}" + 'nodev,noexec,nosuid,relatime' + "nodev,noexec,nosuid,relatime,${opposite}" + "nodev,noexec,nosuid,relatime,${phase},${phase}" + "nodev,noexec,nosuid,relatime,${phase},default_permissions" + ) + for candidate in "${invalid_vfs[@]}"; do + assert_fails _swr_mount_options_are_safe "$phase" "$candidate" "$fs" + done + + invalid_fs=( + "blksize=4096,default_permissions,group_id=0,${phase},user_id=0" + "allow_other,allow_other,blksize=4096,default_permissions,group_id=0,${phase},user_id=0" + 'allow_other,blksize=4096,default_permissions,group_id=0,user_id=0' + "allow_other,blksize=4096,default_permissions,group_id=0,${opposite},user_id=0" + "allow_other,blksize=4096,default_permissions,group_id=0,${phase},${phase},user_id=0" + "allow_other,blksize=4096,group_id=0,${phase},user_id=0" + "allow_other,blksize=4096,default_permissions,default_permissions,group_id=0,${phase},user_id=0" + "allow_other,blksize=4096,default_permissions,group_id=0,${phase}" + "allow_other,blksize=4096,default_permissions,group_id=0,${phase},user_id=1000" + "allow_other,blksize=4096,default_permissions,group_id=0,${phase},user_id=0,user_id=0" + "allow_other,blksize=4096,default_permissions,${phase},user_id=0" + "allow_other,blksize=4096,default_permissions,group_id=1000,${phase},user_id=0" + "allow_other,blksize=4096,default_permissions,group_id=0,group_id=0,${phase},user_id=0" + "allow_other,default_permissions,group_id=0,${phase},user_id=0" + "allow_other,blksize=8192,default_permissions,group_id=0,${phase},user_id=0" + "allow_other,blksize=4096,blksize=4096,default_permissions,group_id=0,${phase},user_id=0" + "allow_other,blksize=4096,default_permissions,group_id=0,${phase},user_id=0,uid=1000" + "allow_other,blksize=4096,default_permissions,group_id=0,nodev,${phase},user_id=0" + "allow_other,blksize=4096,default_permissions,group_id=0,${phase},user_id=0,arbitrary" + ) + for candidate in "${invalid_fs[@]}"; do + assert_fails _swr_mount_options_are_safe "$phase" "$vfs" "$candidate" + done + done +} + +review_mount_option_whitespace_is_lossless() { + local canonical_vfs='nodev,noexec,nosuid,relatime,ro' + local canonical_fs='allow_other,blksize=4096,default_permissions,group_id=0,ro,user_id=0' + local field whitespace candidate json _swr_source='' _swr_target='' + local _swr_fstype='' _swr_vfs='' _swr_fs='' + + # Production breaks caught: read/here-string parsing must not discard a + # suffix, and JSON field framing must not let one field become another. + assert_fails _swr_mount_options_are_safe ro "$canonical_vfs" "$canonical_fs"$'\n' + assert_fails _swr_mount_options_are_safe ro "$canonical_vfs" "$canonical_fs"$'\nunknown' + assert_fails _swr_mount_options_are_safe ro "$canonical_vfs"$'\n' "$canonical_fs" + assert_fails _swr_mount_options_are_safe ro "$canonical_vfs"$'\nunknown' "$canonical_fs" + + for field in vfs fs; do + for whitespace in ' ' $'\t' $'\r' $'\n'; do + if [[ "$field" == vfs ]]; then + candidate="${canonical_vfs/relatime/relatime${whitespace}}" + assert_fails _swr_mount_options_are_safe ro "$candidate" "$canonical_fs" + else + candidate="${canonical_fs/default_permissions/default_permissions${whitespace}}" + assert_fails _swr_mount_options_are_safe ro "$canonical_vfs" "$candidate" + fi + done + done + + for json in \ + '{"filesystems":[{"source":"/dev/fixture recovery","target":"/run/user/1000/mount","fstype":"fuseblk","vfs-options":"nodev,noexec,nosuid,relatime,ro","fs-options":"allow_other,blksize=4096,default_permissions,group_id=0,ro,user_id=0"}]}' \ + '{"filesystems":[{"source":"/dev/fixture","target":"/run/user/1000/mount\tforeign","fstype":"fuseblk","vfs-options":"nodev,noexec,nosuid,relatime,ro","fs-options":"allow_other,blksize=4096,default_permissions,group_id=0,ro,user_id=0"}]}' \ + '{"filesystems":[{"source":"/dev/fixture","target":"/run/user/1000/mount","fstype":"fuseblk\r","vfs-options":"nodev,noexec,nosuid,relatime,ro","fs-options":"allow_other,blksize=4096,default_permissions,group_id=0,ro,user_id=0"}]}' \ + '{"filesystems":[{"source":"/dev/fixture","target":"/run/user/1000/mount","fstype":"fuseblk","vfs-options":"nodev,noexec,nosuid,relatime,ro\nallow_other,blksize=4096,default_permissions,group_id=0,ro,user_id=0","fs-options":""}]}' \ + '{"filesystems":[{"source":"/dev/fixture","target":"/run/user/1000/mount","fstype":"fuseblk","vfs-options":"nodev,noexec,nosuid,relatime,ro","fs-options":"allow_other,blksize=4096,default_permissions,group_id=0,ro,user_id=0\n"}]}'; do + assert_fails _swr_parse_findmnt_json "$json" _swr_source _swr_target \ + _swr_fstype _swr_vfs _swr_fs + done +} + +review_validate_mount_rejects_json_whitespace_reframing() { + local fake_case _swr_created='' + reset_fake_boundary + _swr_create_mountpoint _swr_created || fail 'JSON whitespace mountpoint creation failed' + fake_mountpoint="$_swr_created" + builtin printf '%s\n' ro >"$fake_mount_state" + for fake_case in \ + json_fs_trailing_lf json_fs_trailing_lf_garbage json_vfs_cross_field_reframe; do + builtin printf '%s\n' "$fake_case" >"$fake_case_file" + assert_fails _swr_validate_mount /dev/fixture-recovery-partition "$fake_mountpoint" ro + done + builtin printf '%s\n' valid >"$fake_case_file" + command /usr/bin/rmdir -- "$fake_mountpoint" +} + +review_cycle_uncertain_cleanup() { + local status rc _swr_created='' mounts umounts + for status in 32 124 255; do + reset_fake_boundary + fake_action=mutation + _swr_create_mountpoint _swr_created || fail 'uncertain-cycle mountpoint creation failed' + fake_mountpoint="$_swr_created" + builtin printf '%s\n' "$status" >"$fake_mount_status" + builtin printf '%s\n' 1 >"$fake_mount_uncertain" + builtin printf '%s\n' source_rc1_nonempty_poll2 >"$fake_case_file" + if _swr_run_volume_cycle /dev/fixture-recovery-partition "$fake_mountpoint" >/dev/null 2>&1; then + rc=0 + else + rc=$? + fi + assert_eq 1 "$rc" "cleanup ambiguity did not win after mount status $status" + mounts="$(/usr/bin/grep -Fc $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=10s\t45s\t/usr/bin/ntfs-3g\t-o' "$fake_log")" + umounts="$(/usr/bin/grep -Fc $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=10s\t45s\t/usr/bin/umount\t--' "$fake_log")" + assert_eq 1 "$mounts" "mount status $status triggered a later writable mount" + assert_eq 1 "$umounts" "mount status $status unmount count differs" + assert_eq 2 "$(<"$fake_source_query_count")" "mount status $status source absence query count differs" + assert_eq 0 "$(<"$fake_action_count")" "mount status $status invoked the volume action" + builtin printf '%s\n' absent >"$fake_mount_state" + builtin printf '%s\n' valid >"$fake_case_file" + command /usr/bin/rmdir -- "$fake_mountpoint" + done + + reset_fake_boundary + fake_action=mutation + _swr_create_mountpoint _swr_created || fail 'validation-cycle mountpoint creation failed' + fake_mountpoint="$_swr_created" + builtin printf '%s\n' foreign_source >"$fake_case_file" + _swr_run_volume_cycle /dev/fixture-recovery-partition "$fake_mountpoint" >/dev/null 2>&1 && + fail 'mounted validation failure was accepted' + assert_eq 1 "$(/usr/bin/grep -Fc $'/usr/bin/sudo\t-n\t/usr/bin/timeout\t--signal=TERM\t--kill-after=10s\t45s\t/usr/bin/umount\t--' "$fake_log")" 'validation failure unmount count differs' + assert_eq 2 "$(<"$fake_source_query_count")" 'validation failure source absence query count differs' + assert_eq 0 "$(<"$fake_action_count")" 'validation failure invoked the volume action' + builtin printf '%s\n' valid >"$fake_case_file" + command /usr/bin/rmdir -- "$fake_mountpoint" +} + +review_runtime_root_uses_fixed_command_leaf() { + local direct_marker="$fixture_dir/direct-runtime-readlink" resolved='' + reset_fake_boundary + builtin eval "$production_runtime_root_definition" + function /usr/bin/readlink() { + builtin printf '%s\n' direct >"$direct_marker" + return 98 + } + if ! resolved="$(_swr_runtime_root)"; then + [[ ! -e "$direct_marker" ]] || fail 'runtime root resolution bypassed fixed command leaf' + fail 'intercepted runtime root resolution failed' + fi + [[ ! -e "$direct_marker" ]] || fail 'runtime root resolution bypassed fixed command leaf' + assert_eq /run/user/1000 "$resolved" 'fixed runtime root result differs' + assert_log_line $'/usr/bin/readlink\t-e\t--\t/run/user/1000' 'runtime root fixed command leaf argv differs' +} + +run_review_case() { + local function_name="$1" + shift + if ( "$function_name" "$@" ); then + pass_boundary_case + else + review_failures=$((review_failures + 1)) + fi +} + +test_review_fix_round_one() { + run_review_case review_findmnt_initial_rc1_nonempty + run_review_case review_findmnt_post_poll1_rc1_nonempty + run_review_case review_findmnt_post_poll2_rc1_nonempty + run_review_case review_findmnt_rc1_overflow + run_review_case review_sudo_after_complete_preflight + run_review_case review_sudo_failure_cleans_owned_mountpoint + run_review_case review_preflight_failure_has_no_privileged_call + run_review_case review_mount_option_conflicts + run_review_case review_normalized_mount_option_grammar_both_phases + run_review_case review_mount_option_whitespace_is_lossless + run_review_case review_validate_mount_rejects_json_whitespace_reframing + run_review_case review_cycle_uncertain_cleanup + run_review_case review_runtime_root_uses_fixed_command_leaf + (( review_failures == 0 )) || fail "Task 4 review regressions failed: $review_failures" +} + +review_findmnt_newline_initial() { + local fake_case="$1" label="$2" + reset_fake_boundary + builtin printf '%s\n' "$fake_case" >"$fake_case_file" + _swr_require_unmounted /dev/fixture-recovery-partition && + fail "findmnt RC1 newline-only $label initial output was accepted as absence" + return 0 +} + +review_findmnt_newline_post_unmount() { + local fake_case="$1" label="$2" + reset_fake_boundary + builtin printf '%s\n' ro >"$fake_mount_state" + builtin printf '%s\n' "$fake_case" >"$fake_case_file" + _swr_unmount_volume /dev/fixture-recovery-partition "$fake_mountpoint" >/dev/null 2>&1 && + fail "findmnt RC1 newline-only $label post-unmount output was accepted as absence" + return 0 +} + +review_mount_uncharacterized_extras() { + local field="$1" rejection _swr_created='' + local -a rejections=() + reset_fake_boundary + _swr_create_mountpoint _swr_created || fail 'unknown-extra mountpoint creation failed' + fake_mountpoint="$_swr_created" + builtin printf '%s\n' ro >"$fake_mount_state" + case "$field" in + vfs) rejections=(vfs_permissions vfs_acl vfs_inherit vfs_unknown) ;; + fs) rejections=(fs_permissions fs_acl fs_inherit fs_unknown) ;; + *) fail 'unknown option field selector' ;; + esac + for rejection in "${rejections[@]}"; do + builtin printf '%s\n' "$rejection" >"$fake_case_file" + _swr_validate_mount /dev/fixture-recovery-partition "$fake_mountpoint" ro >/dev/null 2>&1 && + fail "uncharacterized $field mount option was accepted: $rejection" + done + builtin printf '%s\n' valid >"$fake_case_file" + command /usr/bin/rmdir -- "$fake_mountpoint" +} + +review_mount_malformed_csv() { + local rejection _swr_created='' + local -a rejections=( + vfs_trailing_comma fs_trailing_comma + vfs_empty vfs_leading_comma vfs_double_comma + fs_empty fs_leading_comma fs_double_comma + ) + reset_fake_boundary + _swr_create_mountpoint _swr_created || fail 'malformed-CSV mountpoint creation failed' + fake_mountpoint="$_swr_created" + builtin printf '%s\n' ro >"$fake_mount_state" + for rejection in "${rejections[@]}"; do + builtin printf '%s\n' "$rejection" >"$fake_case_file" + _swr_validate_mount /dev/fixture-recovery-partition "$fake_mountpoint" ro >/dev/null 2>&1 && + fail "malformed mount option CSV was accepted: $rejection" + done + builtin printf '%s\n' valid >"$fake_case_file" + command /usr/bin/rmdir -- "$fake_mountpoint" +} + +test_review_fix_round_two() { + run_review_case review_findmnt_newline_initial source_rc1_newline_1 one-LF + run_review_case review_findmnt_newline_initial source_rc1_newline_2 two-LF + run_review_case review_findmnt_newline_initial source_rc1_newline_overflow over-16KiB-LF + run_review_case review_findmnt_newline_post_unmount source_rc1_newline_1 one-LF + run_review_case review_findmnt_newline_post_unmount source_rc1_newline_2 two-LF + run_review_case review_findmnt_newline_post_unmount source_rc1_newline_overflow over-16KiB-LF + run_review_case review_mount_uncharacterized_extras vfs + run_review_case review_mount_uncharacterized_extras fs + run_review_case review_mount_malformed_csv + (( review_failures == 0 )) || fail "Task 4 review round 2 regressions failed: $review_failures" +} + +review_mountpoint_path_is_prearmed_before_attestation() { + local failure="$1" _swr_created='' foreign="$fixture_dir/foreign-mount-target" + mkdir -p -m 0700 -- "$foreign" + _swr_runtime_root() { builtin printf '%s\n' "$fake_runtime_root"; } + _swr_command() { + if [[ "$1:$failure" == /usr/bin/chmod:chmod ]]; then + return 1 + fi + if [[ "$1:$failure" == /usr/bin/stat:attest ]]; then + return 1 + fi + if [[ "$1:$failure" == /usr/bin/chmod:foreign ]]; then + command /usr/bin/rmdir -- "${@: -1}" + command /usr/bin/ln -s -- "$foreign" "${@: -1}" + return 1 + fi + command "$@" + } + _swr_create_mountpoint _swr_created && fail "mountpoint $failure failure was accepted" + [[ "$_swr_created" == "$fake_runtime_root/slack-webhook-recovery."* ]] || + fail "mountpoint $failure path was not prearmed" + if [[ "$failure" == foreign ]]; then + [[ -L "$_swr_created" && "$(command /usr/bin/readlink -- "$_swr_created")" == "$foreign" ]] || + fail 'foreign mountpoint replacement was removed' + elif [[ -d "$_swr_created" ]]; then + failure=cleanup + _swr_remove_mountpoint "$_swr_created" || true + [[ ! -e "$_swr_created" ]] || fail "owned mountpoint $failure residue remained" + fi +} + +review_prepare_publishes_partial_mountpoint() { + local _swr_work='' _swr_device='' _swr_mount='' + _swr_load_recovery_contract() { return 0; } + _swr_resolve_device() { builtin printf -v "$1" '%s' /dev/fixture; } + _swr_require_unmounted() { return 0; } + _swr_runtime_root() { builtin printf '%s\n' /run/user/1000; } + _swr_capture_single_line() { + local name="$1"; shift + case "$*" in + '/usr/bin/stat --format=%u:%g:%a -- /run/user/1000') builtin printf -v "$name" '%s' 1000:1000:700 ;; + '/usr/bin/stat --file-system --format=%T -- /run/user/1000') builtin printf -v "$name" '%s' tmpfs ;; + '/usr/bin/mktemp -d --tmpdir=/run/user/1000 platform-slack-keepass.XXXXXXXX') builtin printf -v "$name" '%s' /run/user/1000/platform-slack-keepass.SYNTHETIC ;; + *) return 1 ;; + esac + } + _swr_command() { return 0; } + _swr_work_root_is_safe() { return 0; } + _swr_create_mountpoint() { + builtin printf -v "$1" '%s' /run/user/1000/slack-webhook-recovery.SYNTHETIC + return 1 + } + _swr_prepare_transaction _swr_work _swr_device _swr_mount /fixture/webhook /fixture/contract && + fail 'partial mountpoint preparation failure was accepted' + assert_eq /run/user/1000/slack-webhook-recovery.SYNTHETIC "$_swr_mount" \ + 'preparation did not publish the partial mountpoint' +} + +review_stop_vault_tears_down_failed_persistent_client() { + local socket="$fixture_dir/stop-failure.sock" vault_pid='' client_pid='' + /usr/bin/python3 -c 'import socket,sys,time; s=socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); time.sleep(30)' "$socket" & + vault_pid=$! + /usr/bin/sleep 0.05 + /usr/bin/sleep 30 & client_pid=$! + trap 'kill -TERM '"$vault_pid"' '"$client_pid"' 2>/dev/null || true; wait '"$vault_pid"' '"$client_pid"' 2>/dev/null || true' EXIT + exec {_SWR_VAULT_CLIENT_READ_FD}/dev/null + _SWR_VAULT_CLIENT_SOCKET="$socket" + _SWR_VAULT_CLIENT_PID="$client_pid" + _swr_vault_request() { return 1; } + _swr_stop_vault "$socket" "$vault_pid" && fail 'failed SHUTDOWN was accepted' + [[ -z "$_SWR_VAULT_CLIENT_SOCKET$_SWR_VAULT_CLIENT_PID$_SWR_VAULT_CLIENT_READ_FD$_SWR_VAULT_CLIENT_WRITE_FD" ]] || + fail 'failed SHUTDOWN retained persistent client state' + kill -0 "$client_pid" 2>/dev/null && fail 'failed SHUTDOWN retained persistent client process' + trap - EXIT +} + +test_start_vault_waits_for_delayed_operator_input() { + # Production break caught: the coordinator kills the healthy vault after + # five seconds even though the vault intentionally withholds its socket + # until both operator inputs have been retained. + local repository="$fixture_dir/delayed-vault-repository" + local runtime="$fixture_dir/delayed-vault-runtime" + local helper="$repository/scripts/libexec/slack-webhook-recovery-vault.py" + local webhook="$fixture_dir/delayed-vault-webhook" input_fifo="$fixture_dir/delayed-vault-webhook.input" + local pre_input="$fixture_dir/delayed-vault-pre-input" helper_pid_file="$runtime/helper.pid" + local saved_repository_root="$SWR_REPOSITORY_ROOT" + local _swr_delayed_pid='' _swr_delayed_socket='' writer_pid='' helper_pid='' start_rc=0 stop_rc=1 + + mkdir -p -m 0700 -- "${helper%/*}" "$runtime" + builtin printf '%s\n' \ + 'import os' \ + 'import socket' \ + 'import sys' \ + 'runtime = sys.argv[sys.argv.index("--runtime-root") + 1]' \ + 'socket_path = os.path.join(runtime, "vault.sock")' \ + 'with open(os.path.join(runtime, "helper.pid"), "w", encoding="ascii") as stream:' \ + ' stream.write(str(os.getpid()))' \ + 'input_path = sys.argv[sys.argv.index("--webhook-file") + 1] + ".input"' \ + 'with open(input_path, "rb", buffering=0) as input_stream:' \ + ' if not input_stream.readline() or not input_stream.readline():' \ + ' raise SystemExit(70)' \ + 'server = socket.socket(socket.AF_UNIX)' \ + 'try:' \ + ' server.bind(socket_path)' \ + ' os.chmod(socket_path, 0o600)' \ + ' server.listen(1)' \ + ' connection, _ = server.accept()' \ + ' with connection:' \ + ' command = b""' \ + ' while not command.endswith(b"\\n"):' \ + ' command += connection.recv(64)' \ + ' if command != b"SHUTDOWN\\n":' \ + ' raise SystemExit(71)' \ + ' connection.sendall(b"stopped\\n")' \ + 'finally:' \ + ' server.close()' \ + ' if os.path.exists(socket_path):' \ + ' os.unlink(socket_path)' >"$helper" + chmod 0600 -- "$helper" + builtin printf '%s\n' synthetic-webhook >"$webhook" + chmod 0600 -- "$webhook" + mkfifo -m 0600 -- "$input_fifo" + + ( + exec 3<>"$input_fifo" + /usr/bin/sleep 5.2 + if [[ -e "$runtime/vault.sock" || -L "$runtime/vault.sock" ]]; then + builtin printf '%s\n' published >"$pre_input" + else + builtin printf '%s\n' absent >"$pre_input" + fi + builtin printf '%s\n' synthetic-app synthetic-input >&3 + ) & + writer_pid=$! + + SWR_REPOSITORY_ROOT="$repository" + set +e + _swr_start_vault _swr_delayed_pid _swr_delayed_socket "$runtime" "$webhook" + start_rc=$? + set -e + SWR_REPOSITORY_ROOT="$saved_repository_root" + wait "$writer_pid" 2>/dev/null || true + + if (( start_rc == 0 )); then + helper_pid="$(<"$helper_pid_file")" + [[ "$(<"$pre_input")" == absent ]] || start_rc=72 + [[ "$_swr_delayed_socket" == "$runtime/vault.sock" && -S "$_swr_delayed_socket" ]] || start_rc=73 + [[ "$_swr_delayed_pid" == "$helper_pid" ]] || start_rc=74 + /usr/bin/sleep 0.1 + kill -0 "$_swr_delayed_pid" 2>/dev/null || start_rc=75 + if (( start_rc == 0 )); then + set +e + _swr_terminate_vault_child "$_swr_delayed_pid" + stop_rc=$? + set -e + fi + elif [[ -f "$helper_pid_file" ]]; then + helper_pid="$(<"$helper_pid_file")" + fi + + if [[ "$helper_pid" =~ ^[1-9][0-9]*$ ]] && kill -0 "$helper_pid" 2>/dev/null; then + kill -TERM "$_swr_delayed_pid" "$helper_pid" 2>/dev/null || true + wait "$_swr_delayed_pid" 2>/dev/null || true + fi + rm -f -- "$input_fifo" "$runtime/vault.sock" + + assert_eq 0 "$start_rc" 'vault startup rejected operator input delayed beyond five seconds' + assert_eq 0 "$stop_rc" "delayed-input vault did not stop cleanly (rc=$stop_rc)" + [[ ! "$helper_pid" =~ ^[1-9][0-9]*$ ]] || + ! kill -0 "$helper_pid" 2>/dev/null || fail 'delayed-input vault helper remained alive' +} + +test_vault_readiness_failure_boundaries() { + # Production breaks caught: unbounded readiness, zombie blindness, + # signal deferral through the full human window, foreign-path deletion, + # and TERM-only cleanup that can orphan a resistant child. + local root="$fixture_dir/vault-readiness-boundaries" socket="$fixture_dir/vault-readiness-boundaries/vault.sock" + local pid='' rc=0 code started_ns ended_ns elapsed_ms + mkdir -m 0700 -- "$root" + + /usr/bin/python3 -c 'import time; time.sleep(30)' & pid=$! + set +e + _swr_wait_vault_ready "$socket" "$pid" 2 + rc=$? + set -e + assert_eq 1 "$rc" 'short vault readiness deadline was accepted' + _swr_terminate_vault_child "$pid" + ! kill -0 "$pid" 2>/dev/null || fail 'deadline cleanup retained vault child' + + /usr/bin/python3 -c 'raise SystemExit(17)' & pid=$! + started_ns="$(/usr/bin/date +%s%N)" + set +e + _swr_wait_vault_ready "$socket" "$pid" 100 + rc=$? + set -e + ended_ns="$(/usr/bin/date +%s%N)" + elapsed_ms=$(((ended_ns - started_ns) / 1000000)) + assert_eq 1 "$rc" 'dead vault child was accepted as waiting' + (( elapsed_ms < 1000 )) || fail 'dead vault child did not fail promptly' + wait "$pid" 2>/dev/null || true + + builtin printf '%s\n' foreign >"$socket" + /usr/bin/python3 -c 'import time; time.sleep(30)' & pid=$! + set +e + _swr_wait_vault_ready "$socket" "$pid" 2 + rc=$? + set -e + assert_eq 1 "$rc" 'foreign vault socket path was accepted' + assert_eq foreign "$(<"$socket")" 'foreign vault socket path was changed' + _swr_terminate_vault_child "$pid" + rm -f -- "$socket" + + for code in 129 130 143; do + /usr/bin/python3 -c 'import time; time.sleep(30)' & pid=$! + SWR_PENDING_SIGNAL="$code" + set +e + _swr_wait_vault_ready "$socket" "$pid" 100 + rc=$? + set -e + assert_eq 2 "$rc" "pending signal $code did not win vault readiness boundary" + _swr_terminate_vault_child "$pid" + ! kill -0 "$pid" 2>/dev/null || fail "pending signal $code retained vault child" + assert_eq "$code" "$SWR_PENDING_SIGNAL" "pending signal $code changed during cleanup" + done + SWR_PENDING_SIGNAL=0 + + /usr/bin/python3 -c 'import signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(30)' & pid=$! + /usr/bin/sleep 0.05 + started_ns="$(/usr/bin/date +%s%N)" + _swr_terminate_vault_child "$pid" + ended_ns="$(/usr/bin/date +%s%N)" + elapsed_ms=$(((ended_ns - started_ns) / 1000000)) + (( elapsed_ms < 3000 )) || fail 'TERM-resistant vault cleanup exceeded bound' + ! kill -0 "$pid" 2>/dev/null || fail 'TERM-resistant vault child remained alive' +} + +test_pending_signal_after_readiness_prevents_publication() { + # Production break caught: a signal delivered after readiness returns but + # before output publication exposes a live socket/PID and is not forwarded + # to the vault until generic SHUTDOWN cleanup. + local repository="$fixture_dir/post-ready-signal-repository" + local helper="$repository/scripts/libexec/slack-webhook-recovery-vault.py" + local runtime webhook saved_repository_root="$SWR_REPOSITORY_ROOT" + local saved_wait_definition signal_name code signum helper_pid rc + local _swr_signal_pid _swr_signal_socket + saved_wait_definition="$(declare -f _swr_wait_vault_ready)" + mkdir -p -m 0700 -- "${helper%/*}" + builtin printf '%s\n' \ + 'import os' \ + 'import signal' \ + 'import sys' \ + 'runtime = sys.argv[sys.argv.index("--runtime-root") + 1]' \ + 'signal_file = os.path.join(runtime, "signal")' \ + 'def finish(signum, _frame):' \ + ' with open(signal_file, "w", encoding="ascii") as stream:' \ + ' stream.write(str(signum))' \ + ' raise SystemExit(128 + signum)' \ + 'for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM):' \ + ' signal.signal(signum, finish)' \ + 'with open(os.path.join(runtime, "helper.pid"), "w", encoding="ascii") as stream:' \ + ' stream.write(str(os.getpid()))' \ + 'signal.pause()' >"$helper" + chmod 0600 -- "$helper" + + for signal_name in HUP INT TERM; do + case "$signal_name" in + HUP) code=129; signum=1 ;; + INT) code=130; signum=2 ;; + TERM) code=143; signum=15 ;; + esac + runtime="$fixture_dir/post-ready-signal-$signal_name" + webhook="$fixture_dir/post-ready-signal-$signal_name-webhook" + mkdir -m 0700 -- "$runtime" + builtin printf synthetic >"$webhook"; chmod 0600 -- "$webhook" + _swr_wait_vault_ready() { + local index + for (( index = 0; index < 100; index++ )); do + [[ -f "$runtime/helper.pid" ]] && break + /usr/bin/sleep 0.01 + done + [[ -f "$runtime/helper.pid" ]] || return 1 + SWR_PENDING_SIGNAL="$code" + return 0 + } + SWR_REPOSITORY_ROOT="$repository" + _swr_signal_pid=sentinel-pid + _swr_signal_socket=sentinel-socket + set +e + _swr_start_vault _swr_signal_pid _swr_signal_socket "$runtime" "$webhook" + rc=$? + set -e + SWR_REPOSITORY_ROOT="$saved_repository_root" + helper_pid="$(<"$runtime/helper.pid")" + if kill -0 "$helper_pid" 2>/dev/null; then + kill -KILL "$helper_pid" 2>/dev/null || true + wait "$helper_pid" 2>/dev/null || true + fi + assert_eq 1 "$rc" "post-ready $signal_name signal was accepted" + assert_eq sentinel-pid "$_swr_signal_pid" "post-ready $signal_name published vault PID" + assert_eq sentinel-socket "$_swr_signal_socket" "post-ready $signal_name published socket" + assert_eq "$signum" "$(<"$runtime/signal")" "post-ready $signal_name was not forwarded" + ! kill -0 "$helper_pid" 2>/dev/null || fail "post-ready $signal_name retained vault child" + SWR_PENDING_SIGNAL=0 + done + eval "$saved_wait_definition" +} + +review_work_root_inventory_precedes_deletion() { + local case_name="$1" root baseline violation='' + _swr_command() { command "$@"; } + root="$(/usr/bin/mktemp -d --tmpdir=/run/user/1000 platform-slack-keepass.XXXXXXXX)" + chmod 0700 -- "$root" + baseline="$root/baseline.kdbx" + builtin printf x >"$baseline"; chmod 0600 -- "$baseline" + case "$case_name" in + unexpected) + builtin printf y >"$root/unexpected"; chmod 0600 -- "$root/unexpected" + _swr_remove_work_root "$root" && violation='unexpected work-root content was accepted' + [[ -f "$baseline" ]] || violation='inventory deleted known evidence before finding unexpected content' + /usr/bin/rm -f -- "$root/unexpected" "$baseline" + ;; + wrong-mode) + builtin printf y >"$root/current.kdbx"; chmod 0644 -- "$root/current.kdbx" + _swr_remove_work_root "$root" && violation='wrong-mode work-root content was accepted' + [[ -f "$baseline" ]] || violation='inventory deleted known evidence before finding wrong mode' + /usr/bin/rm -f -- "$root/current.kdbx" "$baseline" + ;; + *) fail 'unknown topology case' ;; + esac + /usr/bin/rmdir -- "$root" + [[ -z "$violation" ]] || fail "$violation" +} + +review_failed_find_inventory_preserves_work_root() { + local producer="$1" root baseline violation='' + root="$(/usr/bin/mktemp -d --tmpdir=/run/user/1000 platform-slack-keepass.XXXXXXXX)" + chmod 0700 -- "$root" + baseline="$root/baseline.kdbx" + builtin printf x >"$baseline"; chmod 0600 -- "$baseline" + _swr_command() { + if [[ "$1" == /usr/bin/find && "$producer" == root && "$2" == "$root" && "$3" == -mindepth ]]; then + builtin printf '%s\n' baseline.kdbx + return 71 + fi + command "$@" + } + _swr_remove_work_root "$root" && violation="$producer failed inventory was accepted" + [[ -f "$baseline" ]] || violation="$producer failed inventory deleted evidence" + _swr_command() { command "$@"; } + /usr/bin/rm -f -- "$baseline" + /usr/bin/rmdir -- "$root" + [[ -z "$violation" ]] || fail "$violation" +} + +review_work_root_rejects_unexpected_vault_home() { + local root baseline violation='' + root="$(/usr/bin/mktemp -d --tmpdir=/run/user/1000 platform-slack-keepass.XXXXXXXX)" + chmod 0700 -- "$root" + baseline="$root/baseline.kdbx" + builtin printf x >"$baseline"; chmod 0600 -- "$baseline" + mkdir -m 0700 -- "$root/vault-home" + _swr_command() { command "$@"; } + _swr_remove_work_root "$root" && violation='unexpected vault-home was accepted' + [[ -f "$baseline" ]] || violation='unexpected vault-home deleted known evidence' + [[ -d "$root/vault-home" ]] || violation='unexpected vault-home was deleted' + [[ ! -d "$root/vault-home" ]] || /usr/bin/rmdir -- "$root/vault-home" + [[ ! -f "$baseline" ]] || /usr/bin/rm -f -- "$baseline" + [[ ! -d "$root" ]] || /usr/bin/rmdir -- "$root" + [[ -z "$violation" ]] || fail "$violation" +} + +review_nul_inventory_rejects_ambiguous_or_partial_name() { + local inventory_case="$1" root baseline hostile violation='' + root="$(/usr/bin/mktemp -d --tmpdir=/run/user/1000 platform-slack-keepass.XXXXXXXX)" + chmod 0700 -- "$root" + baseline="$root/baseline.kdbx" + builtin printf x >"$baseline"; chmod 0600 -- "$baseline" + hostile="$root/"$'baseline.kdbx\ncurrent.kdbx' + if [[ "$inventory_case" == lf-name ]]; then + builtin printf y >"$hostile"; chmod 0600 -- "$hostile" + _swr_command() { command "$@"; } + else + _swr_command() { + if [[ "$1" == /usr/bin/find && "$2" == "$root" && "$3" == -mindepth ]]; then + builtin printf '%s' baseline.kdbx + return 0 + fi + command "$@" + } + fi + _swr_remove_work_root "$root" && violation="$inventory_case inventory was accepted" + [[ -f "$baseline" ]] || violation="$inventory_case inventory deleted known evidence" + _swr_command() { command "$@"; } + /usr/bin/rm -f -- "$baseline" + [[ ! -e "$hostile" ]] || /usr/bin/rm -f -- "$hostile" + /usr/bin/rmdir -- "$root" + [[ -z "$violation" ]] || fail "$violation" +} + +review_inventory_stops_at_bounded_input_limit() { + local inventory_case="$1" caller="$2" root baseline producer_pid='' violation='' + local complete_marker="$fixture_dir/inventory-${inventory_case}-${caller}.complete" + local pid_marker="$fixture_dir/inventory-${inventory_case}-${caller}.pid" + local -a _swr_inventory=() + root="$(/usr/bin/mktemp -d --tmpdir=/run/user/1000 platform-slack-keepass.XXXXXXXX)" + chmod 0700 -- "$root" + baseline="$root/baseline.kdbx" + builtin printf x >"$baseline"; chmod 0600 -- "$baseline" + _swr_command() { + if [[ "$1" == /usr/bin/find && "$2" == "$root" && "$3" == -mindepth ]]; then + builtin printf '%s' "$BASHPID" >"$pid_marker" + local chunk='' record='' index + case "$inventory_case" in + long-non-nul) + builtin printf -v chunk '%256s' '' + chunk="${chunk// /x}" + for (( index = 0; index < 8192; index++ )); do + builtin printf '%s' "$chunk" || return 74 + done + ;; + count-total-overflow) + for (( index = 0; index < 10000; index++ )); do + builtin printf -v record 'n%0254d' "$index" + builtin printf '%s\0' "$record" || return 75 + done + ;; + *) return 97 ;; + esac + builtin printf complete >"$complete_marker" + return 0 + fi + command "$@" + } + + if [[ "$caller" == parser ]]; then + _swr_inventory_directory _swr_inventory "$root" && + violation="$inventory_case bounded inventory was accepted" + if [[ "$inventory_case" == long-non-nul ]]; then + (( ${#_swr_inventory[@]} == 0 )) || + violation="$inventory_case retained an overlong record" + else + (( ${#_swr_inventory[@]} == 16 )) || + violation="$inventory_case retained ${#_swr_inventory[@]} records after limit" + fi + else + _swr_remove_work_root "$root" && + violation="$inventory_case work-root cleanup accepted failed inventory" + fi + [[ -n "$violation" || ! -e "$complete_marker" ]] || + violation="$inventory_case consumed the entire lazy producer" + producer_pid="$(<"$pid_marker")" + [[ "$producer_pid" =~ ^[1-9][0-9]*$ ]] || + violation="$inventory_case producer PID was not captured" + if [[ "$producer_pid" =~ ^[1-9][0-9]*$ ]] && kill -0 "$producer_pid" 2>/dev/null; then + violation="$inventory_case producer was not terminated and waited" + fi + + [[ -f "$baseline" ]] || violation="$inventory_case inventory deleted evidence" + _swr_command() { command "$@"; } + /usr/bin/rm -f -- "$baseline" "$complete_marker" "$pid_marker" + /usr/bin/rmdir -- "$root" + [[ -z "$violation" ]] || fail "$violation" +} + +test_task5_fix_round_one_boundaries() { + run_review_case review_mountpoint_path_is_prearmed_before_attestation chmod + run_review_case review_mountpoint_path_is_prearmed_before_attestation attest + run_review_case review_mountpoint_path_is_prearmed_before_attestation foreign + run_review_case review_prepare_publishes_partial_mountpoint + run_review_case review_stop_vault_tears_down_failed_persistent_client + run_review_case review_work_root_inventory_precedes_deletion unexpected + run_review_case review_work_root_inventory_precedes_deletion wrong-mode + run_review_case review_failed_find_inventory_preserves_work_root root + run_review_case review_work_root_rejects_unexpected_vault_home + run_review_case review_nul_inventory_rejects_ambiguous_or_partial_name lf-name + run_review_case review_nul_inventory_rejects_ambiguous_or_partial_name partial-frame + (( review_failures == 0 )) || fail "Task 5 fix round one regressions failed: $review_failures" +} + +test_task5_fix_round_four_inventory_boundaries() { + run_review_case review_inventory_stops_at_bounded_input_limit long-non-nul parser + run_review_case review_inventory_stops_at_bounded_input_limit long-non-nul cleanup + run_review_case review_inventory_stops_at_bounded_input_limit count-total-overflow parser + run_review_case review_inventory_stops_at_bounded_input_limit count-total-overflow cleanup + (( review_failures == 0 )) || fail "Task 5 fix round four regressions failed: $review_failures" +} + +transaction_log_event() { + builtin printf '%s\n' "$1" >>"$transaction_log" +} + +_swr_fake_signal_after_event() { + local event="$1" signal_name scenario_event + [[ "$transaction_scenario" == signal-* ]] || return 0 + signal_name="${transaction_scenario#signal-}" + signal_name="${signal_name%%-*}" + scenario_event="${transaction_scenario#signal-${signal_name}-}" + [[ "$scenario_event" == "$event" ]] || return 0 + kill -"$signal_name" "$BASHPID" +} + +reset_transaction_fake() { + : >"$transaction_log" + transaction_phase=ro + transaction_commit_calls=0 + SWR_TRANSACTION_STATE='' + SWR_PENDING_SIGNAL=0 +} + +_swr_fake_prepare_transaction() { + transaction_log_event validate-runtime + builtin printf -v "$1" '%s' /fixture/work-root + builtin printf -v "$2" '%s' /dev/fixture-recovery-partition + builtin printf -v "$3" '%s' /fixture/mountpoint + _swr_fake_signal_after_event validate-runtime +} + +_swr_fake_start_transaction_vault() { + transaction_log_event start-vault-and-retain-inputs + builtin printf -v "$1" '%s' 4242 + builtin printf -v "$2" '%s' /fixture/work-root/vault.sock + _swr_fake_signal_after_event start-vault-and-retain-inputs +} + +_swr_fake_refresh_sudo() { + transaction_log_event sudo-refresh + _swr_fake_signal_after_event sudo-refresh +} + +_swr_fake_probe_transaction_rw() { + transaction_log_event "probe-${transaction_phase}-safety" + [[ "$transaction_scenario:$transaction_phase" == term-rw-probe:rw ]] && kill -TERM "$BASHPID" + _swr_fake_signal_after_event "probe-${transaction_phase}-safety" + return 0 +} + +_swr_fake_mount_transaction_volume() { + transaction_phase="$3" + transaction_log_event "mount-${transaction_phase}" + _swr_fake_signal_after_event "mount-${transaction_phase}" + case "$transaction_scenario:$transaction_phase" in + mount-ro-failure:ro|mount-ro-response-loss-unresolved-source:ro) return 32 ;; + esac +} + +_swr_fake_validate_transaction_mount() { + transaction_log_event "validate-${3}-mount" + [[ "$transaction_scenario:$3" == int-rw-validate:rw ]] && kill -INT "$BASHPID" + _swr_fake_signal_after_event "validate-${3}-mount" + case "$transaction_scenario:$3" in + validate-ro-wrong-uid:ro|validate-ro-wrong-gid:ro|validate-ro-wrong-mode:ro) return 1 ;; + esac + return 0 +} + +_swr_fake_dirfd_snapshot() { + transaction_log_event snapshot + _swr_fake_signal_after_event snapshot +} + +_swr_fake_unmount_transaction_volume() { + transaction_log_event "unmount-${transaction_phase}" + _swr_fake_signal_after_event "unmount-${transaction_phase}" + [[ "$transaction_scenario:$transaction_phase" == hup-final-unmount:rw ]] && kill -HUP "$BASHPID" + case "$transaction_scenario:$transaction_phase" in + mount-ro-response-loss-unresolved-source:ro) return 1 ;; + unmount-ro:ro|unmount-rw:rw|unmount-ambiguous:*) return 1 ;; + esac +} + +_swr_fake_transaction_stably_unmounted() { + case "$transaction_scenario" in + mount-ro-response-loss-unresolved-source|unmount-ro|unmount-rw|unmount-ambiguous) return 1 ;; + esac +} + +_swr_fake_vault_transaction_request() { + local result_name="$1" command="$3" reply='' + case "$command" in + PREPARE) + transaction_log_event classify + _swr_fake_signal_after_event classify + case "$transaction_scenario" in + noop) reply=exact-noop ;; + hup-noop) kill -HUP "$BASHPID"; reply=exact-noop ;; + signal-INT-confirm-update) reply=mismatch ;; + absent|original-drift|backup-collision|backup-fsync|rename-failure|post-rename|final-verification|hup-before|int-before|term-before|hup-during|int-during|term-during|term-rw-probe|int-rw-validate|lost-helper|lost-after-verified|hup-final-unmount|term-cleanup|term-post-cleanup|unmount-ro|unmount-rw|unmount-ambiguous|signal-*) reply=absent ;; + mismatch-no-update|mismatch-update) reply=mismatch ;; + duplicate) reply=ambiguous ;; + wrong-master|retained-input-drift) reply=failed ;; + *) return 97 ;; + esac + ;; + 'BUILD ADD') transaction_log_event build-add; _swr_fake_signal_after_event build-add; reply=candidate-ready ;; + 'BUILD UPDATE') transaction_log_event build-update; _swr_fake_signal_after_event build-update; reply=candidate-ready ;; + VERIFY_CANDIDATE) transaction_log_event verify-candidate; _swr_fake_signal_after_event verify-candidate; reply=verified ;; + 'VERIFY_COMMITTED '*) + transaction_log_event verify-committed-copy + [[ "$transaction_scenario" == final-verification ]] && reply=failed || reply=verified + ;; + *) return 97 ;; + esac + builtin printf -v "$result_name" '%s' "$reply" +} + +_swr_fake_confirm_transaction() { + local expected="$1" + transaction_log_event "confirm-${expected,,}" + _swr_fake_signal_after_event "confirm-${expected,,}" + case "$transaction_scenario:$expected" in + mismatch-no-update:UPDATE) return 1 ;; + hup-before:WRITE) kill -HUP "$BASHPID"; return 1 ;; + int-before:WRITE) kill -INT "$BASHPID"; return 1 ;; + term-before:WRITE) kill -TERM "$BASHPID"; return 1 ;; + esac + [[ "$expected" == WRITE ]] && transaction_phase=rw + return 0 +} + +_swr_fake_dirfd_commit() { + local result_name="$1" + transaction_commit_calls=$((transaction_commit_calls + 1)) + transaction_log_event commit + case "$transaction_scenario" in + original-drift|backup-collision|backup-fsync) + builtin printf -v "$result_name" '%s' precommit-failure + return 1 + ;; + rename-failure|post-rename) + builtin printf -v "$result_name" '%s' committed-but-uncertain + return 1 + ;; + lost-helper) + builtin printf -v "$result_name" '%s' rename-attempted + return 1 + ;; + lost-after-verified) + builtin printf -v "$result_name" '%s' verified-commit + return 0 + ;; + hup-during) kill -HUP "$BASHPID" ;; + int-during) kill -INT "$BASHPID" ;; + term-during) kill -TERM "$BASHPID" ;; + esac + builtin printf -v "$result_name" '%s' verified-commit +} + +_swr_fake_dirfd_snapshot_current() { + transaction_log_event snapshot-current +} + +_swr_fake_stop_transaction_vault() { + transaction_log_event stop-vault + [[ "$transaction_scenario" == term-cleanup ]] && kill -TERM "$BASHPID" + return 0 +} + +_swr_fake_audit_runtime_processes() { transaction_log_event audit-processes; } + +_swr_fake_remove_transaction_work_root() { + transaction_log_event remove-work-root + [[ "$transaction_scenario" == term-post-cleanup ]] && kill -TERM "$BASHPID" + return 0 +} + +assert_transaction_prefix() { + local expected="$1" description="$2" actual + actual="$(/usr/bin/head -n "$(builtin printf '%s\n' "$expected" | /usr/bin/wc -l)" "$transaction_log")" + assert_eq "$expected" "$actual" "$description" +} + +run_transaction_case() { + local scenario="$1" expected_state="$2" expected_output="$3" expected_rc="${4-}" output='' rc=0 + reset_transaction_fake + transaction_scenario="$scenario" + set +e + execute_transaction /fixture/input-webhook /fixture/contract >"$fixture_dir/transaction.output" + rc=$? + set -e + output="$(<"$fixture_dir/transaction.output")" + [[ "$SWR_TRANSACTION_STATE" == "$expected_state" ]] || + fail "$scenario terminal state differs: expected=$expected_state actual=$SWR_TRANSACTION_STATE log=$(/usr/bin/tr '\n' ' ' <"$transaction_log")" + assert_eq "$expected_output" "$output" "$scenario output differs" + if [[ ( "$expected_state" == NOOP || "$expected_state" == VERIFIED_COMMIT ) && + -z "$expected_rc" ]]; then + (( rc == 0 )) || fail "$scenario unexpectedly failed" + else + (( rc != 0 )) || fail "$scenario unexpectedly succeeded" + [[ "$output" != *'SLACK_KEEPASS_RECOVERY=NOOP'* && + "$output" != *'SLACK_KEEPASS_RECOVERY=COMMITTED'* ]] || + fail "$scenario emitted a success token" + fi + [[ -z "$expected_rc" || "$rc" == "$expected_rc" ]] || + fail "$scenario signal status differs: expected=$expected_rc actual=$rc" + (( transaction_commit_calls <= 1 )) || fail "$scenario retried commit" + pass_transaction_case +} + +signal_boundary_case() { + local event="$1" signal_name="${2-TERM}" expected_rc=143 output='' rc=0 later_effects + [[ "$signal_name" == HUP ]] && expected_rc=129 + [[ "$signal_name" == INT ]] && expected_rc=130 + reset_transaction_fake + transaction_scenario="signal-${signal_name}-${event}" + set +e + execute_transaction /fixture/input-webhook /fixture/contract >"$fixture_dir/transaction.output" + rc=$? + set -e + output="$(<"$fixture_dir/transaction.output")" + assert_eq "$expected_rc" "$rc" "signal after $event status differs" + assert_eq '' "$output" "signal after $event emitted output" + assert_eq PRECOMMIT_FAILURE "$SWR_TRANSACTION_STATE" \ + "signal after $event rewrote precommit state" + later_effects="$(/usr/bin/sed -n "0,/^${event}$/d; p" "$transaction_log" | + /usr/bin/grep -Ev '^(unmount-ro|unmount-rw|stop-vault|audit-processes|remove-work-root)$' || true)" + assert_eq '' "$later_effects" "signal after $event allowed a later side effect" +} + +late_post_cleanup_signal_case() { + local output='' rc=0 old_trap_marker="$fixture_dir/old-term-trap" + reset_transaction_fake + transaction_scenario=absent + : >"$old_trap_marker" + builtin trap 'builtin printf old >"$old_trap_marker"' TERM + set -T + builtin trap 'if [[ "$BASH_COMMAND" == *"final_rc == 0 && SWR_PENDING_SIGNAL == 0"* ]]; then builtin trap - DEBUG; kill -TERM "$BASHPID"; fi' DEBUG + set +e + execute_transaction /fixture/input-webhook /fixture/contract >"$fixture_dir/transaction.output" + rc=$? + set -e + builtin trap - DEBUG + set +T + builtin trap - TERM + output="$(<"$fixture_dir/transaction.output")" + assert_eq 143 "$rc" 'post-cleanup/pre-output TERM status differs' + assert_eq '' "$output" 'post-cleanup/pre-output TERM emitted output' + assert_eq VERIFIED_COMMIT "$SWR_TRANSACTION_STATE" \ + 'post-cleanup/pre-output TERM lost verified commit evidence' + assert_eq '' "$(<"$old_trap_marker")" \ + 'post-cleanup/pre-output TERM reached the restored caller trap' +} + +test_transaction_state_machine() { + declare -F execute_transaction >/dev/null || { + plain_fail 'full Slack KeePass transaction state machine is unavailable' + return 1 + } + declare -F cleanup_transaction >/dev/null || fail 'cleanup_transaction is unavailable' + declare -F defer_signal >/dev/null || fail 'defer_signal is unavailable' + + _swr_dirfd_helper() { + if [[ "$1" == commit ]]; then + return 75 + fi + if [[ "$1" == status ]]; then + builtin printf '%s\n' verified-commit + return 0 + fi + return 97 + } + local _swr_durable_status='' + _swr_command() { command "$@"; } + _swr_dirfd_commit _swr_durable_status /fixture/mount /fixture/work backup.kdbx || + fail 'durable verified commit was overridden by lost helper exit status' + _swr_command() { _swr_fake_command "$@"; } + assert_eq verified-commit "$_swr_durable_status" 'durable verified status differs' + pass_transaction_case + + _swr_prepare_transaction() { _swr_fake_prepare_transaction "$@"; } + _swr_start_vault() { _swr_fake_start_transaction_vault "$@"; } + _swr_refresh_sudo() { _swr_fake_refresh_sudo "$@"; } + _swr_probe_rw() { _swr_fake_probe_transaction_rw "$@"; } + _swr_mount_volume_command() { _swr_fake_mount_transaction_volume "$@"; } + _swr_validate_mount() { _swr_fake_validate_transaction_mount "$@"; } + _swr_dirfd_snapshot() { _swr_fake_dirfd_snapshot "$@"; } + _swr_unmount_volume() { _swr_fake_unmount_transaction_volume "$@"; } + _swr_stably_unmounted() { _swr_fake_transaction_stably_unmounted "$@"; } + _swr_vault_request() { _swr_fake_vault_transaction_request "$@"; } + _swr_confirm_token() { _swr_fake_confirm_transaction "$@"; } + _swr_dirfd_commit() { _swr_fake_dirfd_commit "$@"; } + _swr_dirfd_snapshot_current() { _swr_fake_dirfd_snapshot_current "$@"; } + _swr_stop_vault() { _swr_fake_stop_transaction_vault "$@"; } + _swr_audit_runtime_processes() { _swr_fake_audit_runtime_processes "$@"; } + _swr_remove_work_root() { _swr_fake_remove_transaction_work_root "$@"; } + + local initial_order mutation_tail + initial_order=$'validate-runtime\nstart-vault-and-retain-inputs\nsudo-refresh\nprobe-ro-safety\nmount-ro\nvalidate-ro-mount\nsnapshot\nunmount-ro\nclassify' + + run_transaction_case noop NOOP $'SLACK_KEEPASS_RECOVERY=NOOP\nWINDOWS_SSD_UNMOUNTED=PASS\nOFF_HOST_RECOVERY_SATISFIED=NO' + assert_transaction_prefix "$initial_order" 'initial no-op order differs' + + # Production break caught: a known RO mount-command failure with stable + # source absence is precommit failure, while response loss with a retained + # or ambiguous source still requires manual recovery. Reconcile only once. + run_transaction_case mount-ro-failure PRECOMMIT_FAILURE '' + assert_eq \ + $'validate-runtime\nstart-vault-and-retain-inputs\nsudo-refresh\nprobe-ro-safety\nmount-ro\nunmount-ro\nstop-vault\naudit-processes\nremove-work-root' \ + "$(<"$transaction_log")" 'known RO mount failure cleanup trace differs' + assert_eq 1 "$(/usr/bin/grep -Fc unmount-ro "$transaction_log")" \ + 'known RO mount failure reconciliation count differs' + ! /usr/bin/grep -Eq '^(snapshot|classify|probe-rw-safety|mount-rw|commit)$' "$transaction_log" || + fail 'known RO mount failure reached a later transaction effect' + + run_transaction_case mount-ro-response-loss-unresolved-source MANUAL_RECOVERY_REQUIRED \ + 'SLACK_KEEPASS_RECOVERY=MANUAL_RECOVERY_REQUIRED' + assert_eq 1 "$(/usr/bin/grep -Fc unmount-ro "$transaction_log")" \ + 'RO mount response-loss reconciliation count differs' + ! /usr/bin/grep -Eq '^(snapshot|classify|probe-rw-safety|mount-rw|commit)$' "$transaction_log" || + fail 'RO mount response loss reached a later transaction effect' + + local metadata_scenario + for metadata_scenario in \ + validate-ro-wrong-uid validate-ro-wrong-gid validate-ro-wrong-mode; do + run_transaction_case "$metadata_scenario" PRECOMMIT_FAILURE '' + assert_eq 1 "$(/usr/bin/grep -Fc unmount-ro "$transaction_log")" \ + "$metadata_scenario reconciliation count differs" + ! /usr/bin/grep -Eq '^(snapshot|classify|probe-rw-safety|mount-rw|commit)$' "$transaction_log" || + fail "$metadata_scenario reached a later transaction effect" + done + + run_transaction_case absent VERIFIED_COMMIT $'SLACK_KEEPASS_RECOVERY=COMMITTED\nKDBX_PRECHANGE_BACKUP=CREATED\nWINDOWS_SSD_UNMOUNTED=PASS\nOFF_HOST_RECOVERY_SATISFIED=NO' + mutation_tail="$(/usr/bin/sed -n '/^classify$/,$p' "$transaction_log")" + [[ "$mutation_tail" == $'classify\nbuild-add\nverify-candidate\nconfirm-write\nprobe-rw-safety\nmount-rw\nvalidate-rw-mount\ncommit\nsnapshot-current\nverify-committed-copy\nunmount-rw\nstop-vault\naudit-processes\nremove-work-root' ]] || + fail "absent mutation order differs: ${mutation_tail//$'\n'/ -> }" + + run_transaction_case mismatch-no-update PRECOMMIT_FAILURE '' + run_transaction_case mismatch-update VERIFIED_COMMIT $'SLACK_KEEPASS_RECOVERY=COMMITTED\nKDBX_PRECHANGE_BACKUP=CREATED\nWINDOWS_SSD_UNMOUNTED=PASS\nOFF_HOST_RECOVERY_SATISFIED=NO' + mutation_tail="$(/usr/bin/sed -n '/^classify$/,$p' "$transaction_log")" + [[ "$mutation_tail" == $'classify\nconfirm-update\nbuild-update\nverify-candidate\nconfirm-write\nprobe-rw-safety\nmount-rw\nvalidate-rw-mount\ncommit\nsnapshot-current\nverify-committed-copy\nunmount-rw\nstop-vault\naudit-processes\nremove-work-root' ]] || + fail "mismatch mutation order differs: ${mutation_tail//$'\n'/ -> }" + + local scenario state expected + for scenario in duplicate wrong-master retained-input-drift original-drift backup-collision backup-fsync; do + run_transaction_case "$scenario" PRECOMMIT_FAILURE '' + done + run_transaction_case hup-before PRECOMMIT_FAILURE '' 129 + run_transaction_case int-before PRECOMMIT_FAILURE '' 130 + run_transaction_case term-before PRECOMMIT_FAILURE '' 143 + run_transaction_case hup-noop PRECOMMIT_FAILURE '' 129 + run_transaction_case term-rw-probe PRECOMMIT_FAILURE '' 143 + assert_eq 0 "$transaction_commit_calls" 'pending TERM after RW probe reached commit' + run_transaction_case int-rw-validate PRECOMMIT_FAILURE '' 130 + assert_eq 0 "$transaction_commit_calls" 'pending INT after RW validation reached commit' + run_transaction_case lost-after-verified VERIFIED_COMMIT $'SLACK_KEEPASS_RECOVERY=COMMITTED\nKDBX_PRECHANGE_BACKUP=CREATED\nWINDOWS_SSD_UNMOUNTED=PASS\nOFF_HOST_RECOVERY_SATISFIED=NO' + for scenario in rename-failure post-rename final-verification lost-helper; do + run_transaction_case "$scenario" COMMITTED_BUT_UNCERTAIN 'SLACK_KEEPASS_RECOVERY=MANUAL_RECOVERY_REQUIRED' + done + run_transaction_case hup-during COMMITTED_BUT_UNCERTAIN 'SLACK_KEEPASS_RECOVERY=MANUAL_RECOVERY_REQUIRED' 129 + run_transaction_case int-during COMMITTED_BUT_UNCERTAIN 'SLACK_KEEPASS_RECOVERY=MANUAL_RECOVERY_REQUIRED' 130 + run_transaction_case term-during COMMITTED_BUT_UNCERTAIN 'SLACK_KEEPASS_RECOVERY=MANUAL_RECOVERY_REQUIRED' 143 + run_transaction_case hup-final-unmount VERIFIED_COMMIT '' 129 + run_transaction_case term-cleanup VERIFIED_COMMIT '' 143 + run_transaction_case term-post-cleanup VERIFIED_COMMIT '' 143 + late_post_cleanup_signal_case + local signal_event signal_failures=0 + for signal_event in \ + validate-runtime start-vault-and-retain-inputs sudo-refresh probe-ro-safety \ + mount-ro validate-ro-mount snapshot unmount-ro classify build-add \ + verify-candidate confirm-write probe-rw-safety mount-rw validate-rw-mount; do + ( signal_boundary_case "$signal_event" TERM ) || { + builtin printf 'SIGNAL RED: %s\n' "$signal_event" >&2 + signal_failures=$((signal_failures + 1)) + } + done + ( signal_boundary_case confirm-update INT ) || { + builtin printf 'SIGNAL RED: %s\n' confirm-update >&2 + signal_failures=$((signal_failures + 1)) + } + (( signal_failures == 0 )) || fail "$signal_failures signal boundaries failed" + for scenario in unmount-ro unmount-rw unmount-ambiguous; do + run_transaction_case "$scenario" MANUAL_RECOVERY_REQUIRED 'SLACK_KEEPASS_RECOVERY=MANUAL_RECOVERY_REQUIRED' + local unmount_phase="${scenario#unmount-}" + [[ "$unmount_phase" == ambiguous ]] && unmount_phase=ro + assert_eq 1 "$(/usr/bin/grep -Fc "unmount-${unmount_phase}" "$transaction_log")" \ + "$scenario retried an ambiguous unmount" + done +} + +run_worker() { + local fixture_dir="$1" worker_token="$2" + shift 2 + /usr/bin/timeout --signal=TERM --kill-after=5s 30s \ + /usr/bin/bash "$0" --worker "$worker_token" "$fixture_dir" "$@" +} + +audit_fixture_pids() { + local fixture_dir="$1" pid_file pid orphan_count=0 + shopt -s nullglob + for pid_file in "$fixture_dir"/pid.*; do + pid="${pid_file##*.}" + if kill -0 "$pid" 2>/dev/null; then + orphan_count=$((orphan_count + 1)) + fi + done + shopt -u nullglob + printf '%s\n' "$orphan_count" +} + +create_synthetic_kdbx() { + local fixture_dir="$1" database socket_path server_pid attempt + local fixture_master="fixture-$RANDOM-$RANDOM-$$" + local fixture_webhook='https://hooks.' + fixture_webhook+='slack.com/services/T01234567/B01234567/XXXXXXXXXXXXXXXXXXXXXXXX' + runtime_fixture="$(mktemp -d "/run/user/${UID}/slack-webhook-recovery-bash.XXXXXX")" + chmod 0700 -- "$runtime_fixture" + database="$runtime_fixture/baseline.kdbx" + socket_path="$runtime_fixture/vault.sock" + { + builtin printf '%s\n' "$fixture_master" + builtin printf '%s\n' "$fixture_master" + } | HOME=/dev/null XDG_CONFIG_HOME=/dev/null LC_ALL=C.UTF-8 PATH=/usr/bin:/bin \ + /usr/bin/keepassxc-cli db-create -q -p "$database" \ + >"$fixture_dir/db-create.stdout" 2>"$fixture_dir/db-create.stderr" || + fail 'synthetic KeePass database creation failed' + chmod 0600 -- "$database" + builtin printf '%s' "$fixture_webhook" >"$runtime_fixture/input-webhook" + chmod 0600 -- "$runtime_fixture/input-webhook" + if grep -Fq -- "$fixture_master" \ + "$fixture_dir/db-create.stdout" "$fixture_dir/db-create.stderr"; then + fail 'synthetic master appeared in captured KeePassXC output' + fi + + { + builtin printf '%s\n' 'Synthetic Slack App' + builtin printf '%s\n' "$fixture_master" + } | /usr/bin/python3 -c ' +import importlib.util +import os +import sys +spec = importlib.util.spec_from_file_location("swr_vault", sys.argv[1]) +if spec is None or spec.loader is None: + raise SystemExit(2) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +output_fd = os.open(os.devnull, os.O_WRONLY | os.O_CLOEXEC) +try: + module.serve_private_socket( + sys.argv[2], sys.argv[3], input_fd=0, output_fd=output_fd, + accept_timeout=10.0, io_timeout=10.0, child_timeout=20.0, + ) +finally: + os.close(output_fd) +' "$ROOT/scripts/libexec/slack-webhook-recovery-vault.py" \ + "$runtime_fixture" "$runtime_fixture/input-webhook" \ + >"$fixture_dir/vault.stdout" 2>"$fixture_dir/vault.stderr" & + server_pid=$! + builtin printf '' >"$fixture_dir/pid.$server_pid" + for (( attempt = 0; attempt < 100; attempt++ )); do + [[ -S "$socket_path" ]] && break + kill -0 "$server_pid" 2>/dev/null || fail 'imported vault server exited early' + /usr/bin/sleep 0.05 + done + [[ -S "$socket_path" ]] || fail 'imported vault socket did not appear' + [[ "$(stat -c '%a' "$socket_path")" == 600 ]] || fail 'vault socket mode differs' + + if grep -Fq -- "$fixture_master" "/proc/$server_pid/cmdline" \ + "/proc/$server_pid/environ" "$fixture_dir/vault.stdout" \ + "$fixture_dir/vault.stderr"; then + fail 'synthetic master escaped the vault process boundary' + fi + if grep -Fq -- "$fixture_webhook" "/proc/$server_pid/cmdline" \ + "/proc/$server_pid/environ" "$fixture_dir/vault.stdout" \ + "$fixture_dir/vault.stderr"; then + fail 'synthetic webhook escaped the vault process boundary' + fi + + local _swr_reply='' + _swr_vault_request _swr_reply "$socket_path" PREPARE || + fail 'vault PREPARE request failed' + assert_eq absent "$_swr_reply" 'absent database classification differs' + _swr_vault_request _swr_reply "$socket_path" 'BUILD ADD' || + fail 'vault BUILD ADD request failed' + assert_eq candidate-ready "$_swr_reply" 'candidate response differs' + _swr_vault_request _swr_reply "$socket_path" VERIFY_CANDIDATE || + fail 'vault VERIFY_CANDIDATE request failed' + assert_eq verified "$_swr_reply" 'candidate verification response differs' + _swr_vault_request _swr_reply "$socket_path" \ + "VERIFY_COMMITTED $runtime_fixture/candidate.kdbx" || + fail 'vault VERIFY_COMMITTED request failed' + assert_eq verified "$_swr_reply" 'candidate verification response differs' + _swr_stop_vault "$socket_path" "$server_pid" || fail 'vault shutdown failed' + [[ ! -S "$socket_path" ]] || fail 'vault socket remained after shutdown' + unset fixture_master + unset fixture_webhook +} + +cleanup_runtime_fixture() { + if [[ "$runtime_fixture" == "/run/user/${UID}/slack-webhook-recovery-bash."* ]]; then + rm -rf -- "$runtime_fixture" + fi + if [[ "$owned_work_fixture" == /run/user/1000/platform-slack-keepass.* && + -d "$owned_work_fixture" && ! -L "$owned_work_fixture" && -O "$owned_work_fixture" ]]; then + /usr/bin/rmdir -- "$owned_work_fixture" 2>/dev/null || true + fi +} + +cleanup_fixture() { + cleanup_runtime_fixture + [[ "$fixture_dir" == /tmp/slack-webhook-recovery-test.* || "$fixture_dir" == "${TMPDIR:-/tmp}"/slack-webhook-recovery-test.* ]] || return 0 + rm -rf -- "$fixture_dir" +} + +worker_main() { + local worker_token="$1" fixture_dir="$2" + trap cleanup_runtime_fixture EXIT + shift 2 + local expected_token_file="$fixture_dir/worker-token" + [[ -f "$expected_token_file" ]] || fail 'worker token file is absent' + [[ "$( <"$expected_token_file" )" == "$worker_token" ]] || fail 'worker token mismatch' + [[ $# -eq 0 ]] || fail 'focused suite worker accepts no public argv' + + # The production change this catches is widening the closed CLI boundary or + # contacting any external secret/mount path on the default invocation. + [[ -x "$PROD" ]] || fail 'production Slack KeePass recovery helper is absent' + + local output relative_output + output="$(/usr/bin/env -i PATH=/definitely-invalid /usr/bin/bash "$PROD")" || + fail 'default invocation failed' + assert_eq \ + $'SLACK_KEEPASS_RECOVERY_MODE=DRY_RUN\nWINDOWS_SSD_UUID=4EA0196C0C5FA27E\nKDBX_RELATIVE_PATH=HyeonworksRecovery/vault/hyeonworks-recovery.kdbx\nMUTATION=NOT_REQUESTED\nOFF_HOST_RECOVERY_SATISFIED=NO' \ + "$output" \ + 'default output differs' + + relative_output="$( + cd -- "$ROOT" && + /usr/bin/env -i PATH=/definitely-invalid /usr/bin/bash \ + scripts/bootstrap/backup-slack-webhook-recovery.sh + )" || fail 'repository-relative default invocation failed' + assert_eq "$output" "$relative_output" 'repository-relative default output differs' + + /usr/bin/env -i PATH=/usr/bin:/bin /usr/bin/bash "$PROD" --unknown >/dev/null 2>&1 && + fail 'unknown argv was accepted' + /usr/bin/env -i PATH=/usr/bin:/bin /usr/bin/bash "$PROD" --execute --execute >/dev/null 2>&1 && + fail 'duplicate execute argv was accepted' + /usr/bin/env -i PATH=/usr/bin:/bin /usr/bin/bash "$PROD" --execute --slack-webhook-file >/dev/null 2>&1 && + fail 'missing webhook path was accepted' + /usr/bin/env -i PATH=/usr/bin:/bin /usr/bin/bash "$PROD" --execute --slack-webhook-file relative/path >/dev/null 2>&1 && + fail 'relative webhook path was accepted' + /usr/bin/env -i PATH=/usr/bin:/bin /usr/bin/bash "$PROD" --slack-webhook-file /tmp/placeholder --execute >/dev/null 2>&1 && + fail 'reordered argv was accepted' + /usr/bin/env -i PATH=/usr/bin:/bin CDPATH=/tmp /usr/bin/bash "$PROD" >/dev/null 2>&1 && + fail 'CDPATH override was accepted' + /usr/bin/env -i PATH=/usr/bin:/bin BASH_ENV=/dev/null /usr/bin/bash "$PROD" >/dev/null 2>&1 && + fail 'BASH_ENV override was accepted' + /usr/bin/env -i PATH=/usr/bin:/bin ENV=/dev/null /usr/bin/bash "$PROD" >/dev/null 2>&1 && + fail 'ENV override was accepted' + /usr/bin/env -i PATH=/usr/bin:/bin 'BASH_FUNC_printf%%=() { :; }' /usr/bin/bash "$PROD" >/dev/null 2>&1 && + fail 'imported command override was accepted' + /usr/bin/env -i PATH=/usr/bin:/bin SHELLOPTS=braceexpand:hashall /usr/bin/bash "$PROD" >/dev/null 2>&1 && + fail 'inherited SHELLOPTS was accepted' + /usr/bin/env -i PATH=/usr/bin:/bin /usr/bin/bash "$PROD" --execute --slack-webhook-file /tmp/placeholder >/dev/null 2>&1 && + fail 'execute mode without tty was accepted' + + # shellcheck source=/dev/null + SWR_REPOSITORY_ROOT="$ROOT" + source "$LIB" + production_runtime_root_definition="$(declare -f _swr_runtime_root)" + _swr_command() { _swr_fake_command "$@"; } + _swr_runtime_root() { _swr_fake_runtime_root "$@"; } + _swr_volume_action() { _swr_fake_volume_action "$@"; } + local required_boundary + for required_boundary in \ + _swr_load_recovery_contract _swr_resolve_device _swr_require_unmounted \ + _swr_probe_rw _swr_mount_volume _swr_validate_mount _swr_unmount_volume \ + _swr_create_mountpoint _swr_require_mountpoint_unused \ + _swr_run_volume_cycle _swr_open_recovery_boundary; do + declare -F "$required_boundary" >/dev/null || + fail 'exact Windows SSD mount boundary is unavailable' + done + _swr_initial_guard 0 '' '' '' "$PROD" && + fail 'root EUID guard was accepted' + + fake_runtime_root="$fixture_dir/runtime-root" + fake_log="$fixture_dir/fake-command.log" + fake_mount_state="$fixture_dir/fake-mount-state" + fake_probe_status="$fixture_dir/fake-probe-status" + fake_mount_status="$fixture_dir/fake-mount-status" + fake_umount_status="$fixture_dir/fake-umount-status" + fake_umount_after="$fixture_dir/fake-umount-after" + fake_sudo_status="$fixture_dir/fake-sudo-status" + fake_source_query_count="$fixture_dir/fake-source-query-count" + fake_action_count="$fixture_dir/fake-action-count" + fake_mount_uncertain="$fixture_dir/fake-mount-uncertain" + fake_case_file="$fixture_dir/fake-case" + fake_mountpoint="$fake_runtime_root/not-created" + fake_action=noop + mkdir -m 0700 -- "$fake_runtime_root" + + _swr_command /fixture/fake-only-sentinel || + fail 'fixed command leaf was not intercepted after production source' + assert_log_line '/fixture/fake-only-sentinel' 'fake-only command sentinel was not logged' + pass_boundary_case + + owned_work_fixture="$(/usr/bin/mktemp -d --tmpdir=/run/user/1000 platform-slack-keepass.XXXXXXXX)" + /usr/bin/chmod 0700 -- "$owned_work_fixture" + _swr_work_root_is_safe "$owned_work_fixture" || + fail 'owned private tmpfs work root was rejected' + /usr/bin/rmdir -- "$owned_work_fixture" + owned_work_fixture='' + pass_transaction_case + + test_contract_and_device_boundary + test_probe_mount_and_validation_boundary + test_unmount_and_cycle_boundary + test_full_fake_boundary_ordering + test_review_fix_round_one + test_review_fix_round_two + test_task5_fix_round_one_boundaries + test_task5_fix_round_four_inventory_boundaries + test_vault_readiness_failure_boundaries + test_pending_signal_after_readiness_prevents_publication + test_start_vault_waits_for_delayed_operator_input + + create_synthetic_kdbx "$fixture_dir" + declare -F _swr_start_vault >/dev/null || fail 'KeePass database classifier is unavailable' + transaction_log="$fixture_dir/transaction.log" + test_transaction_state_machine + builtin printf 'PASS: %s exact device/probe/mount boundary cases; %s transaction cases; fake-only sentinel=pass; real privileged calls=0\n' \ + "$boundary_cases" "$transaction_cases" +} + +supervisor_main() { + (( $# == 0 )) || fail 'focused suite accepts no public argv' + + fixture_dir="$(mktemp -d "${TMPDIR:-/tmp}/slack-webhook-recovery-test.XXXXXX")" + trap cleanup_fixture EXIT + + local worker_token + printf -v worker_token 'worker-%(%s)T-%s-%s' -1 "$$" "$RANDOM" + printf '%s' "$worker_token" >"$fixture_dir/worker-token" + + local stdout_file="$fixture_dir/stdout" stderr_file="$fixture_dir/stderr" + local started_at finished_at rc orphan_count + printf -v started_at '%(%s)T' -1 + set +e + run_worker "$fixture_dir" "$worker_token" >"$stdout_file" 2>"$stderr_file" + rc=$? + set -e + printf -v finished_at '%(%s)T' -1 + + orphan_count="$(audit_fixture_pids "$fixture_dir")" + if [[ "$orphan_count" != 0 ]]; then + cat "$stderr_file" >&2 + fail 'worker left fixture-owned processes behind' + fi + (( finished_at - started_at <= 30 )) || fail 'worker exceeded supervisor wall bound' + + cat "$stdout_file" + if (( rc == 0 )); then + cleanup_fixture + trap - EXIT + return 0 + fi + cat "$stderr_file" >&2 + cleanup_fixture + trap - EXIT + return "$rc" +} + +if [[ "${1-}" == --worker ]]; then + [[ "${SWR_TEST_WORKER_TOKEN:-}" == '' ]] || fail 'worker token must not arrive from environment' + (( $# >= 3 )) || fail 'worker dispatch is incomplete' + worker_main "$2" "$3" "${@:4}" + exit 0 +fi + +supervisor_main "$@" diff --git a/scripts/validate/test-capture-observability-metric-inventory.sh b/scripts/validate/test-capture-observability-metric-inventory.sh new file mode 100644 index 0000000..2108d4d --- /dev/null +++ b/scripts/validate/test-capture-observability-metric-inventory.sh @@ -0,0 +1,751 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly CAPTURE="$ROOT/scripts/validate/capture-observability-metric-inventory.sh" +readonly REJECT_STATUS=23 +readonly ORIGINAL_PATH="$PATH" + +fail() { + printf 'TEST FAILURE: %s\n' "$*" >&2 + exit 1 +} + +expect_pass() { + local name="$1" output + shift + if ! output="$({ "$@"; } 2>&1)"; then + fail "$name (expected success): $output" + fi + pass_count=$((pass_count + 1)) +} + +expect_reject() { + local name="$1" output rc=0 + shift + output="$({ "$@"; } 2>&1)" || rc=$? + [[ "$rc" == "$REJECT_STATUS" ]] || fail "$name (expected $REJECT_STATUS, got $rc): $output" + [[ "$output" == REJECT:* ]] || fail "$name (unsanitized rejection): $output" + [[ "$output" != *DO-NOT-LEAK-FIXTURE* ]] || fail "$name leaked a fixture value" + pass_count=$((pass_count + 1)) +} + +[[ -f "$CAPTURE" ]] || fail "EXPECTED RED: capture command is missing: $CAPTURE" + +# shellcheck source=/dev/null +source "$CAPTURE" + +fixture_root="$(mktemp -d /tmp/platform-observability-inventory-test.XXXXXX)" +test_helper_pids=() +cleanup() { + local rc=$? + local helper_pid + for helper_pid in "${test_helper_pids[@]}"; do + if [[ "$helper_pid" =~ ^[0-9]+$ ]]; then kill -KILL "$helper_pid" 2>/dev/null || true; fi + done + if [[ "${KEEP_INVENTORY_TEST_FIXTURE:-0}" == 1 ]]; then + printf 'KEPT FIXTURE: %s\n' "$fixture_root" >&2 + exit "$rc" + fi + case "$fixture_root" in + /tmp/platform-observability-inventory-test.*) rm -rf -- "$fixture_root" ;; + *) printf 'TEST FAILURE: refusing unsafe fixture cleanup\n' >&2; rc=1 ;; + esac + exit "$rc" +} +trap cleanup EXIT + +pass_count=0 + +mapfile -t exact_pools <<'POOLS' +podMonitor/platform-data/platform-postgres/0 +serviceMonitor/gitea/gitea/0 +serviceMonitor/keycloak/keycloak/0 +serviceMonitor/kube-system/traefik/0 +serviceMonitor/object-storage/aistor-bucket-usage/0 +serviceMonitor/observability-agent/alloy/0 +serviceMonitor/observability-agent/node-exporter/0 +serviceMonitor/observability/grafana/0 +serviceMonitor/observability/loki/0 +serviceMonitor/observability/observability-core-kube-pr-alertmanager/0 +serviceMonitor/observability/observability-core-kube-pr-alertmanager/1 +serviceMonitor/observability/observability-core-kube-pr-apiserver/0 +serviceMonitor/observability/observability-core-kube-pr-coredns/0 +serviceMonitor/observability/observability-core-kube-pr-kubelet/0 +serviceMonitor/observability/observability-core-kube-pr-kubelet/1 +serviceMonitor/observability/observability-core-kube-pr-kubelet/2 +serviceMonitor/observability/observability-core-kube-pr-operator/0 +serviceMonitor/observability/observability-core-kube-pr-prometheus/0 +serviceMonitor/observability/observability-core-kube-pr-prometheus/1 +serviceMonitor/observability/observability-core-kube-state-metrics/0 +serviceMonitor/observability/tempo/0 +POOLS + +expected_target_selectors="$fixture_root/expected-target-selectors" +expected_series_selectors="$fixture_root/expected-series-selectors" +: >"$expected_target_selectors" +: >"$expected_series_selectors" + +make_targets() { + local destination="$1" pool labels selector series_selector index=0 + local rows="$fixture_root/target-rows.jsonl" + : >"$rows" + for pool in "${exact_pools[@]}"; do + labels="$(jq -cn --arg instance "10.0.0.$((index + 1)):9090" --arg target "target-$index" \ + '{job:"fixture-job",instance:$instance,sensitive_fixture:"DO-NOT-LEAK-FIXTURE",target_fixture:$target}')" + selector="{instance=\"10.0.0.$((index + 1)):9090\",job=\"fixture-job\",sensitive_fixture=\"DO-NOT-LEAK-FIXTURE\",target_fixture=\"target-$index\"}" + if [[ "$pool" == serviceMonitor/object-storage/aistor-bucket-usage/0 ]]; then + labels="$(jq -cn --arg target "target-$index" \ + '{job:"minio-aistor",namespace:"object-storage",service:"minio",endpoint:"http-minio", + container:"minio",instance:"10.42.7.19:9000",pod:"minio-aistor-pool-0-0", + sensitive_fixture:"DO-NOT-LEAK-FIXTURE",target_fixture:$target}')" + selector="{container=\"minio\",endpoint=\"http-minio\",instance=\"10.42.7.19:9000\",job=\"minio-aistor\",namespace=\"object-storage\",pod=\"minio-aistor-pool-0-0\",sensitive_fixture=\"DO-NOT-LEAK-FIXTURE\",service=\"minio\",target_fixture=\"target-$index\"}" + fi + printf '%s\n' "$selector" >>"$expected_target_selectors" + series_selector="{__name__=\"fixture_metric\",${selector#\{}" + printf '%s\n' "$series_selector" >>"$expected_series_selectors" + jq -cn \ + --arg pool "$pool" \ + --arg url "http://10.42.0.$((index + 1)):9090/metrics" \ + --argjson labels "$labels" \ + '{discoveredLabels:$labels,labels:$labels,scrapePool:$pool,scrapeUrl:$url, + globalUrl:$url,lastError:"",lastScrape:"2026-08-12T00:00:00Z", + lastScrapeDuration:0.01,health:"up",scrapeInterval:"30s",scrapeTimeout:"10s"}' \ + >>"$rows" + index=$((index + 1)) + done + jq -cs '{status:"success",data:{activeTargets:.,droppedTargets:[]}}' "$rows" >"$destination" +} + +valid_targets="$fixture_root/targets.json" +make_targets "$valid_targets" +oversized_padding="$fixture_root/oversized-padding" +head -c 8388609 /dev/zero | tr '\000' x >"$oversized_padding" +oversized_targets="$fixture_root/oversized-targets.json" +jq --rawfile padding "$oversized_padding" '. + {padding:$padding}' "$valid_targets" >"$oversized_targets" +expect_pass 'exact 21 active pools accepted' validate_targets_json "$valid_targets" + +post_targets="$fixture_root/post-targets.json" +post_probe_rows="$fixture_root/post-probe-rows.json" +post_substrate_expected_probe_targets | jq '[.[] as $target | $target + { + discoveredLabels:$target.labels,globalUrl:$target.scrapeUrl,lastError:"", + lastScrape:"2026-08-12T00:00:00Z",lastScrapeDuration:0.01, + health:"up",scrapeInterval:"30s",scrapeTimeout:"15s"}]' >"$post_probe_rows" +jq --slurpfile probes "$post_probe_rows" ' + .data.activeTargets += ($probes[0] + [{ + discoveredLabels:{}, + labels:{job:"blackbox-exporter",namespace:"observability",service:"blackbox-exporter", + endpoint:"http",container:"blackbox-exporter",pod:"blackbox-exporter-6d8444ffdb-svsck", + instance:"10.42.0.123:9115"}, + scrapePool:"serviceMonitor/observability/blackbox-exporter/0", + scrapeUrl:"http://10.42.0.123:9115/metrics",globalUrl:"http://10.42.0.123:9115/metrics", + lastError:"",lastScrape:"2026-08-12T00:00:00Z",lastScrapeDuration:0.01, + health:"up",scrapeInterval:"30s",scrapeTimeout:"10s" + }]) +' "$valid_targets" >"$post_targets" +expect_pass 'exact post-substrate 30 targets and multiplicities accepted' \ + validate_targets_json "$post_targets" post-substrate +expect_reject 'post-substrate targets rejected by target-initial phase' \ + validate_targets_json "$post_targets" target-initial +post_missing_probe="$fixture_root/post-missing-probe.json" +jq 'del(.data.activeTargets[] | select(.scrapePool=="probe/observability/platform-public-edge"))' \ + "$post_targets" >"$post_missing_probe" +expect_reject 'post-substrate missing Probe target rejected' \ + validate_targets_json "$post_missing_probe" post-substrate +expect_pass 'Probe query is removed from persisted scrape URL' \ + test "$(sanitize_scrape_url 'http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fexample.test' strip-query)" = \ + 'http://blackbox-exporter.observability.svc.cluster.local:9115/probe' +expect_pass 'UTF-8 Prometheus label key is quoted in selector' \ + test "$(prometheus_selector_for_labels '{"job":"blackbox","observability.hyeonworks.com/probe-group":"public-edge"}')" = \ + '{job="blackbox","observability.hyeonworks.com/probe-group"="public-edge"}' + +mutate_and_reject() { + local name="$1" filter="$2" destination + destination="$fixture_root/${name// /-}.json" + jq "$filter" "$valid_targets" >"$destination" + expect_reject "$name" validate_targets_json "$destination" +} + +mutate_and_reject 'missing pool rejected' 'del(.data.activeTargets[0])' +mutate_and_reject 'extra pool rejected' '.data.activeTargets += [.data.activeTargets[0] | .scrapePool="serviceMonitor/extra/extra/0"]' +mutate_and_reject 'duplicate pool rejected' '.data.activeTargets += [.data.activeTargets[0]]' +mutate_and_reject 'down target rejected' '.data.activeTargets[0].health="down"' +mutate_and_reject 'last error rejected' '.data.activeTargets[0].lastError="DO-NOT-LEAK-FIXTURE"' +mutate_and_reject 'blackbox pool rejected' '.data.activeTargets[0].scrapePool="serviceMonitor/observability/blackbox-exporter/0"' +mutate_and_reject 'Probe pool rejected' '.data.activeTargets[0].scrapePool="probe/observability/public-edge/0"' +dropped_blackbox="$fixture_root/dropped-blackbox.json" +jq '.data.droppedTargets = [{ + "discoveredLabels": { + "__meta_kubernetes_namespace": "observability", + "__meta_kubernetes_service_name": "blackbox-exporter", + "job": "blackbox-exporter" + }, + "labels": {"job": "blackbox-exporter"}, + "scrapePool": "serviceMonitor/observability/blackbox-exporter/0", + "scrapeUrl": "http://blackbox-exporter.observability.svc:9115/metrics", + "globalUrl": "http://blackbox-exporter.observability.svc:9115/metrics", + "lastError": "", + "health": "unknown" +}]' "$valid_targets" >"$dropped_blackbox" +expect_reject 'dropped blackbox identity rejected' validate_targets_json "$dropped_blackbox" +expect_pass 'post-substrate allows blackbox self discovery drops' \ + validate_dropped_targets_json "$dropped_blackbox" post-substrate +dropped_probe="$fixture_root/dropped-probe.json" +jq '.data.droppedTargets = [{ + "discoveredLabels": { + "__meta_kubernetes_namespace": "observability", + "__meta_kubernetes_service_name": "synthetic-exporter", + "monitor_kind": "Probe" + }, + "labels": {"job": "synthetic-checks"}, + "scrapePool": "probe/observability/public-edge/0", + "scrapeUrl": "http://synthetic-exporter.observability.svc:9115/metrics", + "globalUrl": "http://synthetic-exporter.observability.svc:9115/metrics", + "lastError": "", + "health": "unknown" +}]' "$valid_targets" >"$dropped_probe" +expect_reject 'dropped Probe identity rejected' validate_targets_json "$dropped_probe" +legitimate_probe_word="$fixture_root/legitimate-probe-word.json" +jq '.data.droppedTargets = [{ + "discoveredLabels": { + "__meta_kubernetes_pod_annotation_fixture": "{\"livenessProbe\":true}", + "__metrics_path__": "/metrics/probes" + }, + "labels": {"job": "platform-postgres"}, + "scrapePool": "podMonitor/platform-data/platform-postgres/0", + "scrapeUrl": "http://10.42.0.1:9187/metrics/probes", + "globalUrl": "http://10.42.0.1:9187/metrics/probes", + "lastError": "", + "health": "unknown" +}]' "$valid_targets" >"$legitimate_probe_word" +expect_pass 'legitimate dropped metadata containing probe words accepted in active response' \ + validate_targets_json "$legitimate_probe_word" +expect_pass 'legitimate dropped metadata containing probe words accepted in dropped response' \ + validate_dropped_targets_json "$legitimate_probe_word" +empty_dropped_targets="$fixture_root/empty-dropped-targets.json" +printf '%s\n' '{"status":"success","data":{"activeTargets":[],"droppedTargets":[]}}' >"$empty_dropped_targets" +mutate_and_reject 'AIStor wrong job rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.job)="wrong"' +mutate_and_reject 'AIStor wrong namespace rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.namespace)="wrong"' +mutate_and_reject 'AIStor wrong service rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.service)="wrong"' +mutate_and_reject 'AIStor wrong endpoint rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.endpoint)="wrong"' +mutate_and_reject 'AIStor wrong container rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.container)="wrong"' + +expect_pass 'safe HTTP URL accepted' test "$(sanitize_scrape_url 'http://10.42.7.19:9000/minio/metrics/v3/cluster/usage/buckets')" = 'http://10.42.7.19:9000/minio/metrics/v3/cluster/usage/buckets' +expect_pass 'safe HTTPS URL accepted' test "$(sanitize_scrape_url 'https://metrics.example.test/metrics')" = 'https://metrics.example.test/metrics' +for unsafe_url in \ + 'http://user:DO-NOT-LEAK-FIXTURE@host/metrics' \ + 'http://host/metrics?token=DO-NOT-LEAK-FIXTURE' \ + 'http://host/metrics#DO-NOT-LEAK-FIXTURE' \ + 'ftp://host/metrics' \ + 'http://host/space here'; do + expect_reject 'unsafe scrape URL rejected' sanitize_scrape_url "$unsafe_url" +done + +metadata="$fixture_root/metadata.json" +printf '%s\n' '{"status":"success","data":{"z_metric":[{"type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""}],"a_metric":[{"type":"counter","help":"ignored","unit":""}]}}' >"$metadata" +expect_pass 'metadata metric names sorted' test "$(extract_metric_names "$metadata")" = $'a_metric\nz_metric' +metadata_array="$fixture_root/metadata-array.json" +printf '%s\n' '{"status":"success","data":[{"target":{"job":"fixture"},"metric":"z_metric","type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""},{"target":{"job":"fixture"},"metric":"a_metric","type":"counter","help":"ignored","unit":""},{"target":{"job":"fixture"},"metric":"z_metric","type":"gauge","help":"duplicate","unit":""}]}' >"$metadata_array" +expect_pass 'current Prometheus metadata array yields unique sorted metric names' \ + test "$(extract_metric_names "$metadata_array")" = $'a_metric\nz_metric' +bad_metadata_array="$fixture_root/bad-metadata-array.json" +printf '%s\n' '{"status":"success","data":[{"target":{},"metric":"bad metric","type":"gauge","help":"ignored","unit":""}]}' >"$bad_metadata_array" +expect_reject 'invalid current metadata array metric name rejected' extract_metric_names "$bad_metadata_array" +expect_pass 'post-substrate Probe pool accepts probe metrics' \ + validate_metric_names_for_pool $'probe_duration_seconds\nprobe_success' \ + probe/observability/platform-public-edge post-substrate +expect_reject 'target-initial rejects probe metrics' \ + validate_metric_names_for_pool probe_success serviceMonitor/observability/grafana/0 target-initial +expect_reject 'post-substrate non-Probe pool rejects probe metrics' \ + validate_metric_names_for_pool probe_success serviceMonitor/observability/blackbox-exporter/0 post-substrate +expect_reject 'post-substrate Probe pool requires probe_success descriptor' \ + validate_metric_names_for_pool probe_duration_seconds probe/observability/platform-public-edge post-substrate +bad_metadata="$fixture_root/bad-metadata.json" +printf '%s\n' '{"status":"success","data":{"bad metric":[]}}' >"$bad_metadata" +expect_reject 'invalid metadata metric name rejected' extract_metric_names "$bad_metadata" + +rules="$fixture_root/rules.json" +printf '%s\n' '{"status":"success","data":{"groups":[{"name":"fixture","rules":[{"type":"recording","name":"z_metric","query":"DO-NOT-LEAK-FIXTURE"},{"type":"alerting","name":"a_metric"}]}]}}' >"$rules" +expect_reject 'recording-rule collision rejected' reject_recording_rule_collisions "$rules" "$metadata" +printf '%s\n' '{"status":"success","data":{"groups":[{"name":"fixture","rules":[{"type":"recording","name":"recorded_metric"}]}]}}' >"$rules" +expect_pass 'non-colliding recording rule accepted' reject_recording_rule_collisions "$rules" "$metadata" + +series="$fixture_root/series.json" +printf '%s\n' '{"status":"success","data":[{"z_label":"DO-NOT-LEAK-FIXTURE","__name__":"a_metric","a_label":"private-value"},{"__name__":"a_metric","middle":"another-private-value"}]}' >"$series" +expect_pass 'series parser emits only sorted label keys' test "$(extract_series_label_names "$series")" = '["__name__","a_label","middle","z_label"]' +[[ "$(extract_series_label_names "$series")" != *private-value* ]] || fail 'series parser persisted label values' +dotted_series="$fixture_root/dotted-series.json" +printf '%s\n' '{"status":"success","data":[{"__name__":"probe_success","instance":"https://example.test/","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}]}' >"$dotted_series" +expect_pass 'series parser preserves safe quoted Prometheus label names' \ + test "$(extract_series_label_names "$dotted_series")" = \ + '["__name__","instance","job","namespace","observability.hyeonworks.com/probe-group"]' +empty_probe_series="$fixture_root/empty-probe-series.json" +printf '%s\n' '{"status":"success","data":[]}' >"$empty_probe_series" +expect_pass 'target-initial requires probe series absence' \ + validate_probe_series_phase "$empty_probe_series" target-initial +expect_reject 'post-substrate rejects probe series absence' \ + validate_probe_series_phase "$empty_probe_series" post-substrate +expect_pass 'post-substrate accepts observability probe series' \ + validate_probe_series_phase "$dotted_series" post-substrate +expect_reject 'target-initial rejects existing probe series' \ + validate_probe_series_phase "$dotted_series" target-initial + +post_labels="$fixture_root/post-labels.json" +post_substrate_expected_probe_targets | jq '[.[].labels]' >"$post_labels" +for query_kind in success status duration tls; do + query_fixture="$fixture_root/post-query-$query_kind.json" + jq -cn --slurpfile labels "$post_labels" --arg kind "$query_kind" ' + ($labels[0] | if $kind=="tls" then map(select(.instance|startswith("https://"))) else . end) as $selected | + {status:"success",data:{resultType:"vector",result:[$selected[] | + {metric:(. + {__name__:(if $kind=="success" then "probe_success" + elif $kind=="status" then "probe_http_status_code" + elif $kind=="duration" then "probe_duration_seconds" + else "probe_ssl_earliest_cert_expiry" end)}), + value:[1770000000,(if $kind=="success" then "1" + elif $kind=="status" then (if .job=="blackbox-private-edge" then "403" else "200" end) + elif $kind=="duration" then "0.1" else "4102444800" end)]}]}} + ' >"$query_fixture" + expect_pass "post-substrate $query_kind query contract accepted" \ + validate_post_substrate_query_body "$query_fixture" "$query_kind" +done + +large_target_rows="$fixture_root/large-target-rows.jsonl" +awk 'BEGIN { + for (i = 0; i < 20000; i++) + printf "{\"scrape_pool\":\"pool/%d\",\"scrape_url\":\"http://10.42.0.1:9090/metrics\",\"health\":\"up\",\"last_error\":\"\",\"provenance\":\"core\",\"metrics\":[{\"name\":\"fixture_metric\",\"label_names\":[\"__name__\",\"instance\"]}]}\n", i +}' >"$large_target_rows" +large_inventory_contract() { + [[ "$(stat -c %s "$large_target_rows")" -gt 2097152 ]] || return 1 + build_inventory_json "$large_target_rows" target-initial 2026-08-12T00:00:00Z | + jq -e '.phase=="target-initial" and (.targets|length)==20000' >/dev/null +} +expect_pass 'inventory builder reads target rows from file beyond argv size limit' large_inventory_contract + +fake_bin="$fixture_root/bin" +mkdir -p "$fake_bin" +fake_log="$fixture_root/fake-commands.log" +pf_log="$fixture_root/port-forward-pids.log" +: >"$fake_log" +: >"$pf_log" + +cat >"$fake_bin/kubectl" <<'FAKE_KUBECTL' +#!/usr/bin/env bash +set -Eeuo pipefail +printf 'kubectl' >>"$FAKE_LOG" +printf ' <%s>' "$@" >>"$FAKE_LOG" +printf '\n' >>"$FAKE_LOG" +args=" $* " +if [[ "$args" =~ [[:space:]](apply|delete|patch|replace|exec|rollout|scale)[[:space:]] ]]; then + exit 91 +fi +if [[ "$args" == *' config current-context '* ]]; then printf 'default\n'; exit 0; fi +if [[ "$args" == *' config view '* ]]; then printf '%s\n' '{"clusters":[{"cluster":{"server":"https://127.0.0.1:6443"}}]}'; exit 0; fi +if [[ "$args" == *' get --raw=/readyz '* ]]; then printf 'ok\n'; exit 0; fi +if [[ "$args" == *' auth can-i '* ]]; then printf 'yes\n'; exit 0; fi +if [[ "$args" == *' get prometheus '* ]]; then printf '%s\n' '{"apiVersion":"monitoring.coreos.com/v1","kind":"Prometheus","metadata":{"name":"observability-core-kube-pr-prometheus","namespace":"observability"}}'; exit 0; fi +if [[ "$args" == *' get pods '* ]]; then printf '%s\n' '{"items":[{"metadata":{"name":"prometheus-observability-core-kube-pr-prometheus-0","namespace":"observability"},"status":{"phase":"Running","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"prometheus","ready":true}]}}]}'; exit 0; fi +if [[ "$args" == *' port-forward '* ]]; then + printf '%s\n' "$$" >>"$PF_LOG" + pf_stderr_path="$(readlink -- "/proc/$$/fd/2")" + printf '%s\n' "${pf_stderr_path%/*}" >>"$PF_WORKDIR_LOG" + if [[ "${PF_IGNORE_TERM:-0}" == 1 ]]; then + trap 'printf "ignored %s TERM\n" "$$" >>"$PF_LOG"' TERM + else + trap 'printf "stopped %s TERM\n" "$$" >>"$PF_LOG"; exit 0' TERM + fi + trap 'printf "stopped %s INT\n" "$$" >>"$PF_LOG"; exit 0' INT + trap 'printf "stopped %s HUP\n" "$$" >>"$PF_LOG"; exit 0' HUP + if [[ "${PF_NO_ANNOUNCEMENT:-0}" != 1 ]]; then + printf 'Forwarding from 127.0.0.1:43123 -> 9090\n' >&2 + fi + while :; do /usr/bin/sleep 0.05; done +fi +exit 1 +FAKE_KUBECTL + +cat >"$fake_bin/curl" <<'FAKE_CURL' +#!/usr/bin/env bash +set -Eeuo pipefail +[[ "${1-}" == --disable ]] || exit 92 +printf 'curl' >>"$FAKE_LOG" +printf ' <%s>' "$@" >>"$FAKE_LOG" +printf '\n' >>"$FAKE_LOG" +destination='' +url='' +max_filesize='' +all_arguments=" $* " +declare -a data_values=() +declare -a urlencode_values=() +while (($#)); do + case "$1" in + --output) destination="$2"; shift 2 ;; + --max-filesize) max_filesize="$2"; shift 2 ;; + --data) data_values+=("$2"); shift 2 ;; + --data-urlencode) urlencode_values+=("$2"); shift 2 ;; + http://*) url="$1"; shift ;; + *) shift ;; + esac +done +[[ -n "$destination" && -n "$url" ]] || exit 2 +emit_text() { + if [[ "$destination" == - ]]; then printf '%s\n' "$1"; else printf '%s\n' "$1" >"$destination"; fi +} +emit_file() { + if [[ "$destination" == - ]]; then cat -- "$1"; else cp -- "$1" "$destination"; fi +} +case "$url" in + */-/ready) emit_text 'ready' ;; + */api/v1/targets) + [[ "${#data_values[@]}" == 1 ]] || exit 80 + case "${data_values[0]}" in + state=dropped) + if [[ "${FAKE_CURL_MODE:-success}" == dropped_blackbox ]]; then + emit_file "$DROPPED_BLACKBOX_FIXTURE" + else + emit_file "$DROPPED_TARGETS_FIXTURE" + fi + exit 0 + ;; + state=active) ;; + *) exit 81 ;; + esac + if [[ "${FAKE_CURL_MODE:-success}" == hang ]]; then + trap 'exit 128' INT TERM HUP + while :; do /usr/bin/sleep 0.05; done + fi + if [[ "${FAKE_CURL_MODE:-success}" == error ]]; then exit 22; fi + if [[ "${FAKE_CURL_MODE:-success}" == oversized_targets ]]; then + [[ "$max_filesize" =~ ^[0-9]+$ ]] || exit 88 + if (( $(stat -c %s "$OVERSIZED_TARGETS_FIXTURE") > max_filesize )); then exit 63; fi + emit_file "$OVERSIZED_TARGETS_FIXTURE" + else + emit_file "$TARGETS_FIXTURE" + fi + ;; + */api/v1/rules) emit_text '{"status":"success","data":{"groups":[]}}' ;; + */api/v1/targets/metadata) + [[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == match_target=* ]] || exit 84 + [[ "${#data_values[@]}" == 1 && "${data_values[0]}" == 'limit=10000' ]] || exit 85 + target_selector="${urlencode_values[0]#match_target=}" + grep -Fqx -- "$target_selector" "$EXPECTED_TARGET_SELECTORS" || exit 86 + printf '%s\n' "$target_selector" >>"$METADATA_SELECTOR_LOG" + emit_text '{"status":"success","data":{"descriptor_only_metric":[{"type":"gauge","help":"registered without a current series","unit":""}],"fixture_metric":[{"type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""}]}}' + ;; + */api/v1/series) + if [[ "$all_arguments" == *'probe_.+'* ]]; then + [[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == 'match[]={__name__=~"probe_.+"}' ]] || exit 87 + emit_text '{"status":"success","data":[]}' + else + [[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == match\[\]=* ]] || exit 89 + series_selector="${urlencode_values[0]#match[]=}" + if [[ "$series_selector" == *'__name__="descriptor_only_metric"'* ]]; then + fixture_selector="${series_selector/descriptor_only_metric/fixture_metric}" + grep -Fqx -- "$fixture_selector" "$EXPECTED_SERIES_SELECTORS" || exit 90 + emit_text '{"status":"success","data":[]}' + exit 0 + fi + grep -Fqx -- "$series_selector" "$EXPECTED_SERIES_SELECTORS" || exit 90 + if [[ "${FAKE_CURL_MODE:-success}" == all_series_empty ]]; then + emit_text '{"status":"success","data":[]}' + exit 0 + fi + printf '%s\n' "$series_selector" >>"$SERIES_SELECTOR_LOG" + if [[ "${FAKE_CURL_MODE:-success}" == hang_series ]]; then + printf '%s\n' "$$" >>"$CURL_PID_LOG" + : >"$CURL_HANG_MARKER" + trap 'exit 128' INT TERM HUP + while :; do /usr/bin/sleep 0.05; done + fi + emit_text '{"status":"success","data":[{"__name__":"fixture_metric","instance":"10.42.0.1:9090","secret_label":"DO-NOT-LEAK-FIXTURE"}]}' + fi + ;; + *) exit 3 ;; +esac +FAKE_CURL +cat >"$fake_bin/ln" <<'FAKE_LN' +#!/usr/bin/env bash +set -Eeuo pipefail +if [[ "${FAIL_SHA_LINK:-0}" == 1 && "${*: -1}" == */inventory.sha256 ]]; then + exit 1 +fi +exec /usr/bin/ln "$@" +FAKE_LN +chmod 0700 "$fake_bin/kubectl" "$fake_bin/curl" "$fake_bin/ln" + +export PATH="$fake_bin:$ORIGINAL_PATH" +pf_workdir_log="$fixture_root/port-forward-workdirs.log" +curl_pid_log="$fixture_root/curl-pids.log" +metadata_selector_log="$fixture_root/metadata-selectors.log" +series_selector_log="$fixture_root/series-selectors.log" +: >"$pf_workdir_log" +: >"$curl_pid_log" +: >"$metadata_selector_log" +: >"$series_selector_log" +export FAKE_LOG="$fake_log" PF_LOG="$pf_log" PF_WORKDIR_LOG="$pf_workdir_log" +export CURL_PID_LOG="$curl_pid_log" TARGETS_FIXTURE="$valid_targets" OVERSIZED_TARGETS_FIXTURE="$oversized_targets" +export DROPPED_TARGETS_FIXTURE="$empty_dropped_targets" DROPPED_BLACKBOX_FIXTURE="$dropped_blackbox" +export EXPECTED_TARGET_SELECTORS="$expected_target_selectors" EXPECTED_SERIES_SELECTORS="$expected_series_selectors" +export METADATA_SELECTOR_LOG="$metadata_selector_log" SERIES_SELECTOR_LOG="$series_selector_log" + +run_capture() { + local output_dir="$1" + shift + env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" \ + PF_WORKDIR_LOG="$PF_WORKDIR_LOG" CURL_PID_LOG="$CURL_PID_LOG" \ + EXPECTED_TARGET_SELECTORS="$EXPECTED_TARGET_SELECTORS" EXPECTED_SERIES_SELECTORS="$EXPECTED_SERIES_SELECTORS" \ + METADATA_SELECTOR_LOG="$METADATA_SELECTOR_LOG" SERIES_SELECTOR_LOG="$SERIES_SELECTOR_LOG" \ + DROPPED_TARGETS_FIXTURE="$DROPPED_TARGETS_FIXTURE" DROPPED_BLACKBOX_FIXTURE="$DROPPED_BLACKBOX_FIXTURE" \ + OVERSIZED_TARGETS_FIXTURE="$OVERSIZED_TARGETS_FIXTURE" \ + FAKE_CURL_MODE="${FAKE_CURL_MODE:-success}" \ + bash "$CAPTURE" --execute --phase target-initial --output-dir "$output_dir" "$@" +} + +inventory_kubectl_alias() { inventory_kubectl "$@"; } +mutation_log_lines="$(wc -l <"$fake_log")" +for wrapper_name in inventory_kubectl inventory_kubectl_alias; do + for mutation_verb in apply create delete patch replace exec rollout scale; do + expect_reject "$wrapper_name default-denies $mutation_verb" \ + "$wrapper_name" "$mutation_verb" fixture + done +done +[[ "$(wc -l <"$fake_log")" == "$mutation_log_lines" ]] || fail 'denied wrapper mutation reached kubectl' + +ownership_signal_log="$fixture_root/ownership-signals.log" +ownership_ready="$fixture_root/ownership-ready" +bash -c ' + trap '\''printf "TERM\n" >>"$1"; exit 0'\'' TERM + : >"$2" + while :; do /usr/bin/sleep 0.05; done +' bash "$ownership_signal_log" "$ownership_ready" & +ownership_pid=$! +test_helper_pids+=("$ownership_pid") +for ((attempt=0; attempt<40; attempt++)); do [[ -e "$ownership_ready" ]] && break; sleep 0.025; done +[[ -e "$ownership_ready" ]] || fail 'ownership fixture did not start' +ownership_snapshot="$(inventory_process_snapshot "$ownership_pid")" || fail 'ownership fixture snapshot failed' +INVENTORY_PORT_FORWARD_PID="$ownership_pid" +INVENTORY_PORT_FORWARD_START="$((10#${ownership_snapshot#*:} + 1))" +expect_pass 'wrong start-time PID is not signaled' inventory_stop_port_forward +[[ ! -s "$ownership_signal_log" ]] || fail 'wrong start-time process received TERM' +kill -0 "$ownership_pid" 2>/dev/null || fail 'wrong start-time process was killed' +kill -TERM "$ownership_pid" 2>/dev/null || true +for ((attempt=0; attempt<40; attempt++)); do kill -0 "$ownership_pid" 2>/dev/null || break; sleep 0.025; done + +cli_dir="$fixture_root/cli" +mkdir -m 0700 "$cli_dir" +expect_reject 'missing CLI arguments rejected' bash "$CAPTURE" +expect_reject 'missing execute rejected' bash "$CAPTURE" --phase target-initial --output-dir "$cli_dir" +expect_reject 'duplicate argument rejected' bash "$CAPTURE" --execute --phase target-initial --phase target-initial --output-dir "$cli_dir" +expect_reject 'unknown phase rejected' bash "$CAPTURE" --execute --phase future-phase --output-dir "$cli_dir" +expect_reject 'unknown argument rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$cli_dir" --unknown + +relative_dir="${cli_dir#/}" +expect_reject 'relative output path rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$relative_dir" +noncanonical="$fixture_root/../$(basename "$fixture_root")/cli" +expect_reject 'noncanonical output path rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$noncanonical" +chmod 0755 "$cli_dir" +expect_reject 'non-0700 output directory rejected' run_capture "$cli_dir" +chmod 0700 "$cli_dir" +ln -s "$cli_dir" "$fixture_root/cli-link" +expect_reject 'symlink output directory rejected' run_capture "$fixture_root/cli-link" +mkdir -m 0700 "$fixture_root/unsafe-parent" +mkdir -m 0700 "$fixture_root/real-child" +ln -s "$fixture_root/unsafe-parent" "$fixture_root/parent-link" +mkdir -m 0700 "$fixture_root/unsafe-parent/child" +expect_reject 'symlink path component rejected' run_capture "$fixture_root/parent-link/child" +printf 'preserve-me\n' >"$cli_dir/preexisting" +expect_reject 'nonempty output directory rejected without clobber' run_capture "$cli_dir" +[[ "$(cat "$cli_dir/preexisting")" == preserve-me ]] || fail 'preexisting file was modified' +rm "$cli_dir/preexisting" + +empty_series_dir="$fixture_root/empty-series" +mkdir -m 0700 "$empty_series_dir" +FAKE_CURL_MODE=all_series_empty +export FAKE_CURL_MODE +expect_reject 'target with metadata descriptors but no actual series rejected' run_capture "$empty_series_dir" +unset FAKE_CURL_MODE +[[ -z "$(find "$empty_series_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'empty-series response produced accepted output' +: >"$metadata_selector_log" +: >"$series_selector_log" + +success_dir="$fixture_root/success" +mkdir -m 0700 "$success_dir" +expect_pass 'full capture succeeds with fixture APIs' run_capture "$success_dir" +cmp -s <(sort "$expected_target_selectors") <(sort "$metadata_selector_log") || fail 'metadata calls did not cover every exact full-label selector once' +cmp -s <(sort "$expected_series_selectors") <(sort "$series_selector_log") || fail 'series calls did not cover every exact metric selector once' +mapfile -t output_names < <(find "$success_dir" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort) +[[ "${output_names[*]}" == 'inventory.json inventory.sha256' ]] || fail 'capture did not create exactly two files' +for output_name in "${output_names[@]}"; do + [[ -f "$success_dir/$output_name" && ! -L "$success_dir/$output_name" ]] || fail "$output_name is not a regular non-symlink" + [[ "$(stat -c %a "$success_dir/$output_name")" == 600 ]] || fail "$output_name mode is not 0600" +done +[[ "$(jq -cS . "$success_dir/inventory.json")"$'\n' == "$(cat "$success_dir/inventory.json")"$'\n' ]] || fail 'inventory is not canonical jq -cS JSON' +(cd "$success_dir" && sha256sum --check inventory.sha256 >/dev/null) || fail 'inventory hash does not bind JSON bytes' +jq -e ' + .schema == "platform-observability-metric-inventory/v1" and + .phase == "target-initial" and .context == "default" and + .api_server == "https://127.0.0.1:6443" and + .prometheus == "observability/observability-core-kube-pr-prometheus" and + .preexisting_verify_only == ["grafana"] and + (.targets | length == 21) and + ([.targets[].scrape_pool] == ([.targets[].scrape_pool] | sort)) and + (all(.targets[]; .health == "up" and .last_error == "" and + .metrics == [{"label_names":["__name__","instance","secret_label"],"name":"fixture_metric"}])) and + ([.targets[] | select(.scrape_pool=="serviceMonitor/observability/grafana/0") | .provenance] == ["preexisting-verify-only"]) and + ([.targets[] | select(.scrape_pool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .provenance] == ["task6-owned"]) +' "$success_dir/inventory.json" >/dev/null || fail 'inventory schema/provenance/content mismatch' +if rg -q 'DO-NOT-LEAK-FIXTURE|private-value|help|query|fragment|exemplar' "$success_dir/inventory.json"; then + fail 'inventory persisted a prohibited value or field' +fi +success_pf_pid="$(head -n 1 "$pf_log")" +if kill -0 "$success_pf_pid" 2>/dev/null; then fail 'port-forward survived success'; fi + +oversized_dir="$fixture_root/oversized-response" +mkdir -m 0700 "$oversized_dir" +FAKE_CURL_MODE=oversized_targets +export FAKE_CURL_MODE +expect_reject 'Prometheus response above the byte ceiling rejected' run_capture "$oversized_dir" +unset FAKE_CURL_MODE +[[ -z "$(find "$oversized_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'oversized response produced accepted output' + +dropped_live_dir="$fixture_root/dropped-live-response" +mkdir -m 0700 "$dropped_live_dir" +FAKE_CURL_MODE=dropped_blackbox +export FAKE_CURL_MODE +expect_reject 'live dropped blackbox response rejected with exact active set' run_capture "$dropped_live_dir" +unset FAKE_CURL_MODE +[[ -z "$(find "$dropped_live_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'dropped blackbox response produced accepted output' + +sigkill_dir="$fixture_root/sigkill" +mkdir -m 0700 "$sigkill_dir" +sigkill_marker="$fixture_root/sigkill-series-started" +sigkill_pf_lines="$(wc -l <"$pf_log")" +sigkill_workdir_lines="$(wc -l <"$pf_workdir_log")" +sigkill_curl_lines="$(wc -l <"$curl_pid_log")" +env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" PF_WORKDIR_LOG="$PF_WORKDIR_LOG" \ + CURL_PID_LOG="$CURL_PID_LOG" CURL_HANG_MARKER="$sigkill_marker" \ + TARGETS_FIXTURE="$TARGETS_FIXTURE" OVERSIZED_TARGETS_FIXTURE="$OVERSIZED_TARGETS_FIXTURE" \ + FAKE_CURL_MODE=hang_series bash "$CAPTURE" --execute --phase target-initial --output-dir "$sigkill_dir" \ + >/dev/null 2>&1 & +sigkill_capture_pid=$! +for ((attempt=0; attempt<200; attempt++)); do + [[ -e "$sigkill_marker" ]] && break + kill -0 "$sigkill_capture_pid" 2>/dev/null || break + sleep 0.025 +done +[[ -e "$sigkill_marker" ]] || fail 'SIGKILL residue fixture did not reach the series request' +sigkill_workdir="$(sed -n "$((sigkill_workdir_lines + 1))p" "$pf_workdir_log")" +sigkill_pf_pid="$(sed -n "$((sigkill_pf_lines + 1))p" "$pf_log")" +sigkill_curl_pid="$(sed -n "$((sigkill_curl_lines + 1))p" "$curl_pid_log")" +[[ "$sigkill_workdir" == /tmp/platform-observability-inventory-capture.* && -d "$sigkill_workdir" && ! -L "$sigkill_workdir" ]] || fail 'SIGKILL fixture work directory was not identified safely' +kill -KILL "$sigkill_capture_pid" 2>/dev/null || true +wait "$sigkill_capture_pid" 2>/dev/null || true +for owned_pid in "$sigkill_curl_pid" "$sigkill_pf_pid"; do + [[ "$owned_pid" =~ ^[0-9]+$ ]] || fail 'SIGKILL fixture lost an owned child PID' + kill -KILL "$owned_pid" 2>/dev/null || true + for ((attempt=0; attempt<40; attempt++)); do + kill -0 "$owned_pid" 2>/dev/null || break + sleep 0.025 + done +done +raw_residue="$(find "$sigkill_workdir" -type f \( -name 'targets*.json' -o -name 'rules*.json' -o -name 'metadata*.json' -o -name 'series*.json' \) -print)" +secret_residue=0 +if rg -q 'DO-NOT-LEAK-FIXTURE' "$sigkill_workdir"; then secret_residue=1; fi +find "$sigkill_workdir" -depth -delete +[[ -z "$raw_residue" ]] || fail "SIGKILL left raw Prometheus response files: $raw_residue" +[[ "$secret_residue" == 0 ]] || fail 'SIGKILL residue persisted a raw label/help value' +pass_count=$((pass_count + 1)) + +stubborn_pf_dir="$fixture_root/stubborn-port-forward" +mkdir -m 0700 "$stubborn_pf_dir" +stubborn_lines="$(wc -l <"$pf_log")" +set +e +timeout --preserve-status --signal=TERM --kill-after=0.5s 5s \ + env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" PF_IGNORE_TERM=1 \ + bash "$CAPTURE" --execute --phase target-initial --output-dir "$stubborn_pf_dir" >/dev/null 2>&1 +stubborn_rc=$? +set -e +stubborn_pf_pid="$(sed -n "$((stubborn_lines + 1))p" "$pf_log")" +[[ "$stubborn_pf_pid" =~ ^[0-9]+$ ]] || fail 'TERM-ignoring fixture did not start a port-forward' +stubborn_survived_production_cleanup=0 +if kill -0 "$stubborn_pf_pid" 2>/dev/null; then + stubborn_survived_production_cleanup=1 + kill -KILL "$stubborn_pf_pid" 2>/dev/null || true + for ((attempt=0; attempt<40; attempt++)); do + kill -0 "$stubborn_pf_pid" 2>/dev/null || break + sleep 0.025 + done +fi +[[ "$stubborn_rc" == 0 ]] || fail "TERM-ignoring port-forward cleanup exceeded its bound (status $stubborn_rc)" +grep -Fqx "ignored $stubborn_pf_pid TERM" "$pf_log" || fail 'TERM-ignoring fixture did not receive TERM before escalation' +[[ "$stubborn_survived_production_cleanup" == 0 ]] || fail 'TERM-ignoring port-forward survived production cleanup' +pass_count=$((pass_count + 1)) + +silent_pf_dir="$fixture_root/silent-port-forward" +mkdir -m 0700 "$silent_pf_dir" +PF_NO_ANNOUNCEMENT=1 +export PF_NO_ANNOUNCEMENT +expect_pass 'readiness does not depend on port-forward status text' run_capture "$silent_pf_dir" +unset PF_NO_ANNOUNCEMENT + +publish_failure_dir="$fixture_root/publish-failure" +mkdir -m 0700 "$publish_failure_dir" +FAIL_SHA_LINK=1 +export FAIL_SHA_LINK +expect_reject 'second-file publication failure removes only run-owned output' run_capture "$publish_failure_dir" +unset FAIL_SHA_LINK +[[ -z "$(find "$publish_failure_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'publication failure left a partial inventory' + +error_dir="$fixture_root/error" +mkdir -m 0700 "$error_dir" +error_pf_lines="$(wc -l <"$pf_log")" +FAKE_CURL_MODE=error +export FAKE_CURL_MODE +expect_reject 'ordinary API error is sanitized' run_capture "$error_dir" +[[ -z "$(find "$error_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'ordinary error left partial output' +error_pf_pid="$(sed -n "$((error_pf_lines + 1))p" "$pf_log")" +[[ "$error_pf_pid" =~ ^[0-9]+$ ]] || fail 'ordinary error did not start a port-forward' +if kill -0 "$error_pf_pid" 2>/dev/null; then fail 'port-forward survived ordinary error'; fi +unset FAKE_CURL_MODE + +for signal in INT TERM HUP; do + signal_dir="$fixture_root/signal-${signal,,}" + mkdir -m 0700 "$signal_dir" + before_lines="$(wc -l <"$pf_log")" + set +e + timeout --preserve-status --signal="$signal" --kill-after=2s 0.3s \ + env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" FAKE_CURL_MODE=hang \ + bash "$CAPTURE" --execute --phase target-initial --output-dir "$signal_dir" >/dev/null 2>&1 + signal_rc=$? + set -e + [[ "$signal_rc" != 0 ]] || fail "$signal unexpectedly succeeded" + [[ -z "$(find "$signal_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail "$signal left partial output" + signal_pf_pid="$(sed -n "$((before_lines + 1))p" "$pf_log")" + [[ "$signal_pf_pid" =~ ^[0-9]+$ ]] || fail "$signal did not start a port-forward" + if kill -0 "$signal_pf_pid" 2>/dev/null; then fail "port-forward survived $signal"; fi + pass_count=$((pass_count + 1)) +done + +if awk ' + /kubectl/ { + line=$0 + if (line ~ / <(apply|delete|patch|replace|exec|rollout|scale)>/) exit 1 + if (line ~ / / && line !~ / /) exit 1 + } +' "$fake_log"; then :; else fail 'dynamic command trace contains a mutation verb'; fi + +# Whole-source proof complements the runtime trace. Removing line boundaries +# prevents a backslash-newline, command prefix, or wrapper alias from hiding a +# mutation verb. The exact RBAC question is the sole create exception. +[[ "$(rg -Fxc " 'create pods/portforward'; do" "$CAPTURE")" == 1 ]] || fail 'port-forward RBAC exception is not pinned exactly once' +mutation_free_source="$(sed "s/'create pods\\/portforward'/'RBAC_PORTFORWARD'/" "$CAPTURE")" +if rg -q '(^|[^[:alnum:]_])(apply|create|delete|patch|replace|exec|rollout|scale)($|[^[:alnum:]_])' <<<"$mutation_free_source"; then + fail 'whole-source proof found a Kubernetes mutation verb' +fi +[[ "$(rg -c '^[[:space:]]*kubectl --request-timeout=' "$CAPTURE")" == 2 ]] || fail 'kubectl invocation count escaped the wrapper/port-forward allowlist' +if rg -q '^[[:space:]]*(alias[[:space:]]+[^=]+=.*kubectl|[A-Za-z_][A-Za-z0-9_]*=.*kubectl)' "$CAPTURE"; then + fail 'kubectl executable alias bypasses the default-deny wrapper' +fi +pass_count=$((pass_count + 1)) + +printf 'PASS: %d target-initial inventory assertions\n' "$pass_count" diff --git a/scripts/validate/test-configure-keycloak-grafana-oidc.sh b/scripts/validate/test-configure-keycloak-grafana-oidc.sh new file mode 100755 index 0000000..c91b9df --- /dev/null +++ b/scripts/validate/test-configure-keycloak-grafana-oidc.sh @@ -0,0 +1,651 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly SCRIPT="$ROOT/scripts/bootstrap/configure-keycloak-grafana-oidc.sh" +readonly NOW_UTC='2026-08-12T07:00:00Z' +readonly CLIENT_SECRET='GrafanaFixtureClientSecret-123456789' +WORK='' +ASSERTIONS=0 + +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } +pass() { ASSERTIONS=$((ASSERTIONS + 1)); printf 'PASS: %s\n' "$1"; } +cleanup() { + trap - EXIT HUP INT TERM + case "$WORK" in /tmp/platform-grafana-oidc-test.??????) rm -rf -- "$WORK" ;; esac +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +write_executable() { + local path=$1 + shift + printf '%s\n' "$@" >"$path" + chmod 0755 "$path" +} + +make_fakes() { + local fixture=$1 + mkdir -p "$fixture/bin" "$fixture/state" "$fixture/evidence-parent" + chmod 0700 "$fixture" "$fixture/state" "$fixture/evidence-parent" + chmod 0755 "$fixture/bin" + : >"$fixture/commands.log" + + write_executable "$fixture/bin/encryption" \ + '#!/usr/bin/env bash' \ + 'set -Eeuo pipefail' \ + 'printf '\''encryption'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'if [[ -n "${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" ]]; then /usr/bin/env | LC_ALL=C /usr/bin/sort >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; printf '\''--\n'\'' >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; fi' \ + 'n=0; [[ ! -f "$PLATFORM_TEST_ENCRYPTION_COUNT" ]] || read -r n <"$PLATFORM_TEST_ENCRYPTION_COUNT"' \ + 'n=$((n + 1)); printf '\''%s\n'\'' "$n" >"$PLATFORM_TEST_ENCRYPTION_COUNT"' \ + '[[ "$n" != "${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" ]]' + write_executable "$fixture/bin/restore" \ + '#!/usr/bin/env bash' \ + 'set -Eeuo pipefail' \ + 'printf '\''restore'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'if [[ -n "${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" ]]; then /usr/bin/env | LC_ALL=C /usr/bin/sort >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; printf '\''--\n'\'' >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; fi' \ + 'n=0; [[ ! -f "$PLATFORM_TEST_RESTORE_COUNT" ]] || read -r n <"$PLATFORM_TEST_RESTORE_COUNT"' \ + 'n=$((n + 1)); printf '\''%s\n'\'' "$n" >"$PLATFORM_TEST_RESTORE_COUNT"' \ + 'if [[ "$n" == 1 && -f "$PLATFORM_TEST_STATE/remove-evidence-parent" ]]; then /usr/bin/mv -- "${PLATFORM_TEST_STATE%/state}/evidence-parent" "${PLATFORM_TEST_STATE%/state}/evidence-parent-missing"; fi' \ + '[[ "$n" != "${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" ]]' + write_executable "$fixture/bin/sudo" \ + '#!/usr/bin/env bash' \ + 'set -Eeuo pipefail' \ + 'printf '\''sudo'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'if [[ "${1:-}" == -v ]]; then : >"$PLATFORM_TEST_SUDO_REFRESHED"; exit 0; fi' \ + '[[ -f "$PLATFORM_TEST_SUDO_REFRESHED" ]] || exit 92' \ + '[[ "${1:-}" != --non-interactive && "${1:-}" != -n ]] || shift' \ + 'exec "$@"' + + cat >"$fixture/bin/keycloak-api" <<'PY' +#!/usr/bin/env python3 +import http.server, json, os, pathlib, signal, sys, time, urllib.parse + +port = int(sys.argv[1]); root = pathlib.Path(os.environ["PLATFORM_TEST_STATE"]) +state_path = root / "keycloak.json"; log = pathlib.Path(os.environ["PLATFORM_TEST_COMMAND_LOG"]) + +def load(): return json.loads(state_path.read_text()) +def save(value): state_path.write_text(json.dumps(value, sort_keys=True)) +def public_client(item): return {k:v for k,v in item.items() if k != "_secret"} +def maybe_fault(label, handler): + fault = os.environ.get("PLATFORM_TEST_FAULT", "") + marker = root / ("fault-fired-" + label) + if marker.exists(): + return False + if fault == "timeout-" + label: + marker.write_text("1") + time.sleep(5) + return True + if fault == "loss-" + label: + marker.write_text("1") + handler.close_connection = True + return True + if fault == "http500-" + label: + marker.write_text("1") + handler.send_response(500); handler.end_headers(); return True + if fault == "drift-secret-" + label: + marker.write_text("1") + secret_path=root/"secret.json" + item=json.loads(secret_path.read_text()) + item["metadata"]["uid"]="secret-concurrent-replacement" + item["metadata"]["resourceVersion"]="99" + secret_path.write_text(json.dumps(item,sort_keys=True)) + handler.send_response(500); handler.end_headers(); return True + if fault.startswith("signal-") and fault.endswith("-" + label): + marker.write_text("1") + sig = fault.split("-", 2)[1] + os.kill(int(os.environ["PLATFORM_TEST_TARGET_PID"]), getattr(signal, "SIG" + sig)) + handler.close_connection = True + return True + return False + +class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): return + def body(self): + length = int(self.headers.get("Content-Length", "0")); data = self.rfile.read(length) + return json.loads(data) if data else None + def send_json(self, code, value): + payload = json.dumps(value, separators=(",", ":")).encode() + self.send_response(code); self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))); self.end_headers(); self.wfile.write(payload) + def empty(self, code): self.send_response(code); self.end_headers() + def parts(self): return urllib.parse.urlparse(self.path), urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) + def do_GET(self): + parsed, query = self.parts(); path = parsed.path; state = load() + if path.endswith("/.well-known/openid-configuration"): + return self.send_json(200, {"issuer":"https://id.learn.hyeonworks.com/realms/hyeonworks"}) + if path.endswith("/clients"): + matches=[public_client(x) for x in state["clients"] if x.get("clientId") == query.get("clientId",[""])[0]] + return self.send_json(200, matches[:int(query.get("max",["2"])[0])]) + if "/clients/" in path and path.endswith("/client-secret"): + cid=path.split("/clients/",1)[1].split("/",1)[0] + matches=[x for x in state["clients"] if x["id"] == cid] + return self.send_json(200,{"type":"secret","value":matches[0]["_secret"]}) if len(matches)==1 else self.empty(404) + if "/clients/" in path and path.endswith("/protocol-mappers/models"): + cid=path.split("/clients/",1)[1].split("/",1)[0] + return self.send_json(200,state["mappers"].get(cid,[])) + if "/clients/" in path: + cid=path.split("/clients/",1)[1].split("/",1)[0]; matches=[public_client(x) for x in state["clients"] if x["id"]==cid] + return self.send_json(200,matches[0]) if len(matches)==1 else self.empty(404) + if path.endswith("/groups") and "/users/" not in path: + search=query.get("search",[""])[0] + matches=[x for x in state["groups"] if x.get("name")==search or x.get("path")=="/"+search] + return self.send_json(200,matches[:int(query.get("max",["2"])[0])]) + if "/groups/" in path: + gid=path.split("/groups/",1)[1].split("/",1)[0]; matches=[x for x in state["groups"] if x["id"]==gid] + return self.send_json(200,matches[0]) if len(matches)==1 else self.empty(404) + if path.endswith("/users"): + username=query.get("username",[""])[0]; matches=[{"id":x["id"],"username":x["username"]} for x in state["users"] if x["username"]==username] + return self.send_json(200,matches[:int(query.get("max",["2"])[0])]) + if "/users/" in path and path.endswith("/groups"): + uid=path.split("/users/",1)[1].split("/",1)[0]; users=[x for x in state["users"] if x["id"]==uid] + groups=[x for x in state["groups"] if x["id"] in users[0]["groups"]] if len(users)==1 else [] + return self.send_json(200,groups[:int(query.get("max",["101"])[0])]) + self.empty(404) + def do_POST(self): + with log.open("a",encoding="utf-8") as stream: stream.write("keycloak POST "+self.path+"\n") + parsed,_=self.parts(); path=parsed.path + if path.endswith("/protocol/openid-connect/token"): + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + return self.send_json(200,{"access_token":"fixture-admin-token","token_type":"Bearer"}) + body=self.body(); state=load() + if path.endswith("/clients"): + item=body; item["id"]="client-created"; item["_secret"]=os.environ["PLATFORM_TEST_CLIENT_SECRET"] + state["clients"].append(item); state["mappers"][item["id"]]=[]; save(state) + if maybe_fault("client-create",self): return + self.send_response(201); self.send_header("Location",self.path+"/"+item["id"]); self.end_headers(); return + if path.endswith("/groups"): + item=body; item["id"]="group-created-"+str(len(state["groups"])+1); item["path"]="/"+item["name"] + state["groups"].append(item); save(state) + if maybe_fault("group-create",self): return + self.send_response(201); self.send_header("Location",self.path+"/"+item["id"]); self.end_headers(); return + if path.endswith("/protocol-mappers/models"): + cid=path.split("/clients/",1)[1].split("/",1)[0]; item=body; item["id"]="mapper-created" + state["mappers"].setdefault(cid,[]).append(item); save(state) + if maybe_fault("mapper-create",self): return + self.send_response(201); self.send_header("Location",self.path+"/"+item["id"]); self.end_headers(); return + self.empty(404) + def do_PUT(self): + with log.open("a",encoding="utf-8") as stream: stream.write("keycloak PUT "+self.path+"\n") + parsed,_=self.parts(); path=parsed.path; body=self.body(); state=load() + if "/protocol-mappers/models/" in path: + cid=path.split("/clients/",1)[1].split("/",1)[0]; mid=path.rsplit("/",1)[1] + for i,item in enumerate(state["mappers"].get(cid,[])): + if item["id"]==mid: body["id"]=mid; state["mappers"][cid][i]=body; save(state); break + if maybe_fault("mapper-put",self): return + return self.empty(204) + if "/clients/" in path: + cid=path.split("/clients/",1)[1].split("/",1)[0] + for i,item in enumerate(state["clients"]): + if item["id"]==cid: + secret=body.pop("secret",item["_secret"]); body["id"]=cid; body["_secret"]=secret; state["clients"][i]=body; save(state); break + if maybe_fault("client-put",self): return + return self.empty(204) + if "/groups/" in path and "/users/" not in path: + gid=path.split("/groups/",1)[1].split("/",1)[0] + for i,item in enumerate(state["groups"]): + if item["id"]==gid: + body["id"]=gid; body["path"]="/"+body["name"]; state["groups"][i]=body; save(state); break + if maybe_fault("group-put",self): return + return self.empty(204) + if "/users/" in path and "/groups/" in path: + uid=path.split("/users/",1)[1].split("/",1)[0]; gid=path.rsplit("/",1)[1] + for user in state["users"]: + if user["id"]==uid and gid not in user["groups"]: user["groups"].append(gid) + save(state) + label="admin-member-put" if gid.startswith("group-admin") else "viewer-member-put" + if maybe_fault(label,self): return + return self.empty(204) + self.empty(404) + def do_DELETE(self): + with log.open("a",encoding="utf-8") as stream: stream.write("keycloak DELETE "+self.path+"\n") + parsed,_=self.parts(); path=parsed.path; state=load() + if "/users/" in path and "/groups/" in path: + uid=path.split("/users/",1)[1].split("/",1)[0]; gid=path.rsplit("/",1)[1] + for user in state["users"]: + if user["id"]==uid and gid in user["groups"]: user["groups"].remove(gid) + save(state); return self.empty(204) + if "/protocol-mappers/models/" in path: + cid=path.split("/clients/",1)[1].split("/",1)[0]; mid=path.rsplit("/",1)[1] + if os.environ.get("PLATFORM_TEST_ROLLBACK_FAULT","")=="noop-mapper-delete": + (root/"rollback-fault-fired-noop-mapper-delete").write_text("1") + return self.empty(204) + state["mappers"][cid]=[x for x in state["mappers"].get(cid,[]) if x["id"]!=mid]; save(state); return self.empty(204) + if "/clients/" in path: + cid=path.split("/clients/",1)[1].split("/",1)[0]; state["clients"]=[x for x in state["clients"] if x["id"]!=cid]; state["mappers"].pop(cid,None); save(state); return self.empty(204) + if "/groups/" in path: + gid=path.rsplit("/",1)[1] + if os.environ.get("PLATFORM_TEST_ROLLBACK_FAULT","")=="noop-group-delete": + (root/"rollback-fault-fired-noop-group-delete").write_text("1") + return self.empty(204) + state["groups"]=[x for x in state["groups"] if x["id"]!=gid] + for user in state["users"]: user["groups"]=[x for x in user["groups"] if x!=gid] + save(state); return self.empty(204) + self.empty(404) + +try: + server=http.server.ThreadingHTTPServer(("127.0.0.1",port),Handler) + server.serve_forever() +except BaseException as error: + (root/"keycloak-api-error").write_text(repr(error)) + raise +PY + chmod 0755 "$fixture/bin/keycloak-api" + /usr/bin/python3 -m py_compile "$fixture/bin/keycloak-api" + + cat >"$fixture/bin/kube-api" <<'PY' +#!/usr/bin/env python3 +import base64, http.server, json, os, pathlib, socketserver, sys +sock=sys.argv[1]; root=pathlib.Path(os.environ["PLATFORM_TEST_STATE"]); path=root/"secret.json" +def load(): return json.loads(path.read_text()) if path.exists() else None +class Server(socketserver.UnixStreamServer): pass +class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self,*_): return + def payload(self): + n=int(self.headers.get("Content-Length","0")); raw=self.rfile.read(n); return json.loads(raw) if raw else None + def send_json(self,code,item): + raw=json.dumps(item,separators=(",",":")).encode(); self.send_response(code); self.send_header("Content-Type","application/json"); self.send_header("Content-Length",str(len(raw))); self.end_headers(); self.wfile.write(raw) + def do_GET(self): + item=load() + if self.path.endswith("/grafana-keycloak-oidc"): + return self.send_json(200,item) if item else self.send_json(404,{"kind":"Status","reason":"NotFound"}) + self.send_json(405,{"kind":"Status"}) + def do_POST(self): + if path.exists(): return self.send_json(409,{"kind":"Status","reason":"AlreadyExists"}) + item=self.payload(); item["metadata"]["uid"]="secret-created-uid"; item["metadata"]["resourceVersion"]="1"; path.write_text(json.dumps(item,sort_keys=True)) + fault=os.environ.get("PLATFORM_TEST_FAULT","") + if fault=="loss-secret-create": self.close_connection=True; return + if fault=="conflict-secret-create": path.unlink(); return self.send_json(409,{"kind":"Status","reason":"Conflict"}) + self.send_json(201,item) + def do_DELETE(self): + item=load(); options=self.payload() or {}; pre=(options.get("preconditions") or {}) + if not item: return self.send_json(404,{"kind":"Status","reason":"NotFound"}) + if pre.get("uid")!=item["metadata"]["uid"] or pre.get("resourceVersion")!=item["metadata"]["resourceVersion"]: + return self.send_json(409,{"kind":"Status","reason":"Conflict"}) + path.unlink(); self.send_json(200,{"kind":"Status","status":"Success"}) +with Server(sock,Handler) as server: server.serve_forever() +PY + chmod 0755 "$fixture/bin/kube-api" + /usr/bin/python3 -m py_compile "$fixture/bin/kube-api" + + cat >"$fixture/bin/kubectl" <<'PY' +#!/usr/bin/env python3 +import base64,json,os,pathlib,sys +args=sys.argv[1:]; log=pathlib.Path(os.environ["PLATFORM_TEST_COMMAND_LOG"]) +with log.open("a") as f: f.write("kubectl "+" ".join(args)+"\n") +if "--request-timeout=5s" not in args: raise SystemExit(84) +args=[x for x in args if x!="--request-timeout=5s"] +def opt(name): + for i,x in enumerate(args): + if x==name and i+1 8080",flush=True) + os.execv(os.environ["PLATFORM_TEST_KEYCLOAK_API"],[os.environ["PLATFORM_TEST_KEYCLOAK_API"],port]) +if args and args[0]=="proxy": + sock=opt("--unix-socket"); os.execv(os.environ["PLATFORM_TEST_KUBE_API"],[os.environ["PLATFORM_TEST_KUBE_API"],sock]) +raise SystemExit(83) +PY + chmod 0755 "$fixture/bin/kubectl" +} + +seed_state() { + local fixture=$1 profile=${2:-absent} + python3 -I -S - "$fixture/state/keycloak.json" "$profile" "$CLIENT_SECRET" <<'PY' +import json,pathlib,sys +path,profile,secret=sys.argv[1:] +client={"id":"client-existing","clientId":"grafana","name":"Drifted Grafana","enabled":False,"protocol":"openid-connect","publicClient":True,"standardFlowEnabled":False,"implicitFlowEnabled":True,"directAccessGrantsEnabled":True,"serviceAccountsEnabled":True,"authorizationServicesEnabled":True,"fullScopeAllowed":True,"rootUrl":"https://wrong.invalid","baseUrl":"https://wrong.invalid","redirectUris":["https://wrong.invalid/cb"],"webOrigins":["+"],"attributes":{},"_secret":secret} +groups=[{"id":"group-admin-existing","name":"platform-observability-admins","path":"/platform-observability-admins"},{"id":"group-viewer-existing","name":"platform-observability-viewers","path":"/platform-observability-viewers"}] +mapper={"id":"mapper-existing","name":"grafana-groups","protocol":"openid-connect","protocolMapper":"oidc-hardcoded-claim-mapper","consentRequired":True,"config":{"claim.name":"wrong"}} +state={"clients":[],"groups":[],"mappers":{},"users":[{"id":"user-admin","username":"admin-user","groups":[]},{"id":"user-viewer","username":"viewer-user","groups":[]}]} +if profile in {"existing","exact-secret","preexisting-admin","duplicate-client","duplicate-user","duplicate-group","mapper-duplicate"}: + state["clients"]=[client]; state["groups"]=groups; state["mappers"]={"client-existing":[mapper]} +if profile=="exact-secret": + state["clients"][0].update({"name":"Grafana","description":"Grafana confidential OIDC client managed by the platform bootstrap","enabled":True,"protocol":"openid-connect","clientAuthenticatorType":"client-secret","publicClient":False,"standardFlowEnabled":True,"implicitFlowEnabled":False,"directAccessGrantsEnabled":False,"serviceAccountsEnabled":False,"authorizationServicesEnabled":False,"consentRequired":False,"fullScopeAllowed":False,"rootUrl":"https://grafana.learn.hyeonworks.com","baseUrl":"https://grafana.learn.hyeonworks.com","redirectUris":["https://grafana.learn.hyeonworks.com/login/generic_oauth"],"webOrigins":["https://grafana.learn.hyeonworks.com"],"attributes":{"post.logout.redirect.uris":"https://grafana.learn.hyeonworks.com/*","oauth2.device.authorization.grant.enabled":"false","oidc.ciba.grant.enabled":"false"}}) + state["mappers"]["client-existing"]=[{"id":"mapper-existing","name":"grafana-groups","protocol":"openid-connect","protocolMapper":"oidc-group-membership-mapper","consentRequired":False,"config":{"claim.name":"groups","full.path":"true","id.token.claim":"true","access.token.claim":"true","userinfo.token.claim":"true"}}] +if profile=="preexisting-admin": state["users"][0]["groups"]=["group-admin-existing"] +if profile=="duplicate-client": state["clients"].append(dict(client,id="client-second")) +if profile=="duplicate-user": state["users"].append({"id":"user-admin-second","username":"admin-user","groups":[]}) +if profile=="duplicate-group": state["groups"].append({"id":"group-admin-second","name":"platform-observability-admins","path":"/platform-observability-admins"}) +if profile=="mapper-duplicate": state["mappers"]["client-existing"].append(dict(mapper,id="mapper-second",name="other",config={"claim.name":"groups"})) +pathlib.Path(path).write_text(json.dumps(state,sort_keys=True)) +PY +} + +seed_secret() { + local fixture=$1 secret=${2:-$CLIENT_SECRET} + python3 -I -S - "$fixture/state/secret.json" "$secret" <<'PY' +import base64,json,pathlib,sys +path,secret=sys.argv[1:] +item={"apiVersion":"v1","kind":"Secret","type":"Opaque","metadata":{"namespace":"observability","name":"grafana-keycloak-oidc","uid":"secret-existing-uid","resourceVersion":"7"},"data":{"client-id":base64.b64encode(b"grafana").decode(),"client-secret":base64.b64encode(secret.encode()).decode()}} +pathlib.Path(path).write_text(json.dumps(item,sort_keys=True)) +PY +} + +new_fixture() { + local name=$1 profile=${2:-absent} fixture + fixture="$WORK/$name" + mkdir -p "$fixture"; make_fakes "$fixture"; seed_state "$fixture" "$profile"; printf '%s\n' "$fixture" +} + +run_script() { + local fixture=$1; shift + env \ + PATH="$fixture/bin:$PATH" \ + PLATFORM_GRAFANA_OIDC_CONFIRMATIONS="${PLATFORM_TEST_CONFIRMATIONS:-}" \ + PLATFORM_GRAFANA_OIDC_NOW_UTC="$NOW_UTC" \ + PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \ + PLATFORM_TEST_STATE="$fixture/state" \ + PLATFORM_TEST_SUDO_REFRESHED="$fixture/sudo-refreshed" \ + PLATFORM_TEST_ENCRYPTION_COUNT="$fixture/encryption-count" \ + PLATFORM_TEST_RESTORE_COUNT="$fixture/restore-count" \ + PLATFORM_TEST_ENCRYPTION_FAIL_AT="${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" \ + PLATFORM_TEST_RESTORE_FAIL_AT="${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" \ + PLATFORM_TEST_VALIDATOR_ENV_LOG="${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" \ + PLATFORM_TEST_CONTEXT="${PLATFORM_TEST_CONTEXT:-default}" \ + PLATFORM_TEST_FAULT="${PLATFORM_TEST_FAULT:-}" \ + PLATFORM_TEST_ROLLBACK_FAULT="${PLATFORM_TEST_ROLLBACK_FAULT:-}" \ + PLATFORM_TEST_CLIENT_SECRET="$CLIENT_SECRET" \ + PLATFORM_TEST_KEYCLOAK_API="$fixture/bin/keycloak-api" \ + PLATFORM_TEST_KUBE_API="$fixture/bin/kube-api" \ + bash -c 'source "$1"; shift; platform_grafana_oidc_fixture_main "$@"' \ + platform-grafana-oidc-test "$SCRIPT" "$fixture" "$@" +} + +assert_exact_contract() { + local fixture=$1 + python3 -I -S - "$fixture/state/keycloak.json" "$fixture/state/secret.json" "$CLIENT_SECRET" <<'PY' || return 1 +import base64,json,pathlib,sys +kc=json.loads(pathlib.Path(sys.argv[1]).read_text()); secret=json.loads(pathlib.Path(sys.argv[2]).read_text()); expected=sys.argv[3] +assert len(kc["clients"])==1 +c=kc["clients"][0] +assert c["clientId"]=="grafana" and c["publicClient"] is False and c["standardFlowEnabled"] is True +assert all(c[x] is False for x in ("implicitFlowEnabled","directAccessGrantsEnabled","serviceAccountsEnabled","authorizationServicesEnabled")) +assert c["rootUrl"]==c["baseUrl"]=="https://grafana.learn.hyeonworks.com" +assert c["redirectUris"]==["https://grafana.learn.hyeonworks.com/login/generic_oauth"] +assert c["webOrigins"]==["https://grafana.learn.hyeonworks.com"] and c["fullScopeAllowed"] is False +assert c["attributes"]["post.logout.redirect.uris"]=="https://grafana.learn.hyeonworks.com/*" +assert c["attributes"]["oauth2.device.authorization.grant.enabled"]=="false" and c["attributes"]["oidc.ciba.grant.enabled"]=="false" +assert c["_secret"]==expected +assert sorted(x["path"] for x in kc["groups"])==["/platform-observability-admins","/platform-observability-viewers"] +m=kc["mappers"][c["id"]]; assert len(m)==1 +assert m[0]["name"]=="grafana-groups" and m[0]["protocolMapper"]=="oidc-group-membership-mapper" +assert m[0]["config"]=={"access.token.claim":"true","claim.name":"groups","full.path":"true","id.token.claim":"true","userinfo.token.claim":"true"} +assert secret["type"]=="Opaque" and sorted(secret["data"])==["client-id","client-secret"] +assert base64.b64decode(secret["data"]["client-id"])==b"grafana" +assert base64.b64decode(secret["data"]["client-secret"]).decode()==expected +PY +} + +WORK="$(mktemp -d /tmp/platform-grafana-oidc-test.XXXXXX)"; chmod 0700 "$WORK" + +[[ -f "$SCRIPT" ]] || fail 'production Grafana OIDC bootstrap is absent' + +fixture="$(new_fixture dry-run)" +output="$(run_script "$fixture")" || fail 'no-argument dry-run failed' +grep -Fq 'GRAFANA_OIDC_DRY_RUN=PASS' <<<"$output" || fail 'dry-run marker absent' +[[ ! -s "$fixture/commands.log" ]] || fail 'dry-run crossed a system boundary' +[[ "$output" != *"$CLIENT_SECRET"* ]] || fail 'dry-run leaked a payload' +pass 'no-argument mode is payload-free and read-only' + +fixture="$(new_fixture boundary)" +for argv in '--admin admin-user' '--execute --check-recovery-evidence' '--check-recovery-evidence --viewer viewer-user' '--execute --admin' '--execute --admin admin-user --admin admin-user'; do + read -r -a args <<<"$argv" + if run_script "$fixture" "${args[@]}" >"$fixture/out" 2>&1; then fail "unsafe CLI accepted: $argv"; fi + [[ ! -s "$fixture/commands.log" ]] || fail "invalid CLI crossed a boundary: $argv" +done +if PLATFORM_GRAFANA_OIDC_TEST_MODE=1 bash "$SCRIPT" >"$fixture/direct" 2>&1; then fail 'production entrypoint accepted an environment test bypass'; fi +pass 'CLI modes are closed and production rejects test overrides' + +fixture="$(new_fixture last-gate)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_RESTORE_FAIL_AT=2 \ + run_script "$fixture" --execute >"$fixture/out" 2>&1; then fail 'last restore gate failure was accepted'; fi +[[ "$(grep -c '^encryption --expect-reencrypted$' "$fixture/commands.log")" == 2 ]] || { + sed -n '1,200p' "$fixture/out" >&2; sed -n '1,240p' "$fixture/commands.log" >&2 + [[ ! -f "$fixture/state/keycloak-api-error" ]] || cat "$fixture/state/keycloak-api-error" >&2 + fail 'encryption gate was not fresh twice' +} +[[ "$(grep -c '^restore --check$' "$fixture/commands.log")" == 2 ]] || { + sed -n '1,200p' "$fixture/out" >&2; sed -n '1,240p' "$fixture/commands.log" >&2 + fail 'restore gate was not fresh twice' +} +python3 -I -S - "$fixture/state/keycloak.json" <<'PY' || fail 'last gate mutated Keycloak' +import json,pathlib,sys +x=json.loads(pathlib.Path(sys.argv[1]).read_text()); assert x["clients"]==[] and x["groups"]==[] +PY +[[ ! -e "$fixture/state/secret.json" ]] || fail 'last gate mutated Kubernetes' +pass 'both validator pairs run in fresh processes and last-gate failure is zero-mutation' + +fixture="$(new_fixture missing-evidence-parent)" +: >"$fixture/state/remove-evidence-parent" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' \ + run_script "$fixture" --execute >"$fixture/out" 2>&1; then + fail 'missing recovery-evidence parent was accepted' +fi +! grep -Fq 'Type APPLY default:' "$fixture/out" || + fail 'missing recovery-evidence parent reached confirmation' +! grep -Eq '^keycloak (POST|PUT|DELETE) /admin/realms/hyeonworks/' "$fixture/commands.log" || + fail 'missing recovery-evidence parent reached Keycloak mutation' +[[ ! -e "$fixture/state/secret.json" ]] || + fail 'missing recovery-evidence parent reached Kubernetes Secret mutation' +pass 'missing recovery-evidence parent fails closed before object mutation' + +fixture="$(new_fixture create)" +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' run_script "$fixture" --execute >"$fixture/out" 2>&1 || { sed -n '1,220p' "$fixture/out" >&2; fail 'create path failed'; } +assert_exact_contract "$fixture" || fail 'created contract differs' +[[ "$(<"$fixture/out")" != *"$CLIENT_SECRET"* ]] || fail 'success output leaked client secret' +! grep -Fq "$CLIENT_SECRET" "$fixture/commands.log" || fail 'client secret reached argv log' +grep -Fq 'GRAFANA_OIDC_TRANSACTION=PASS' "$fixture/out" || fail 'success marker absent' +marker="$fixture/evidence-parent/recovery-evidence/keycloak.env" +[[ -f "$marker" && ! -L "$marker" && "$(stat -c '%u:%a:%h' "$marker")" == "$(id -u):600:1" ]] || fail 'marker metadata differs' +[[ "$(wc -l <"$marker" | tr -d '[:space:]')" == 4 ]] || fail 'marker key set differs' +grep -Fqx 'schema=platform-observability-recovery-evidence-v1' "$marker" || fail 'marker schema differs' +grep -Fqx 'context=default' "$marker" || fail 'marker context differs' +grep -Fqx 'resource=keycloak/hyeonworks/client/grafana' "$marker" || fail 'marker resource differs' +grep -Fqx "checked_at_utc=$NOW_UTC" "$marker" || fail 'marker timestamp differs' +pass 'create path establishes the exact client, groups, mapper, Secret, and evidence marker' + +fixture="$(new_fixture hostile-curlrc)" +mkdir -m 0700 -- "$fixture/curl-home" +printf 'trace-ascii = "%s"\n' "$fixture/curl-trace" >"$fixture/curl-home/.curlrc" +chmod 0600 -- "$fixture/curl-home/.curlrc" +CURL_HOME="$fixture/curl-home" \ + PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' \ + run_script "$fixture" --execute >"$fixture/out" 2>&1 || { + sed -n '1,220p' "$fixture/out" >&2 + fail 'hostile curl default-config fixture did not complete' + } +[[ ! -e "$fixture/curl-trace" ]] || fail 'curl loaded a user default config and wrote a credential trace' +pass 'curl ignores user default configuration across credential-bearing requests' + +marker="$fixture/evidence-parent/recovery-evidence/keycloak.env" +: >"$fixture/commands.log" +run_script "$fixture" --check-recovery-evidence >"$fixture/check" 2>&1 || fail 'fresh evidence rejected' +grep -Fq 'KEYCLOAK_RECOVERY_EVIDENCE=PASS' "$fixture/check" || fail 'check marker absent' +! grep -q '^encryption\|^restore' "$fixture/commands.log" || fail 'check mode ran mutation gates' +grep -q '^sudo ' "$fixture/commands.log" || fail 'check did not use narrow sudo' +if PLATFORM_TEST_CONTEXT=other run_script "$fixture" --check-recovery-evidence >"$fixture/wrong" 2>&1; then fail 'wrong context accepted evidence'; fi +ln -- "$marker" "$marker.second" +if run_script "$fixture" --check-recovery-evidence >"$fixture/link" 2>&1; then fail 'multiply linked marker accepted'; fi +unlink -- "$marker.second" +cp -- "$marker" "$fixture/marker-valid" +sed -i 's/checked_at_utc=.*/checked_at_utc=2026-06-01T00:00:00Z/' "$marker" +if run_script "$fixture" --check-recovery-evidence >"$fixture/stale" 2>&1; then fail 'stale recovery evidence accepted'; fi +cp -- "$fixture/marker-valid" "$marker" +printf 'extra=forbidden\n' >>"$marker" +if run_script "$fixture" --check-recovery-evidence >"$fixture/extra" 2>&1; then fail 'extra recovery evidence field accepted'; fi +cp -- "$fixture/marker-valid" "$marker" +mv -- "$fixture/evidence-parent/recovery-evidence" "$fixture/evidence-parent/recovery-evidence-real" +ln -s -- recovery-evidence-real "$fixture/evidence-parent/recovery-evidence" +if run_script "$fixture" --check-recovery-evidence >"$fixture/symlink" 2>&1; then fail 'symlink evidence directory accepted'; fi +unlink -- "$fixture/evidence-parent/recovery-evidence" +mv -- "$fixture/evidence-parent/recovery-evidence-real" "$fixture/evidence-parent/recovery-evidence" +pass 'evidence check binds context, age, schema, and no-follow metadata' + +fixture="$(new_fixture existing existing)"; seed_secret "$fixture" +before_secret="$(sha256sum "$fixture/state/secret.json" | awk '{print $1}')" +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' run_script "$fixture" --execute --admin admin-user --viewer viewer-user >"$fixture/out" 2>&1 || { + sed -n '1,240p' "$fixture/out" >&2; sed -n '1,300p' "$fixture/commands.log" >&2 + fail 'existing update failed' +} +assert_exact_contract "$fixture" || fail 'existing update contract differs' +[[ "$(sha256sum "$fixture/state/secret.json" | awk '{print $1}')" == "$before_secret" ]] || fail 'exact existing Secret was rotated or rewritten' +python3 -I -S - "$fixture/state/keycloak.json" <<'PY' || fail 'requested memberships absent' +import json,pathlib,sys +x=json.loads(pathlib.Path(sys.argv[1]).read_text()); ids={g["path"]:g["id"] for g in x["groups"]}; users={u["username"]:u for u in x["users"]} +assert ids["/platform-observability-admins"] in users["admin-user"]["groups"] +assert ids["/platform-observability-viewers"] in users["viewer-user"]["groups"] +PY +pass 'existing client is declaratively updated without credential rotation and memberships are optional' + +fixture="$(new_fixture exact-noop exact-secret)"; seed_secret "$fixture" +before_keycloak="$(sha256sum "$fixture/state/keycloak.json" | awk '{print $1}')" +before_exact_secret="$(sha256sum "$fixture/state/secret.json" | awk '{print $1}')" +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' run_script "$fixture" --execute >"$fixture/out" 2>&1 || { + sed -n '1,240p' "$fixture/out" >&2; fail 'exact-existing no-op failed' +} +[[ "$(sha256sum "$fixture/state/keycloak.json" | awk '{print $1}')" == "$before_keycloak" ]] || fail 'exact-existing Keycloak state was rewritten' +[[ "$(sha256sum "$fixture/state/secret.json" | awk '{print $1}')" == "$before_exact_secret" ]] || fail 'exact-existing Secret was rewritten' +! grep -q '^keycloak PUT ' "$fixture/commands.log" || fail 'exact-existing Keycloak state was PUT' +[[ -f "$fixture/evidence-parent/recovery-evidence/keycloak.env" ]] || fail 'exact-existing no-op omitted recovery evidence' +pass 'exact existing live-style state is a no-op except recovery evidence validation' + +for profile in duplicate-client duplicate-group duplicate-user mapper-duplicate; do + fixture="$(new_fixture "$profile" "$profile")"; seed_secret "$fixture" + args=(--execute); [[ "$profile" != duplicate-user ]] || args+=(--admin admin-user) + if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' run_script "$fixture" "${args[@]}" >"$fixture/out" 2>&1; then fail "$profile ambiguity accepted"; fi + ! grep -q 'request=\(POST\|PUT\|DELETE\)' "$fixture/commands.log" || fail "$profile reached mutation" +done +pass 'duplicate client, group, mapper, and user states fail before mutation' + +fixture="$(new_fixture mismatch existing)"; seed_secret "$fixture" 'DifferentClientSecret-123456789' +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' run_script "$fixture" --execute >"$fixture/out" 2>&1; then fail 'mismatched existing Secret accepted'; fi +python3 -I -S - "$fixture/state/keycloak.json" <<'PY' || fail 'mismatch mutated client' +import json,pathlib,sys +x=json.loads(pathlib.Path(sys.argv[1]).read_text()); assert x["clients"][0]["name"]=="Drifted Grafana" +PY +pass 'existing Secret/client mismatch refuses rotation with zero mutation' + +for malformed_client_id in trailing-newline trailing-nul; do + fixture="$(new_fixture "client-id-$malformed_client_id" existing)"; seed_secret "$fixture" + python3 -I -S - "$fixture/state/secret.json" "$malformed_client_id" <<'PY' +import base64,json,pathlib,sys +path=pathlib.Path(sys.argv[1]); variant=sys.argv[2] +item=json.loads(path.read_text()) +payload={"trailing-newline":b"grafana\n","trailing-nul":b"grafana\0"}[variant] +item["data"]["client-id"]=base64.b64encode(payload).decode() +path.write_text(json.dumps(item,sort_keys=True)) +PY + cp -- "$fixture/state/keycloak.json" "$fixture/before.json" + if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' \ + run_script "$fixture" --execute >"$fixture/out" 2>&1; then + fail "$malformed_client_id OIDC Secret client-id was accepted" + fi + cmp --silent -- "$fixture/before.json" "$fixture/state/keycloak.json" || + fail "$malformed_client_id OIDC Secret reached Keycloak mutation" +done +pass 'OIDC Secret client-id must decode to the exact grafana byte sequence' + +fixture="$(new_fixture rollback preexisting-admin)"; seed_secret "$fixture" +cp -- "$fixture/state/keycloak.json" "$fixture/before.json" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT=http500-viewer-member-put \ + run_script "$fixture" --execute --admin admin-user --viewer viewer-user >"$fixture/out" 2>&1; then fail 'membership failure accepted'; fi +cmp --silent "$fixture/before.json" "$fixture/state/keycloak.json" || fail 'rollback did not restore exact Keycloak prestate' +[[ "$(sha256sum "$fixture/state/secret.json" | awk '{print $1}')" == "$before_secret" ]] || fail 'rollback rewrote prior Secret' +grep -Fq 'GRAFANA_OIDC_ROLLBACK=PASS' "$fixture/out" || fail 'rollback marker absent' +pass 'rollback preserves preexisting membership and removes only transaction-added membership' + +for rollback_fault in noop-group-delete noop-mapper-delete; do + fixture="$(new_fixture "$rollback_fault")" + if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' \ + PLATFORM_TEST_FAULT=http500-viewer-member-put \ + PLATFORM_TEST_ROLLBACK_FAULT="$rollback_fault" \ + run_script "$fixture" --execute --viewer viewer-user >"$fixture/out" 2>&1; then + fail "$rollback_fault trigger was accepted" + fi + [[ -f "$fixture/state/rollback-fault-fired-$rollback_fault" ]] || + fail "$rollback_fault fixture did not intercept the rollback DELETE" + ! grep -Fq 'GRAFANA_OIDC_ROLLBACK=PASS' "$fixture/out" || + fail "$rollback_fault produced a false rollback PASS" + grep -Fq 'GRAFANA_OIDC_ROLLBACK=FAIL' "$fixture/out" || + fail "$rollback_fault omitted rollback failure evidence" + grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/out" || + fail "$rollback_fault omitted manual recovery evidence" +done +pass 'created Keycloak object rollback requires exact post-delete absence' + +for fault in loss-group-put loss-client-create loss-client-put loss-mapper-put loss-secret-create conflict-secret-create timeout-client-create; do + fixture="$(new_fixture "fault-$fault")" + if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT="$fault" run_script "$fixture" --execute >"$fixture/out" 2>&1; then fail "$fault accepted"; fi + python3 -I -S - "$fixture/state/keycloak.json" <<'PY' || fail "$fault left Keycloak objects" +import json,pathlib,sys +x=json.loads(pathlib.Path(sys.argv[1]).read_text()); assert x["clients"]==[] and x["groups"]==[] +PY + [[ ! -e "$fixture/state/secret.json" ]] || fail "$fault left transaction Secret" +done +pass 'response loss and Kubernetes conflict classify ownership and reverse exact transaction state' + +fixture="$(new_fixture update-response-loss existing)"; seed_secret "$fixture"; cp -- "$fixture/state/keycloak.json" "$fixture/before.json" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT=loss-client-put \ + run_script "$fixture" --execute >"$fixture/out" 2>&1; then fail 'lost client update response was accepted'; fi +cmp --silent -- "$fixture/before.json" "$fixture/state/keycloak.json" || fail 'lost update response did not restore exact client prestate' +grep -Fq 'GRAFANA_OIDC_ROLLBACK=PASS' "$fixture/out" || { + sed -n '1,240p' "$fixture/out" >&2; fail 'lost update response omitted rollback proof' +} +pass 'lost Keycloak update response is classified and exact prior client state is restored' + +for sig in HUP INT TERM; do + fixture="$(new_fixture "signal-${sig,,}" existing)"; seed_secret "$fixture"; cp -- "$fixture/state/keycloak.json" "$fixture/before.json" + if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT="signal-$sig-viewer-member-put" \ + run_script "$fixture" --execute --admin admin-user --viewer viewer-user >"$fixture/out" 2>&1; then fail "$sig was accepted"; fi + cmp --silent -- "$fixture/before.json" "$fixture/state/keycloak.json" || fail "$sig did not restore exact Keycloak prestate" + grep -Fq 'GRAFANA_OIDC_ROLLBACK=PASS' "$fixture/out" || fail "$sig did not complete the rollback path" + ! grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/out" || fail "$sig exact ownership was misclassified as ambiguous" +done +pass 'HUP, INT, and TERM classify exact ownership and restore the transaction prestate' + +fixture="$(new_fixture exit-fallback existing)"; seed_secret "$fixture"; cp -- "$fixture/state/keycloak.json" "$fixture/before.json" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT=signal-USR1-viewer-member-put \ + run_script "$fixture" --execute --admin admin-user --viewer viewer-user >"$fixture/out" 2>&1; then fail 'unhandled fatal signal was accepted'; fi +cmp --silent -- "$fixture/before.json" "$fixture/state/keycloak.json" || fail 'EXIT fallback did not restore exact Keycloak prestate' +grep -Fq 'GRAFANA_OIDC_ROLLBACK=PASS' "$fixture/out" || fail 'EXIT fallback omitted rollback proof' +pass 'EXIT fallback classifies pending ownership and restores exact prestate' + +fixture="$(new_fixture secret-uid-conflict)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT=drift-secret-viewer-member-put \ + run_script "$fixture" --execute --viewer viewer-user >"$fixture/out" 2>&1; then fail 'concurrent Secret replacement was accepted'; fi +[[ -f "$fixture/state/secret.json" ]] || fail 'concurrent Secret replacement was deleted' +python3 -I -S - "$fixture/state/secret.json" <<'PY' || fail 'concurrent Secret replacement identity was not preserved' +import json,pathlib,sys +item=json.loads(pathlib.Path(sys.argv[1]).read_text()) +assert item["metadata"]["uid"]=="secret-concurrent-replacement" and item["metadata"]["resourceVersion"]=="99" +PY +grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/out" || fail 'Secret UID/RV conflict omitted manual recovery marker' +pass 'Secret UID and resourceVersion preconditions preserve a concurrent replacement' + +for line in $(grep '^kubectl ' "$fixture/commands.log"); do :; done +grep '^kubectl ' "$fixture/commands.log" | grep -vq -- '--request-timeout=5s' && fail 'an unbounded kubectl invocation was observed' +[[ "$(<"$fixture/out")" != *"$CLIENT_SECRET"* ]] || fail 'fault output leaked payload' +! grep -Fq "$CLIENT_SECRET" "$fixture/commands.log" || fail 'fault argv leaked payload' +pass 'kubectl and curl boundaries are bounded and payloads stay out of output and argv' + +printf 'KEYCLOAK GRAFANA OIDC TEST PASS (%s assertions)\n' "$ASSERTIONS" diff --git a/scripts/validate/test-create-observability-secrets.sh b/scripts/validate/test-create-observability-secrets.sh new file mode 100755 index 0000000..de4d46e --- /dev/null +++ b/scripts/validate/test-create-observability-secrets.sh @@ -0,0 +1,1728 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly SCRIPT="$ROOT/scripts/bootstrap/create-observability-secrets.sh" +readonly NOW_UTC='2026-08-12T07:00:00Z' +readonly ADMIN_USER='breakglass-admin' +readonly ADMIN_PASSWORD='GrafanaPassword-123456789' +readonly SLACK_URL='https://hooks.slack.com/services/T123/B456/S789' + +WORK='' +ASSERTIONS=0 + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +pass() { + ASSERTIONS=$((ASSERTIONS + 1)) + printf 'PASS: %s\n' "$1" +} + +cleanup() { + trap - EXIT HUP INT TERM + case "$WORK" in + /tmp/platform-observability-secrets-test.??????) + rm -rf -- "$WORK" + ;; + esac +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +write_executable() { + local path=$1 + shift + printf '%s\n' "$@" >"$path" + chmod 0755 "$path" +} + +make_fakes() { + local fixture=$1 + mkdir -p "$fixture/bin" "$fixture/state" "$fixture/evidence-parent" + chmod 0700 "$fixture" "$fixture/state" "$fixture/evidence-parent" + chmod 0755 "$fixture/bin" + : >"$fixture/commands.log" + + write_executable "$fixture/bin/encryption" \ + '#!/usr/bin/env bash' \ + 'set -Eeuo pipefail' \ + 'if [[ -n "${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" ]]; then /usr/bin/env | LC_ALL=C /usr/bin/sort >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; printf '\''%s\n'\'' '\''--'\'' >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; fi' \ + 'printf '\''encryption'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'count=0; [[ ! -f "$PLATFORM_TEST_ENCRYPTION_COUNT" ]] || read -r count <"$PLATFORM_TEST_ENCRYPTION_COUNT"' \ + 'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_ENCRYPTION_COUNT"' \ + 'if [[ "$count" == 2 && -n "${PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE:-}" ]]; then printf '\''%s'\'' '\''MutatedPassword-987654321'\'' >"$PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE"; fi' \ + '[[ "$count" != "${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" ]]' + + write_executable "$fixture/bin/restore" \ + '#!/usr/bin/env bash' \ + 'set -Eeuo pipefail' \ + 'if [[ -n "${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" ]]; then /usr/bin/env | LC_ALL=C /usr/bin/sort >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; printf '\''%s\n'\'' '\''--'\'' >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; fi' \ + 'printf '\''restore'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'count=0; [[ ! -f "$PLATFORM_TEST_RESTORE_COUNT" ]] || read -r count <"$PLATFORM_TEST_RESTORE_COUNT"' \ + 'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_RESTORE_COUNT"' \ + '[[ "$count" != "${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" ]]' + + write_executable "$fixture/bin/sudo" \ + '#!/usr/bin/env bash' \ + 'set -Eeuo pipefail' \ + 'printf '\''sudo'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \ + 'if [[ "${1:-}" == -v ]]; then : >"$PLATFORM_TEST_SUDO_REFRESHED"; exit 0; fi' \ + '[[ -f "$PLATFORM_TEST_SUDO_REFRESHED" ]] || exit 92' \ + '[[ "${1:-}" != -n && "${1:-}" != --non-interactive ]] || shift' \ + 'command=${1:-}; shift || :' \ + 'case "$command" in' \ + ' /usr/bin/python3)' \ + ' if [[ "${4:-}" == platform-observability-marker-helper && ( "${5:-}" == check-dir || "${5:-}" == ensure-dir ) && ( -n "${PLATFORM_TEST_HIERARCHY_FAULT:-}" || -n "${PLATFORM_TEST_HIERARCHY_TRACE:-}" ) ]]; then' \ + ' exec "$PLATFORM_TEST_HIERARCHY_SHIM" "$@"' \ + ' fi' \ + ' if [[ "${4:-}" == platform-observability-marker-helper && "${5:-}" == install ]]; then' \ + ' count=0; [[ ! -f "$PLATFORM_TEST_MARKER_MOVE_COUNT" ]] || read -r count <"$PLATFORM_TEST_MARKER_MOVE_COUNT"' \ + ' count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_MARKER_MOVE_COUNT"' \ + ' [[ "$count" != "${PLATFORM_TEST_MARKER_MOVE_FAIL_AT:-0}" ]] || exit 75' \ + ' if /usr/bin/python3 "$@"; then marker_rc=0; else marker_rc=$?; fi' \ + ' (( marker_rc == 0 )) || exit "$marker_rc"' \ + ' [[ "$count" != "${PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT:-0}" ]] || exit 76' \ + ' if [[ "$count" == "${PLATFORM_TEST_MARKER_SIGNAL_AT:-0}" ]]; then /bin/kill -"${PLATFORM_TEST_MARKER_SIGNAL:-HUP}" "$PPID"; fi' \ + ' exit 0' \ + ' fi' \ + ' exec /usr/bin/python3 "$@" ;;' \ + ' /usr/bin/install|/bin/install|install)' \ + ' args=(); while (( $# > 0 )); do case "$1" in -o|-g) shift 2 ;; *) args+=("$1"); shift ;; esac; done' \ + ' exec /usr/bin/install "${args[@]}" ;;' \ + ' /usr/bin/stat|/bin/stat|stat)' \ + ' format=; path=; while (( $# > 0 )); do case "$1" in --format=*) format=${1#*=}; shift ;; --format) format=$2; shift 2 ;; --) shift ;; *) path=$1; shift ;; esac; done' \ + ' [[ "$format" == "%u:%g:%a:%F" ]] || exit 93' \ + ' metadata=$(/usr/bin/stat --format="%a:%F" -- "$path")' \ + ' printf '\''0:0:%s\n'\'' "$metadata" ;;' \ + ' /bin/mv|/usr/bin/mv)' \ + ' count=0; [[ ! -f "$PLATFORM_TEST_MARKER_MOVE_COUNT" ]] || read -r count <"$PLATFORM_TEST_MARKER_MOVE_COUNT"' \ + ' count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_MARKER_MOVE_COUNT"' \ + ' [[ "$count" != "${PLATFORM_TEST_MARKER_MOVE_FAIL_AT:-0}" ]] || exit 75' \ + ' exec /bin/mv "$@" ;;' \ + ' *) exec "$command" "$@" ;;' \ + 'esac' + + cat >"$fixture/bin/hierarchy-python" <<'PY' +#!/usr/bin/env python3 +import os +import pathlib +import sys + +arguments = sys.argv[1:] +if arguments[:3] != ["-I", "-S", "-"]: + raise SystemExit(90) +sys.argv = ["-"] + arguments[3:] + +fixture = pathlib.Path(os.environ["PLATFORM_TEST_STATE"]).parent +fault = os.environ.get("PLATFORM_TEST_HIERARCHY_FAULT", "") +trace_raw = os.environ.get("PLATFORM_TEST_HIERARCHY_TRACE", "") +phase = None +component = None +target = None +if fault: + try: + phase, component = fault.split(":", 1) + except ValueError: + raise SystemExit(91) + if phase not in { + "wrong-uid", + "wrong-gid", + "wrong-mode", + "replace-after-open", + "replace-after-create", + "replace-after-reopen", + "replace-before-attestation", + "replace-before-safe-absence", + "concurrent-create-exact", + "concurrent-create-unsafe", + }: + raise SystemExit(91) + if component not in {"evidence-parent", "platform", "recovery-evidence"}: + raise SystemExit(91) + if component == "evidence-parent": + target = fixture / component + elif component == "platform": + target = fixture / "evidence-parent" / component + else: + target = fixture / "evidence-parent" / "platform" / component +if not fault and not trace_raw: + raise SystemExit(91) +trace_path = pathlib.Path(trace_raw) if trace_raw else None +managed_paths = { + fixture / "evidence-parent": "evidence-parent", + fixture / "evidence-parent" / "platform": "platform", + fixture / "evidence-parent" / "platform" / "recovery-evidence": "recovery-evidence", +} + +real_open = os.open +real_mkdir = os.mkdir +real_fstat = os.fstat +real_fsync = os.fsync +real_rename = os.rename +real_chmod = os.chmod +created = set() +reopened = set() +attested = set() +fsynced = set() +injected = False + + +def fd_path(descriptor): + try: + return pathlib.Path(os.readlink(f"/proc/self/fd/{descriptor}")) + except OSError: + return None + + +def call_path(path, directory_fd): + decoded = pathlib.Path(os.fsdecode(path)) + if decoded.is_absolute(): + return decoded + parent = fd_path(directory_fd) + if parent is None: + return decoded + return parent / decoded + + +def replace_target(): + backup = target.with_name(f".{target.name}.{phase}.original") + real_rename(target, backup) + real_mkdir(target, 0o700) + real_chmod(target, 0o755 if phase == "replace-after-create" else 0o700) + pathlib.Path(os.environ["PLATFORM_TEST_HIERARCHY_FAULT_LOG"]).write_text( + f"{phase}:{component}\n", + encoding="utf-8", + ) + + +def create_concurrent_target(created_path): + global injected + injected = True + real_mkdir(created_path, 0o700) + real_chmod(created_path, 0o755 if phase == "concurrent-create-unsafe" else 0o700) + created.add(created_path) + trace_event("concurrent-create", created_path) + pathlib.Path(os.environ["PLATFORM_TEST_HIERARCHY_FAULT_LOG"]).write_text( + f"{phase}:{component}\n", + encoding="utf-8", + ) + + +def trace_event(event, path): + if trace_path is None or path not in managed_paths: + return + with trace_path.open("a", encoding="utf-8") as stream: + stream.write(f"{event}:{managed_paths[path]}\n") + + +def shim_open(path, flags, mode=0o777, *, dir_fd=None): + global injected + opened_path = call_path(path, dir_fd) + try: + if dir_fd is None: + descriptor = real_open(path, flags, mode) + else: + descriptor = real_open(path, flags, mode, dir_fd=dir_fd) + except FileNotFoundError: + safe_absence_child = None + if component == "evidence-parent": + safe_absence_child = target / "platform" + elif component == "platform": + safe_absence_child = target / "recovery-evidence" + if phase == "replace-before-safe-absence" and opened_path == safe_absence_child and not injected: + injected = True + replace_target() + raise + should_replace = ( + opened_path == target + and not injected + and ( + phase == "replace-after-open" + or (phase == "replace-after-reopen" and target in created) + ) + ) + if should_replace: + injected = True + replace_target() + if opened_path in created and opened_path not in reopened: + reopened.add(opened_path) + trace_event("reopen", opened_path) + return descriptor + + +def shim_mkdir(path, mode=0o777, *, dir_fd=None): + global injected + created_path = call_path(path, dir_fd) + if ( + phase in {"concurrent-create-exact", "concurrent-create-unsafe"} + and created_path == target + and not injected + ): + create_concurrent_target(created_path) + if dir_fd is None: + result = real_mkdir(path, mode) + else: + result = real_mkdir(path, mode, dir_fd=dir_fd) + created.add(created_path) + trace_event("mkdir", created_path) + if phase == "replace-after-create" and created_path == target and not injected: + injected = True + replace_target() + return result + + +def shim_fsync(descriptor): + parent = fd_path(descriptor) + for created_path in reversed(tuple(created)): + if created_path.parent == parent and created_path not in fsynced: + fsynced.add(created_path) + trace_event("fsync-parent", created_path) + break + return real_fsync(descriptor) + + +def shim_fstat(descriptor): + global injected + opened_path = fd_path(descriptor) + if ( + phase == "replace-before-attestation" + and opened_path == target + and target in created + and not injected + ): + injected = True + replace_target() + metadata = real_fstat(descriptor) + if opened_path in created and opened_path not in attested: + attested.add(opened_path) + trace_event("attest", opened_path) + if opened_path != target or injected or phase not in {"wrong-uid", "wrong-gid", "wrong-mode"}: + return metadata + injected = True + values = list(metadata) + if phase == "wrong-uid": + values[4] = metadata.st_uid + 1 + elif phase == "wrong-gid": + values[5] = metadata.st_gid + 1 + else: + values[0] = (metadata.st_mode & ~0o7777) | 0o755 + return os.stat_result(values) + + +os.open = shim_open +os.mkdir = shim_mkdir +os.fstat = shim_fstat +os.fsync = shim_fsync +source = sys.stdin.read() +exec(compile(source, "", "exec"), {"__name__": "__main__"}) +PY + chmod 0755 "$fixture/bin/hierarchy-python" + + cat >"$fixture/bin/kubectl" <<'PY' +#!/usr/bin/env python3 +import base64 +import http.server +import json +import os +import pathlib +import signal +import socketserver +import sys +import time + +args = sys.argv[1:] +state = pathlib.Path(os.environ["PLATFORM_TEST_STATE"]) +log = pathlib.Path(os.environ["PLATFORM_TEST_COMMAND_LOG"]) +with log.open("a", encoding="utf-8") as stream: + stream.write("kubectl " + " ".join(args) + "\n") + +required_timeout = os.environ.get("PLATFORM_TEST_REQUIRE_REQUEST_TIMEOUT", "0") == "1" +if required_timeout and args and "--request-timeout=5s" not in args: + raise SystemExit(84) +required_process_timeout = os.environ.get("PLATFORM_TEST_REQUIRE_PROCESS_TIMEOUT", "0") == "1" +if required_process_timeout: + parent_cmdline = pathlib.Path(f"/proc/{os.getppid()}/cmdline").read_bytes().split(b"\0") + if not parent_cmdline or parent_cmdline[0] != b"/usr/bin/timeout": + raise SystemExit(85) + +def option(name, default=None): + for i, value in enumerate(args): + if value == name and i + 1 < len(args): + return args[i + 1] + if value.startswith(name + "="): + return value.split("=", 1)[1] + return default + +def normalized(): + output = [] + skip = False + for i, value in enumerate(args): + if skip: + skip = False + continue + if value in {"--namespace", "-n"}: + skip = True + continue + if value.startswith("--namespace="): + continue + if value.startswith("--request-timeout="): + continue + output.append(value) + return output + +def secret_path(name): + return state / f"secret__observability__{name}.json" + +def load_secret(name): + path = secret_path(name) + return json.loads(path.read_text()) if path.exists() else None + +def save_secret(name, item): + secret_path(name).write_text(json.dumps(item), encoding="utf-8") + +nargs = normalized() +if os.environ.get("PLATFORM_TEST_HANG_KUBECTL", "0") == "1" and nargs and nargs[0] != "proxy": + import time + time.sleep(30) +if nargs and nargs[0] == "proxy": + sock = option("--unix-socket") + if not sock: + raise SystemExit(81) + (state / "proxy-pid").write_text(str(os.getpid())) + + class Server(socketserver.UnixStreamServer): + allow_reuse_address = False + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + return + + def do_DELETE(self): + name = self.path.rsplit("/", 1)[-1] + length = int(self.headers.get("Content-Length", "0")) + try: + body = json.loads(self.rfile.read(length)) + expected_uid = body["preconditions"]["uid"] + except Exception: + self.send_response(400); self.end_headers(); return + item = load_secret(name) + if item is None: + self.send_response(404); self.end_headers(); return + if item["metadata"]["uid"] != expected_uid: + self.send_response(409); self.end_headers(); return + secret_path(name).unlink() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"kind":"Status","status":"Success"}') + + with Server(sock, Handler) as server: + server.serve_forever() + raise SystemExit(0) + +if nargs == ["config", "current-context"]: + print(os.environ.get("PLATFORM_TEST_CONTEXT", "default")) + raise SystemExit(0) +if nargs[:2] == ["get", "namespace"] and nargs[2:] == ["observability"]: + raise SystemExit(0) +if len(nargs) >= 3 and nargs[:2] == ["get", "secret"]: + name = nargs[2] + item = load_secret(name) + ignore = "--ignore-not-found" in args + if item is None: + if ignore: + raise SystemExit(0) + raise SystemExit(1) + output = option("--output", option("-o", "")) + if output == "json" and os.environ.get("PLATFORM_TEST_POSTCREATE_EXTRA_KEY", "0") == "1" and item["metadata"]["uid"].startswith("uid-owned-"): + item["data"]["extra"] = base64.b64encode(b"drift").decode() + if output in {"json", "jsonpath={.metadata.uid}"}: + if output == "json": + print(json.dumps(item)) + else: + print(item["metadata"]["uid"], end="") + elif output == "name": + print("secret/" + name) + else: + raise SystemExit(82) + raise SystemExit(0) +if nargs and nargs[0] == "create" and option("--filename"): + item = json.loads(pathlib.Path(option("--filename")).read_text()) + name = item["metadata"]["name"] + count_file = pathlib.Path(os.environ["PLATFORM_TEST_CREATE_COUNT"]) + count = int(count_file.read_text()) if count_file.exists() else 0 + count += 1 + count_file.write_text(str(count)) + if secret_path(name).exists(): + raise SystemExit(1) + item["metadata"]["uid"] = f"uid-owned-{count}" + ambiguous = count == int(os.environ.get("PLATFORM_TEST_CREATE_AMBIGUOUS_AT", "0")) + failing = count == int(os.environ.get("PLATFORM_TEST_CREATE_FAIL_AT", "0")) + if ambiguous or not failing: + save_secret(name, item) + signal_at = int(os.environ.get("PLATFORM_TEST_CREATE_SIGNAL_AT", "0")) + if count == signal_at: + signal_name = os.environ.get("PLATFORM_TEST_CREATE_SIGNAL", "HUP") + signal_number = getattr(signal, "SIG" + signal_name) + timeout_parent = os.getppid() + timeout_status = pathlib.Path(f"/proc/{timeout_parent}/stat").read_text().split() + shell_parent = int(timeout_status[3]) + os.kill(shell_parent, signal_number) + time.sleep(0.2) + raise SystemExit(74) + if failing or ambiguous: + if os.environ.get("PLATFORM_TEST_DRIFT_UID_ON_CREATE_FAILURE", "0") == "1": + for path in state.glob("secret__observability__*.json"): + current = json.loads(path.read_text()) + current["metadata"]["uid"] = "uid-external-replacement" + path.write_text(json.dumps(current)) + raise SystemExit(73) + if option("--output", option("-o", "")) == "json": + print(json.dumps(item)) + sys.stdout.flush() + else: + print(f"secret/{name} created") + sys.stdout.flush() + signal_after_response_at = int(os.environ.get("PLATFORM_TEST_CREATE_SIGNAL_AFTER_RESPONSE_AT", "0")) + if count == signal_after_response_at: + signal_name = os.environ.get("PLATFORM_TEST_CREATE_SIGNAL", "HUP") + signal_number = getattr(signal, "SIG" + signal_name) + timeout_parent = os.getppid() + timeout_status = pathlib.Path(f"/proc/{timeout_parent}/stat").read_text().split() + shell_parent = int(timeout_status[3]) + os.kill(shell_parent, signal_number) + time.sleep(0.2) + raise SystemExit(74) + raise SystemExit(0) +raise SystemExit(83) +PY + chmod 0755 "$fixture/bin/kubectl" +} + +new_fixture() { + local name=$1 fixture + fixture="$WORK/$name" + mkdir -p "$fixture" + make_fakes "$fixture" + printf '%s' "$ADMIN_USER" >"$fixture/admin-user" + printf '%s' "$ADMIN_PASSWORD" >"$fixture/admin-password" + printf '%s' "$SLACK_URL" >"$fixture/slack-url" + chmod 0600 "$fixture/admin-user" "$fixture/admin-password" "$fixture/slack-url" + printf '%s\n' "$fixture" +} + +create_exact_evidence_hierarchy() { + local fixture=$1 + mkdir -m 0700 -- "$fixture/evidence-parent/platform" + mkdir -m 0700 -- "$fixture/evidence-parent/platform/recovery-evidence" +} + +prepare_managed_component_parent() { + local fixture=$1 component=$2 + case "$component" in + evidence-parent|platform) ;; + recovery-evidence) mkdir -m 0700 -- "$fixture/evidence-parent/platform" ;; + *) fail "unknown managed component: $component" ;; + esac +} + +managed_component_path() { + local fixture=$1 component=$2 + case "$component" in + evidence-parent) printf '%s\n' "$fixture/evidence-parent" ;; + platform) printf '%s\n' "$fixture/evidence-parent/platform" ;; + recovery-evidence) printf '%s\n' "$fixture/evidence-parent/platform/recovery-evidence" ;; + *) return 1 ;; + esac +} + +seed_secret() { + local fixture=$1 name=$2 type=$3 keys=$4 + python3 -I -S - "$fixture" "$name" "$type" "$keys" <<'PY' +import base64, json, pathlib, sys +fixture, name, secret_type, keys = sys.argv[1:] +values = { + "admin-user": "breakglass-admin", + "admin-password": "GrafanaPassword-123456789", + "url": "https://hooks.slack.com/services/T123/B456/S789", + "extra": "not-secret-fixture", +} +item = { + "apiVersion": "v1", "kind": "Secret", "type": secret_type, + "metadata": {"namespace": "observability", "name": name, "uid": "uid-existing-" + name}, + "data": {key: base64.b64encode(values[key].encode()).decode() for key in keys.split(",") if key}, +} +path = pathlib.Path(fixture) / "state" / f"secret__observability__{name}.json" +path.write_text(json.dumps(item)) +PY +} + +run_script() { + local fixture=$1 + shift + env \ + PATH="$fixture/bin:$PATH" \ + PLATFORM_OBSERVABILITY_SECRETS_CONFIRMATIONS="${PLATFORM_TEST_CONFIRMATIONS:-}" \ + PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC="$NOW_UTC" \ + PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE="${PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE:-}" \ + PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \ + PLATFORM_TEST_STATE="$fixture/state" \ + PLATFORM_TEST_SUDO_REFRESHED="$fixture/sudo-refreshed" \ + PLATFORM_TEST_ENCRYPTION_COUNT="$fixture/encryption-count" \ + PLATFORM_TEST_RESTORE_COUNT="$fixture/restore-count" \ + PLATFORM_TEST_CREATE_COUNT="$fixture/create-count" \ + PLATFORM_TEST_MARKER_MOVE_COUNT="$fixture/marker-move-count" \ + PLATFORM_TEST_ENCRYPTION_FAIL_AT="${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" \ + PLATFORM_TEST_RESTORE_FAIL_AT="${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" \ + PLATFORM_TEST_CREATE_FAIL_AT="${PLATFORM_TEST_CREATE_FAIL_AT:-0}" \ + PLATFORM_TEST_CREATE_AMBIGUOUS_AT="${PLATFORM_TEST_CREATE_AMBIGUOUS_AT:-0}" \ + PLATFORM_TEST_CREATE_SIGNAL_AT="${PLATFORM_TEST_CREATE_SIGNAL_AT:-0}" \ + PLATFORM_TEST_CREATE_SIGNAL_AFTER_RESPONSE_AT="${PLATFORM_TEST_CREATE_SIGNAL_AFTER_RESPONSE_AT:-0}" \ + PLATFORM_TEST_CREATE_SIGNAL="${PLATFORM_TEST_CREATE_SIGNAL:-HUP}" \ + PLATFORM_TEST_DRIFT_UID_ON_CREATE_FAILURE="${PLATFORM_TEST_DRIFT_UID_ON_CREATE_FAILURE:-0}" \ + PLATFORM_TEST_POSTCREATE_EXTRA_KEY="${PLATFORM_TEST_POSTCREATE_EXTRA_KEY:-0}" \ + PLATFORM_TEST_MARKER_MOVE_FAIL_AT="${PLATFORM_TEST_MARKER_MOVE_FAIL_AT:-0}" \ + PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT="${PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT:-0}" \ + PLATFORM_TEST_MARKER_SIGNAL_AT="${PLATFORM_TEST_MARKER_SIGNAL_AT:-0}" \ + PLATFORM_TEST_MARKER_SIGNAL="${PLATFORM_TEST_MARKER_SIGNAL:-HUP}" \ + PLATFORM_TEST_HIERARCHY_FAULT="${PLATFORM_TEST_HIERARCHY_FAULT:-}" \ + PLATFORM_TEST_HIERARCHY_TRACE="${PLATFORM_TEST_HIERARCHY_TRACE:-}" \ + PLATFORM_TEST_HIERARCHY_SHIM="$fixture/bin/hierarchy-python" \ + PLATFORM_TEST_HIERARCHY_FAULT_LOG="$fixture/hierarchy-fault.log" \ + PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE="${PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE:-}" \ + PLATFORM_TEST_CONTEXT="${PLATFORM_TEST_CONTEXT:-default}" \ + PLATFORM_TEST_VALIDATOR_ENV_LOG="${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" \ + PLATFORM_TEST_REQUIRE_REQUEST_TIMEOUT="${PLATFORM_TEST_REQUIRE_REQUEST_TIMEOUT:-0}" \ + PLATFORM_TEST_REQUIRE_PROCESS_TIMEOUT="${PLATFORM_TEST_REQUIRE_PROCESS_TIMEOUT:-0}" \ + PLATFORM_TEST_HANG_KUBECTL="${PLATFORM_TEST_HANG_KUBECTL:-0}" \ + bash -c 'source "$1"; shift; platform_observability_secrets_fixture_main "$@"' \ + platform-observability-secrets-test "$SCRIPT" "$fixture" "$@" +} + +run_internal_root_helper() { + local fixture=$1 action=$2 directory=$3 basename=$4 + local source=${5:-} expected_kind=${6:-} expected_source=${7:-} + env \ + PATH="$fixture/bin:$PATH" \ + PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC="$NOW_UTC" \ + PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \ + PLATFORM_TEST_STATE="$fixture/state" \ + PLATFORM_TEST_SUDO_REFRESHED="$fixture/sudo-refreshed" \ + PLATFORM_TEST_MARKER_MOVE_COUNT="$fixture/marker-move-count" \ + PLATFORM_TEST_MARKER_MOVE_FAIL_AT=0 \ + PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT=0 \ + PLATFORM_TEST_MARKER_SIGNAL_AT=0 \ + PLATFORM_TEST_HIERARCHY_FAULT="${PLATFORM_TEST_HIERARCHY_FAULT:-}" \ + PLATFORM_TEST_HIERARCHY_TRACE="${PLATFORM_TEST_HIERARCHY_TRACE:-}" \ + PLATFORM_TEST_HIERARCHY_SHIM="$fixture/bin/hierarchy-python" \ + PLATFORM_TEST_HIERARCHY_FAULT_LOG="$fixture/hierarchy-fault.log" \ + bash -c ' + source "$1" + TEST_MODE=true + configure_test_boundaries "$2" + sudo_refresh + directory=$4 + basename=$5 + [[ "$directory" != @configured-anchor ]] || directory=$EVIDENCE_HIERARCHY_ANCHOR + [[ "$basename" != @configured-profile ]] || basename=$EVIDENCE_HIERARCHY_PROFILE + root_marker_helper /dev/null "$3" "$directory" "$basename" \ + "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$6" "$7" "$8" "$EUID" + ' platform-observability-hierarchy-helper-test "$SCRIPT" "$fixture" "$action" "$directory" "$basename" \ + "$source" "$expected_kind" "$expected_source" +} + +run_hierarchy_action() { + local fixture=$1 action=$2 + run_internal_root_helper "$fixture" "$action" @configured-anchor @configured-profile +} + +assert_slack_risk_hierarchy_failure() { + local fixture=$1 label=$2 fault=${3:-} + if PLATFORM_TEST_HIERARCHY_FAULT="$fault" \ + PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + fail "$label was accepted" + fi + [[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || + fail "$label published a Slack risk marker" + [[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || + fail "$label left a Slack Secret" + ! grep -Fq 'OBSERVABILITY_SECRETS_TRANSACTION=PASS' "$fixture/output" || + fail "$label emitted a transaction success marker" + pass "$label fails closed before marker publication" +} + +WORK="$(mktemp -d /tmp/platform-observability-secrets-test.XXXXXX)" +chmod 0700 "$WORK" + +[[ -f "$SCRIPT" ]] || fail 'production script is absent' + +fixture="$(new_fixture dry-run)" +output="$(run_script "$fixture")" || fail 'dry-run failed' +[[ "$output" == *'OBSERVABILITY_SECRETS_DRY_RUN=PASS'* ]] || fail 'dry-run marker absent' +[[ ! -s "$fixture/commands.log" ]] || fail 'dry-run crossed a system boundary' +[[ "$output" != *"$ADMIN_PASSWORD"* && "$output" != *"$SLACK_URL"* ]] || fail 'dry-run leaked payload' +pass 'dry-run is fixed, payload-free, and read-only' + +fixture="$(new_fixture direct-test-mode)" +if env \ + PLATFORM_OBSERVABILITY_SECRETS_TEST_MODE=1 \ + PLATFORM_OBSERVABILITY_SECRETS_KUBECTL_BIN="$fixture/bin/kubectl" \ + PLATFORM_OBSERVABILITY_SECRETS_SUDO_BIN="$fixture/bin/sudo" \ + PLATFORM_OBSERVABILITY_SECRETS_ENCRYPTION_SCRIPT="$fixture/bin/encryption" \ + PLATFORM_OBSERVABILITY_SECRETS_RESTORE_SCRIPT="$fixture/bin/restore" \ + PLATFORM_OBSERVABILITY_SECRETS_EVIDENCE_DIR="$fixture/evidence-parent/platform/recovery-evidence" \ + PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC="$NOW_UTC" \ + bash "$SCRIPT" >"$fixture/direct-output" 2>&1; then + fail 'direct production execution accepted environment-only test mode' +fi +if PLATFORM_OBSERVABILITY_SECRETS_KUBECTL_BIN="$fixture/bin/kubectl" \ + bash "$SCRIPT" >"$fixture/override-output" 2>&1; then + fail 'production execution silently ignored a boundary override' +fi +if PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE=first-write \ + bash "$SCRIPT" >"$fixture/marker-fault-override-output" 2>&1; then + fail 'production execution accepted a marker fault injection override' +fi +pass 'production entrypoint rejects test mode and all boundary overrides' + +for argv in \ + '--execute' \ + '--grafana-admin' \ + '--execute --grafana-admin --grafana-admin-user-file /tmp/a' \ + '--execute --slack-webhook --slack-webhook-file relative' \ + '--check-grafana-recovery-evidence --execute --grafana-admin' \ + '--accept-no-off-host-slack-recovery' \ + '--check-slack-deployment-evidence --accept-no-off-host-slack-recovery' \ + '--execute --grafana-admin --grafana-admin-user-file /tmp/a --grafana-admin-password-file /tmp/b --accept-no-off-host-slack-recovery'; do + fixture="$(new_fixture argv)" + read -r -a args <<<"$argv" + if run_script "$fixture" "${args[@]}" >"$fixture/output" 2>&1; then + fail "invalid argv accepted: $argv" + fi + [[ "$(PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC="$NOW_UTC" \ + bash -c 'source "$1"; shift; platform_observability_secrets_fixture_main "$@"' \ + platform-observability-secrets-test "$SCRIPT" "$fixture" "${args[@]}" >/dev/null 2>&1; printf %s "$?")" == 2 ]] || + fail "invalid argv did not return usage RC 2: $argv" + [[ ! -s "$fixture/commands.log" ]] || fail "invalid argv crossed a system boundary: $argv" +done +pass 'closed CLI rejects partial, relative, and mixed modes before boundaries' + +fixture="$(new_fixture insecure-file)" +chmod 0644 "$fixture/admin-password" +if run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'insecure input mode accepted' +fi +[[ ! -s "$fixture/commands.log" ]] || fail 'insecure input reached a system boundary' +pass 'input metadata is gated before external or payload access' + +fixture="$(new_fixture input-symlink)" +mv -- "$fixture/admin-user" "$fixture/admin-user-real" +ln -s -- admin-user-real "$fixture/admin-user" +if run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'symlink input accepted' +fi +[[ ! -s "$fixture/commands.log" ]] || fail 'symlink input reached a system boundary' +pass 'symlink payload input is rejected before external access' + +fixture="$(new_fixture input-hardlink)" +ln -- "$fixture/admin-password" "$fixture/admin-password-second-name" +if run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'multiply-linked payload input accepted' +fi +[[ ! -s "$fixture/commands.log" ]] || fail 'multiply-linked input reached a system boundary' +pass 'payload input must have exactly one hard link' + +fixture="$(new_fixture short-password)" +printf '%s' '1234567890123456789' >"$fixture/admin-password" +if run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'short Grafana password accepted' +fi +! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'short password reached create' +[[ "$(<"$fixture/output")" != *'Type APPLY'* ]] || fail 'short password reached confirmation' +pass 'Grafana password shorter than 20 bytes is rejected before prompt' + +fixture="$(new_fixture first-gate)" +if PLATFORM_TEST_ENCRYPTION_FAIL_AT=1 run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'failed initial encryption validator accepted' +fi +[[ "$(<"$fixture/commands.log")" == 'encryption --expect-reencrypted' ]] || + fail 'initial encryption failure did not stop at the fresh validator process' +[[ "$(<"$fixture/output")" != *'Type APPLY'* ]] || fail 'initial validator failure reached confirmation' +pass 'initial encryption gate is zero-read and zero-mutation' + +fixture="$(new_fixture validator-env)" +: >"$fixture/validator-env.log" +printf '%s\n' 'if [[ "${1:-}" == --expect-reencrypted || "${1:-}" == --check ]]; then printf INJECTED >"$PLATFORM_TEST_STATE/injected"; fi' >"$fixture/bash-env" +if ! BASH_ENV=/dev/fd/9 \ + PLATFORM_TEST_VALIDATOR_ENV_LOG="$fixture/validator-env.log" \ + PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1 \ + 9< <(printf '%s\n' 'if [[ "${1:-}" == --expect-reencrypted || "${1:-}" == --check ]]; then printf INJECTED >"$PLATFORM_TEST_STATE/injected"; fi'); then + sed -n '1,180p' "$fixture/output" >&2 + sed -n '1,220p' "$fixture/commands.log" >&2 + fail 'safe validator environment transaction failed' +fi +[[ ! -e "$fixture/state/injected" ]] || fail 'validator child sourced inherited BASH_ENV' +! grep -Eq '^(BASH_ENV|ENV|CDPATH|SHELLOPTS)=' "$fixture/validator-env.log" || + fail 'validator child inherited a shell injection variable' +pass 'fresh validator processes receive only an exact safe environment' + +fixture="$(new_fixture kubectl-timeout)" +if ! PLATFORM_TEST_REQUIRE_REQUEST_TIMEOUT=1 \ + PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'kubectl calls omitted the exact bounded request timeout' +fi +pass 'every non-proxy kubectl API call carries the bounded request timeout' + +fixture="$(new_fixture kubectl-process-timeout)" +if ! PLATFORM_TEST_REQUIRE_PROCESS_TIMEOUT=1 \ + PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'kubectl processes were not enclosed by an exact process deadline' +fi +fixture="$(new_fixture kubectl-proxy-process-timeout)" +if PLATFORM_TEST_REQUIRE_PROCESS_TIMEOUT=1 PLATFORM_TEST_CREATE_FAIL_AT=2 \ + PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" \ + --slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then + fail 'process-timeout proxy rollback fault was unexpectedly accepted' +fi +grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || + fail 'rollback proxy was not enclosed by a process deadline' +grep -Eq '^kubectl .* proxy ' "$fixture/commands.log" || fail 'proxy deadline regression did not reach proxy' +proxy_pid="$(<"$fixture/state/proxy-pid")" +if /bin/kill -0 "$proxy_pid" 2>/dev/null; then + fail 'rollback returned while the kubectl proxy process was still alive' +fi +pass 'every kubectl process, including proxy, is enclosed by a process deadline' + +fixture="$(new_fixture kubectl-hang)" +hang_started="$(date +%s)" +if PLATFORM_TEST_HANG_KUBECTL=1 run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'hanging kubectl fake was accepted' +fi +hang_elapsed=$(( $(date +%s) - hang_started )) +(( hang_elapsed <= 6 )) || fail 'hanging kubectl process exceeded its fixed deadline' +[[ "$(<"$fixture/output")" != *'Type APPLY'* ]] || fail 'hanging kubectl process reached confirmation' +pass 'a kubectl process that ignores request timeout is forcibly bounded' + +fixture="$(new_fixture create-response)" +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1 || + fail 'create response transaction failed' +grep -Eq '^kubectl .*create --filename=.*create-grafana.json --output=json' "$fixture/commands.log" || + fail 'Secret creation did not consume one JSON create response' +create_line="$(grep -n '^kubectl .*create --filename=.*create-grafana.json ' "$fixture/commands.log" | cut -d: -f1)" +uid_get_line="$(grep -nF 'jsonpath={.metadata.uid}' "$fixture/commands.log" | head -1 | cut -d: -f1 || true)" +[[ -z "$uid_get_line" || "$uid_get_line" -lt "$create_line" ]] || + fail 'transaction ownership was adopted from a separate post-create UID GET' +pass 'Secret UID and payload ownership come from the exact create response' + +fixture="$(new_fixture malformed-slack)" +printf '%s' 'https://hooks.slack.com/services/T1/B2' >"$fixture/slack-url" +if run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then + fail 'malformed Slack URL accepted' +fi +! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'bad Slack URL reached create' +[[ "$(<"$fixture/output")" != *'Type APPLY'* ]] || fail 'bad Slack URL reached confirmation' +pass 'Slack input requires the exact prefix and three path components' + +fixture="$(new_fixture schema-drift)" +seed_secret "$fixture" grafana-admin kubernetes.io/basic-auth 'admin-user,admin-password' +if run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'wrong existing Secret type accepted' +fi +! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'schema drift reached create' +pass 'existing Secret requires exact Opaque type and key set' + +fixture="$(new_fixture payload-drift)" +seed_secret "$fixture" grafana-admin Opaque 'admin-user,admin-password' +printf '%s' 'DifferentPassword-123456789' >"$fixture/admin-password" +if run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'existing payload drift accepted' +fi +! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'payload drift reached create' +pass 'existing payload mismatch refuses implicit rotation' + +fixture="$(new_fixture last-gate)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + PLATFORM_TEST_RESTORE_FAIL_AT=2 run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'failed last restore gate accepted' +fi +[[ "$(grep -c '^encryption --expect-reencrypted$' "$fixture/commands.log")" == 2 ]] || + fail 'encryption validator was not a fresh two-call boundary' +[[ "$(grep -c '^restore --check$' "$fixture/commands.log")" == 2 ]] || + fail 'restore validator was not a fresh two-call boundary' +! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'last gate failure reached create' +[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence" ]] || fail 'last gate failure mutated the root evidence tree' +pass 'immediate pre-create validators fail with zero Secret mutation' + +fixture="$(new_fixture both-success)" +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" \ + --slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1 || { + sed -n '1,240p' "$fixture/output" >&2 + fail 'combined create transaction failed' + } +[[ -f "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'Grafana Secret absent' +[[ -f "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || fail 'Slack Secret absent' +[[ "$(grep -c '^encryption --expect-reencrypted$' "$fixture/commands.log")" == 2 ]] || fail 'combined transaction encryption gate count changed' +[[ "$(grep -c '^restore --check$' "$fixture/commands.log")" == 2 ]] || fail 'combined transaction restore gate count changed' +[[ "$(grep -c '^kubectl .*create --filename=' "$fixture/commands.log")" == 2 ]] || fail 'combined transaction did not create exactly two Secrets' +grep -Fq 'OBSERVABILITY_GRAFANA_ADMIN=CREATE_CONFIRMED' "$fixture/output" || fail 'Grafana success marker absent' +grep -Fq 'OBSERVABILITY_SLACK_WEBHOOK=CREATE_CONFIRMED' "$fixture/output" || fail 'Slack success marker absent' +[[ "$(<"$fixture/output")" != *"$ADMIN_PASSWORD"* && "$(<"$fixture/output")" != *"$SLACK_URL"* ]] || fail 'success output leaked payload' +! grep -Fq "$ADMIN_PASSWORD" "$fixture/commands.log" || fail 'password leaked to argv log' +! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail 'webhook leaked to argv log' +for marker in grafana-local slack; do + path="$fixture/evidence-parent/platform/recovery-evidence/$marker.env" + [[ -f "$path" && ! -L "$path" ]] || fail "$marker recovery marker absent" + [[ "$(stat -c '%a' "$path")" == 600 ]] || fail "$marker recovery marker mode changed" + [[ "$(wc -l <"$path" | tr -d '[:space:]')" == 4 ]] || fail "$marker recovery marker key count changed" +done +grep -Fqx 'schema=platform-observability-recovery-evidence-v1' "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" || fail 'marker schema changed' +grep -Fqx 'context=default' "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" || fail 'marker context changed' +grep -Fqx 'resource=observability/grafana-admin' "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" || fail 'Grafana marker resource changed' +grep -Fqx "checked_at_utc=$NOW_UTC" "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" || fail 'marker timestamp changed' +grep -Fqx 'resource=observability/alertmanager-slack-webhook' "$fixture/evidence-parent/platform/recovery-evidence/slack.env" || fail 'Slack marker resource changed' +pass 'combined create is one payload-safe transaction with exact evidence markers' + +: >"$fixture/commands.log" +run_script "$fixture" --check-grafana-recovery-evidence >"$fixture/check-output" 2>&1 || { + sed -n '1,160p' "$fixture/check-output" >&2 + sed -n '1,240p' "$fixture/commands.log" >&2 + fail 'fresh Grafana recovery marker rejected' +} +run_script "$fixture" --check-slack-recovery-evidence >>"$fixture/check-output" 2>&1 || { + sed -n '1,200p' "$fixture/check-output" >&2 + sed -n '1,280p' "$fixture/commands.log" >&2 + fail 'fresh Slack recovery marker rejected' +} +grep -Fq 'GRAFANA_RECOVERY_EVIDENCE=PASS' "$fixture/check-output" || fail 'Grafana check marker absent' +grep -Fq 'SLACK_RECOVERY_EVIDENCE=PASS' "$fixture/check-output" || fail 'Slack check marker absent' +! grep -q '^encryption\|^restore' "$fixture/commands.log" || fail 'evidence-only check crossed encryption validators' +[[ "$(grep -c '^kubectl --request-timeout=5s config current-context$' "$fixture/commands.log")" == 2 ]] || fail 'evidence check did not bind current-user context' +grep -q '^sudo ' "$fixture/commands.log" || fail 'evidence checker did not use narrow sudo' +pass 'standalone recovery checks use only narrow sudo and accept exact fresh evidence' + +if PLATFORM_TEST_CONTEXT=wrong-context run_script "$fixture" --check-grafana-recovery-evidence >"$fixture/context-output" 2>&1; then + fail 'recovery evidence accepted under the wrong current-user context' +fi +pass 'recovery evidence is bound to the live current-user context' + +mv -- "$fixture/evidence-parent/platform/recovery-evidence" "$fixture/evidence-parent/platform/recovery-evidence-real" +ln -s -- recovery-evidence-real "$fixture/evidence-parent/platform/recovery-evidence" +if run_script "$fixture" --check-grafana-recovery-evidence >"$fixture/symlink-output" 2>&1; then + fail 'recovery evidence under a symlink directory was accepted' +fi +unlink -- "$fixture/evidence-parent/platform/recovery-evidence" +mv -- "$fixture/evidence-parent/platform/recovery-evidence-real" "$fixture/evidence-parent/platform/recovery-evidence" +pass 'recovery checker rejects a symlink evidence directory' + +ln -- "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" \ + "$fixture/evidence-parent/platform/recovery-evidence/grafana-local-second-name.env" +if run_script "$fixture" --check-grafana-recovery-evidence >"$fixture/hardlink-output" 2>&1; then + fail 'multiply-linked recovery evidence was accepted' +fi +unlink -- "$fixture/evidence-parent/platform/recovery-evidence/grafana-local-second-name.env" +pass 'recovery evidence must have exactly one hard link' + +printf 'extra=forbidden\n' >>"$fixture/evidence-parent/platform/recovery-evidence/slack.env" +if run_script "$fixture" --check-slack-recovery-evidence >"$fixture/extra-output" 2>&1; then + fail 'recovery evidence with an extra key was accepted' +fi +sed -i '$d' "$fixture/evidence-parent/platform/recovery-evidence/slack.env" +pass 'recovery checker rejects an extra marker field' + +sed -i "s/checked_at_utc=.*/checked_at_utc=2026-07-01T00:00:00Z/" "$fixture/evidence-parent/platform/recovery-evidence/slack.env" +if run_script "$fixture" --check-slack-recovery-evidence >"$fixture/stale-output" 2>&1; then + fail 'stale Slack evidence accepted' +fi +pass 'recovery evidence older than 30 days is rejected' + +fixture="$(new_fixture reuse)" +seed_secret "$fixture" grafana-admin Opaque 'admin-user,admin-password' +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1 || fail 'exact existing Grafana Secret not reusable' +! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'exact existing Secret was rotated' +grep -Fq 'OBSERVABILITY_GRAFANA_ADMIN=REUSED_UNCHANGED' "$fixture/output" || fail 'reuse marker absent' +pass 'exact existing contract is reused without rotation' + +fixture="$(new_fixture input-race)" +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE="$fixture/admin-password" \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1 || fail 'immutable input transaction failed' +python3 -I -S - "$fixture/state/secret__observability__grafana-admin.json" "$ADMIN_PASSWORD" <<'PY' || +import base64, json, pathlib, sys +item = json.loads(pathlib.Path(sys.argv[1]).read_text()) +if base64.b64decode(item["data"]["admin-password"]).decode() != sys.argv[2]: + raise SystemExit(1) +PY + fail 'last-gate input replacement changed the created Secret payload' +pass 'create consumes the private pre-confirmation snapshot, not a raced input path' + +fixture="$(new_fixture rollback)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \ + PLATFORM_TEST_CREATE_FAIL_AT=2 run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" \ + --slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then + fail 'second create failure accepted' +fi +[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'owned Grafana Secret was not UID-precondition rolled back' +[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || fail 'failed Slack create unexpectedly exists' +grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || fail 'rollback success marker absent' +pass 'failed combined create rolls back only captured transaction UID' + +fixture="$(new_fixture marker-rollback)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \ + PLATFORM_TEST_MARKER_MOVE_FAIL_AT=2 run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" \ + --slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then + fail 'second marker install failure accepted' +fi +[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'marker failure left Grafana Secret' +[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || fail 'marker failure left Slack Secret' +[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] || fail 'marker failure left first transaction marker' +[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || fail 'marker failure left failed transaction marker' +grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || fail 'marker failure rollback marker absent' +pass 'combined transaction rolls Secrets and prior marker writes back together' + +fixture="$(new_fixture marker-post-rename)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT=1 run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'post-rename marker failure accepted' +fi +[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || + fail 'post-rename marker failure left the transaction Secret' +[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] || + fail 'post-rename marker failure left an untracked committed marker' +grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || + fail 'post-rename marker failure did not complete truthful rollback' +pass 'post-rename failure is classified and rolls the committed marker back' + +fixture="$(new_fixture marker-post-rename-prior)" +create_exact_evidence_hierarchy "$fixture" +printf '%s\n' \ + 'schema=platform-observability-recovery-evidence-v1' \ + 'context=default' \ + 'resource=observability/grafana-admin' \ + 'checked_at_utc=2026-08-01T00:00:00Z' \ + >"$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" +chmod 0600 -- "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" +cp -- "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" "$fixture/prior-marker-copy" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT=1 run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'post-rename failure over a prior marker was accepted' +fi +cmp --silent -- "$fixture/prior-marker-copy" \ + "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" || + fail 'post-rename classification did not restore the exact prior marker' +[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || + fail 'prior-marker rollback left the transaction Secret' +pass 'post-rename rollback restores an exact prior marker, not just absence' + +fixture="$(new_fixture marker-hup)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + PLATFORM_TEST_MARKER_SIGNAL_AT=1 PLATFORM_TEST_MARKER_SIGNAL=HUP \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'HUP during the post-rename window was accepted' +fi +[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || + fail 'HUP left the transaction Secret untracked' +[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] || + fail 'HUP left the committed marker untracked' +grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || + fail 'HUP did not run the transaction rollback path' +pass 'HUP closes the post-rename marker ownership window' + +for marker_signal in INT TERM; do + fixture="$(new_fixture "marker-${marker_signal,,}")" + if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + PLATFORM_TEST_MARKER_SIGNAL_AT=1 PLATFORM_TEST_MARKER_SIGNAL="$marker_signal" \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail "$marker_signal during the post-rename window was accepted" + fi + [[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || + fail "$marker_signal left the transaction Secret untracked" + [[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] || + fail "$marker_signal left the committed marker untracked" + grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || + fail "$marker_signal did not run the transaction rollback path" +done +pass 'INT and TERM close the post-rename marker ownership window' + +fixture="$(new_fixture marker-exit-fallback)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + PLATFORM_TEST_MARKER_SIGNAL_AT=1 PLATFORM_TEST_MARKER_SIGNAL=USR1 \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'unexpected process exit during the post-rename window was accepted' +fi +[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || + fail 'EXIT fallback left the transaction Secret untracked' +[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] || + fail 'EXIT fallback left the committed marker untracked' +grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || + fail 'EXIT fallback did not run the transaction rollback path' +pass 'EXIT fallback classifies and rolls back an unhandled fatal signal' + +fixture="$(new_fixture postcreate-drift)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + PLATFORM_TEST_POSTCREATE_EXTRA_KEY=1 run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'post-create schema drift accepted' +fi +[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'post-create validation failure bypassed rollback' +grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || fail 'post-create rollback marker absent' +pass 'post-create contract failure enters the UID-precondition rollback path' + +fixture="$(new_fixture ambiguous)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \ + PLATFORM_TEST_CREATE_AMBIGUOUS_AT=2 run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" \ + --slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then + fail 'ambiguous create outcome accepted' +fi +[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'known-owned earlier Secret was not rolled back' +[[ -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || fail 'ambiguous resource was destructively deleted' +grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" || fail 'ambiguous create manual marker absent' +pass 'ambiguous create preserves unknown ownership and requests manual recovery' + +for create_signal in HUP INT TERM; do + fixture="$(new_fixture "create-signal-${create_signal,,}")" + if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + PLATFORM_TEST_CREATE_SIGNAL_AT=1 PLATFORM_TEST_CREATE_SIGNAL="$create_signal" \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail "$create_signal before the JSON create response was accepted" + fi + [[ -e "$fixture/state/secret__observability__grafana-admin.json" ]] || + fail "$create_signal destructively removed an ambiguously-owned Secret" + [[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] || + fail "$create_signal unexpectedly committed recovery evidence" + grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" || + fail "$create_signal ambiguous create omitted the manual recovery marker" +done +pass 'HUP, INT, and TERM never adopt a Secret without its exact JSON create response' + +fixture="$(new_fixture create-signal-after-response)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \ + PLATFORM_TEST_CREATE_SIGNAL_AFTER_RESPONSE_AT=1 PLATFORM_TEST_CREATE_SIGNAL=HUP \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then + fail 'HUP after the complete JSON create response was accepted' +fi +[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || + fail 'complete create-response ownership was not UID-precondition rolled back' +grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || + fail 'complete create-response signal did not complete rollback' +! grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" || + fail 'complete create-response ownership was incorrectly classified ambiguous' +pass 'a complete JSON create response closes the signal ownership window' + +fixture="$(new_fixture uid-drift)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \ + PLATFORM_TEST_CREATE_FAIL_AT=2 PLATFORM_TEST_DRIFT_UID_ON_CREATE_FAILURE=1 \ + run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" \ + --slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then + fail 'UID drift rollback accepted' +fi +[[ -e "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'replacement UID was deleted' +grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" || fail 'UID precondition conflict manual marker absent' +pass 'UID precondition prevents deletion of a concurrent replacement' + +fixture="$(new_fixture slack-risk-fully-absent-hierarchy)" +rmdir -- "$fixture/evidence-parent" +if PLATFORM_TEST_HIERARCHY_TRACE="$fixture/hierarchy.trace" \ + PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + hierarchy_rc=0 +else + hierarchy_rc=$? +fi +if (( hierarchy_rc != 0 )); then + [[ ! -e "$fixture/evidence-parent" ]] || + fail 'failed fully absent hierarchy transaction created evidence hierarchy' + ! grep -Fq 'OBSERVABILITY_SECRETS_TRANSACTION=PASS' "$fixture/output" || + fail 'failed fully absent hierarchy transaction emitted a success marker' + fail 'fully absent managed hierarchy blocked the Slack risk transaction' +fi +[[ -d "$fixture/evidence-parent/platform/recovery-evidence" ]] || + fail 'fully absent managed hierarchy was not created' +[[ -f "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || + fail 'fully absent managed hierarchy omitted the Slack risk marker' +expected_managed_metadata="$(printf '%s\n%s\n%s' \ + "$EUID:$(id -g):700" "$EUID:$(id -g):700" "$EUID:$(id -g):700")" +[[ "$(stat -c '%u:%g:%a' -- \ + "$fixture/evidence-parent" \ + "$fixture/evidence-parent/platform" \ + "$fixture/evidence-parent/platform/recovery-evidence")" == \ + "$expected_managed_metadata" ]] || + fail 'fully absent managed hierarchy metadata is not exact' +[[ "$(<"$fixture/hierarchy.trace")" == $'mkdir:evidence-parent\nreopen:evidence-parent\nattest:evidence-parent\nfsync-parent:evidence-parent\nmkdir:platform\nreopen:platform\nattest:platform\nfsync-parent:platform\nmkdir:recovery-evidence\nreopen:recovery-evidence\nattest:recovery-evidence\nfsync-parent:recovery-evidence' ]] || + fail 'fully absent hierarchy mkdir, reopen, attestation, and parent-fsync ordering changed' +grep -Fq 'OBSERVABILITY_SECRETS_TRANSACTION=PASS' "$fixture/output" || + fail 'fully absent managed hierarchy transaction success marker absent' +pass 'Slack risk transaction safely creates a fully absent managed hierarchy' + +fixture="$(new_fixture slack-risk-fully-present-hierarchy)" +create_exact_evidence_hierarchy "$fixture" +present_parent_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent")" +present_platform_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform")" +present_leaf_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform/recovery-evidence")" +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1 || + fail 'fully present exact hierarchy blocked the Slack risk transaction' +[[ "$(stat -c '%d:%i' -- "$fixture/evidence-parent")" == "$present_parent_identity" && + "$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform")" == "$present_platform_identity" && + "$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform/recovery-evidence")" == "$present_leaf_identity" ]] || + fail 'fully present exact hierarchy was replaced' +[[ -f "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || + fail 'fully present exact hierarchy omitted the Slack risk marker' +grep -Fq 'OBSERVABILITY_SECRETS_TRANSACTION=PASS' "$fixture/output" || + fail 'fully present exact hierarchy transaction success marker absent' +pass 'Slack risk transaction preserves a fully present exact hierarchy' + +for hierarchy_action in check-dir ensure-dir; do + fixture="$(new_fixture "hierarchy-helper-arbitrary-anchor-${hierarchy_action}")" + mkdir -m 0700 -- "$fixture/arbitrary-directory" + mkdir -m 0700 -- "$fixture/arbitrary-directory/evidence-parent" + mkdir -m 0700 -- "$fixture/arbitrary-directory/evidence-parent/platform" + mkdir -m 0700 -- "$fixture/arbitrary-directory/evidence-parent/platform/recovery-evidence" + if run_internal_root_helper "$fixture" "$hierarchy_action" \ + "$fixture/arbitrary-directory" fixture-hierarchy >/dev/null 2>&1; then + fail "$hierarchy_action accepted an arbitrary hierarchy anchor" + fi + [[ -d "$fixture/arbitrary-directory/evidence-parent/platform/recovery-evidence" ]] || + fail "$hierarchy_action mutated the arbitrary hierarchy anchor" + + fixture="$(new_fixture "hierarchy-helper-arbitrary-profile-${hierarchy_action}")" + create_exact_evidence_hierarchy "$fixture" + if run_internal_root_helper "$fixture" "$hierarchy_action" \ + @configured-anchor arbitrary-hierarchy >/dev/null 2>&1; then + fail "$hierarchy_action accepted an arbitrary hierarchy profile" + fi +done + +fixture="$(new_fixture hierarchy-helper-arbitrary-read)" +mkdir -m 0700 -- "$fixture/arbitrary-directory" +printf '%s\n' 'non-secret-marker-fixture' >"$fixture/arbitrary-directory/slack.env" +chmod 0600 -- "$fixture/arbitrary-directory/slack.env" +if run_internal_root_helper "$fixture" read \ + "$fixture/arbitrary-directory" slack.env >/dev/null 2>&1; then + fail 'read accepted an arbitrary marker directory' +fi + +fixture="$(new_fixture hierarchy-helper-arbitrary-install)" +mkdir -m 0700 -- "$fixture/arbitrary-directory" +printf '%s\n' 'non-secret-marker-fixture' >"$fixture/marker-source" +chmod 0600 -- "$fixture/marker-source" +if run_internal_root_helper "$fixture" install \ + "$fixture/arbitrary-directory" slack.env "$fixture/marker-source" absent '' >/dev/null 2>&1; then + fail 'install accepted an arbitrary marker directory' +fi +[[ ! -e "$fixture/arbitrary-directory/slack.env" ]] || + fail 'rejected arbitrary install mutated its marker directory' + +fixture="$(new_fixture hierarchy-helper-arbitrary-unlink)" +mkdir -m 0700 -- "$fixture/arbitrary-directory" +printf '%s\n' 'non-secret-marker-fixture' >"$fixture/marker-source" +cp -- "$fixture/marker-source" "$fixture/arbitrary-directory/slack.env" +chmod 0600 -- "$fixture/marker-source" "$fixture/arbitrary-directory/slack.env" +if run_internal_root_helper "$fixture" unlink-exact \ + "$fixture/arbitrary-directory" slack.env "$fixture/marker-source" file \ + "$fixture/marker-source" >/dev/null 2>&1; then + fail 'unlink-exact accepted an arbitrary marker directory' +fi +cmp --silent -- "$fixture/marker-source" "$fixture/arbitrary-directory/slack.env" || + fail 'rejected arbitrary unlink changed its exact marker' +pass 'every helper action rejects arbitrary hierarchy profiles and directories' + +for managed_component in evidence-parent platform recovery-evidence; do + fixture="$(new_fixture "hierarchy-symlink-${managed_component}")" + prepare_managed_component_parent "$fixture" "$managed_component" + managed_path="$(managed_component_path "$fixture" "$managed_component")" + if [[ "$managed_component" == evidence-parent ]]; then + rmdir -- "$managed_path" + fi + mkdir -m 0700 -- "${managed_path}-target" + ln -s -- "$(basename -- "${managed_path}-target")" "$managed_path" + assert_slack_risk_hierarchy_failure "$fixture" "symlink at managed component $managed_component" + + fixture="$(new_fixture "hierarchy-nondirectory-${managed_component}")" + prepare_managed_component_parent "$fixture" "$managed_component" + managed_path="$(managed_component_path "$fixture" "$managed_component")" + if [[ "$managed_component" == evidence-parent ]]; then + rmdir -- "$managed_path" + fi + : >"$managed_path" + chmod 0700 -- "$managed_path" + assert_slack_risk_hierarchy_failure "$fixture" "non-directory at managed component $managed_component" + + fixture="$(new_fixture "hierarchy-wrong-mode-${managed_component}")" + prepare_managed_component_parent "$fixture" "$managed_component" + managed_path="$(managed_component_path "$fixture" "$managed_component")" + [[ -e "$managed_path" ]] || mkdir -m 0700 -- "$managed_path" + chmod 0755 -- "$managed_path" + assert_slack_risk_hierarchy_failure "$fixture" "wrong mode at managed component $managed_component" + + for metadata_fault in wrong-uid wrong-gid; do + fixture="$(new_fixture "hierarchy-${metadata_fault}-${managed_component}")" + create_exact_evidence_hierarchy "$fixture" + assert_slack_risk_hierarchy_failure "$fixture" \ + "$metadata_fault at managed component $managed_component" \ + "$metadata_fault:$managed_component" + done +done + +for managed_component in evidence-parent platform recovery-evidence; do + fixture="$(new_fixture "hierarchy-replace-after-open-${managed_component}")" + create_exact_evidence_hierarchy "$fixture" + if PLATFORM_TEST_HIERARCHY_FAULT="replace-after-open:$managed_component" \ + run_hierarchy_action "$fixture" check-dir >/dev/null 2>&1; then + fail "check-dir accepted replacement after open at managed component $managed_component" + fi + [[ "$(<"$fixture/hierarchy-fault.log")" == "replace-after-open:$managed_component" ]] || + fail "check-dir did not reach replacement after open at managed component $managed_component" + pass "check-dir binds the opened $managed_component descriptor to its current parent entry" + + for replacement_phase in replace-after-create replace-after-reopen replace-before-attestation; do + fixture="$(new_fixture "hierarchy-${replacement_phase}-${managed_component}")" + prepare_managed_component_parent "$fixture" "$managed_component" + if [[ "$managed_component" == evidence-parent ]]; then + rmdir -- "$fixture/evidence-parent" + fi + if PLATFORM_TEST_HIERARCHY_FAULT="$replacement_phase:$managed_component" \ + run_hierarchy_action "$fixture" ensure-dir >/dev/null 2>&1; then + fail "ensure-dir accepted $replacement_phase at managed component $managed_component" + fi + [[ "$(<"$fixture/hierarchy-fault.log")" == "$replacement_phase:$managed_component" ]] || + fail "ensure-dir did not reach $replacement_phase at managed component $managed_component" + [[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || + fail "ensure-dir $replacement_phase published a marker" + pass "ensure-dir rejects $replacement_phase at managed component $managed_component" + done +done + +for prefix_component in evidence-parent platform; do + fixture="$(new_fixture "hierarchy-safe-absence-race-${prefix_component}")" + if [[ "$prefix_component" == platform ]]; then + mkdir -m 0700 -- "$fixture/evidence-parent/platform" + fi + if PLATFORM_TEST_HIERARCHY_FAULT="replace-before-safe-absence:$prefix_component" \ + run_hierarchy_action "$fixture" check-dir >/dev/null 2>&1; then + safe_absence_rc=0 + else + safe_absence_rc=$? + fi + (( safe_absence_rc != 0 )) || + fail "check-dir accepted prefix replacement before safe absence at $prefix_component" + (( safe_absence_rc != 4 )) || + fail "check-dir misclassified replaced prefix $prefix_component as safe absence" + [[ "$(<"$fixture/hierarchy-fault.log")" == "replace-before-safe-absence:$prefix_component" ]] || + fail "check-dir did not reach the safe-absence race at $prefix_component" + pass "check-dir revalidates $prefix_component before reporting a missing child" +done + +for managed_component in evidence-parent platform recovery-evidence; do + fixture="$(new_fixture "hierarchy-concurrent-exact-${managed_component}")" + prepare_managed_component_parent "$fixture" "$managed_component" + if [[ "$managed_component" == evidence-parent ]]; then + rmdir -- "$fixture/evidence-parent" + fi + PLATFORM_TEST_HIERARCHY_FAULT="concurrent-create-exact:$managed_component" \ + PLATFORM_TEST_HIERARCHY_TRACE="$fixture/hierarchy.trace" \ + run_hierarchy_action "$fixture" ensure-dir >/dev/null 2>&1 || + fail "ensure-dir rejected exact concurrent creation at $managed_component" + [[ "$(<"$fixture/hierarchy-fault.log")" == "concurrent-create-exact:$managed_component" ]] || + fail "ensure-dir did not reach exact concurrent creation at $managed_component" + grep -Fqx "fsync-parent:$managed_component" "$fixture/hierarchy.trace" || + fail "ensure-dir omitted parent fsync after exact concurrent creation at $managed_component" + [[ "$(stat -c '%u:%g:%a' -- "$(managed_component_path "$fixture" "$managed_component")")" == \ + "$EUID:$(id -g):700" ]] || + fail "ensure-dir accepted wrong exact-concurrent metadata at $managed_component" + pass "ensure-dir accepts and parent-fsyncs exact concurrent creation at $managed_component" + + fixture="$(new_fixture "hierarchy-concurrent-unsafe-${managed_component}")" + prepare_managed_component_parent "$fixture" "$managed_component" + if [[ "$managed_component" == evidence-parent ]]; then + rmdir -- "$fixture/evidence-parent" + fi + if PLATFORM_TEST_HIERARCHY_FAULT="concurrent-create-unsafe:$managed_component" \ + run_hierarchy_action "$fixture" ensure-dir >/dev/null 2>&1; then + fail "ensure-dir accepted unsafe concurrent creation at $managed_component" + fi + [[ "$(<"$fixture/hierarchy-fault.log")" == "concurrent-create-unsafe:$managed_component" ]] || + fail "ensure-dir did not reach unsafe concurrent creation at $managed_component" + pass "ensure-dir rejects unsafe concurrent creation at $managed_component" +done + +fixture="$(new_fixture slack-risk-create-failure-fully-absent-hierarchy)" +rmdir -- "$fixture/evidence-parent" +if PLATFORM_TEST_CREATE_FAIL_AT=1 \ + PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + fail 'Slack create failure was accepted with a fully absent hierarchy' +fi +[[ ! -e "$fixture/evidence-parent" ]] || + fail 'Slack create failure mutated the fully absent hierarchy' +[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || + fail 'Slack create failure left a Secret with a fully absent hierarchy' +[[ -f "$fixture/create-count" && "$(<"$fixture/create-count")" == 1 ]] || + fail 'Slack create failure did not reach the pre-ensure Secret create boundary' +! grep -Fq 'OBSERVABILITY_SECRETS_TRANSACTION=PASS' "$fixture/output" || + fail 'Slack create failure emitted a transaction success marker' +pass 'Secret create failure leaves the fully absent hierarchy absent' + +fixture="$(new_fixture slack-risk-final-component-absent-hierarchy)" +mkdir -m 0700 -- "$fixture/evidence-parent/platform" +partial_parent_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent")" +partial_platform_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform")" +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1 || + fail 'final-component-absent hierarchy blocked the Slack risk transaction' +[[ "$(stat -c '%d:%i' -- "$fixture/evidence-parent")" == "$partial_parent_identity" && + "$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform")" == "$partial_platform_identity" && + -f "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || + fail 'final-component-absent hierarchy changed an existing prefix or omitted its marker' +pass 'Slack risk transaction creates only the missing final managed component' + +fixture="$(new_fixture slack-risk-partially-absent-hierarchy)" +partial_parent_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent")" +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1 || + fail 'partially absent hierarchy blocked the Slack risk transaction' +[[ "$(stat -c '%d:%i' -- "$fixture/evidence-parent")" == "$partial_parent_identity" ]] || + fail 'partially absent hierarchy replaced its existing parent' +expected_managed_metadata="$(printf '%s\n%s\n%s' \ + "$EUID:$(id -g):700" "$EUID:$(id -g):700" "$EUID:$(id -g):700")" +[[ "$(stat -c '%u:%g:%a' -- \ + "$fixture/evidence-parent" \ + "$fixture/evidence-parent/platform" \ + "$fixture/evidence-parent/platform/recovery-evidence")" == \ + "$expected_managed_metadata" ]] || + fail 'partially absent hierarchy metadata changed' +[[ -f "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || fail 'risk marker absent' +[[ "$(wc -l <"$fixture/evidence-parent/platform/recovery-evidence/slack.env" | tr -d '[:space:]')" == 8 ]] || + fail 'risk marker schema line count changed' +grep -Fqx 'schema=platform-observability-slack-risk-acceptance-v1' \ + "$fixture/evidence-parent/platform/recovery-evidence/slack.env" || fail 'risk marker schema absent' +pass 'Slack risk acceptance creates an exact marker in a partially absent hierarchy' + +if run_script "$fixture" --check-slack-recovery-evidence >"$fixture/strict-risk-output" 2>&1; then + fail 'strict recovery checker accepted risk evidence' +fi +run_script "$fixture" --check-slack-deployment-evidence >"$fixture/deployment-risk-output" 2>&1 || + fail 'deployment checker rejected risk evidence' +[[ "$(<"$fixture/deployment-risk-output")" == 'SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED' ]] || + fail 'deployment checker risk verdict changed' +pass 'strict checker rejects and deployment checker accepts exact risk evidence' + +: >"$fixture/marker-move-count" +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/risk-noop-output" 2>&1 || + fail 'exact risk marker did not permit a verified Slack no-op' +[[ ! -s "$fixture/marker-move-count" ]] || fail 'exact risk marker was rewritten' +pass 'exact risk marker is a verified no-rewrite Slack no-op' + +fixture="$(new_fixture slack-risk-existing)" +seed_secret "$fixture" alertmanager-slack-webhook Opaque url +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1 || + fail 'existing exact Slack Secret did not accept risk marker installation' +[[ ! -e "$fixture/create-count" ]] || fail 'existing exact Slack Secret was created during risk marker installation' +[[ "$(cat "$fixture/marker-move-count")" == 1 ]] || fail 'absent risk marker was not installed exactly once' +pass 'existing exact Slack Secret installs only an absent risk marker' + +fixture="$(new_fixture slack-risk-marker-failure)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + PLATFORM_TEST_MARKER_MOVE_FAIL_AT=1 run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + fail 'risk marker installation failure was accepted' +fi +[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || + fail 'risk marker failure left a newly-created Slack Secret' +[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || + fail 'risk marker failure left a marker' +pass 'risk marker write failure rolls back only the newly-created Slack Secret' + +fixture="$(new_fixture slack-risk-first-write-red)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE=first-write \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + printf 'TEST FAILURE: helper first-write-loop failure boundary is unavailable\n' >&2 + exit 1 +fi +[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" && + ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" && + -z "$(find "$fixture/evidence-parent" -name '.slack.env.new.*' -print -quit)" ]] || + fail 'first-write-loop failure changed Slack Secret or marker state' +[[ -f "$fixture/marker-move-count" && "$(cat "$fixture/marker-move-count")" == 1 ]] || { + printf 'TEST FAILURE: helper first-write-loop failure did not reach marker install\n' >&2 + exit 1 +} +! grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" || + fail 'first-write-loop failure incorrectly requested manual recovery' +! grep -Fq "$SLACK_URL" "$fixture/output" || fail 'first-write-loop failure leaked payload to output' +! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail 'first-write-loop failure leaked payload to command log' +pass 'first-write-loop helper failure preserves absent marker state and rolls back owned Slack Secret' + +fixture="$(new_fixture slack-strict-pre-rename-red)" +create_exact_evidence_hierarchy "$fixture" +printf '%s\n' \ + 'schema=platform-observability-slack-risk-acceptance-v1' \ + 'context=default' \ + 'resource=observability/alertmanager-slack-webhook' \ + 'reason=off-host-slack-recovery-deferred' \ + 'accepted_by_uid=1000' \ + 'approval_method=interactive-exact-confirmation' \ + 'approval_ref=2026-08-14-observability-slack-recovery-risk-acceptance-design' \ + "checked_at_utc=$NOW_UTC" >"$fixture/evidence-parent/platform/recovery-evidence/slack.env" +chmod 0600 -- "$fixture/evidence-parent/platform/recovery-evidence/slack.env" +cp -- "$fixture/evidence-parent/platform/recovery-evidence/slack.env" "$fixture/prior-slack-marker" +seed_secret "$fixture" alertmanager-slack-webhook Opaque url +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY SLACK default' \ + PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE=pre-rename \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then + printf 'TEST FAILURE: helper pre-rename failure boundary is unavailable\n' >&2 + exit 1 +fi +[[ -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || + fail 'pre-rename failure removed an existing Slack Secret' +cmp --silent -- "$fixture/prior-slack-marker" "$fixture/evidence-parent/platform/recovery-evidence/slack.env" || + fail 'pre-rename failure changed the prior exact marker' +[[ -z "$(find "$fixture/evidence-parent" -name '.slack.env.new.*' -print -quit)" ]] || + fail 'pre-rename failure left helper temporary residue' +! grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" || + fail 'pre-rename failure incorrectly requested manual recovery' +! grep -Fq "$SLACK_URL" "$fixture/output" || fail 'pre-rename failure leaked payload to output' +! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail 'pre-rename failure leaked payload to command log' +pass 'post-fsync pre-rename helper failure preserves prior marker and existing Slack Secret' + +fixture="$(new_fixture slack-risk-post-rename-loss-red)" +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT=1 run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + fail 'risk post-rename response loss was accepted' +fi +if [[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" || + ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || + ! grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output"; then + fail 'risk post-rename response loss did not preserve the marker and Secret' +fi +[[ "$(cat "$fixture/create-count")" == 1 && "$(cat "$fixture/marker-move-count")" == 1 ]] || + fail 'risk post-rename response loss mutation count changed' +[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/.slack.env.new."* ]] || + fail 'risk post-rename response loss left temporary marker residue' +! grep -Fq "$SLACK_URL" "$fixture/output" || fail 'risk post-rename response loss leaked payload to output' +! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail 'risk post-rename response loss leaked payload to command log' +pass 'risk post-rename response loss preserves marker and Secret for manual recovery' + +fixture="$(new_fixture slack-risk-transition)" +create_exact_evidence_hierarchy "$fixture" +printf '%s\n' \ + 'schema=platform-observability-slack-risk-acceptance-v1' \ + 'context=default' \ + 'resource=observability/alertmanager-slack-webhook' \ + 'reason=off-host-slack-recovery-deferred' \ + 'accepted_by_uid=1000' \ + 'approval_method=interactive-exact-confirmation' \ + 'approval_ref=2026-08-14-observability-slack-recovery-risk-acceptance-design' \ + "checked_at_utc=$NOW_UTC" >"$fixture/evidence-parent/platform/recovery-evidence/slack.env" +chmod 0600 -- "$fixture/evidence-parent/platform/recovery-evidence/slack.env" +seed_secret "$fixture" alertmanager-slack-webhook Opaque url +PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY SLACK default' \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1 || + fail 'strict Slack recovery did not promote risk evidence' +grep -Fqx 'schema=platform-observability-recovery-evidence-v1' \ + "$fixture/evidence-parent/platform/recovery-evidence/slack.env" || fail 'risk marker was not promoted to recovery' +pass 'strict Slack recovery atomically promotes risk evidence' + +fixture="$(new_fixture slack-risk-noop-and-downgrade)" +create_exact_evidence_hierarchy "$fixture" +printf '%s\n' \ + 'schema=platform-observability-recovery-evidence-v1' \ + 'context=default' \ + 'resource=observability/alertmanager-slack-webhook' \ + "checked_at_utc=$NOW_UTC" >"$fixture/evidence-parent/platform/recovery-evidence/slack.env" +chmod 0600 -- "$fixture/evidence-parent/platform/recovery-evidence/slack.env" +seed_secret "$fixture" alertmanager-slack-webhook Opaque url +if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + fail 'risk path downgraded recovery evidence' +fi +grep -Fqx 'schema=platform-observability-recovery-evidence-v1' \ + "$fixture/evidence-parent/platform/recovery-evidence/slack.env" || fail 'risk downgrade changed recovery marker' +pass 'risk path rejects recovery-marker downgrade before mutation' + +for confirmation_case in wrong partial eof; do + fixture="$(new_fixture "slack-risk-confirmation-${confirmation_case}")" + rmdir -- "$fixture/evidence-parent" + case "$confirmation_case" in + wrong) confirmations=$'WRONG\nACCEPT NO OFF-HOST SLACK RECOVERY default' ;; + partial) confirmations=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY' ;; + eof) confirmations='APPLY default' ;; + esac + if PLATFORM_TEST_CONFIRMATIONS="$confirmations" run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + fail "risk $confirmation_case confirmation was accepted" + fi + [[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" && + ! -e "$fixture/evidence-parent" && ! -e "$fixture/create-count" ]] || + fail "risk $confirmation_case confirmation mutated Secret or fully absent hierarchy state" + ! grep -Fq "$SLACK_URL" "$fixture/output" || fail "risk $confirmation_case confirmation leaked payload to output" + ! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail "risk $confirmation_case confirmation leaked payload to command log" + pass "risk $confirmation_case confirmation has zero mutation and no payload residue" +done + +fixture="$(new_fixture slack-risk-combined-reject)" +if run_script "$fixture" --execute --grafana-admin \ + --grafana-admin-user-file "$fixture/admin-user" \ + --grafana-admin-password-file "$fixture/admin-password" \ + --slack-webhook --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + fail 'combined Grafana and Slack risk interface was accepted' +fi +[[ ! -s "$fixture/commands.log" && ! -e "$fixture/evidence-parent/platform/recovery-evidence" ]] || + fail 'combined Grafana and Slack risk rejection crossed a mutation boundary' +pass 'combined Grafana and Slack risk interface is rejected before contact' + +fixture="$(new_fixture slack-risk-missing-file)" +if run_script "$fixture" --execute --slack-webhook \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + fail 'risk interface without a Slack file was accepted' +fi +[[ ! -s "$fixture/commands.log" && ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" && + ! -e "$fixture/evidence-parent/platform/recovery-evidence" ]] || + fail 'risk interface without a Slack file crossed a mutation boundary' +! grep -Fq "$SLACK_URL" "$fixture/output" || fail 'risk missing file leaked payload to output' +pass 'risk interface without a Slack file is rejected before contact' + +for risk_signal in HUP INT TERM; do + fixture="$(new_fixture "slack-risk-create-signal-${risk_signal,,}")" + if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + PLATFORM_TEST_CREATE_SIGNAL_AT=1 PLATFORM_TEST_CREATE_SIGNAL="$risk_signal" \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + fail "risk $risk_signal during transaction activation was accepted" + fi + [[ -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" && + ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || + fail "risk $risk_signal transaction-activation state was not preserved" + [[ "$(cat "$fixture/create-count")" == 1 && ! -e "$fixture/marker-move-count" ]] || + fail "risk $risk_signal transaction-activation mutation count changed" + grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" || + fail "risk $risk_signal transaction-activation omitted manual recovery" + ! grep -Fq "$SLACK_URL" "$fixture/output" || fail "risk $risk_signal signal leaked payload to output" + ! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail "risk $risk_signal signal leaked payload to command log" + pass "risk $risk_signal during transaction activation preserves ambiguous Secret only" +done + +for risk_signal in HUP INT TERM; do + fixture="$(new_fixture "slack-risk-marker-signal-${risk_signal,,}")" + if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \ + PLATFORM_TEST_MARKER_SIGNAL_AT=1 PLATFORM_TEST_MARKER_SIGNAL="$risk_signal" \ + run_script "$fixture" --execute --slack-webhook \ + --slack-webhook-file "$fixture/slack-url" \ + --accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then + fail "risk $risk_signal after marker activation was accepted" + fi + [[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" && + ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || + fail "risk $risk_signal post-marker signal left a committed resource" + [[ "$(cat "$fixture/create-count")" == 1 && "$(cat "$fixture/marker-move-count")" == 1 ]] || + fail "risk $risk_signal post-marker signal mutation count changed" + grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || + fail "risk $risk_signal post-marker signal omitted rollback" + ! grep -Fq "$SLACK_URL" "$fixture/output" || fail "risk $risk_signal signal leaked payload to output" + ! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail "risk $risk_signal signal leaked payload to command log" + pass "risk $risk_signal after marker activation rolls back known transaction state" +done + +printf 'OBSERVABILITY SECRETS TEST PASS (%s assertions)\n' "$ASSERTIONS" diff --git a/scripts/validate/test-host-nginx-observability-source.sh b/scripts/validate/test-host-nginx-observability-source.sh new file mode 100644 index 0000000..83c4674 --- /dev/null +++ b/scripts/validate/test-host-nginx-observability-source.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)" +readonly DENY="$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf" +readonly FULL="$ROOT/infrastructure/networking/host-nginx/learn-services-observability.conf" + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +[[ -f "$DENY" && ! -L "$DENY" ]] || fail 'deny-guard source is missing' +[[ -f "$FULL" && ! -L "$FULL" ]] || fail 'full observability source is missing' + +python3 - "$DENY" "$FULL" <<'PY' +import pathlib +import sys + +deny_path, full_path = map(pathlib.Path, sys.argv[1:]) +deny = deny_path.read_text() +full = full_path.read_text() +old = " proxy_pass http://127.0.0.1:1;" +new = " proxy_pass http://127.0.0.1:30080;" + +if deny.count(old) != 1: + raise SystemExit("FAIL: deny guard does not have one closed Grafana upstream") +if deny.count(new) != 4: + raise SystemExit("FAIL: deny guard existing service upstream set changed") +expected = deny.replace(old, new, 1) +if full != expected: + raise SystemExit("FAIL: full candidate differs from deny guard outside the Grafana upstream") +if full.count(old) != 0 or full.count(new) != 5: + raise SystemExit("FAIL: full candidate Grafana upstream is not exact") + +required = ( + "ssl_reject_handshake on;", + "server_name git.learn.hyeonworks.com;", + "server_name id.learn.hyeonworks.com;", + "server_name storage-admin.learn.hyeonworks.com;", + "server_name db-admin.learn.hyeonworks.com;", + "server_name grafana.learn.hyeonworks.com;", + "location = /metrics {", + "allow 192.168.0.0/24;", + "allow 100.64.0.0/10;", + "deny all;", +) +for token in required: + if token not in full: + raise SystemExit(f"FAIL: full candidate lacks {token}") +if full.count("location = /metrics {") != 2: + raise SystemExit("FAIL: Gitea and Grafana metrics guards are not exact") +PY + +printf 'HOST NGINX OBSERVABILITY SOURCE CONTRACT PASS\n' diff --git a/scripts/validate/test-k3s-local-recovery-a1.sh b/scripts/validate/test-k3s-local-recovery-a1.sh new file mode 100644 index 0000000..3470dae --- /dev/null +++ b/scripts/validate/test-k3s-local-recovery-a1.sh @@ -0,0 +1,3057 @@ +#!/usr/bin/env bash + +# Source-safe RED coverage for the A1 payload verifier. +set -Eeuo pipefail + +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly A1_LIBRARY_PATH="${REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery-a1.sh" +readonly A1_HELPER_PATH="${REPOSITORY_ROOT}/scripts/libexec/k3s-local-recovery-a1-check.sh" +readonly AUTHORITY_DESIGN_PATH="/home/donghyeon/workspace/docs/platform/plans/2026-08-09-k3s-luks-header-proof-design.md" +readonly AUTHORITY_PLAN_PATH="/home/donghyeon/workspace/docs/platform/plans/2026-08-09-k3s-luks-header-proof-implementation.md" +readonly AUTHORITY_DESIGN_SHA256='eb9500f949c97e4099a6ff36edb38ad5cea7c62fdbe4f7bae822b4be3129b4da' +readonly AUTHORITY_PLAN_SHA256='9f5c5cfdc769177db06ed46b88c4847b54b2d278c56b65e3180b19cbadf72fa6' + +fail() { + printf 'TEST FAILURE: %s\n' "$*" >&2 + exit 1 +} + +# This guard intentionally runs before mktemp, sourcing, or fixture allocation. +[[ -f "$A1_LIBRARY_PATH" && ! -L "$A1_LIBRARY_PATH" && -r "$A1_LIBRARY_PATH" ]] || + fail 'A1 library missing' +[[ -f "$A1_HELPER_PATH" && ! -L "$A1_HELPER_PATH" && + -r "$A1_HELPER_PATH" ]] || fail 'A1 helper missing' + +[[ "$(/usr/bin/sha256sum -- "$AUTHORITY_DESIGN_PATH")" == \ + "${AUTHORITY_DESIGN_SHA256} ${AUTHORITY_DESIGN_PATH}" ]] || + fail 'A1 authority design drift' +[[ "$(/usr/bin/sha256sum -- "$AUTHORITY_PLAN_PATH")" == \ + "${AUTHORITY_PLAN_SHA256} ${AUTHORITY_PLAN_PATH}" ]] || + fail 'A1 authority plan drift' + +a1_static_source_is_safe() { + local source_path="${1-}" kind="${2-}" + (( $# == 2 )) || return 97 + [[ "$kind" == library || "$kind" == helper ]] || return 97 + [[ -f "$source_path" && ! -L "$source_path" ]] || return 1 + /usr/bin/awk -v kind="$kind" ' + function trim(value) { + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + return value + } + function is_absolute_external(value) { + return value ~ /\/(usr\/)?(s?bin)\/[A-Za-z0-9._+:-]+/ + } + function is_command_position_variable(value, prefix) { + prefix = "(^|(&&|\\|\\||;)[[:space:]]*)(((if|while|until)[[:space:]]+)?(![[:space:]]+)?)" + return value ~ (prefix "[\"\047]?\\$\\{?[A-Za-z0-9_@*#?-]") + } + function is_bare_sensitive_command(value, prefix, names) { + prefix = "(^|(&&|\\|\\||;)[[:space:]]*)(((if|while|until)[[:space:]]+)?(![[:space:]]+)?)" + names = "(sudo|findmnt|blockdev|mount|umount|losetup|cryptsetup|sqlite3|readlink|lsblk|stat|sha256sum|find|od|wc|unlink|env|bash)" + return value ~ (prefix "[\"\047]?" names "([\"\047]?[[:space:]]|[\"\047]?$)") + } + function is_allowed_command_token(token, seam) { + if (token == seam || token == "k3slra1_check_main") return 1 + if (token ~ /^_k3slr(a1|a1h)?_[A-Za-z0-9_]+$/) return 1 + return token ~ /^(:|printf|read|mapfile|return|shift|break|continue|declare|local|readonly|unset|export|trap|set|source|true|false|test)$/ + } + function is_single_seam_command(value, seam, probe, direct_regex, + conditional_regex, substitution_regex) { + probe = value + sub(/[[:space:]]*;[[:space:]]*then$/, "", probe) + sub(/[[:space:]]+\|\|[[:space:]]+return[[:space:]]+[0-9]+$/, "", probe) + if (probe ~ /&&|\|\||;|\|/) return 0 + direct_regex = "^" seam "[[:space:]]+\/(usr\/)?(s?bin)\/" + conditional_regex = "^(if|while|until)[[:space:]]+(![[:space:]]+)?" seam "[[:space:]]+\/(usr\/)?(s?bin)\/" + substitution_regex = "\\$\\(" seam "[[:space:]]+\/(usr\/)?(s?bin)\/" + return (probe ~ direct_regex || probe ~ conditional_regex || + probe ~ substitution_regex) + } + function process_substitutions_are_closed(value) { + if (value !~ /[<>]\(/) return 1 + if (kind == "library") { + if (value ~ /^if[[:space:]]+_k3slra1_command[[:space:]]+\/(usr\/)?(s?bin)\/[A-Za-z0-9._+:-]+[^<>]*2>[[:space:]]+>\(_k3slra1_stderr_marker\);[[:space:]]+then$/) + return 1 + if (value ~ /^(findmnt|metadata|fd-metadata|mapper-stat|size-stat|wc|od|sha256|readlink|lsblk|blockdev)\)[[:space:]]+mapfile[[:space:]]+-d[[:space:]]+\047\047[[:space:]]+-t[[:space:]]+records[[:space:]]+<[[:space:]]+<\(_k3slra1_(findmnt|metadata|fd_metadata|mapper_stat|size_stat|wc|od|sha256|readlink|lsblk|blockdev)_packet[[:space:]]+"\$argument"\)[[:space:]]+;;$/) + return 1 + if (value ~ /^mapfile[[:space:]]+-d[[:space:]]+\047\047[[:space:]]+-t[[:space:]]+entries[[:space:]]+<[[:space:]]+<\(_k3slra1_find_mount_children_packet[[:space:]]+"\$mount_root"\)$/) + return 1 + if (value ~ /^mapfile[[:space:]]+-d[[:space:]]+\047\047[[:space:]]+-t[[:space:]]+inventory[[:space:]]+<[[:space:]]+<\(_k3slra1_find_regular_packet[[:space:]]+"\$pre_root"\)$/) + return 1 + } else { + if (value ~ /^if[[:space:]]+_k3slra1h_command[[:space:]]+\/(usr\/)?(s?bin)\/[A-Za-z0-9._+:-]+[^<>]*2>[[:space:]]+>\(_k3slra1h_stderr_marker\);[[:space:]]+then$/) + return 1 + if (value ~ /^(metadata|readlink)\)[[:space:]]+mapfile[[:space:]]+-d[[:space:]]+\047\047[[:space:]]+-t[[:space:]]+records[[:space:]]+<[[:space:]]+<\(_k3slra1h_(metadata|readlink)_packet[[:space:]]+"\$argument"\)[[:space:]]+;;$/) + return 1 + } + return 0 + } + function commands_are_closed(value, function_name, seam, count, item_index, + part, token, parts) { + if (function_name == seam && value == "\"$@\"") return 1 + if (is_single_seam_command(value, seam)) return 1 + if (kind == "library" && function_name == "_k3slra1_sqlite_quick_check") { + if (value == "trap \047_capture_signal=130; if (( transaction_rc == 0 )); then transaction_rc=130; fi\047 INT") return 1 + if (value == "trap \047_capture_signal=143; if (( transaction_rc == 0 )); then transaction_rc=143; fi\047 TERM") return 1 + if (value == "if (( _capture_signal != 0 && transaction_rc == 0 )); then") return 1 + } + if (value ~ /^(\[\[|\(\(|case[[:space:]]|for[[:space:]]|select[[:space:]])/) return 1 + count = split(value, parts, /&&|\|\||;/) + for (item_index = 1; item_index <= count; item_index++) { + part = trim(parts[item_index]) + sub(/^(if|while|until)[[:space:]]+/, "", part) + sub(/^![[:space:]]*/, "", part) + if (part == "" || part ~ /^(then|do|done|fi|else|esac|\{|\})$/) continue + if (part ~ /^IFS=[^[:space:]]*[[:space:]]+read[[:space:]]/) { + sub(/^IFS=[^[:space:]]*[[:space:]]+/, "", part) + } + if (part ~ /^(\[\[|\(\()/) continue + if (part == "umask 077") continue + token = part + sub(/[[:space:]].*$/, "", token) + if (token ~ /\)$/) continue + if (token ~ /^[A-Za-z_][A-Za-z0-9_]*=/) { + if (part == token) continue + return 0 + } + if (!is_allowed_command_token(token, seam)) return 0 + } + return 1 + } + function inspect_body(value, function_name, probe, seam, external_ok, + direct_regex, conditional_regex, substitution_regex) { + seam = (kind == "library" ? "_k3slra1_command" : "_k3slra1h_command") + if (value ~ /`/) return 0 + if (!process_substitutions_are_closed(value)) return 0 + if (value ~ /^(readonly|local)[[:space:]]/ && value ~ /\$\(/) return 0 + probe = value + gsub("\\$\\(" seam "[[:space:]]+", "", probe) + if (probe ~ /\$\(/) return 0 + if (value ~ /(^|[;[:space:]])(eval|builtin|command|exec)[[:space:]]/) return 0 + if (value ~ /(^|[;[:space:]])source[[:space:]]/) { + if (kind != "helper" || + (value != "source \"$shared_library\" || return 1" && + value != "source \"$a1_library\" || return 1")) return 0 + } + if (value == "source \"$shared_library\" || return 1") shared_source++ + if (value == "source \"$a1_library\" || return 1") a1_source++ + if (value == "_k3slr_load_contract \"$contract_file\" || return 1") contract_call++ + if (function_name == seam && value == "\"$@\"") return 1 + if (is_command_position_variable(value) || is_bare_sensitive_command(value)) return 0 + if (!commands_are_closed(value, function_name, seam)) return 0 + if (is_absolute_external(value)) { + if (value ~ /(&&|\|\||;|\|)[[:space:]]*[\"\047]?\/(usr\/)?(s?bin)\//) + return 0 + direct_regex = "^" seam "[[:space:]]+\/(usr\/)?(s?bin)\/[A-Za-z0-9._+:-]+" + conditional_regex = "^(if|while|until)[[:space:]]+(![[:space:]]+)?" seam "[[:space:]]+\/(usr\/)?(s?bin)\/[A-Za-z0-9._+:-]+" + substitution_regex = "\\$\\(" seam "[[:space:]]+\/(usr\/)?(s?bin)\/[A-Za-z0-9._+:-]+" + external_ok = (value ~ direct_regex || value ~ conditional_regex || + value ~ substitution_regex) + if (!external_ok) return 0 + } + if (value ~ /_k3slra1_command[[:space:]]+\/usr\/bin\/sudo([[:space:]]|$)/) + sudo_count++ + return 1 + } + { + raw = $0 + value = trim(raw) + if (value == "" || value ~ /^#/) next + if (value ~ /\/home\/donghyeon\/workspace/) bad = 1 + if (value ~ /lib\/k3s-local-recovery\.sh/) shared_path++ + if (value ~ /lib\/k3s-local-recovery-a1\.sh/) a1_path++ + if (value ~ /contract\/local-recovery\.env/) contract_path++ + if (in_function) { + if ((!subshell_function && raw == "}") || + (subshell_function && raw == ")")) { + in_function = 0 + function_name = "" + subshell_function = 0 + next + } + if (!inspect_body(value, function_name)) bad = 1 + next + } + if (kind == "library" && + value == "_k3slra1_sqlite_quick_check() (") { + function_name = "_k3slra1_sqlite_quick_check" + subshell_function = 1 + in_function = 1 + next + } + if (value ~ /^[A-Za-z_][A-Za-z0-9_]*\(\)[[:space:]]*\{$/) { + function_name = value + sub(/\(\)[[:space:]]*\{$/, "", function_name) + in_function = 1 + next + } + if (value ~ /^readonly[[:space:]]+/) { + if (is_absolute_external(value) || + value ~ /=(\047)?(sudo|findmnt|blockdev|mount|umount|losetup|cryptsetup|sqlite3|readlink|lsblk|stat|sha256sum|find|od|wc|unlink|env|bash)(\047)?$/) { + bad = 1 + next + } + if (value ~ /^readonly[[:space:]]+[A-Za-z_][A-Za-z0-9_]*=(\047[^\047]*\047|[A-Za-z0-9_.:+\/-]+)$/) next + } + if (kind == "helper") { + if (guard == 0 && value == "if [[ \"${BASH_SOURCE[0]}\" == \"$0\" ]]; then") { guard = 1; next } + if (guard == 1 && value == "_k3slra1h_direct_signal_rc=0") { guard = 2; next } + if (guard == 2 && value == "_k3slra1h_direct_rc=0") { guard = 3; next } + if (guard == 3 && value == "trap \047_k3slra1h_direct_signal_rc=130; if (( _k3slra1h_direct_rc == 0 )); then _k3slra1h_direct_rc=130; fi\047 INT") { guard = 4; next } + if (guard == 4 && value == "trap \047_k3slra1h_direct_signal_rc=143; if (( _k3slra1h_direct_rc == 0 )); then _k3slra1h_direct_rc=143; fi\047 TERM") { guard = 5; next } + if (guard == 5 && value == "if k3slra1_check_main \"$@\"; then") { guard = 6; next } + if (guard == 6 && value == "_k3slra1h_direct_rc=0") { guard = 7; next } + if (guard == 7 && value == "else") { guard = 8; next } + if (guard == 8 && value == "_k3slra1h_direct_rc=$?") { guard = 9; next } + if (guard == 9 && value == "fi") { guard = 10; next } + if (guard == 10 && value == "if (( _k3slra1h_direct_signal_rc != 0 && _k3slra1h_direct_rc == 0 )); then") { guard = 11; next } + if (guard == 11 && value == "_k3slra1h_direct_rc=$_k3slra1h_direct_signal_rc") { guard = 12; next } + if (guard == 12 && value == "fi") { guard = 13; next } + if (guard == 13 && value == "exit \"$_k3slra1h_direct_rc\"") { guard = 14; next } + if (guard == 14 && value == "fi") { guard = 15; next } + } + bad = 1 + } + END { + if (in_function || bad) exit 1 + if (kind == "library" && (guard != 0 || sudo_count != 1)) exit 1 + if (kind == "helper" && (guard != 15 || shared_source != 1 || + a1_source != 1 || contract_call != 1 || shared_path < 1 || + a1_path < 1 || contract_path < 1)) exit 1 + } + ' "$source_path" +} + +a1_production_static_preflight() { + # No production source operation occurs until both files pass this scanner. + a1_static_source_is_safe "$A1_LIBRARY_PATH" library || return 41 + a1_static_source_is_safe "$A1_HELPER_PATH" helper || return 42 +} + +# No production API or source operation is allowed before this conservative +# external-call preflight succeeds. +preflight_rc=0 +if a1_production_static_preflight; then + : +else + preflight_rc=$? +fi +case "$preflight_rc" in + 0) ;; + 41) fail 'A1 library static preflight failed' ;; + 42) fail 'A1 helper static preflight failed' ;; + *) fail 'A1 production static preflight failed' ;; +esac + +# shellcheck source=/dev/null +source "$A1_LIBRARY_PATH" + +require_function() { + local name="${1-}" + (( $# == 1 )) || fail 'invalid require_function invocation' + declare -F "$name" >/dev/null || fail "A1 API missing: ${name}" +} + +for required_api in \ + _k3slra1_verify_payload \ + _k3slra1_verify_payload_root \ + _k3slra1_select_pre_bundle \ + _k3slra1_verify_manifest \ + _k3slra1_capture_manifest_hex \ + _k3slra1_select_sqlite_layout \ + _k3slra1_sqlite_quick_check; do + require_function "$required_api" +done + +readonly A1_TARGET_COUNT=80 +readonly A1_TARGET_BYTES=22857829 +readonly A1_ANCHOR_BYTES=22851685 +readonly A1_BUNDLE_NAME='k3s-secrets-encryption-20260808T000000Z' +readonly A1_MAIN_MAPPER='/dev/mapper/k3s-recovery' +readonly A1_PROOF_MAPPER='/dev/mapper/k3s-recovery-proof' +readonly A1_MAIN_CANONICAL_MAPPER='/dev/dm-8' +readonly A1_PROOF_CANONICAL_MAPPER='/dev/dm-7' +readonly A1_MAIN_MAJOR_MINOR='253:8' +readonly A1_PROOF_MAJOR_MINOR='253:7' +readonly A1_MOUNT_OPTIONS='ro,nosuid,nodev,noexec,relatime,noload' + +fixture_root="$(mktemp -d "${TMPDIR:-/tmp}/k3s-local-recovery-a1-test.XXXXXXXX")" +cleanup() { + case "$fixture_root" in + /tmp/k3s-local-recovery-a1-test.*|"${TMPDIR:-/tmp}"/k3s-local-recovery-a1-test.*) + rm -rf -- "$fixture_root" + ;; + *) + printf 'TEST CLEANUP FAILURE: unexpected fixture root\n' >&2 + return 1 + ;; + esac +} +trap cleanup EXIT + +assert_index=0 +assert_succeeds() { + local description="${1-}" stdout_file stderr_file rc=0 + shift || fail 'assert_succeeds requires a description' + (( assert_index += 1 )) + stdout_file="${fixture_root}/assert-${assert_index}.stdout" + stderr_file="${fixture_root}/assert-${assert_index}.stderr" + if "$@" >"$stdout_file" 2>"$stderr_file"; then + rc=0 + else + rc=$? + fi + (( rc == 0 )) || fail "$description returned rc ${rc}" + [[ ! -s "$stdout_file" && ! -s "$stderr_file" ]] || + fail "$description leaked public output" + rm -- "$stdout_file" "$stderr_file" +} + +assert_fails() { + local description="${1-}" stdout_file stderr_file rc=0 + shift || fail 'assert_fails requires a description' + (( assert_index += 1 )) + stdout_file="${fixture_root}/assert-${assert_index}.stdout" + stderr_file="${fixture_root}/assert-${assert_index}.stderr" + if "$@" >"$stdout_file" 2>"$stderr_file"; then + rc=0 + else + rc=$? + fi + (( rc != 0 )) || fail "$description unexpectedly succeeded" + (( rc != 97 )) || fail "$description returned fixture/model rc 97" + [[ ! -s "$stdout_file" && ! -s "$stderr_file" ]] || + fail "$description leaked public output" + rm -- "$stdout_file" "$stderr_file" +} + +fixture_path_is_owned() { + local path="${1-}" + (( $# == 1 )) || return 97 + case "$path" in + "$case_root"|"$case_root"/*) return 0 ;; + *) return 97 ;; + esac +} + +expected_mapper_for_role() { + local role="${1-}" + (( $# == 1 )) || return 97 + case "$role" in + proof) printf '%s\n' "$A1_PROOF_MAPPER" ;; + original-baseline|original-final) printf '%s\n' "$A1_MAIN_MAPPER" ;; + *) return 97 ;; + esac +} + +expected_canonical_mapper_for_role() { + local role="${1-}" + (( $# == 1 )) || return 97 + case "$role" in + proof) printf '%s\n' "$A1_PROOF_CANONICAL_MAPPER" ;; + original-baseline|original-final) printf '%s\n' "$A1_MAIN_CANONICAL_MAPPER" ;; + *) return 97 ;; + esac +} + +expected_major_minor_for_role() { + local role="${1-}" + (( $# == 1 )) || return 97 + case "$role" in + proof) printf '%s\n' "$A1_PROOF_MAJOR_MINOR" ;; + original-baseline|original-final) printf '%s\n' "$A1_MAIN_MAJOR_MINOR" ;; + *) return 97 ;; + esac +} + +canonical_mapper_for_alias() { + local mapper_alias="${1-}" + (( $# == 1 )) || return 97 + if [[ "$current_case" == canonical_node_wrong_namespace ]]; then + case "$mapper_alias" in + "$A1_PROOF_MAPPER"|"$A1_MAIN_MAPPER") printf '/dev/loop7\n'; return 0 ;; + esac + fi + case "$mapper_alias" in + "$A1_PROOF_MAPPER") printf '%s\n' "$A1_PROOF_CANONICAL_MAPPER" ;; + "$A1_MAIN_MAPPER") printf '%s\n' "$A1_MAIN_CANONICAL_MAPPER" ;; + /dev/mapper/a1-source-mismatch) printf '/dev/dm-99\n' ;; + /dev/mapper/not-owned) printf '/dev/dm-98\n' ;; + *) return 97 ;; + esac +} + +declare -a fixture_targets=() +case_root='' +mount_root='' +bundle_root='' +pre_root='' +capture_root='' +manifest_path='' +anchor_path='' +fixture_database_relative='' +current_case='' +current_role='' +findmnt_calls=0 +readlink_calls=0 +lsblk_calls=0 +blockdev_calls=0 +sqlite_calls=0 +sqlite_unlink_stdout_calls=0 +sqlite_unlink_stderr_calls=0 +command_trace='' +hostile_capture_surface='' +hostile_capture_mode='' +hostile_capture_file='' +hostile_capture_valid='' +hostile_capture_family='' +hostile_violation_trace='' +helper_modeled_target='' +helper_modeled_mutation='' +readonly A1_METADATA_FORMAT='%d:%i|%h|%s|%y|%F|%u:%g|%a' + +trace_stage() { + local stage="${1-}" + (( $# == 1 )) || return 97 + [[ "$stage" =~ ^[a-z0-9-]+$ && -n "$command_trace" ]] || return 97 + printf '%s\n' "$stage" >>"$command_trace" +} + +trace_count() { + local stage="${1-}" count + (( $# == 1 )) || return 97 + count="$(/usr/bin/awk -v wanted="$stage" '$0 == wanted { count++ } END { print count + 0 }' \ + "$command_trace")" || return 97 + printf '%s\n' "$count" +} + +emit_hostile_capture() { + local valid="${1-}" + (( $# == 1 )) || return 97 + case "$hostile_capture_mode" in + stdout-nonzero) + printf '%s\n' "$valid" + return 23 + ;; + raw-nul) + printf '%s\0\n' "$valid" + ;; + extra-trailing-lf) + printf '%s\n\n' "$valid" + ;; + nonempty-stderr) + printf 'hostile-capture-stderr\n' >&2 + printf '%s\n' "$valid" + ;; + *) return 97 ;; + esac +} + +build_manifest() { + local target + : >"$manifest_path" + ( + cd -- "$pre_root" + for target in "${fixture_targets[@]}"; do + /usr/bin/sha256sum -- "$target" + done + ) >"$manifest_path" +} + +build_payload() { + local layout="${1-}" index suffix database_relative + (( $# == 1 )) || return 97 + case "$layout" in + canonical) database_relative='./datastore/state.db' ;; + legacy) database_relative='./rootfs/var/lib/rancher/k3s/server/db/state.db' ;; + *) return 97 ;; + esac + + mount_root="${case_root}/mount" + bundle_root="${mount_root}/${A1_BUNDLE_NAME}" + pre_root="${bundle_root}/pre" + capture_root="${case_root}/capture" + manifest_path="${pre_root}/verification.manifest" + anchor_path="${pre_root}/payload/anchor.bin" + fixture_database_relative="$database_relative" + mkdir -p -- \ + "${pre_root}/$(dirname -- "${database_relative#./}")" \ + "${pre_root}/payload" \ + "${capture_root}/home" + chmod 0700 -- "$mount_root" "$bundle_root" "$pre_root" "$capture_root" "${capture_root}/home" + + fixture_targets=( + "$database_relative" + "${database_relative}-wal" + "${database_relative}-shm" + './payload/anchor.bin' + './payload/plus+allowed' + ) + /usr/bin/truncate --size=4096 -- "${pre_root}/${database_relative#./}" + /usr/bin/truncate --size=1024 -- "${pre_root}/${database_relative#./}-wal" + /usr/bin/truncate --size=1024 -- "${pre_root}/${database_relative#./}-shm" + /usr/bin/truncate --size="$A1_ANCHOR_BYTES" -- "$anchor_path" + : >"${pre_root}/payload/plus+allowed" + for (( index = 1; index <= 75; index += 1 )); do + printf -v suffix '%02d' "$index" + : >"${pre_root}/payload/dummy-${suffix}" + fixture_targets+=("./payload/dummy-${suffix}") + done + (( ${#fixture_targets[@]} == A1_TARGET_COUNT )) || return 97 + build_manifest + ( + cd -- "$pre_root" + chmod 0600 -- "${fixture_targets[@]}" verification.manifest + ) + /usr/bin/find "$mount_root" -type d -exec chmod 0700 -- '{}' + +} + +model_test_owned_stat() { + local requested_format="${1-}" requested_path="${2-}" physical_path + local actual_dev actual_inode actual_nlink actual_size actual_mtime actual_type actual_mode + local modeled_dev=4242 modeled_inode modeled_nlink modeled_uid=0 modeled_gid=0 modeled_mode + local modeled_type + local sqlite_trace_count=0 + (( $# == 2 )) || return 97 + physical_path="$requested_path" + case "$requested_path" in + /proc/[0-9]*/fd/[0-9]*) + physical_path="$(/usr/bin/readlink --canonicalize-existing -- "$requested_path")" || return 97 + ;; + esac + fixture_path_is_owned "$physical_path" || return 97 + IFS='|' read -r actual_dev actual_inode actual_nlink actual_size actual_mtime actual_type actual_mode < <( + /usr/bin/stat --format='%d|%i|%h|%s|%y|%F|%a' -- "$physical_path" + ) || return 97 + modeled_inode="$actual_inode" + modeled_nlink="$actual_nlink" + modeled_mode="$actual_mode" + modeled_type="$actual_type" + [[ "$current_case" == bundle_wrong_owner && "$physical_path" == "$bundle_root" ]] && modeled_uid=1000 + [[ "$current_case" == manifest_wrong_mode && "$physical_path" == "$manifest_path" ]] && modeled_mode=644 + [[ "$current_case" == payload_wrong_device && "$physical_path" == "$anchor_path" ]] && modeled_dev=4243 + if [[ "$current_case" == heterogeneous_target_metadata ]]; then + case "$physical_path" in + "${pre_root}/payload/dummy-01") + modeled_uid=1000 + modeled_gid=1001 + modeled_mode=640 + ;; + "${pre_root}/payload/dummy-02") + modeled_uid=65534 + modeled_gid=65534 + modeled_mode=400 + ;; + "${pre_root}/payload/dummy-03") + modeled_uid=0 + modeled_gid=44 + modeled_mode=644 + ;; + esac + fi + if [[ "$current_case" == manifest_duplicate_inode && + "$physical_path" == "${pre_root}/payload/dummy-05" ]]; then + modeled_inode="$(/usr/bin/stat --format='%i' -- "${pre_root}/payload/dummy-04")" || return 97 + modeled_nlink=1 + fi + if [[ "$current_case" == sqlite_capture_wrong_owner && + "$physical_path" == "${capture_root}/sqlite.stdout" ]]; then + modeled_uid=1000 + fi + [[ "$current_case" == sqlite_capture_wrong_mode && + "$physical_path" == "${capture_root}/sqlite.stdout" ]] && modeled_mode=644 + if [[ "$current_case" == sqlite_wal_drift_after_child && + "$physical_path" == "${pre_root}/${fixture_database_relative#./}-wal" ]]; then + sqlite_trace_count="$(trace_count sqlite)" || return 97 + if (( sqlite_trace_count >= 1 )); then + actual_mtime='2099-01-01 00:00:00.000000000 +0000' + fi + fi + if [[ "$current_case" == sqlite_capture_parent_drift && + "$requested_path" == "$capture_root" ]]; then + sqlite_trace_count="$(trace_count sqlite)" || return 97 + if (( sqlite_trace_count >= 1 )); then + modeled_inode="$((modeled_inode + 1000000))" + fi + fi + case "$current_case" in + sqlite_layout_db_identity_drift) + if [[ "$physical_path" == "${pre_root}/${fixture_database_relative#./}" ]]; then + trace_stage layout-db-stat || return 97 + modeled_inode="$((modeled_inode + 2000000))" + fi + ;; + sqlite_layout_wal_identity_drift) + if [[ "$physical_path" == "${pre_root}/${fixture_database_relative#./}-wal" ]]; then + trace_stage layout-wal-stat || return 97 + modeled_inode="$((modeled_inode + 2000000))" + fi + ;; + sqlite_layout_shm_identity_drift) + if [[ "$physical_path" == "${pre_root}/${fixture_database_relative#./}-shm" ]]; then + trace_stage layout-shm-stat || return 97 + modeled_inode="$((modeled_inode + 2000000))" + fi + ;; + esac + if [[ -n "$helper_modeled_target" && "$physical_path" == "$helper_modeled_target" ]]; then + case "$helper_modeled_mutation" in + wrong-owner) modeled_uid=1000 ;; + wrong-mode) modeled_mode=755 ;; + wrong-device) modeled_dev=4243 ;; + nlink-two) modeled_nlink=2 ;; + wrong-type) modeled_type='directory' ;; + '') ;; + *) return 97 ;; + esac + fi + case "$current_case" in + helper-lib-wrong-owner) + [[ "$physical_path" != "${capture_root}/lib" ]] || modeled_uid=1000 + ;; + helper-libexec-wrong-owner) + [[ "$physical_path" != "${capture_root}/libexec" ]] || modeled_uid=1000 + ;; + helper-contract-wrong-owner) + [[ "$physical_path" != "${capture_root}/contract" ]] || modeled_uid=1000 + ;; + helper-lib-wrong-mode) + [[ "$physical_path" != "${capture_root}/lib" ]] || modeled_mode=755 + ;; + helper-libexec-wrong-mode) + [[ "$physical_path" != "${capture_root}/libexec" ]] || modeled_mode=755 + ;; + helper-contract-wrong-mode) + [[ "$physical_path" != "${capture_root}/contract" ]] || modeled_mode=755 + ;; + esac + + case "$requested_format" in + "$A1_METADATA_FORMAT") + printf '%s:%s|%s|%s|%s|%s|%s:%s|%s\n' "$modeled_dev" "$modeled_inode" \ + "$modeled_nlink" "$actual_size" "$actual_mtime" "$modeled_type" \ + "$modeled_uid" "$modeled_gid" "$modeled_mode" + ;; + '%d:%i') printf '%s:%s\n' "$modeled_dev" "$modeled_inode" ;; + '%i') printf '%s\n' "$modeled_inode" ;; + '%h') printf '%s\n' "$modeled_nlink" ;; + '%s') printf '%s\n' "$actual_size" ;; + '%y') printf '%s\n' "$actual_mtime" ;; + '%F') printf '%s\n' "$modeled_type" ;; + '%u:%g') printf '%s:%s\n' "$modeled_uid" "$modeled_gid" ;; + '%a') printf '%s\n' "$modeled_mode" ;; + *) return 97 ;; + esac +} + +replace_manifest_path() { + local old_path="${1-}" new_path="${2-}" line record_path temporary + (( $# == 2 )) || return 97 + temporary="${manifest_path}.rewrite" + : >"$temporary" + while IFS= read -r line; do + record_path="${line:66}" + if [[ "$record_path" == "$old_path" ]]; then + printf '%s %s\n' "${line:0:64}" "$new_path" >>"$temporary" + else + printf '%s\n' "$line" >>"$temporary" + fi + done <"$manifest_path" + mv -- "$temporary" "$manifest_path" +} + +replace_fixture_target() { + local old_path="${1-}" new_path="${2-}" index + (( $# == 2 )) || return 97 + for index in "${!fixture_targets[@]}"; do + if [[ "${fixture_targets[$index]}" == "$old_path" ]]; then + fixture_targets[$index]="$new_path" + return 0 + fi + done + return 97 +} + +mutate_fixture() { + local mutation="${1-}" first_line first_hash first_path upper_hash + local long_a long_b long_tail long_path old_wal replacement other_database + (( $# == 1 )) || return 97 + case "$mutation" in + canonical_pass|legacy_pass|original_baseline_pass|manifest_plus_path_pass|\ + mount_missing_noload|mapping_not_readonly|\ + mount_wrong_source|mount_wrong_target|mount_wrong_fstype|mount_wrong_major_minor|\ + mount_missing_vfs_ro|mount_forbidden_vfs_rw|mount_evidence_drift|\ + mapper_canonical_mismatch|source_expected_canonical_mismatch|mapper_not_block|mapper_major_minor_mismatch|\ + mapping_evidence_drift|mount_third_evidence_drift|mapping_third_evidence_drift|\ + payload_write_attempt|manifest_drift_after_sqlite|sqlite_wal_drift_after_child|\ + sqlite_quick_check_extra_line|sqlite_quick_check_error|\ + sqlite_quick_check_stderr|sqlite_quick_check_nul|\ + sqlite_capture_path_swap|sqlite_capture_parent_drift|\ + sqlite_capture_wrong_owner|sqlite_capture_wrong_mode|sqlite_capture_cleanup_failure|\ + sqlite_capture_int_cleanup|sqlite_capture_term_cleanup|\ + bundle_wrong_owner|manifest_wrong_mode|payload_wrong_device|manifest_duplicate_inode|\ + proof_mapper_swap|baseline_mapper_swap|final_mapper_swap|\ + mount_evidence_multiline|mount_evidence_cr|mount_evidence_nul|mount_evidence_stderr|\ + findmnt_valid_record_stderr|readlink_valid_record_stderr|lsblk_valid_record_stderr|\ + blockdev_multiline|blockdev_cr|blockdev_nul|blockdev_stderr|blockdev_valid_record_stderr) + ;; + unrelated_lost_found_pass) + mkdir -- "${mount_root}/lost+found" + ;; + preexisting_unmanifested_file) + : >"${pre_root}/payload/preexisting-unmanifested" + ;; + manifest_path_512_bytes_pass) + printf -v long_a '%0250d' 0 + printf -v long_b '%0250d' 0 + long_a="${long_a//0/a}" + long_b="${long_b//0/b}" + long_tail='12345678' + long_path="./${long_a}/${long_b}/${long_tail}" + [[ ${#long_path} -eq 512 ]] || return 97 + mkdir -p -- "${pre_root}/${long_a}/${long_b}" + mv -- "${pre_root}/payload/plus+allowed" "${pre_root}/${long_path#./}" + replace_fixture_target './payload/plus+allowed' "$long_path" + build_manifest + ;; + bundle_none) + rm -rf -- "$bundle_root" + ;; + bundle_two) + cp -a -- "$bundle_root" "${mount_root}/k3s-secrets-encryption-20260808T000001Z" + ;; + matching_bundle_symlink) + ln -s -- "$A1_BUNDLE_NAME" \ + "${mount_root}/k3s-secrets-encryption-20260808T000001Z" + ;; + bundle_bad_name) + mv -- "$bundle_root" "${mount_root}/k3s-secrets-encryption-20260808t000000z" + bundle_root="${mount_root}/k3s-secrets-encryption-20260808t000000z" + pre_root="${bundle_root}/pre" + manifest_path="${pre_root}/verification.manifest" + anchor_path="${pre_root}/payload/anchor.bin" + ;; + manifest_missing) + rm -- "$manifest_path" + ;; + manifest_no_final_lf) + /usr/bin/truncate --size=-1 -- "$manifest_path" + ;; + manifest_cr) + IFS= read -r first_line <"$manifest_path" + { + printf '%s\r\n' "$first_line" + tail -n +2 -- "$manifest_path" + } >"${manifest_path}.rewrite" + mv -- "${manifest_path}.rewrite" "$manifest_path" + ;; + manifest_uppercase_hash) + IFS= read -r first_line <"$manifest_path" + first_hash="${first_line:0:64}" + first_path="${first_line:66}" + upper_hash="${first_hash^^}" + { + printf '%s %s\n' "$upper_hash" "$first_path" + tail -n +2 -- "$manifest_path" + } >"${manifest_path}.rewrite" + mv -- "${manifest_path}.rewrite" "$manifest_path" + ;; + manifest_one_space_separator) + IFS= read -r first_line <"$manifest_path" + { + printf '%s %s\n' "${first_line:0:64}" "${first_line:66}" + tail -n +2 -- "$manifest_path" + } >"${manifest_path}.rewrite" + mv -- "${manifest_path}.rewrite" "$manifest_path" + ;; + manifest_tab_separator) + IFS= read -r first_line <"$manifest_path" + { + printf '%s\t%s\n' "${first_line:0:64}" "${first_line:66}" + tail -n +2 -- "$manifest_path" + } >"${manifest_path}.rewrite" + mv -- "${manifest_path}.rewrite" "$manifest_path" + ;; + manifest_parent_escape) + replace_manifest_path './payload/anchor.bin' './payload/../payload/anchor.bin' + ;; + manifest_absolute_path) + replace_manifest_path './payload/anchor.bin' '/payload/anchor.bin' + ;; + manifest_double_slash) + replace_manifest_path './payload/anchor.bin' './payload//anchor.bin' + ;; + manifest_dot_segment) + replace_manifest_path './payload/anchor.bin' './payload/./anchor.bin' + ;; + manifest_trailing_slash) + replace_manifest_path './payload/anchor.bin' './payload/anchor.bin/' + ;; + manifest_invalid_character) + replace_manifest_path './payload/anchor.bin' './payload/anchor:bin' + ;; + manifest_path_513_bytes) + printf -v long_a '%0250d' 0 + printf -v long_b '%0250d' 0 + long_a="${long_a//0/a}" + long_b="${long_b//0/b}" + long_tail='123456789' + long_path="./${long_a}/${long_b}/${long_tail}" + [[ ${#long_path} -eq 513 ]] || return 97 + mkdir -p -- "${pre_root}/${long_a}/${long_b}" + mv -- "${pre_root}/payload/plus+allowed" "${pre_root}/${long_path#./}" + replace_fixture_target './payload/plus+allowed' "$long_path" + build_manifest + ;; + manifest_too_large|manifest_65537_bytes) + /usr/bin/truncate --size=65537 -- "$manifest_path" + ;; + manifest_nul) + /usr/bin/truncate --size=-1 -- "$manifest_path" + printf '\0\n' >>"$manifest_path" + ;; + manifest_duplicate_path) + replace_manifest_path './payload/dummy-02' './payload/dummy-01' + ;; + manifest_self_target) + replace_manifest_path './payload/dummy-01' './verification.manifest' + ;; + manifest_hash_mismatch) + printf x >>"$anchor_path" + ;; + manifest_target_symlink) + rm -- "${pre_root}/payload/dummy-01" + ln -s -- plus+allowed "${pre_root}/payload/dummy-01" + ;; + manifest_target_hardlink_alias) + rm -- "${pre_root}/payload/dummy-02" + ln -- "${pre_root}/payload/dummy-01" "${pre_root}/payload/dummy-02" + ;; + manifest_target_nlink_two) + ln -- "${pre_root}/payload/dummy-03" "${bundle_root}/unlisted-hardlink" + ;; + target_count_79) + head -n 79 -- "$manifest_path" >"${manifest_path}.rewrite" + mv -- "${manifest_path}.rewrite" "$manifest_path" + ;; + target_count_81) + : >"${pre_root}/payload/dummy-81" + fixture_targets+=('./payload/dummy-81') + build_manifest + ;; + target_bytes_22857828) + /usr/bin/truncate --size=$((A1_ANCHOR_BYTES - 1)) -- "$anchor_path" + build_manifest + ;; + target_bytes_22857830) + /usr/bin/truncate --size=$((A1_ANCHOR_BYTES + 1)) -- "$anchor_path" + build_manifest + ;; + sqlite_mixed_layout) + if [[ "$current_case" == legacy* ]]; then + other_database="${pre_root}/datastore/state.db" + else + other_database="${pre_root}/rootfs/var/lib/rancher/k3s/server/db/state.db" + fi + mkdir -p -- "$(dirname -- "$other_database")" + : >"$other_database" + : >"${other_database}-wal" + : >"${other_database}-shm" + ;; + sqlite_alternate_partial_layout) + if [[ "$fixture_database_relative" == './datastore/state.db' ]]; then + other_database="${pre_root}/rootfs/var/lib/rancher/k3s/server/db/state.db" + else + other_database="${pre_root}/datastore/state.db" + fi + mkdir -p -- "$(dirname -- "$other_database")" + : >"$other_database" + ;; + sqlite_missing_wal) + old_wal="${fixture_database_relative}-wal" + replacement='./payload/sqlite-wal-replacement' + rm -- "${pre_root}/${old_wal#./}" + /usr/bin/truncate --size=1024 -- "${pre_root}/${replacement#./}" + replace_fixture_target "$old_wal" "$replacement" + build_manifest + ;; + sqlite_missing_shm) + old_wal="${fixture_database_relative}-shm" + replacement='./payload/sqlite-shm-replacement' + rm -- "${pre_root}/${old_wal#./}" + /usr/bin/truncate --size=1024 -- "${pre_root}/${replacement#./}" + replace_fixture_target "$old_wal" "$replacement" + build_manifest + ;; + sqlite_not_in_manifest) + replacement='./payload/sqlite-db-replacement' + /usr/bin/truncate --size=4096 -- "${pre_root}/${replacement#./}" + replace_fixture_target "$fixture_database_relative" "$replacement" + build_manifest + ;; + sqlite_inode_not_manifest_target) + replacement="${bundle_root}/unlisted-sqlite-inode" + cp -- "${pre_root}/${fixture_database_relative#./}" "$replacement" + rm -- "${pre_root}/${fixture_database_relative#./}" + ln -- "$replacement" "${pre_root}/${fixture_database_relative#./}" + ;; + sqlite_sidecar_symlink) + replacement="${pre_root}/payload/sqlite-sidecar-target" + /usr/bin/truncate --size=1024 -- "$replacement" + rm -- "${pre_root}/${fixture_database_relative#./}-wal" + ln -s -- "$replacement" \ + "${pre_root}/${fixture_database_relative#./}-wal" + ;; + sqlite_sidecar_hardlink) + replacement="${pre_root}/payload/sqlite-sidecar-target" + /usr/bin/truncate --size=1024 -- "$replacement" + rm -- "${pre_root}/${fixture_database_relative#./}-wal" + ln -- "$replacement" "${pre_root}/${fixture_database_relative#./}-wal" + ;; + sqlite_capture_alias) + ln -s -- sqlite.stderr "${capture_root}/sqlite.stdout" + ;; + sqlite_capture_preexisting) + : >"${capture_root}/sqlite.stdout" + chmod 0600 -- "${capture_root}/sqlite.stdout" + ;; + sqlite_capture_stderr_preexisting) + # Injected at the stderr-open boundary, after stdout is created and pinned. + ;; + sqlite_capture_stderr_fifo) + mkfifo -- "${capture_root}/sqlite.stderr" + ;; + sqlite_capture_stderr_symlink) + : >"${capture_root}/home/sqlite-stderr-target" + chmod 0600 -- "${capture_root}/home/sqlite-stderr-target" + ln -s -- home/sqlite-stderr-target "${capture_root}/sqlite.stderr" + ;; + sqlite_layout_unmanifested_symlink|sqlite_layout_unmanifested_fifo|sqlite_layout_unmanifested_directory) + old_wal="${fixture_database_relative}-wal" + replacement='./payload/sqlite-layout-wal-replacement' + mv -- "${pre_root}/${old_wal#./}" "${pre_root}/${replacement#./}" + replace_fixture_target "$old_wal" "$replacement" + build_manifest + case "$mutation" in + sqlite_layout_unmanifested_symlink) + ln -s -- "${pre_root}/${replacement#./}" "${pre_root}/${old_wal#./}" + ;; + sqlite_layout_unmanifested_fifo) + mkfifo -- "${pre_root}/${old_wal#./}" + ;; + sqlite_layout_unmanifested_directory) + mkdir -- "${pre_root}/${old_wal#./}" + ;; + esac + ;; + payload_component_symlink) + mv -- "${pre_root}/payload" "${pre_root}/payload.real" + ln -s -- payload.real "${pre_root}/payload" + anchor_path="${pre_root}/payload/anchor.bin" + ;; + *) + return 97 + ;; + esac +} + +# The only command model installed for payload verification. It delegates +# read-only parsing of test-owned files, models mount/mapping observations, and +# rejects every unregistered binary with rc 97. +install_payload_command_model() { + _k3slra1_stderr_open_boundary() { + trace_stage stderr-open-attempt + if [[ "$current_case" == sqlite_capture_stderr_preexisting ]]; then + : >"${capture_root}/sqlite.stderr" + chmod 0600 -- "${capture_root}/sqlite.stderr" + fi + } + + _k3slra1_command() { + local binary="${1-}" expected_mapper expected_canonical expected_mm modeled_source + local option path observed_canonical + (( $# >= 1 )) || return 97 + expected_mapper="$(expected_mapper_for_role "$current_role")" || return 97 + expected_canonical="$(expected_canonical_mapper_for_role "$current_role")" || return 97 + expected_mm="$(expected_major_minor_for_role "$current_role")" || return 97 + [[ "$current_case" != canonical_node_wrong_namespace ]] || expected_canonical='/dev/loop7' + modeled_source="$expected_mapper" + case "$current_case" in + proof_mapper_swap) modeled_source="$A1_MAIN_MAPPER" ;; + baseline_mapper_swap|final_mapper_swap) modeled_source="$A1_PROOF_MAPPER" ;; + source_expected_canonical_mismatch) modeled_source='/dev/mapper/a1-source-mismatch' ;; + mount_wrong_source) modeled_source='/dev/mapper/not-owned' ;; + esac + case "$binary" in + /usr/bin/findmnt) + (( $# == 7 )) && [[ "${2-}" == --noheadings && "${3-}" == --raw && + "${4-}" == --mountpoint && "${5-}" == "$mount_root" && + "${6-}" == --output && + "${7-}" == 'SOURCE,TARGET,FSTYPE,VFS-OPTIONS,FS-OPTIONS,MAJ:MIN' ]] || return 97 + trace_stage findmnt || return 97 + findmnt_calls="$(trace_count findmnt)" || return 97 + local observed_source="$modeled_source" observed_fstype=ext4 + local observed_target="$mount_root" observed_vfs='ro,nosuid,nodev,noexec,relatime' + local observed_fs=noload observed_mm="$expected_mm" + [[ "$current_case" == mount_missing_noload ]] && + observed_fs='errors=remount-ro' + [[ "$current_case" == mount_wrong_source ]] && observed_source='/dev/mapper/not-owned' + [[ "$current_case" == mount_wrong_target ]] && observed_target="${mount_root}-other" + [[ "$current_case" == mount_wrong_fstype ]] && observed_fstype=xfs + [[ "$current_case" == mount_wrong_major_minor ]] && observed_mm='253:99' + [[ "$current_case" == mount_missing_vfs_ro ]] && + observed_vfs='nosuid,nodev,noexec,relatime' + [[ "$current_case" == mount_forbidden_vfs_rw ]] && + observed_vfs='rw,nosuid,nodev,noexec,relatime' + [[ "$current_case" == mount_evidence_drift && $findmnt_calls -ge 2 ]] && + observed_mm='253:8' + [[ "$current_case" == mount_third_evidence_drift && $findmnt_calls -ge 3 ]] && + observed_mm='253:8' + case "$current_case" in + mount_evidence_multiline) + printf '%s %s %s %s %s %s\n%s %s %s %s %s %s\n' \ + "$observed_source" "$observed_target" "$observed_fstype" "$observed_vfs" "$observed_fs" "$observed_mm" \ + "$observed_source" "$observed_target" "$observed_fstype" "$observed_vfs" "$observed_fs" "$observed_mm" + return 0 + ;; + mount_evidence_cr) + printf '%s %s %s %s %s %s\r\n' "$observed_source" "$observed_target" \ + "$observed_fstype" "$observed_vfs" "$observed_fs" "$observed_mm" + return 0 + ;; + mount_evidence_nul) + printf '%s %s %s %s %s %s\0\n' "$observed_source" "$observed_target" \ + "$observed_fstype" "$observed_vfs" "$observed_fs" "$observed_mm" + return 0 + ;; + mount_evidence_stderr) + printf 'mount-canary\n' >&2 + ;; + findmnt_valid_record_stderr) + printf '%s %s %s %s %s %s\n' "$observed_source" "$observed_target" \ + "$observed_fstype" "$observed_vfs" "$observed_fs" "$observed_mm" >&2 + return 0 + ;; + esac + if [[ "$hostile_capture_surface" == findmnt ]]; then + hostile_model_output findmnt \ + "$observed_source $observed_target $observed_fstype $observed_vfs $observed_fs $observed_mm" + return $? + fi + printf '%s %s %s %s %s %s\n' "$observed_source" "$observed_target" \ + "$observed_fstype" "$observed_vfs" "$observed_fs" "$observed_mm" + ;; + /usr/sbin/blockdev) + (( $# == 3 )) && [[ "${2-}" == --getro && "${3-}" == "$expected_canonical" ]] || return 97 + trace_stage blockdev || return 97 + blockdev_calls="$(trace_count blockdev)" || return 97 + if [[ "$hostile_capture_surface" == blockdev ]]; then + hostile_model_output blockdev 1 + return $? + fi + if [[ "$current_case" == mapping_not_readonly || + ( "$current_case" == mapping_evidence_drift && $blockdev_calls -ge 2 ) || + ( "$current_case" == mapping_third_evidence_drift && $blockdev_calls -ge 3 ) ]]; then + printf '0\n' + else + case "$current_case" in + blockdev_multiline) printf '1\n1\n' ;; + blockdev_cr) printf '1\r\n' ;; + blockdev_nul) printf '1\0\n' ;; + blockdev_stderr) printf 'block-canary\n' >&2; printf '1\n' ;; + blockdev_valid_record_stderr) printf '1\n' >&2 ;; + *) printf '1\n' ;; + esac + fi + ;; + /usr/bin/readlink) + if (( $# == 4 )) && [[ "${2-}" == --canonicalize-existing && "${3-}" == -- ]] && + [[ "${4-}" == "$modeled_source" || "${4-}" == "$expected_mapper" ]]; then + trace_stage readlink || return 97 + readlink_calls="$(trace_count readlink)" || return 97 + observed_canonical="$(canonical_mapper_for_alias "${4-}")" || return 97 + if [[ "$current_case" == mapper_canonical_mismatch ]] && + (( readlink_calls % 2 == 0 )); then + observed_canonical='/dev/dm-99' + fi + if [[ "$current_case" == readlink_valid_record_stderr ]]; then + printf '%s\n' "$observed_canonical" >&2 + return 0 + fi + printf '%s\n' "$observed_canonical" + return 0 + fi + (( $# == 4 )) && [[ "${2-}" == --canonicalize-existing && "${3-}" == -- ]] || return 97 + fixture_path_is_owned "${4-}" || return 97 + /usr/bin/readlink --canonicalize-existing -- "${4-}" + ;; + /usr/bin/lsblk) + (( $# == 8 )) && [[ "${2-}" == --noheadings && "${3-}" == --nodeps && + "${4-}" == --raw && "${5-}" == --output && "${6-}" == 'MAJ:MIN' && + "${7-}" == -- && "${8-}" == "$expected_canonical" ]] || return 97 + trace_stage lsblk || return 97 + lsblk_calls="$(trace_count lsblk)" || return 97 + case "$current_case" in + mapper_major_minor_mismatch) printf '253:99\n' ;; + lsblk_valid_record_stderr) printf '%s\n' "$expected_mm" >&2 ;; + *) + if [[ "$hostile_capture_surface" == lsblk ]]; then + hostile_model_output lsblk "$expected_mm" + return $? + fi + printf '%s\n' "$expected_mm" + ;; + esac + ;; + /usr/bin/stat) + if [[ "${!#}" == "$expected_canonical" ]]; then + trace_stage mapper-stat || return 97 + case "${2-}" in + --format='%F|%Hr:%Lr') + if [[ "$hostile_capture_surface" == mapper-stat ]]; then + emit_hostile_capture "block special file|${expected_mm}" + return $? + else + [[ "$current_case" == mapper_not_block ]] && + printf 'regular file|%s\n' "$expected_mm" || printf 'block special file|%s\n' "$expected_mm" + fi + ;; + --format='%F') + [[ "$current_case" == mapper_not_block ]] && + printf 'regular file\n' || printf 'block special file\n' + ;; + --format='%Hr:%Lr') printf '%s\n' "$expected_mm" ;; + *) return 97 ;; + esac + return 0 + fi + if [[ "${!#}" == /proc/[0-9]*/fd/[0-9]* ]]; then + (( $# == 5 )) && [[ "${2-}" == --dereference && "${3-}" == --format=* && + "${4-}" == -- ]] || return 97 + case "${5-}" in + */fd/6) trace_stage fd6-stat || return 97 ;; + */fd/7) trace_stage fd7-stat || return 97 ;; + */fd/8) trace_stage fd8-stat || return 97 ;; + *) return 97 ;; + esac + model_test_owned_stat "${3#--format=}" "${5-}" + return 0 + fi + (( $# == 4 )) && [[ "${2-}" == --format=* && "${3-}" == -- ]] || return 97 + model_test_owned_stat "${2#--format=}" "${4-}" + ;; + /usr/bin/sha256sum) + (( $# == 3 )) && [[ "${2-}" == -- ]] || return 97 + fixture_path_is_owned "${3-}" || return 97 + /usr/bin/sha256sum -- "${3-}" + ;; + /usr/bin/od) + (( $# == 6 )) && [[ "${2-}" == -An && "${3-}" == -v && "${4-}" == -tx1 && + "${5-}" == -- ]] || return 97 + fixture_path_is_owned "${6-}" || return 97 + /usr/bin/od -An -v -tx1 -- "${6-}" + ;; + /usr/bin/wc) + (( $# == 4 )) && [[ "${2-}" == -c && "${3-}" == -- ]] || return 97 + fixture_path_is_owned "${4-}" || return 97 + /usr/bin/wc -c -- "${4-}" + ;; + /usr/bin/find) + if (( $# == 7 )) && [[ "${2-}" == "$mount_root" && "${3-}" == -mindepth && + "${4-}" == 1 && "${5-}" == -maxdepth && "${6-}" == 1 && "${7-}" == -print0 ]]; then + if [[ "$hostile_capture_surface" == bundle-find ]]; then + hostile_bundle_find_output + return $? + fi + /usr/bin/find "$mount_root" -mindepth 1 -maxdepth 1 -print0 + elif (( $# == 6 )) && [[ "${2-}" == "$pre_root" && "${3-}" == -xdev && + "${4-}" == -type && "${5-}" == f && "${6-}" == -print0 ]]; then + if [[ "$hostile_capture_surface" == inventory-find ]]; then + hostile_inventory_find_output + return $? + fi + /usr/bin/find "$pre_root" -xdev -type f -print0 + else + return 97 + fi + ;; + /usr/bin/env) + local expected_database="${pre_root}/${fixture_database_relative#./}" + (( $# == 16 )) && [[ "${2-}" == -i && + "${3-}" == 'PATH=/usr/sbin:/usr/bin:/sbin:/bin' && "${4-}" == 'LC_ALL=C' && + "${5-}" == "HOME=${capture_root}/home" && "${6-}" == /usr/bin/sqlite3 && + "${7-}" == -safe && "${8-}" == -nofollow && "${9-}" == -readonly && + "${10-}" == -batch && "${11-}" == -bail && "${12-}" == -noheader && + "${13-}" == -init && "${14-}" == /dev/null && + "${15-}" == "$expected_database" && + "${16-}" == 'PRAGMA query_only=ON; PRAGMA quick_check;' ]] || return 97 + trace_stage sqlite || return 97 + sqlite_calls="$(trace_count sqlite)" || return 97 + case "$current_case" in + sqlite_capture_int_cleanup) + kill -INT "$BASHPID" + return 130 + ;; + sqlite_capture_term_cleanup) + kill -TERM "$BASHPID" + return 143 + ;; + sqlite_capture_term_cleanup_failure) + kill -TERM "$BASHPID" + return 143 + ;; + esac + case "$current_case" in + sqlite_capture_path_swap) + mv -- "${capture_root}/sqlite.stdout" "${capture_root}/sqlite.stdout.detached" + : >"${capture_root}/sqlite.stdout" + chmod 0600 -- "${capture_root}/sqlite.stdout" + ;; + sqlite_capture_stdout_hardlink_swap) + ln -- "${capture_root}/sqlite.stdout" "${capture_root}/sqlite.stdout.detached" + ;; + sqlite_capture_stderr_hardlink_swap) + ln -- "${capture_root}/sqlite.stderr" "${capture_root}/sqlite.stderr.detached" + ;; + manifest_drift_after_sqlite) + /usr/bin/truncate --size=$((A1_ANCHOR_BYTES + 1)) -- "$anchor_path" + ;; + payload_write_attempt) + : >"${pre_root}/payload/unexpected-write" + ;; + esac + case "$current_case" in + sqlite_quick_check_extra_line) printf 'ok\nextra\n' ;; + sqlite_quick_check_stderr) printf 'sqlite-canary\n' >&2; printf 'ok\n' ;; + sqlite_quick_check_nul) printf 'ok\0\n' ;; + sqlite_quick_check_error) return 23 ;; + *) printf 'ok\n' ;; + esac + ;; + /usr/bin/unlink) + (( $# == 2 )) || return 97 + case "${2-}" in + "${capture_root}/sqlite.stdout") trace_stage unlink-stdout || return 97 ;; + "${capture_root}/sqlite.stderr") trace_stage unlink-stderr || return 97 ;; + *) return 97 ;; + esac + if [[ "$current_case" == sqlite_capture_cleanup_failure || + "$current_case" == sqlite_capture_term_cleanup_failure ]]; then + return 31 + fi + if [[ "$current_case" == sqlite_partial_unlink_noop ]]; then + return 0 + fi + /usr/bin/unlink "${2-}" + ;; + *) + return 97 + ;; + esac + } +} + +hostile_model_output() { + local key="${1-}" valid="${2-}" + (( $# == 2 )) || return 97 + if [[ "$hostile_capture_surface" == "$key" ]]; then + trace_stage "hostile-${key}" || return 97 + emit_hostile_capture "$valid" + else + printf '%s\n' "$valid" + fi +} + +hostile_inventory_find_output() { + trace_stage hostile-inventory-find || return 97 + case "$hostile_capture_mode" in + stdout-nonzero) + /usr/bin/find "$pre_root" -xdev -type f -print0 + return 23 + ;; + raw-nul) + /usr/bin/find "$pre_root" -xdev -type f -print0 + printf '%s\0' "$manifest_path" + ;; + extra-trailing-lf) + /usr/bin/find "$pre_root" -xdev -type f -print0 + printf '%s\n\0' "${pre_root}/payload/not-an-inventory-path" + ;; + nonempty-stderr) + /usr/bin/find "$pre_root" -xdev -type f -print0 + printf 'hostile-inventory-stderr\n' >&2 + ;; + *) return 97 ;; + esac +} + +hostile_bundle_find_output() { + trace_stage hostile-bundle-find || return 97 + case "$hostile_capture_mode" in + stdout-nonzero) + /usr/bin/find "$mount_root" -mindepth 1 -maxdepth 1 -print0 + return 23 + ;; + raw-nul) + /usr/bin/find "$mount_root" -mindepth 1 -maxdepth 1 -print0 + printf '%s\0' "$bundle_root" + ;; + extra-trailing-lf) + printf '%s\n\0' "$bundle_root" + ;; + nonempty-stderr) + /usr/bin/find "$mount_root" -mindepth 1 -maxdepth 1 -print0 + printf 'hostile-bundle-stderr\n' >&2 + ;; + *) return 97 ;; + esac +} + +install_hostile_library_command_model() { + _k3slra1_command() { + local valid='' key='' + case "${1-}" in + /usr/bin/stat) + if (( $# == 5 )) && [[ "${2-}" == --dereference && + "${3-}" == "--format=${A1_METADATA_FORMAT}" && "${4-}" == -- ]]; then + [[ "${5-}" == /proc/[0-9]*/fd/[0-9]* ]] || return 97 + valid="$(model_test_owned_stat "$A1_METADATA_FORMAT" "${5-}")" || return 97 + hostile_model_output fd-stat "$valid" + return + fi + (( $# == 4 )) && [[ "${3-}" == -- && "${4-}" == "$hostile_capture_file" ]] || return 97 + case "${2-}" in + "--format=${A1_METADATA_FORMAT}") + valid="$(model_test_owned_stat "$A1_METADATA_FORMAT" "${4-}")" || return 97 + key=metadata-stat + ;; + --format='%s') + valid="$(/usr/bin/stat --format='%s' -- "${4-}")" || return 97 + key="${hostile_capture_family}-stat" + ;; + *) return 97 ;; + esac + hostile_model_output "$key" "$valid" + ;; + /usr/bin/readlink) + (( $# == 4 )) && [[ "${2-}" == --canonicalize-existing && "${3-}" == -- && + "${4-}" == "$hostile_capture_file" ]] || return 97 + valid="$(/usr/bin/readlink --canonicalize-existing -- "${4-}")" || return 97 + hostile_model_output canonical-readlink "$valid" + ;; + /usr/bin/sha256sum) + (( $# == 3 )) && [[ "${2-}" == -- && "${3-}" == "$hostile_capture_file" ]] || return 97 + valid="$(/usr/bin/sha256sum -- "${3-}")" || return 97 + hostile_model_output sha256 "$valid" + ;; + /usr/bin/wc) + (( $# == 4 )) && [[ "${2-}" == -c && "${3-}" == -- && + "${4-}" == "$hostile_capture_file" ]] || return 97 + valid="$(/usr/bin/wc -c -- "${4-}")" || return 97 + hostile_model_output "${hostile_capture_family}-wc" "$valid" + ;; + /usr/bin/od) + (( $# == 6 )) && [[ "${2-}" == -An && "${3-}" == -v && "${4-}" == -tx1 && + "${5-}" == -- && "${6-}" == "$hostile_capture_file" ]] || return 97 + valid="$(/usr/bin/od -An -v -tx1 -- "${6-}")" || return 97 + hostile_model_output "${hostile_capture_family}-od" "$valid" + ;; + *) return 97 ;; + esac + } +} + +install_hostile_helper_command_model() { + _k3slra1h_command() { + local valid='' + case "${1-}" in + /usr/bin/stat) + (( $# == 4 )) && [[ "${2-}" == "--format=${A1_METADATA_FORMAT}" && + "${3-}" == -- && "${4-}" == "$hostile_capture_file" ]] || return 97 + valid="$(model_test_owned_stat "$A1_METADATA_FORMAT" "${4-}")" || return 97 + hostile_model_output helper-stat "$valid" + ;; + /usr/bin/readlink) + (( $# == 4 )) && [[ "${2-}" == --canonicalize-existing && "${3-}" == -- && + "${4-}" == "$hostile_capture_file" ]] || return 97 + valid="$(/usr/bin/readlink --canonicalize-existing -- "${4-}")" || return 97 + hostile_model_output helper-readlink "$valid" + ;; + *) return 97 ;; + esac + } +} + +hostile_capture_api_rejects() { + local surface="${1-}" mode="${2-}" result='sentinel' rc=0 before=0 after=0 + local trace_key='' + local identity='sentinel' nlink='sentinel' size='sentinel' mtime='sentinel' + local type='sentinel' owner='sentinel' permissions='sentinel' fd_path='' + local count='count-sentinel' bytes='bytes-sentinel' snapshot='snapshot-sentinel' + local stdout_sink="${case_root}/hostile-api.stdout" stderr_sink="${case_root}/hostile-api.stderr" + (( $# == 2 )) || return 97 + current_case=hostile-capture + current_role=proof + hostile_capture_surface="$surface" + hostile_capture_mode="$mode" + hostile_capture_family='' + case "$surface" in + metadata-stat|canonical-readlink|sha256) hostile_capture_file="$anchor_path" ;; + fd-stat) hostile_capture_file="$anchor_path" ;; + manifest-stat|manifest-wc|manifest-od) + hostile_capture_file="$manifest_path" + hostile_capture_family=manifest + ;; + file-stat|file-wc|file-od) + hostile_capture_file="${capture_root}/hex-probe" + hostile_capture_family=file + ;; + helper-stat|helper-readlink) hostile_capture_file="$anchor_path" ;; + mapper-stat|findmnt|lsblk|blockdev|bundle-find|inventory-find) ;; + *) return 97 ;; + esac + trace_key="hostile-${surface}" + [[ "$surface" != mapper-stat ]] || trace_key=mapper-stat + : >"$stdout_sink" + : >"$stderr_sink" + before="$(trace_count "$trace_key")" || return 97 + case "$surface" in + metadata-stat) + install_hostile_library_command_model + if _k3slra1_metadata "$hostile_capture_file" result >"$stdout_sink" 2>"$stderr_sink"; then rc=0; else rc=$?; fi + [[ "$result" == sentinel ]] || return 1 + ;; + canonical-readlink) + install_hostile_library_command_model + if _k3slra1_canonical_path "$hostile_capture_file" result >"$stdout_sink" 2>"$stderr_sink"; then rc=0; else rc=$?; fi + [[ "$result" == sentinel ]] || return 1 + ;; + fd-stat) + install_hostile_library_command_model + exec 9<"$hostile_capture_file" + fd_path="/proc/${BASHPID}/fd/9" + if _k3slra1_fd_metadata_fields "$fd_path" identity nlink size mtime type owner permissions >"$stdout_sink" 2>"$stderr_sink"; then rc=0; else rc=$?; fi + exec 9>&- + [[ "$identity|$nlink|$size|$mtime|$type|$owner|$permissions" == \ + 'sentinel|sentinel|sentinel|sentinel|sentinel|sentinel|sentinel' ]] || return 1 + ;; + manifest-stat|manifest-wc|manifest-od) + install_hostile_library_command_model + if _k3slra1_capture_manifest_hex "$hostile_capture_file" result >"$stdout_sink" 2>"$stderr_sink"; then rc=0; else rc=$?; fi + [[ "$result" == sentinel ]] || return 1 + ;; + sha256) + install_hostile_library_command_model + if _k3slra1_sha256_record "$hostile_capture_file" result >"$stdout_sink" 2>"$stderr_sink"; then rc=0; else rc=$?; fi + [[ "$result" == sentinel ]] || return 1 + ;; + file-stat|file-wc|file-od) + install_hostile_library_command_model + if _k3slra1_capture_file_hex "$hostile_capture_file" 16 result >"$stdout_sink" 2>"$stderr_sink"; then rc=0; else rc=$?; fi + [[ "$result" == sentinel ]] || return 1 + ;; + helper-stat) + source "$A1_HELPER_PATH" + install_hostile_helper_command_model + if _k3slra1h_capture_metadata "$hostile_capture_file" result >"$stdout_sink" 2>"$stderr_sink"; then rc=0; else rc=$?; fi + [[ "$result" == sentinel ]] || return 1 + ;; + helper-readlink) + source "$A1_HELPER_PATH" + install_hostile_helper_command_model + if _k3slra1h_capture_canonical "$hostile_capture_file" result >"$stdout_sink" 2>"$stderr_sink"; then rc=0; else rc=$?; fi + [[ "$result" == sentinel ]] || return 1 + ;; + mapper-stat|findmnt|lsblk|blockdev) + current_case=hostile-mapper-stat + current_role=proof + install_payload_command_model + if _k3slra1_capture_mount_mapping_evidence "$mount_root" proof result >"$stdout_sink" 2>"$stderr_sink"; then rc=0; else rc=$?; fi + [[ "$result" == sentinel ]] || return 1 + ;; + inventory-find) + current_case=hostile-inventory-find + current_role=proof + install_payload_command_model + if _k3slra1_analyze_manifest "$bundle_root" count bytes snapshot >"$stdout_sink" 2>"$stderr_sink"; then rc=0; else rc=$?; fi + [[ "$count|$bytes|$snapshot" == \ + 'count-sentinel|bytes-sentinel|snapshot-sentinel' ]] || return 1 + ;; + bundle-find) + current_case=hostile-bundle-find + current_role=proof + install_payload_command_model + if _k3slra1_select_pre_bundle "$mount_root" result >"$stdout_sink" 2>"$stderr_sink"; then rc=0; else rc=$?; fi + [[ "$result" == sentinel ]] || return 1 + ;; + esac + after="$(trace_count "$trace_key")" || return 97 + (( after == before + 1 )) || return 97 + (( rc != 0 && rc != 97 )) || return 1 + [[ ! -s "$stdout_sink" && ! -s "$stderr_sink" ]] +} + +test_command_capture_hostile_matrix() ( + local surface mode violation_count=0 + case_root="${fixture_root}/hostile-capture-${BASHPID}" + mkdir -- "$case_root" + current_case=hostile-capture + current_role=proof + command_trace="${case_root}/command.trace" + hostile_violation_trace="${case_root}/violations" + : >"$command_trace" + : >"$hostile_violation_trace" + build_payload canonical + printf 'ok\n' >"${capture_root}/hex-probe" + chmod 0600 -- "${capture_root}/hex-probe" + for surface in metadata-stat canonical-readlink mapper-stat findmnt lsblk blockdev bundle-find inventory-find fd-stat manifest-stat manifest-wc manifest-od sha256 file-stat file-wc file-od helper-stat helper-readlink; do + for mode in stdout-nonzero raw-nul extra-trailing-lf nonempty-stderr; do + if hostile_capture_api_rejects "$surface" "$mode"; then + : + else + printf '%s:%s\n' "$surface" "$mode" >>"$hostile_violation_trace" + (( violation_count += 1 )) + fi + done + done + (( violation_count == 0 )) +) + +test_dynamic_output_aliases_are_closed() ( + local rc=0 mount_record='mount-sentinel' + local path='path-sentinel' mf_nlink='nlink-sentinel' mf_size='size-sentinel' + local mf_mtime='mtime-sentinel' mf_type='type-sentinel' + local mf_owner='owner-sentinel' mf_mode='mode-sentinel' + local metadata='metadata-sentinel' ff_nlink='fd-nlink-sentinel' + local ff_size='fd-size-sentinel' ff_mtime='fd-mtime-sentinel' + local ff_type='fd-type-sentinel' ff_owner='fd-owner-sentinel' + local ff_mode='fd-mode-sentinel' canonical_mount='bundle-sentinel' + local manifest='manifest-sentinel' am_bytes='analyze-bytes-sentinel' + local am_snapshot='analyze-snapshot-sentinel' bundle='count-sentinel' + local vm_bytes='verify-bytes-sentinel' record='helper-sentinel' + local size='capture-sentinel' target='helper-canonical-sentinel' + local _selected_value='selected-internal-sentinel' + local _count_value='count-internal-sentinel' + local _bytes_value='bytes-internal-sentinel' + local _snapshot_value='snapshot-internal-sentinel' + local _count_result='count-result-sentinel' + local _bytes_result='bytes-result-sentinel' + local _snapshot_result='snapshot-result-sentinel' + local alias_name='' expected_alias='' + case_root="${fixture_root}/dynamic-alias-${BASHPID}" + mkdir -- "$case_root" + current_case=dynamic-output-alias + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + + if _k3slra1_capture_mount_mapping_evidence "$mount_root" proof mount_record; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$mount_record" == mount-sentinel && ! -s "$command_trace" ]] || return 1 + + : >"$command_trace" + if _k3slra1_metadata_fields "$anchor_path" path mf_nlink mf_size mf_mtime mf_type mf_owner mf_mode; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$path|$mf_nlink|$mf_size|$mf_mtime|$mf_type|$mf_owner|$mf_mode" == \ + 'path-sentinel|nlink-sentinel|size-sentinel|mtime-sentinel|type-sentinel|owner-sentinel|mode-sentinel' ]] || return 1 + [[ ! -s "$command_trace" ]] || return 1 + + : >"$command_trace" + exec 9<"$anchor_path" + if _k3slra1_fd_metadata_fields "/proc/${BASHPID}/fd/9" metadata ff_nlink ff_size ff_mtime ff_type ff_owner ff_mode; then rc=0; else rc=$?; fi + exec 9>&- + (( rc != 0 && rc != 97 )) || return 1 + [[ "$metadata|$ff_nlink|$ff_size|$ff_mtime|$ff_type|$ff_owner|$ff_mode" == \ + 'metadata-sentinel|fd-nlink-sentinel|fd-size-sentinel|fd-mtime-sentinel|fd-type-sentinel|fd-owner-sentinel|fd-mode-sentinel' ]] || return 1 + [[ ! -s "$command_trace" ]] || return 1 + + : >"$command_trace" + if _k3slra1_select_pre_bundle "$mount_root" canonical_mount; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$canonical_mount" == bundle-sentinel && ! -s "$command_trace" ]] || return 1 + + : >"$command_trace" + if _k3slra1_analyze_manifest "$bundle_root" manifest am_bytes am_snapshot; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$manifest|$am_bytes|$am_snapshot" == \ + 'manifest-sentinel|analyze-bytes-sentinel|analyze-snapshot-sentinel' ]] || return 1 + [[ ! -s "$command_trace" ]] || return 1 + + for alias_name in _selected_value _count_value _bytes_value _snapshot_value; do + : >"$command_trace" + expected_alias="${!alias_name}" + if _k3slra1_analyze_manifest "$bundle_root" "$alias_name" am_bytes am_snapshot; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "${!alias_name}" == "$expected_alias" && ! -s "$command_trace" ]] || return 1 + done + + : >"$command_trace" + if _k3slra1_verify_manifest "$bundle_root" bundle vm_bytes; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$bundle|$vm_bytes" == 'count-sentinel|verify-bytes-sentinel' && ! -s "$command_trace" ]] || return 1 + + for alias_name in _count_result _bytes_result _snapshot_result; do + : >"$command_trace" + expected_alias="${!alias_name}" + if _k3slra1_verify_manifest "$bundle_root" "$alias_name" vm_bytes; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "${!alias_name}" == "$expected_alias" && ! -s "$command_trace" ]] || return 1 + done + + : >"$command_trace" + path='layout-sentinel' + if _k3slra1_select_sqlite_layout "$bundle_root" path; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$path" == layout-sentinel && ! -s "$command_trace" ]] || return 1 + + : >"$command_trace" + path='guard-sentinel' + if _k3slra1_metadata "$anchor_path" path; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$path" == guard-sentinel && ! -s "$command_trace" ]] || return 1 + if _k3slra1_canonical_path "$anchor_path" path; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$path" == guard-sentinel && ! -s "$command_trace" ]] || return 1 + if _k3slra1_capture_manifest_hex "$manifest_path" size; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$size" == capture-sentinel && ! -s "$command_trace" ]] || return 1 + printf 'ok\n' >"${capture_root}/alias-probe" + chmod 0600 -- "${capture_root}/alias-probe" + if _k3slra1_capture_file_hex "${capture_root}/alias-probe" 16 size; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$size" == capture-sentinel && ! -s "$command_trace" ]] || return 1 + + source "$A1_HELPER_PATH" + install_helper_command_model + if _k3slra1h_capture_metadata "$anchor_path" record; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$record" == helper-sentinel && ! -s "$command_trace" ]] || return 1 + if _k3slra1h_capture_canonical "$anchor_path" target; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$target" == helper-canonical-sentinel && ! -s "$command_trace" ]] +) + +test_mapper_stat_precedes_lsblk() ( + local evidence='unchanged' rc=0 + case_root="${fixture_root}/mapper-stat-order-${BASHPID}" + mkdir -- "$case_root" + current_case=mapper_not_block + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + if _k3slra1_capture_mount_mapping_evidence "$mount_root" proof evidence; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$evidence" == unchanged ]] || return 1 + [[ "$(trace_count findmnt)" == 1 && "$(trace_count readlink)" == 2 && + "$(trace_count mapper-stat)" == 1 && "$(trace_count lsblk)" == 0 && + "$(trace_count blockdev)" == 0 ]] +) + +test_canonical_mapper_namespace_is_closed() ( + local evidence='unchanged' rc=0 + case_root="${fixture_root}/mapper-namespace-${BASHPID}" + mkdir -- "$case_root" + current_case=canonical_node_wrong_namespace + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + if _k3slra1_capture_mount_mapping_evidence "$mount_root" proof evidence; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$evidence" == unchanged ]] || return 1 + [[ "$(trace_count findmnt)" == 1 && "$(trace_count readlink)" == 2 && + "$(trace_count mapper-stat)" == 0 && "$(trace_count lsblk)" == 0 && + "$(trace_count blockdev)" == 0 ]] +) + +run_case() ( + local case_name="${1-}" layout=canonical mutation verify_rc=0 + local direct_count='' direct_bytes='' direct_layout='unchanged' + local actual_wal_path='' actual_wal_before='' actual_wal_after='' + (( $# == 1 )) || return 97 + umask 022 + current_case="$case_name" + current_role=proof + case "$case_name" in + legacy_pass|legacy_sqlite_mixed_layout) layout=legacy ;; + manifest_plus_path_pass|manifest_path_512_bytes_pass|unrelated_lost_found_pass) + current_role=original-final + ;; + baseline_mapper_swap) current_role=original-baseline ;; + final_mapper_swap) current_role=original-final ;; + original_baseline_pass) current_role=original-baseline ;; + esac + case_root="${fixture_root}/case-${case_name}-${BASHPID}" + mkdir -- "$case_root" + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload "$layout" + mutation="$case_name" + [[ "$case_name" == legacy_sqlite_mixed_layout ]] && mutation=sqlite_mixed_layout + mutate_fixture "$mutation" + install_payload_command_model + + case "$case_name" in + sqlite_layout_unmanifested_symlink|sqlite_layout_unmanifested_fifo|sqlite_layout_unmanifested_directory) + _k3slra1_verify_manifest "$bundle_root" direct_count direct_bytes || return 97 + [[ "$direct_count" == "$A1_TARGET_COUNT" && "$direct_bytes" == "$A1_TARGET_BYTES" ]] || return 97 + _k3slra1_select_sqlite_layout "$bundle_root" direct_layout && return 97 + [[ "$direct_layout" == unchanged ]] || return 97 + ;; + esac + + if [[ "$case_name" == sqlite_wal_drift_after_child ]]; then + actual_wal_path="${pre_root}/${fixture_database_relative#./}-wal" + actual_wal_before="$( + /usr/bin/stat --format='%s|%y|%i' -- "$actual_wal_path" + /usr/bin/sha256sum -- "$actual_wal_path" + )" || return 97 + fi + + if _k3slra1_verify_payload_root "$mount_root" "$current_role" "$capture_root"; then + verify_rc=0 + else + verify_rc=$? + fi + + if [[ "$case_name" == sqlite_wal_drift_after_child ]]; then + actual_wal_after="$( + /usr/bin/stat --format='%s|%y|%i' -- "$actual_wal_path" + /usr/bin/sha256sum -- "$actual_wal_path" + )" || return 97 + [[ "$actual_wal_after" == "$actual_wal_before" ]] || return 97 + fi + + findmnt_calls="$(trace_count findmnt)" || return 97 + readlink_calls="$(trace_count readlink)" || return 97 + lsblk_calls="$(trace_count lsblk)" || return 97 + blockdev_calls="$(trace_count blockdev)" || return 97 + sqlite_calls="$(trace_count sqlite)" || return 97 + sqlite_unlink_stdout_calls="$(trace_count unlink-stdout)" || return 97 + sqlite_unlink_stderr_calls="$(trace_count unlink-stderr)" || return 97 + + case "$case_name" in + canonical_pass|legacy_pass|original_baseline_pass|manifest_plus_path_pass|manifest_path_512_bytes_pass|\ + unrelated_lost_found_pass) + (( findmnt_calls == 3 )) || return 97 + (( readlink_calls == 6 )) || return 97 + (( lsblk_calls == 3 )) || return 97 + (( blockdev_calls == 3 )) || return 97 + (( sqlite_calls == 1 )) || return 97 + (( sqlite_unlink_stdout_calls == 1 && sqlite_unlink_stderr_calls == 1 )) || return 97 + [[ "$(trace_count fd6-stat)" == 2 && "$(trace_count fd7-stat)" == 2 && + "$(trace_count fd8-stat)" == 2 ]] || return 97 + ;; + mount_missing_noload|mount_wrong_target|mount_wrong_fstype|\ + mount_wrong_major_minor|mount_missing_vfs_ro|mount_forbidden_vfs_rw|\ + mount_evidence_multiline|mount_evidence_cr|mount_evidence_nul|mount_evidence_stderr|\ + mapper_not_block|mapper_major_minor_mismatch|blockdev_multiline|blockdev_cr|\ + blockdev_nul|blockdev_stderr) + (( findmnt_calls >= 1 )) || return 97 + ;; + findmnt_valid_record_stderr) + (( findmnt_calls == 1 && readlink_calls == 0 && lsblk_calls == 0 && + blockdev_calls == 0 )) || return 97 + ;; + readlink_valid_record_stderr) + (( findmnt_calls == 1 && readlink_calls == 1 )) || return 97 + (( lsblk_calls == 0 && blockdev_calls == 0 )) || return 97 + ;; + lsblk_valid_record_stderr) + (( findmnt_calls == 1 && readlink_calls == 2 && lsblk_calls == 1 )) || return 97 + (( blockdev_calls == 0 )) || return 97 + ;; + blockdev_valid_record_stderr) + (( findmnt_calls == 1 && readlink_calls == 2 && lsblk_calls == 1 && + blockdev_calls == 1 )) || return 97 + ;; + mount_wrong_source|proof_mapper_swap|baseline_mapper_swap|final_mapper_swap|mapper_canonical_mismatch|\ + source_expected_canonical_mismatch) + (( findmnt_calls == 1 && readlink_calls == 2 )) || return 97 + (( lsblk_calls == 0 && blockdev_calls == 0 && sqlite_calls == 0 )) || return 97 + ;; + mapping_not_readonly) + (( blockdev_calls >= 1 )) || return 97 + ;; + mount_evidence_drift) + (( findmnt_calls >= 2 )) || return 97 + ;; + mapping_evidence_drift) + (( blockdev_calls >= 2 )) || return 97 + ;; + mount_third_evidence_drift) + (( findmnt_calls == 3 && sqlite_calls == 1 )) || return 97 + (( sqlite_unlink_stdout_calls == 1 && sqlite_unlink_stderr_calls == 1 )) || return 97 + ;; + mapping_third_evidence_drift) + (( blockdev_calls == 3 && sqlite_calls == 1 )) || return 97 + (( sqlite_unlink_stdout_calls == 1 && sqlite_unlink_stderr_calls == 1 )) || return 97 + ;; + payload_write_attempt|manifest_drift_after_sqlite|sqlite_wal_drift_after_child) + (( findmnt_calls == 2 && readlink_calls == 4 && lsblk_calls == 2 && + blockdev_calls == 2 )) || return 97 + (( sqlite_calls == 1 )) || return 97 + (( sqlite_unlink_stdout_calls == 1 && sqlite_unlink_stderr_calls == 1 )) || return 97 + ;; + sqlite_quick_check_extra_line|sqlite_quick_check_error|sqlite_quick_check_stderr|\ + sqlite_quick_check_nul|sqlite_capture_cleanup_failure) + (( sqlite_calls == 1 )) || return 97 + (( sqlite_unlink_stdout_calls == 1 && sqlite_unlink_stderr_calls == 1 )) || return 97 + ;; + sqlite_capture_wrong_owner|sqlite_capture_wrong_mode) + (( sqlite_calls == 0 )) || return 97 + (( sqlite_unlink_stdout_calls == 1 && sqlite_unlink_stderr_calls == 1 )) || return 97 + ;; + sqlite_capture_alias|sqlite_capture_preexisting) + (( sqlite_calls == 0 && sqlite_unlink_stdout_calls == 0 && + sqlite_unlink_stderr_calls == 0 )) || return 97 + ;; + sqlite_capture_stderr_preexisting) + (( sqlite_calls == 0 && sqlite_unlink_stdout_calls == 1 && + sqlite_unlink_stderr_calls == 0 )) || return 97 + [[ "$(trace_count fd6-stat)" == 1 && "$(trace_count fd7-stat)" == 1 && + "$(trace_count fd8-stat)" == 0 ]] || return 97 + [[ "$(/usr/bin/tail -n 4 -- "$command_trace")" == \ + $'fd6-stat\nfd7-stat\nstderr-open-attempt\nunlink-stdout' ]] || return 97 + [[ ! -e "${capture_root}/sqlite.stdout" && ! -L "${capture_root}/sqlite.stdout" && + -f "${capture_root}/sqlite.stderr" && ! -L "${capture_root}/sqlite.stderr" ]] || return 97 + ;; + sqlite_capture_stderr_fifo|sqlite_capture_stderr_symlink) + (( findmnt_calls == 2 && readlink_calls == 4 && lsblk_calls == 2 && + blockdev_calls == 2 && sqlite_calls == 0 )) || return 97 + (( sqlite_unlink_stdout_calls == 0 && sqlite_unlink_stderr_calls == 0 )) || return 97 + [[ "$(trace_count stderr-open-attempt)" == 0 && "$(trace_count fd6-stat)" == 0 && + "$(trace_count fd7-stat)" == 0 && "$(trace_count fd8-stat)" == 0 ]] || return 97 + [[ ! -e "${capture_root}/sqlite.stdout" && ! -L "${capture_root}/sqlite.stdout" ]] || return 97 + [[ -e "${capture_root}/sqlite.stderr" || -L "${capture_root}/sqlite.stderr" ]] || return 97 + ;; + sqlite_capture_path_swap) + (( sqlite_calls == 1 && sqlite_unlink_stdout_calls == 0 && + sqlite_unlink_stderr_calls == 1 )) || return 97 + [[ -f "${capture_root}/sqlite.stdout" && + -f "${capture_root}/sqlite.stdout.detached" ]] || return 97 + ;; + sqlite_capture_parent_drift) + (( sqlite_calls == 1 && sqlite_unlink_stdout_calls == 1 && + sqlite_unlink_stderr_calls == 1 )) || return 97 + ;; + manifest_duplicate_path) + (( findmnt_calls == 1 && sqlite_calls == 0 )) || return 97 + ;; + preexisting_unmanifested_file) + (( findmnt_calls == 1 && readlink_calls == 2 && lsblk_calls == 1 && + blockdev_calls == 1 )) || return 97 + (( sqlite_calls == 0 && sqlite_unlink_stdout_calls == 0 && + sqlite_unlink_stderr_calls == 0 )) || return 97 + ;; + bundle_*|matching_bundle_symlink|manifest_*|target_*|sqlite_mixed_layout|\ + legacy_sqlite_mixed_layout|sqlite_alternate_partial_layout|sqlite_missing_wal|\ + sqlite_missing_shm|sqlite_not_in_manifest|sqlite_inode_not_manifest_target|\ + sqlite_sidecar_symlink|sqlite_sidecar_hardlink|sqlite_layout_unmanifested_*|\ + bundle_wrong_owner|manifest_wrong_mode|\ + payload_wrong_device|payload_component_symlink) + (( sqlite_calls == 0 && sqlite_unlink_stdout_calls == 0 && + sqlite_unlink_stderr_calls == 0 )) || return 97 + ;; + esac + return "$verify_rc" +) + +test_direct_api_contract() ( + local selected='unchanged' count='unchanged' bytes='unchanged' layout='unchanged' + local invalid_rc=0 + case_root="${fixture_root}/direct-api-${BASHPID}" + mkdir -- "$case_root" + current_case=direct_api + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + + _k3slra1_select_pre_bundle "$mount_root" selected || return 1 + [[ "$selected" == "$bundle_root" ]] || return 1 + _k3slra1_verify_manifest "$bundle_root" count bytes || return 1 + [[ "$count" == "$A1_TARGET_COUNT" && "$bytes" == "$A1_TARGET_BYTES" ]] || return 1 + _k3slra1_select_sqlite_layout "$bundle_root" layout || return 1 + [[ "$layout" == canonical ]] || return 1 + + selected=unchanged + if _k3slra1_select_pre_bundle "$mount_root" '_k3slra1_select_bundle'; then + return 1 + else + invalid_rc=$? + fi + (( invalid_rc == 1 )) || return 1 + [[ "$selected" == unchanged ]] || return 1 + count=unchanged + if _k3slra1_verify_manifest "$bundle_root" count count; then + return 1 + else + invalid_rc=$? + fi + (( invalid_rc == 1 )) || return 1 + [[ "$count" == unchanged ]] || return 1 + if _k3slra1_verify_payload "$mount_root" proof; then + return 1 + else + invalid_rc=$? + fi + (( invalid_rc == 1 )) || return 1 + if _k3slra1_verify_payload "$mount_root" unsupported "$capture_root"; then + return 1 + else + invalid_rc=$? + fi + (( invalid_rc == 1 )) || return 1 + return 0 +) + +test_command_model_requires_canonical_argv() ( + local expected_canonical expected_mm fd_path fd_metadata='' observed_mm='' invalid_rc=0 + case_root="${fixture_root}/command-model-${BASHPID}" + mkdir -- "$case_root" + current_case=canonical_pass + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + expected_canonical="$(expected_canonical_mapper_for_role "$current_role")" || return 97 + expected_mm="$(expected_major_minor_for_role "$current_role")" || return 97 + : >"${capture_root}/sqlite.stdout" + chmod 0600 -- "${capture_root}/sqlite.stdout" + exec 7<>"${capture_root}/sqlite.stdout" + fd_path="/proc/${BASHPID}/fd/7" + if _k3slra1_command /usr/bin/stat --format="$A1_METADATA_FORMAT" -- "$fd_path"; then + return 1 + else + invalid_rc=$? + fi + (( invalid_rc == 97 )) || return 1 + fd_metadata="$(_k3slra1_command /usr/bin/stat --dereference --format="$A1_METADATA_FORMAT" -- "$fd_path")" || return 1 + [[ "$fd_metadata" == *'|regular file|0:0|600' || + "$fd_metadata" == *'|regular empty file|0:0|600' ]] || return 1 + if _k3slra1_command /usr/bin/lsblk --noheadings --nodeps --output 'MAJ:MIN' -- "$expected_canonical"; then + return 1 + else + invalid_rc=$? + fi + (( invalid_rc == 97 )) || return 1 + observed_mm="$(_k3slra1_command /usr/bin/lsblk --noheadings --nodeps --raw --output 'MAJ:MIN' -- "$expected_canonical")" || return 1 + [[ "$observed_mm" == "$expected_mm" ]] || return 1 + [[ "$(trace_count lsblk)" == 1 ]] || return 1 + exec 7>&- +) + +test_sqlite_signal_cleanup_contract() ( + local signal_name="${1-}" expected_case='' quick_rc=0 + (( $# == 1 )) || return 97 + case "$signal_name" in + INT) expected_case=sqlite_capture_int_cleanup ;; + TERM) expected_case=sqlite_capture_term_cleanup ;; + *) return 97 ;; + esac + case_root="${fixture_root}/sqlite-signal-${signal_name}-${BASHPID}" + mkdir -- "$case_root" + current_case="$expected_case" + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + if _k3slra1_sqlite_quick_check "$bundle_root" canonical "$capture_root"; then + quick_rc=0 + else + quick_rc=$? + fi + (( quick_rc != 0 && quick_rc != 97 )) || return 1 + [[ "$(trace_count sqlite)" == 1 ]] || return 1 + [[ "$(trace_count unlink-stdout)" == 1 ]] || return 1 + [[ "$(trace_count unlink-stderr)" == 1 ]] || return 1 + [[ ! -e "${capture_root}/sqlite.stdout" && ! -e "${capture_root}/sqlite.stderr" ]] +) + +test_root_signal_status_contract() ( + local case_name="${1-}" expected_rc="${2-}" root_rc=0 + (( $# == 2 )) || return 97 + [[ "$case_name" == sqlite_capture_term_cleanup || + "$case_name" == sqlite_capture_term_cleanup_failure ]] || return 97 + [[ "$expected_rc" == 1 || "$expected_rc" == 143 ]] || return 97 + case_root="${fixture_root}/root-signal-${case_name}-${BASHPID}" + mkdir -- "$case_root" + current_case="$case_name" + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + if _k3slra1_verify_payload_root "$mount_root" proof "$capture_root"; then + root_rc=0 + else + root_rc=$? + fi + (( root_rc == expected_rc )) || return 1 + [[ "$(trace_count sqlite)" == 1 ]] || return 1 + [[ "$(trace_count unlink-stdout)" == 1 ]] || return 1 + [[ "$(trace_count unlink-stderr)" == 1 ]] || return 1 + [[ "$(trace_count findmnt)" == 2 ]] || return 1 + if (( expected_rc == 143 )); then + [[ ! -e "${capture_root}/sqlite.stdout" && ! -e "${capture_root}/sqlite.stderr" ]] + else + [[ -f "${capture_root}/sqlite.stdout" && -f "${capture_root}/sqlite.stderr" ]] + fi +) + +test_sqlite_preserves_shell_state() ( + local case_name expected_success rc=0 before_umask after_umask + local before_int before_term after_int after_term + local state_trace="${fixture_root}/sqlite-shell-state-${BASHPID}.trace" + : >"$state_trace" + for case_name in canonical_pass sqlite_quick_check_error sqlite_capture_int_cleanup sqlite_capture_term_cleanup; do + case_root="${fixture_root}/sqlite-shell-state-${case_name}-${BASHPID}" + mkdir -- "$case_root" + current_case="$case_name" + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + umask 027 + trap 'printf "caller-int\n" >>"$state_trace"' INT + trap 'printf "caller-term\n" >>"$state_trace"' TERM + before_umask="$(umask)" + before_int="$(trap -p INT)" + before_term="$(trap -p TERM)" + if _k3slra1_sqlite_quick_check "$bundle_root" canonical "$capture_root"; then rc=0; else rc=$?; fi + after_umask="$(umask)" + after_int="$(trap -p INT)" + after_term="$(trap -p TERM)" + [[ "$before_umask" == "$after_umask" && "$before_int" == "$after_int" && + "$before_term" == "$after_term" ]] || return 1 + expected_success=0 + [[ "$case_name" != canonical_pass ]] || expected_success=1 + if (( expected_success == 1 )); then + (( rc == 0 )) || return 1 + else + (( rc != 0 && rc != 97 )) || return 1 + fi + trap - INT TERM + done + [[ ! -s "$state_trace" ]] +) + +test_sqlite_hardlink_swap_is_rejected() ( + local stream="${1-}" rc=0 detached='' + (( $# == 1 )) || return 97 + case "$stream" in + stdout) current_case=sqlite_capture_stdout_hardlink_swap ;; + stderr) current_case=sqlite_capture_stderr_hardlink_swap ;; + *) return 97 ;; + esac + case_root="${fixture_root}/sqlite-hardlink-${stream}-${BASHPID}" + mkdir -- "$case_root" + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + if _k3slra1_sqlite_quick_check "$bundle_root" canonical "$capture_root"; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$(trace_count sqlite)" == 1 && "$(trace_count fd6-stat)" == 2 && + "$(trace_count fd7-stat)" == 2 && "$(trace_count fd8-stat)" == 2 && + "$(trace_count unlink-stdout)" == 1 && + "$(trace_count unlink-stderr)" == 1 ]] || return 1 + [[ ! -e "${capture_root}/sqlite.stdout" && ! -e "${capture_root}/sqlite.stderr" ]] || return 1 + detached="${capture_root}/sqlite.${stream}.detached" + if [[ "$stream" == stdout ]]; then + detached="${capture_root}/sqlite.stdout.detached" + else + detached="${capture_root}/sqlite.stderr.detached" + fi + [[ -f "$detached" && "$(/usr/bin/stat --format='%h' -- "$detached")" == 1 ]] +) + +test_sqlite_layout_identity_binding() ( + local member suffix trace_name selected_path before after before_hash after_hash + local snapshot='' count='' bytes='' layout='unchanged' rc=0 + for member in db wal shm; do + case_root="${fixture_root}/sqlite-layout-binding-${member}-${BASHPID}" + mkdir -- "$case_root" + current_case=canonical_pass + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + _k3slra1_analyze_manifest "$bundle_root" count bytes snapshot || return 1 + [[ "$count" == "$A1_TARGET_COUNT" && "$bytes" == "$A1_TARGET_BYTES" ]] || return 1 + case "$member" in + db) suffix=''; trace_name=layout-db-stat ;; + wal) suffix=-wal; trace_name=layout-wal-stat ;; + shm) suffix=-shm; trace_name=layout-shm-stat ;; + *) return 97 ;; + esac + selected_path="${pre_root}/${fixture_database_relative#./}${suffix}" + before="$(/usr/bin/stat --format='%d:%i|%h|%s|%y|%F|%u:%g|%a' -- "$selected_path")" || return 97 + before_hash="$(/usr/bin/sha256sum -- "$selected_path")" || return 97 + [[ "$before" == *'|1|'* ]] || return 97 + current_case="sqlite_layout_${member}_identity_drift" + layout=unchanged + if _k3slra1_select_sqlite_layout_from_snapshot "$bundle_root" "$snapshot" layout; then rc=0; else rc=$?; fi + after="$(/usr/bin/stat --format='%d:%i|%h|%s|%y|%F|%u:%g|%a' -- "$selected_path")" || return 97 + after_hash="$(/usr/bin/sha256sum -- "$selected_path")" || return 97 + (( rc != 0 && rc != 97 )) || return 1 + [[ "$layout" == unchanged && "$before" == "$after" && + "$before_hash" == "$after_hash" ]] || return 1 + [[ "$(trace_count "$trace_name")" == 1 ]] || return 1 + done +) + +test_manifest_accepts_heterogeneous_target_metadata() ( + local count='unchanged' bytes='unchanged' + local identity='' nlink='' size='' mtime='' type='' owner='' mode='' + case_root="${fixture_root}/heterogeneous-targets-${BASHPID}" + mkdir -- "$case_root" + current_case=heterogeneous_target_metadata + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + _k3slra1_verify_manifest "$bundle_root" count bytes || return 1 + [[ "$count" == "$A1_TARGET_COUNT" && "$bytes" == "$A1_TARGET_BYTES" ]] || return 1 + _k3slra1_metadata_fields "${pre_root}/payload/dummy-01" \ + identity nlink size mtime type owner mode || return 1 + [[ "$nlink|$type|$owner|$mode" == '1|regular file|1000:1001|640' ]] || return 1 + _k3slra1_metadata_fields "${pre_root}/payload/dummy-02" \ + identity nlink size mtime type owner mode || return 1 + [[ "$nlink|$type|$owner|$mode" == '1|regular file|65534:65534|400' ]] || return 1 + _k3slra1_metadata_fields "${pre_root}/payload/dummy-03" \ + identity nlink size mtime type owner mode || return 1 + [[ "$nlink|$type|$owner|$mode" == '1|regular file|0:44|644' ]] +) + +test_partial_stdout_unlink_requires_absence() ( + local stdout_path identity='' nlink='' size='' mtime='' type='' owner='' mode='' rc=0 + case_root="${fixture_root}/partial-unlink-${BASHPID}" + mkdir -- "$case_root" + current_case=sqlite_partial_unlink_noop + current_role=proof + command_trace="${case_root}/command.trace" + : >"$command_trace" + build_payload canonical + install_payload_command_model + stdout_path="${capture_root}/sqlite.stdout" + : >"$stdout_path" + chmod 0600 -- "$stdout_path" + _k3slra1_metadata_fields "$stdout_path" identity nlink size mtime type owner mode || return 1 + if _k3slra1_remove_partial_capture "$stdout_path" "${identity%%:*}" "$identity"; then + rc=0 + else + rc=$? + fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$(trace_count unlink-stdout)" == 1 ]] || return 1 + [[ -f "$stdout_path" && ! -L "$stdout_path" ]] +) + +bounded_manifest_reader_accepts_65536_bytes() ( + local boundary_root="${fixture_root}/manifest-boundary-${BASHPID}" + local boundary_file="${boundary_root}/manifest" captured='unchanged' + mkdir -- "$boundary_root" + printf '%65535s\n' '' | tr ' ' a >"$boundary_file" + [[ "$(/usr/bin/stat --format='%s' -- "$boundary_file")" == 65536 ]] || return 97 + _k3slra1_capture_manifest_hex "$boundary_file" captured || return 1 + [[ ${#captured} -eq 131072 && "${captured: -2}" == 0a ]] +) + +bounded_manifest_reader_rejects_65537_bytes() ( + local boundary_root="${fixture_root}/manifest-overflow-${BASHPID}" rc=0 + local boundary_file="${boundary_root}/manifest" captured='unchanged' + mkdir -- "$boundary_root" + printf '%65536s\n' '' | tr ' ' a >"$boundary_file" + [[ "$(/usr/bin/stat --format='%s' -- "$boundary_file")" == 65537 ]] || return 97 + if _k3slra1_capture_manifest_hex "$boundary_file" captured; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$captured" == unchanged ]] +) + +fixed_mount_name_for_role() { + local role="${1-}" + (( $# == 1 )) || return 97 + case "$role" in + proof) printf 'proof-mount\n' ;; + original-baseline) printf 'original-baseline-mount\n' ;; + original-final) printf 'original-final-mount\n' ;; + *) return 97 ;; + esac +} + +test_parent_helper_dispatch() ( + local dispatch_count=0 dispatch_failure=0 role mount_name modeled_mount modeled_capture + local modeled_helper invalid_rc=0 + modeled_capture="${fixture_root}/parent-dispatch-${BASHPID}" + modeled_helper="${modeled_capture}/libexec/k3s-local-recovery-a1-check.sh" + mkdir -p -- "${modeled_capture}/libexec" + : >"$modeled_helper" + + _k3slra1_command() { + (( $# == 15 )) || return 97 + [[ "${1-}" == /usr/bin/sudo && "${2-}" == --non-interactive && "${3-}" == -- && + "${4-}" == /usr/bin/env && "${5-}" == -i && + "${6-}" == 'PATH=/usr/sbin:/usr/bin:/sbin:/bin' && "${7-}" == 'LC_ALL=C' && + "${8-}" == /usr/bin/bash && "${9-}" == --noprofile && "${10-}" == --norc && + "${11-}" == -- && "${12-}" == "$modeled_helper" && + "${13-}" == "$modeled_mount" && "${14-}" == "$role" && + "${15-}" == "$modeled_capture" ]] || return 97 + (( dispatch_count += 1 )) + (( dispatch_failure == 0 )) + } + + export BASH_ENV='/fixture/must-not-cross-env-i' + export -f fixture_path_is_owned + for role in original-baseline proof original-final; do + mount_name="$(fixed_mount_name_for_role "$role")" || return 1 + modeled_mount="${modeled_capture}/${mount_name}" + mkdir -- "$modeled_mount" + _k3slra1_verify_payload "$modeled_mount" "$role" "$modeled_capture" || return 1 + done + (( dispatch_count == 3 )) || return 1 + + if _k3slra1_verify_payload "$modeled_mount" unsupported "$modeled_capture"; then + return 1 + else + invalid_rc=$? + fi + (( invalid_rc == 1 )) || return 1 + if _k3slra1_verify_payload "$modeled_mount" proof; then + return 1 + else + invalid_rc=$? + fi + (( invalid_rc == 1 )) || return 1 + (( dispatch_count == 3 )) || return 1 + dispatch_failure=1 + role=proof + modeled_mount="${modeled_capture}/proof-mount" + if _k3slra1_verify_payload "$modeled_mount" "$role" "$modeled_capture"; then + return 1 + else + invalid_rc=$? + fi + (( invalid_rc == 1 )) || return 1 + (( dispatch_count == 4 )) || return 1 +) + +test_parent_helper_output_is_suppressed() ( + local modeled_capture="${fixture_root}/parent-quiet-${BASHPID}" + local modeled_mount="${modeled_capture}/proof-mount" + local modeled_helper="${modeled_capture}/libexec/k3s-local-recovery-a1-check.sh" + local public_stdout="${modeled_capture}/public.stdout" + local public_stderr="${modeled_capture}/public.stderr" + local dispatch_count=0 rc=0 + mkdir -p -- "${modeled_capture}/libexec" "$modeled_mount" + : >"$modeled_helper" + _k3slra1_command() { + (( $# == 15 )) || return 97 + [[ "${1-}" == /usr/bin/sudo && "${2-}" == --non-interactive && "${3-}" == -- && + "${4-}" == /usr/bin/env && "${5-}" == -i && + "${6-}" == 'PATH=/usr/sbin:/usr/bin:/sbin:/bin' && "${7-}" == 'LC_ALL=C' && + "${8-}" == /usr/bin/bash && "${9-}" == --noprofile && "${10-}" == --norc && + "${11-}" == -- && "${12-}" == "$modeled_helper" && + "${13-}" == "$modeled_mount" && "${14-}" == proof && + "${15-}" == "$modeled_capture" ]] || return 97 + (( dispatch_count += 1 )) + printf 'hostile-sudo-stdout\n' + printf 'hostile-sudo-stderr\n' >&2 + return 23 + } + if _k3slra1_verify_payload "$modeled_mount" proof "$modeled_capture" \ + >"$public_stdout" 2>"$public_stderr"; then + rc=0 + else + rc=$? + fi + (( rc == 23 && dispatch_count == 1 )) || return 1 + [[ ! -s "$public_stdout" && ! -s "$public_stderr" ]] +) + +test_helper_source_guard() ( + local source_stdout="${fixture_root}/helper-source.stdout" + local source_stderr="${fixture_root}/helper-source.stderr" + local source_trace="${fixture_root}/helper-source.trace" + : >"$source_trace" + if ( + set -Eeuo pipefail + # shellcheck source=/dev/null + source "$A1_HELPER_PATH" + declare -F k3slra1_check_main >/dev/null + printf 'source-only\n' >>"$source_trace" + ) >"$source_stdout" 2>"$source_stderr"; then + : + else + return 1 + fi + [[ ! -s "$source_stdout" && ! -s "$source_stderr" ]] || return 1 + [[ "$(<"$source_trace")" == source-only ]] || return 1 +) + +test_helper_source_preserves_signal_traps() ( + local trap_trace="${fixture_root}/helper-source-traps-${BASHPID}.trace" + local before_int before_term after_int after_term + : >"$trap_trace" + trap 'printf "caller-int\n" >>"$trap_trace"' INT + trap 'printf "caller-term\n" >>"$trap_trace"' TERM + before_int="$(trap -p INT)" + before_term="$(trap -p TERM)" + source "$A1_HELPER_PATH" + after_int="$(trap -p INT)" + after_term="$(trap -p TERM)" + trap - INT TERM + [[ "$before_int" == "$after_int" && "$before_term" == "$after_term" ]] || return 1 + [[ ! -s "$trap_trace" ]] +) + +write_direct_term_helper_fixture() { + local staged_root="${1-}" nested_rc="${2-}" helper temporary a1 + (( $# == 2 )) || return 97 + [[ "$nested_rc" == 0 || "$nested_rc" == 1 ]] || return 97 + write_staged_helper_dependencies "$staged_root" + helper="${staged_root}/libexec/k3s-local-recovery-a1-check.sh" + temporary="${helper}.rewrite" + a1="${staged_root}/lib/k3s-local-recovery-a1.sh" + /usr/bin/awk ' + $0 == "if [[ \"${BASH_SOURCE[0]}\" == \"$0\" ]]; then" { + print "_k3slra1h_euid_is_root() {" + print " :" + print "}" + print "_k3slra1h_load_staged_dependencies() {" + print " (( $# == 1 )) || return 97" + print " source \"${1}/lib/k3s-local-recovery-a1.sh\"" + print "}" + } + { print } + ' "$helper" >"$temporary" + mv -- "$temporary" "$helper" + chmod 0600 -- "$helper" + printf '%s\n' \ + 'readonly K3SLRA1_MAIN_MAPPING_NAME=k3s-recovery' \ + '_k3slra1_verify_payload_root() {' \ + ' (( $# == 3 )) || return 97' \ + ' printf "nested-check-ready\n" >>"$K3SLRA1_TERM_TRACE"' \ + ' /usr/bin/sleep 0.35' \ + ' printf "nested-check-cleanup\n" >>"$K3SLRA1_TERM_TRACE"' \ + " return ${nested_rc}" \ + '}' >"$a1" + chmod 0600 -- "$a1" +} + +test_helper_direct_term_waits_for_nested_cleanup() ( + local nested_rc="${1-}" expected_rc="${2-}" + local modeled_capture="${fixture_root}/helper-direct-term-${nested_rc}-${BASHPID}" + local modeled_mount="${modeled_capture}/proof-mount" + local helper="${modeled_capture}/libexec/k3s-local-recovery-a1-check.sh" + local term_trace="${modeled_capture}/term.trace" + local public_stdout="${modeled_capture}/public.stdout" + local public_stderr="${modeled_capture}/public.stderr" + local helper_pid='' watchdog_pid='' ready=0 rc=0 iteration + (( $# == 2 )) || return 97 + [[ "$nested_rc" == 0 || "$nested_rc" == 1 ]] || return 97 + [[ "$expected_rc" == 1 || "$expected_rc" == 143 ]] || return 97 + write_direct_term_helper_fixture "$modeled_capture" "$nested_rc" + mkdir -p -- "$modeled_mount" + chmod 0700 -- "$modeled_mount" + : >"$term_trace" + export K3SLRA1_TERM_TRACE="$term_trace" + /usr/bin/bash --noprofile --norc -- "$helper" "$modeled_mount" proof "$modeled_capture" \ + >"$public_stdout" 2>"$public_stderr" & + helper_pid=$! + for (( iteration = 0; iteration < 100; iteration += 1 )); do + if [[ "$(<"$term_trace")" == nested-check-ready ]]; then + ready=1 + break + fi + /usr/bin/sleep 0.01 + done + if (( ready != 1 )); then + kill -KILL "$helper_pid" 2>/dev/null || : + wait "$helper_pid" 2>/dev/null || : + return 1 + fi + ( + /usr/bin/sleep 3 + kill -KILL "$helper_pid" 2>/dev/null || : + ) & + watchdog_pid=$! + kill -TERM "$helper_pid" || return 1 + if wait "$helper_pid"; then rc=0; else rc=$?; fi + kill -TERM "$watchdog_pid" 2>/dev/null || : + wait "$watchdog_pid" 2>/dev/null || : + (( rc == expected_rc )) || return 1 + [[ "$(<"$term_trace")" == $'nested-check-ready\nnested-check-cleanup' ]] || return 1 + [[ ! -s "$public_stdout" && ! -s "$public_stderr" ]] +) + +test_helper_nonroot_main_is_closed() ( + local helper_stdout="${fixture_root}/helper-nonroot.stdout" + local helper_stderr="${fixture_root}/helper-nonroot.stderr" + local downstream_trace="${fixture_root}/helper-nonroot.downstream" + # shellcheck source=/dev/null + source "$A1_HELPER_PATH" + # This is not a successful skip: an EUID 0 runner must provide a separate + # non-root/user-namespace fixture instead of silently passing this contract. + (( EUID != 0 )) || return 77 + : >"$downstream_trace" + _k3slra1h_load_staged_dependencies() { + printf 'loader\n' >>"$downstream_trace" + return 97 + } + _k3slra1_verify_payload_root() { + printf 'transaction\n' >>"$downstream_trace" + return 97 + } + local helper_rc=0 + if k3slra1_check_main /fixture/mount proof /fixture/capture \ + >"$helper_stdout" 2>"$helper_stderr"; then + return 1 + else + helper_rc=$? + fi + (( helper_rc == 1 )) || return 1 + [[ ! -s "$helper_stdout" && ! -s "$helper_stderr" && + ! -s "$downstream_trace" ]] +) + +write_staged_helper_dependencies() { + local staged_root="${1-}" mode="${2-success}" shared a1 contract + local shared_failure='' a1_failure='' contract_failure='' mapping_name=k3s-recovery + (( $# == 1 || $# == 2 )) || return 97 + case "$mode" in + success) ;; + shared-failure) shared_failure='return 23' ;; + a1-failure) a1_failure='return 23' ;; + contract-failure) contract_failure=' return 23' ;; + mapping-mismatch) mapping_name='unexpected-mapping' ;; + *) return 97 ;; + esac + shared="${staged_root}/lib/k3s-local-recovery.sh" + a1="${staged_root}/lib/k3s-local-recovery-a1.sh" + contract="${staged_root}/contract/local-recovery.env" + mkdir -p -- "${staged_root}/lib" "${staged_root}/libexec" "${staged_root}/contract" + printf '%s\n' \ + 'printf "shared\n" >>"$K3SLRA1_STAGE_TRACE"' \ + "$shared_failure" \ + '_k3slr_load_contract() {' \ + ' (( $# == 1 )) || return 97' \ + ' [[ "$1" == "$K3SLRA1_EXPECTED_CONTRACT" ]] || return 97' \ + ' printf "contract\n" >>"$K3SLRA1_STAGE_TRACE"' \ + "$contract_failure" \ + " K3SLR_MAPPING_NAME=${mapping_name}" \ + ' K3SLR_PROOF_MAPPING_NAME=k3s-recovery-proof' \ + '}' >"$shared" + printf '%s\n' \ + 'printf "a1\n" >>"$K3SLRA1_STAGE_TRACE"' \ + "$a1_failure" \ + 'readonly K3SLRA1_MAIN_MAPPING_NAME=k3s-recovery' \ + '_k3slra1_verify_payload_root() {' \ + ' (( $# == 3 )) || return 97' \ + ' [[ "$1" == "$K3SLRA1_EXPECTED_MOUNT" && "$2" == proof &&' \ + ' "$3" == "$K3SLRA1_EXPECTED_CAPTURE" ]] || return 97' \ + ' printf "transaction\n" >>"$K3SLRA1_STAGE_TRACE"' \ + '}' >"$a1" + printf 'fixture-contract=1\n' >"$contract" + cp -- "$A1_HELPER_PATH" "${staged_root}/libexec/k3s-local-recovery-a1-check.sh" + chmod 0600 -- "$shared" "$a1" "$contract" \ + "${staged_root}/libexec/k3s-local-recovery-a1-check.sh" + /usr/bin/find "$staged_root" -type d -exec chmod 0700 -- '{}' + +} + +install_helper_command_model() { + _k3slra1h_command() { + local modeled='' + case "${1-}" in + /usr/bin/stat) + (( $# == 4 )) && [[ "${2-}" == --format=* && "${3-}" == -- ]] || return 97 + modeled="$(model_test_owned_stat "${2#--format=}" "${4-}")" || return 97 + if [[ "$current_case" == helper-staged-file-extra-field && + "${4-}" == "${capture_root}/lib/k3s-local-recovery.sh" ]]; then + modeled="${modeled/|regular file|/|unexpected|regular file|}" + fi + printf '%s\n' "$modeled" + ;; + /usr/bin/readlink) + (( $# == 4 )) && [[ "${2-}" == --canonicalize-existing && "${3-}" == -- ]] || return 97 + fixture_path_is_owned "${4-}" || return 97 + /usr/bin/readlink --canonicalize-existing -- "${4-}" + ;; + /usr/bin/sha256sum) + (( $# == 3 )) && [[ "${2-}" == -- ]] || return 97 + fixture_path_is_owned "${3-}" || return 97 + /usr/bin/sha256sum -- "${3-}" + ;; + *) return 97 ;; + esac + } +} + +test_helper_compositional_main() ( + local modeled_capture="${fixture_root}/helper-root-${BASHPID}" + local modeled_mount="${modeled_capture}/proof-mount" + local stage_trace="${fixture_root}/helper-root-${BASHPID}.trace" + case_root="$modeled_capture" + mount_root="$modeled_mount" + capture_root="$modeled_capture" + current_case=helper_composition + current_role=proof + write_staged_helper_dependencies "$modeled_capture" + mkdir -p -- "$modeled_mount" "${modeled_capture}/home" + : >"$stage_trace" + chmod 0700 -- "$modeled_mount" "${modeled_capture}/home" + export K3SLRA1_STAGE_TRACE="$stage_trace" + export K3SLRA1_EXPECTED_CONTRACT="${modeled_capture}/contract/local-recovery.env" + export K3SLRA1_EXPECTED_MOUNT="$modeled_mount" + export K3SLRA1_EXPECTED_CAPTURE="$modeled_capture" + # shellcheck source=/dev/null + source "$A1_HELPER_PATH" + declare -F _k3slra1h_load_staged_dependencies >/dev/null || return 1 + + _k3slra1h_euid_is_root() { + (( $# == 0 )) || return 97 + printf 'root-gate\n' >>"$stage_trace" + } + install_helper_command_model + + k3slra1_check_main "$modeled_mount" proof "$modeled_capture" || return 1 + [[ "$(<"$stage_trace")" == $'root-gate\nshared\na1\ncontract\ntransaction' ]] || return 1 + : >"$stage_trace" + local invalid_rc=0 + if k3slra1_check_main "$modeled_mount" proof "$modeled_capture" extra; then + return 1 + else + invalid_rc=$? + fi + (( invalid_rc == 1 )) || return 1 + [[ ! -s "$stage_trace" ]] || return 1 +) + +test_helper_staged_failure_is_closed() ( + local mode="${1-}" expected_trace='' failure_rc=0 + local modeled_capture="${fixture_root}/helper-failure-${mode}-${BASHPID}" + local modeled_mount="${modeled_capture}/proof-mount" + local stage_trace="${fixture_root}/helper-failure-${mode}-${BASHPID}.trace" + (( $# == 1 )) || return 97 + case "$mode" in + shared-failure) expected_trace=$'root-gate\nshared' ;; + a1-failure) expected_trace=$'root-gate\nshared\na1' ;; + contract-failure|mapping-mismatch) expected_trace=$'root-gate\nshared\na1\ncontract' ;; + *) return 97 ;; + esac + case_root="$modeled_capture" + mount_root="$modeled_mount" + capture_root="$modeled_capture" + current_case=helper_composition + current_role=proof + write_staged_helper_dependencies "$modeled_capture" "$mode" + mkdir -p -- "$modeled_mount" "${modeled_capture}/home" + : >"$stage_trace" + chmod 0700 -- "$modeled_mount" "${modeled_capture}/home" + export K3SLRA1_STAGE_TRACE="$stage_trace" + export K3SLRA1_EXPECTED_CONTRACT="${modeled_capture}/contract/local-recovery.env" + export K3SLRA1_EXPECTED_MOUNT="$modeled_mount" + export K3SLRA1_EXPECTED_CAPTURE="$modeled_capture" + source "$A1_HELPER_PATH" + _k3slra1h_euid_is_root() { + (( $# == 0 )) || return 97 + printf 'root-gate\n' >>"$stage_trace" + } + install_helper_command_model + if k3slra1_check_main "$modeled_mount" proof "$modeled_capture"; then + return 1 + else + failure_rc=$? + fi + (( failure_rc == 1 )) || return 1 + [[ "$(<"$stage_trace")" == "$expected_trace" ]] +) + +test_helper_intermediate_directories_are_pinned() ( + local directory_kind mutation modeled_capture modeled_mount stage_trace rc=0 + local violations="${fixture_root}/helper-directory-violations-${BASHPID}" + : >"$violations" + for directory_kind in lib libexec contract; do + for mutation in wrong-owner wrong-mode wrong-device symlink; do + modeled_capture="${fixture_root}/helper-dir-${directory_kind}-${mutation}-${BASHPID}" + modeled_mount="${modeled_capture}/proof-mount" + stage_trace="${modeled_capture}.trace" + case_root="$modeled_capture" + mount_root="$modeled_mount" + capture_root="$modeled_capture" + current_role=proof + current_case=helper-directory-negative + helper_modeled_target="${modeled_capture}/${directory_kind}" + helper_modeled_mutation="$mutation" + write_staged_helper_dependencies "$modeled_capture" + mkdir -p -- "$modeled_mount" "${modeled_capture}/home" + chmod 0700 -- "$modeled_mount" "${modeled_capture}/home" + if [[ "$mutation" == symlink ]]; then + helper_modeled_mutation='' + mv -- "${modeled_capture}/${directory_kind}" "${modeled_capture}/${directory_kind}.real" + ln -s -- "${directory_kind}.real" "${modeled_capture}/${directory_kind}" + fi + : >"$stage_trace" + export K3SLRA1_STAGE_TRACE="$stage_trace" + export K3SLRA1_EXPECTED_CONTRACT="${modeled_capture}/contract/local-recovery.env" + export K3SLRA1_EXPECTED_MOUNT="$modeled_mount" + export K3SLRA1_EXPECTED_CAPTURE="$modeled_capture" + source "$A1_HELPER_PATH" + _k3slra1h_euid_is_root() { + (( $# == 0 )) || return 97 + printf 'root-gate\n' >>"$stage_trace" + } + install_helper_command_model + if k3slra1_check_main "$modeled_mount" proof "$modeled_capture"; then rc=0; else rc=$?; fi + if (( rc == 0 || rc == 97 )) || [[ "$(<"$stage_trace")" != root-gate ]]; then + printf '%s:%s\n' "$directory_kind" "$mutation" >>"$violations" + fi + done + done + [[ ! -s "$violations" ]] +) + +test_helper_staged_files_are_pinned() ( + local file_kind mutation modeled_capture modeled_mount staged_file stage_trace rc=0 + local violations="${fixture_root}/helper-file-violations-${BASHPID}" + : >"$violations" + for file_kind in shared a1 contract helper; do + for mutation in wrong-owner wrong-mode wrong-device nlink-two wrong-type symlink; do + modeled_capture="${fixture_root}/helper-file-${file_kind}-${mutation}-${BASHPID}" + modeled_mount="${modeled_capture}/proof-mount" + stage_trace="${modeled_capture}.trace" + case "$file_kind" in + shared) staged_file="${modeled_capture}/lib/k3s-local-recovery.sh" ;; + a1) staged_file="${modeled_capture}/lib/k3s-local-recovery-a1.sh" ;; + contract) staged_file="${modeled_capture}/contract/local-recovery.env" ;; + helper) staged_file="${modeled_capture}/libexec/k3s-local-recovery-a1-check.sh" ;; + *) return 97 ;; + esac + case_root="$modeled_capture" + mount_root="$modeled_mount" + capture_root="$modeled_capture" + current_role=proof + current_case=helper-file-negative + helper_modeled_target="$staged_file" + helper_modeled_mutation="$mutation" + write_staged_helper_dependencies "$modeled_capture" + mkdir -p -- "$modeled_mount" "${modeled_capture}/home" + chmod 0700 -- "$modeled_mount" "${modeled_capture}/home" + if [[ "$mutation" == symlink ]]; then + helper_modeled_mutation='' + mv -- "$staged_file" "${staged_file}.real" + ln -s -- "$(basename -- "$staged_file").real" "$staged_file" + fi + : >"$stage_trace" + export K3SLRA1_STAGE_TRACE="$stage_trace" + export K3SLRA1_EXPECTED_CONTRACT="${modeled_capture}/contract/local-recovery.env" + export K3SLRA1_EXPECTED_MOUNT="$modeled_mount" + export K3SLRA1_EXPECTED_CAPTURE="$modeled_capture" + source "$A1_HELPER_PATH" + _k3slra1h_euid_is_root() { + (( $# == 0 )) || return 97 + printf 'root-gate\n' >>"$stage_trace" + } + install_helper_command_model + if k3slra1_check_main "$modeled_mount" proof "$modeled_capture"; then rc=0; else rc=$?; fi + if (( rc == 0 || rc == 97 )) || [[ "$(<"$stage_trace")" != root-gate ]]; then + printf '%s:%s\n' "$file_kind" "$mutation" >>"$violations" + fi + done + done + [[ ! -s "$violations" ]] +) + +test_helper_staged_file_metadata_is_exact() ( + local modeled_capture="${fixture_root}/helper-file-fields-${BASHPID}" + local modeled_mount="${modeled_capture}/proof-mount" + local stage_trace="${modeled_capture}.trace" rc=0 + case_root="$modeled_capture" + mount_root="$modeled_mount" + capture_root="$modeled_capture" + current_case=helper-staged-file-extra-field + current_role=proof + write_staged_helper_dependencies "$modeled_capture" + mkdir -p -- "$modeled_mount" "${modeled_capture}/home" + chmod 0700 -- "$modeled_mount" "${modeled_capture}/home" + : >"$stage_trace" + export K3SLRA1_STAGE_TRACE="$stage_trace" + export K3SLRA1_EXPECTED_CONTRACT="${modeled_capture}/contract/local-recovery.env" + export K3SLRA1_EXPECTED_MOUNT="$modeled_mount" + export K3SLRA1_EXPECTED_CAPTURE="$modeled_capture" + source "$A1_HELPER_PATH" + _k3slra1h_euid_is_root() { + (( $# == 0 )) || return 97 + printf 'root-gate\n' >>"$stage_trace" + } + install_helper_command_model + if k3slra1_check_main "$modeled_mount" proof "$modeled_capture"; then rc=0; else rc=$?; fi + (( rc != 0 && rc != 97 )) || return 1 + [[ "$(<"$stage_trace")" == root-gate ]] +) + +test_helper_xtrace_is_rejected() ( + local public_stdout="${fixture_root}/helper-xtrace.stdout" + local public_stderr="${fixture_root}/helper-xtrace.stderr" + local downstream_trace="${fixture_root}/helper-xtrace.downstream" + local xtrace_sink="${fixture_root}/helper-xtrace.trace" rc=0 + : >"$downstream_trace" + : >"$xtrace_sink" + source "$A1_HELPER_PATH" + _k3slra1h_euid_is_root() { + (( $# == 0 )) || return 97 + printf 'root-gate\n' >>"$downstream_trace" + } + _k3slra1h_load_staged_dependencies() { + printf 'loader\n' >>"$downstream_trace" + return 97 + } + _k3slra1_verify_payload_root() { + printf 'transaction\n' >>"$downstream_trace" + return 97 + } + exec 9>"$xtrace_sink" + if ( BASH_XTRACEFD=9; set -x; k3slra1_check_main /fixture/proof-mount proof /fixture ) \ + >"$public_stdout" 2>"$public_stderr"; then + rc=0 + else + rc=$? + fi + exec 9>&- + (( rc != 0 && rc != 97 )) || return 1 + [[ "$(<"$downstream_trace")" == root-gate ]] || return 1 + [[ -s "$xtrace_sink" ]] || return 1 + [[ ! -s "$public_stdout" && ! -s "$public_stderr" ]] +) + +test_static_preflight_adversaries() ( + local scanner_root="${fixture_root}/static-scanner-${BASHPID}" + local valid_source quoted_absolute_source readonly_command_source indirect_source + local bare_unknown_source unsafe_umask_source valid_subshell_source + local generic_subshell_source library_input_process_source library_stderr_process_source + local helper_input_process_source helper_guard_trap_source + mkdir -- "$scanner_root" + valid_source="${scanner_root}/valid-library.sh" + quoted_absolute_source="${scanner_root}/quoted-absolute.sh" + readonly_command_source="${scanner_root}/readonly-command.sh" + indirect_source="${scanner_root}/indirect-command.sh" + bare_unknown_source="${scanner_root}/bare-unknown-command.sh" + unsafe_umask_source="${scanner_root}/unsafe-umask.sh" + valid_subshell_source="${scanner_root}/valid-subshell.sh" + generic_subshell_source="${scanner_root}/generic-subshell.sh" + library_input_process_source="${scanner_root}/library-input-process.sh" + library_stderr_process_source="${scanner_root}/library-stderr-process.sh" + helper_input_process_source="${scanner_root}/helper-input-process.sh" + helper_guard_trap_source="${scanner_root}/helper-guard-trap.sh" + + printf '%s\n' \ + 'readonly SAFE=1' \ + '_k3slra1_command() {' \ + ' "$@"' \ + '}' \ + 'dispatch() {' \ + ' _k3slra1_command /usr/bin/sudo --non-interactive -- /usr/bin/env -i /usr/bin/bash' \ + '}' >"$valid_source" + a1_static_source_is_safe "$valid_source" library || return 1 + + cp -- "$valid_source" "$valid_subshell_source" + printf '%s\n' \ + '_k3slra1_sqlite_quick_check() (' \ + ' return 1' \ + ')' >>"$valid_subshell_source" + a1_static_source_is_safe "$valid_subshell_source" library || return 1 + + cp -- "$valid_source" "$generic_subshell_source" + printf '%s\n' \ + 'generic_subshell() (' \ + ' return 1' \ + ')' >>"$generic_subshell_source" + a1_static_source_is_safe "$generic_subshell_source" library && return 1 + + cp -- "$valid_source" "$quoted_absolute_source" + printf '%s\n' \ + 'quoted_bypass() {' \ + ' "/usr/bin/sudo" --version' \ + '}' >>"$quoted_absolute_source" + a1_static_source_is_safe "$quoted_absolute_source" library && return 1 + + { + printf '%s\n' "readonly SUDO='/usr/bin/sudo'" + tail -n +2 -- "$valid_source" + } >"$readonly_command_source" + a1_static_source_is_safe "$readonly_command_source" library && return 1 + + cp -- "$valid_source" "$indirect_source" + printf '%s\n' \ + 'indirect_bypass() {' \ + ' local runner=sudo' \ + ' "$runner" --version' \ + '}' >>"$indirect_source" + a1_static_source_is_safe "$indirect_source" library && return 1 + + cp -- "$valid_source" "$bare_unknown_source" + printf '%s\n' \ + 'unknown_bypass() {' \ + ' curl https://invalid.example' \ + '}' >>"$bare_unknown_source" + a1_static_source_is_safe "$bare_unknown_source" library && return 1 + + cp -- "$valid_source" "$unsafe_umask_source" + printf '%s\n' \ + 'unsafe_umask() {' \ + ' umask 000' \ + '}' >>"$unsafe_umask_source" + a1_static_source_is_safe "$unsafe_umask_source" library && return 1 + + cp -- "$valid_source" "$library_input_process_source" + printf '%s\n' \ + 'input_process_bypass() {' \ + " mapfile -d '' -t records < <(_k3slra1_metadata_packet \"\$producer\")" \ + '}' >>"$library_input_process_source" + a1_static_source_is_safe "$library_input_process_source" library && return 1 + + cp -- "$valid_source" "$library_stderr_process_source" + printf '%s\n' \ + 'stderr_process_bypass() {' \ + ' if _k3slra1_command /usr/bin/stat -- "$path" 2> >(_k3slra1_wrong_marker); then' \ + ' :' \ + ' fi' \ + '}' >>"$library_stderr_process_source" + a1_static_source_is_safe "$library_stderr_process_source" library && return 1 + + cp -- "$A1_HELPER_PATH" "$helper_input_process_source" + printf '%s\n' \ + 'helper_process_bypass() {' \ + " mapfile -d '' -t records < <(_k3slra1h_metadata_packet \"\$target\" extra)" \ + '}' >>"$helper_input_process_source" + a1_static_source_is_safe "$helper_input_process_source" helper && return 1 + + /usr/bin/awk ' + $0 == " trap \047_k3slra1h_direct_signal_rc=130; if (( _k3slra1h_direct_rc == 0 )); then _k3slra1h_direct_rc=130; fi\047 INT" { + print " trap \047printf hostile-guard\047 INT" + next + } + { print } + ' "$A1_HELPER_PATH" >"$helper_guard_trap_source" + a1_static_source_is_safe "$helper_guard_trap_source" helper && return 1 + return 0 +) + +test_real_env_i_sentinel_probe() ( + local probe_root="${fixture_root}/env-i-${BASHPID}" probe_script bash_env_file sentinel output + probe_script="${probe_root}/probe.sh" + bash_env_file="${probe_root}/hostile-bash-env" + sentinel="${probe_root}/sentinel" + mkdir -- "$probe_root" + printf 'printf leak >%q\n' "$sentinel" >"$bash_env_file" + printf '%s\n' \ + '[[ -z "${BASH_ENV+x}" ]] || exit 91' \ + 'declare -F k3slra1_hostile_export >/dev/null && exit 92' \ + 'printf "env-clean\n"' >"$probe_script" + chmod 0700 -- "$probe_script" + k3slra1_hostile_export() { :; } + export -f k3slra1_hostile_export + BASH_ENV="$bash_env_file" output="$( + /usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin LC_ALL=C \ + /usr/bin/bash --noprofile --norc -- "$probe_script" + )" || return 1 + [[ "$output" == env-clean && ! -e "$sentinel" ]] +) + +# Keep the first RED focused and cheap: this compositional matrix allocates one +# synthetic payload and cannot reach live commands because every seam is fake. +assert_succeeds 'all capture APIs preserve status, NUL, LF, stderr, and outputs' \ + test_command_capture_hostile_matrix + +readonly -a expected_failure_cases=( + bundle_none + bundle_two + bundle_bad_name + matching_bundle_symlink + manifest_missing + manifest_no_final_lf + manifest_nul + manifest_cr + manifest_uppercase_hash + manifest_one_space_separator + manifest_tab_separator + manifest_parent_escape + manifest_absolute_path + manifest_double_slash + manifest_dot_segment + manifest_trailing_slash + manifest_invalid_character + manifest_path_513_bytes + manifest_65537_bytes + manifest_duplicate_path + manifest_self_target + manifest_hash_mismatch + manifest_target_symlink + manifest_target_hardlink_alias + manifest_target_nlink_two + manifest_duplicate_inode + manifest_drift_after_sqlite + sqlite_wal_drift_after_child + preexisting_unmanifested_file + target_count_79 + target_count_81 + target_bytes_22857828 + target_bytes_22857830 + sqlite_mixed_layout + legacy_sqlite_mixed_layout + sqlite_alternate_partial_layout + sqlite_missing_wal + sqlite_missing_shm + sqlite_not_in_manifest + sqlite_inode_not_manifest_target + sqlite_sidecar_symlink + sqlite_sidecar_hardlink + sqlite_layout_unmanifested_symlink + sqlite_layout_unmanifested_fifo + sqlite_layout_unmanifested_directory + sqlite_quick_check_extra_line + sqlite_quick_check_error + sqlite_quick_check_stderr + sqlite_quick_check_nul + sqlite_capture_alias + sqlite_capture_preexisting + sqlite_capture_stderr_preexisting + sqlite_capture_stderr_fifo + sqlite_capture_stderr_symlink + sqlite_capture_path_swap + sqlite_capture_parent_drift + sqlite_capture_wrong_owner + sqlite_capture_wrong_mode + sqlite_capture_cleanup_failure + mount_missing_noload + mount_wrong_source + mount_wrong_target + mount_wrong_fstype + mount_wrong_major_minor + mount_missing_vfs_ro + mount_forbidden_vfs_rw + mount_evidence_multiline + mount_evidence_cr + mount_evidence_nul + mount_evidence_stderr + findmnt_valid_record_stderr + readlink_valid_record_stderr + lsblk_valid_record_stderr + mount_evidence_drift + mount_third_evidence_drift + proof_mapper_swap + baseline_mapper_swap + final_mapper_swap + mapper_canonical_mismatch + source_expected_canonical_mismatch + mapper_not_block + mapper_major_minor_mismatch + mapping_not_readonly + mapping_evidence_drift + mapping_third_evidence_drift + blockdev_multiline + blockdev_cr + blockdev_nul + blockdev_stderr + blockdev_valid_record_stderr + bundle_wrong_owner + manifest_wrong_mode + payload_wrong_device + payload_component_symlink + payload_write_attempt +) + +for fixture_case in "${expected_failure_cases[@]}"; do + assert_fails "$fixture_case is rejected" run_case "$fixture_case" +done + +assert_succeeds 'canonical payload is accepted' run_case canonical_pass +assert_succeeds 'legacy payload is accepted' run_case legacy_pass +assert_succeeds 'original baseline role is accepted with the main mapper' \ + run_case original_baseline_pass +assert_succeeds 'plus path is accepted' run_case manifest_plus_path_pass +assert_succeeds '512-byte path field is accepted' run_case manifest_path_512_bytes_pass +assert_succeeds 'unrelated lost+found directory is accepted' run_case unrelated_lost_found_pass +assert_succeeds 'direct A1 APIs preserve strict output contracts' test_direct_api_contract +assert_succeeds 'dynamic output aliases fail before mutation or command dispatch' \ + test_dynamic_output_aliases_are_closed +assert_succeeds 'mapper block-type stat precedes lsblk and blockdev' \ + test_mapper_stat_precedes_lsblk +assert_succeeds 'canonical mapper nodes are restricted to the dm namespace' \ + test_canonical_mapper_namespace_is_closed +assert_succeeds 'command model requires dereferenced FD stat and raw canonical lsblk argv' \ + test_command_model_requires_canonical_argv +assert_succeeds 'INT during SQLite check cleans both owned captures' \ + test_sqlite_signal_cleanup_contract INT +assert_succeeds 'TERM during SQLite check cleans both owned captures' \ + test_sqlite_signal_cleanup_contract TERM +assert_succeeds 'TERM status propagates through the real root SQLite path' \ + test_root_signal_status_contract sqlite_capture_term_cleanup 143 +assert_succeeds 'cleanup failure takes precedence over TERM status' \ + test_root_signal_status_contract sqlite_capture_term_cleanup_failure 1 +assert_succeeds 'SQLite quick-check preserves caller umask and signal traps' \ + test_sqlite_preserves_shell_state +assert_succeeds 'stdout hardlink drift is rejected after child FD re-pin' \ + test_sqlite_hardlink_swap_is_rejected stdout +assert_succeeds 'stderr hardlink drift is rejected after child FD re-pin' \ + test_sqlite_hardlink_swap_is_rejected stderr +assert_succeeds 'SQLite layout members remain bound to manifest-time identities' \ + test_sqlite_layout_identity_binding +assert_succeeds 'manifest targets preserve heterogeneous ownership and modes' \ + test_manifest_accepts_heterogeneous_target_metadata +assert_succeeds 'partial stdout cleanup verifies unlink postcondition' \ + test_partial_stdout_unlink_requires_absence +assert_succeeds 'bounded manifest reader accepts exactly 65536 bytes' \ + bounded_manifest_reader_accepts_65536_bytes +assert_succeeds 'bounded manifest reader rejects 65537 bytes without output mutation' \ + bounded_manifest_reader_rejects_65537_bytes +assert_succeeds 'parent dispatches only the fixed env-clean root helper' test_parent_helper_dispatch +assert_succeeds 'parent suppresses hostile root-helper stdout and stderr' \ + test_parent_helper_output_is_suppressed +assert_succeeds 'sourcing helper never auto-runs main' test_helper_source_guard +assert_succeeds 'sourcing helper preserves caller signal traps' \ + test_helper_source_preserves_signal_traps +assert_succeeds 'direct helper TERM waits for nested cleanup and exits 143' \ + test_helper_direct_term_waits_for_nested_cleanup 0 143 +assert_succeeds 'direct helper preserves nested cleanup failure over TERM' \ + test_helper_direct_term_waits_for_nested_cleanup 1 1 +assert_succeeds 'direct non-root helper main is fail-closed' test_helper_nonroot_main_is_closed +assert_succeeds 'actual helper loader composes staged shared, A1, contract, and transaction' \ + test_helper_compositional_main +assert_succeeds 'staged shared source failure stops before A1 and transaction' \ + test_helper_staged_failure_is_closed shared-failure +assert_succeeds 'staged A1 source failure stops before contract and transaction' \ + test_helper_staged_failure_is_closed a1-failure +assert_succeeds 'staged contract parser failure stops before transaction' \ + test_helper_staged_failure_is_closed contract-failure +assert_succeeds 'contract main mapping mismatch stops before transaction' \ + test_helper_staged_failure_is_closed mapping-mismatch +assert_succeeds 'helper pins every staged intermediate directory' \ + test_helper_intermediate_directories_are_pinned +assert_succeeds 'helper pins owner, mode, device, nlink, type, and path for staged files' \ + test_helper_staged_files_are_pinned +assert_succeeds 'helper staged metadata parser rejects extra fields' \ + test_helper_staged_file_metadata_is_exact +assert_succeeds 'helper rejects xtrace before staged loading or transaction' \ + test_helper_xtrace_is_rejected +assert_succeeds 'static preflight rejects quoted, readonly, indirect, and unknown commands' \ + test_static_preflight_adversaries +assert_succeeds 'real sudo-free env-i boundary removes hostile shell sentinels' \ + test_real_env_i_sentinel_probe + +printf 'PASS: strict A1 payload fixtures\n' diff --git a/scripts/validate/test-k3s-local-recovery-fmask.sh b/scripts/validate/test-k3s-local-recovery-fmask.sh new file mode 100644 index 0000000..a7b91a4 --- /dev/null +++ b/scripts/validate/test-k3s-local-recovery-fmask.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Regression test for the NTFS mask used by the recovery volume. The 0177 +# file mask removes owner execute plus every group/other permission, producing +# mode 0600 for the KeePass database and LUKS container. +set -Eeuo pipefail + +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly VALIDATOR_PATH="${REPOSITORY_ROOT}/scripts/validate/k3s-local-recovery.sh" + +# shellcheck source=/dev/null +source "$VALIDATOR_PATH" + +readonly LIVE_SAFE_OPTIONS='rw,nosuid,nodev,noexec,relatime,uid=1000,gid=1000,dmask=0077,fmask=0177,iocharset=utf8' + +if ! _k3slrv_outer_masks_secure "$LIVE_SAFE_OPTIONS"; then + printf '%s\n' \ + 'TEST FAILURE: recovery validator rejected secure fmask=0177' >&2 + exit 1 +fi + +printf '%s\n' 'PASS: recovery validator accepts secure fmask=0177' diff --git a/scripts/validate/test-k3s-local-recovery-header-proof.sh b/scripts/validate/test-k3s-local-recovery-header-proof.sh new file mode 100644 index 0000000..ade040e --- /dev/null +++ b/scripts/validate/test-k3s-local-recovery-header-proof.sh @@ -0,0 +1,797 @@ +#!/usr/bin/env bash + +# Source-safe TDD coverage for the standalone K3s recovery LUKS header proof. +set -Eeuo pipefail + +readonly K3SLRH_TEST_DIRECTORY="${BASH_SOURCE[0]%/*}" +readonly K3SLRH_REPOSITORY_ROOT="$(cd -- "${K3SLRH_TEST_DIRECTORY}/../.." && pwd -P)" +readonly K3SLRH_SHARED_LIBRARY="${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery.sh" +readonly K3SLRH_A1_LIBRARY="${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery-a1.sh" +readonly K3SLRH_A1_HELPER="${K3SLRH_REPOSITORY_ROOT}/scripts/libexec/k3s-local-recovery-a1-check.sh" +readonly K3SLRH_PROOF_LIBRARY="${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery-header-proof.sh" +readonly K3SLRH_WRAPPER="${K3SLRH_REPOSITORY_ROOT}/scripts/bootstrap/prove-k3s-local-recovery-header.sh" +readonly K3SLRH_CONTRACT="${K3SLRH_REPOSITORY_ROOT}/infrastructure/security/k3s/local-recovery.env" +readonly K3SLRH_VALIDATOR="${K3SLRH_REPOSITORY_ROOT}/scripts/validate/k3s-local-recovery.sh" +readonly K3SLRH_AUTHORITY_DESIGN='/home/donghyeon/workspace/docs/platform/plans/2026-08-09-k3s-luks-header-proof-design.md' +readonly K3SLRH_AUTHORITY_PLAN='/home/donghyeon/workspace/docs/platform/plans/2026-08-09-k3s-luks-header-proof-implementation.md' + +readonly K3SLRH_AUTHORITY_DESIGN_SHA256='0d1546795da59490956ba2e260bf12df210083514ae55c0551de632d54e278ec' +readonly K3SLRH_AUTHORITY_PLAN_SHA256='126f7fa43fb857dd9ff753aac4ce6528f7a227d7151d688f8cce2b6e0ff8fbc9' +readonly K3SLRH_EXPECTED_SHARED_SHA256='a206d81539e6c37fa085457ba01cff9de57de50f7235e3a4085989780e99585d' +readonly K3SLRH_EXPECTED_A1_SHA256='8b39fc46229ebd4c77ff20040f942c4ef7b756d564d344a8ecdd11a7428e2968' +readonly K3SLRH_EXPECTED_A1_HELPER_SHA256='06831fa35943a328ecd183d3c6f721b24c4607b2f349abf6201a12e690d08ebe' +readonly K3SLRH_EXPECTED_CONTRACT_SHA256='c045f42c2dc3eda09c37a123bcc05d88d94f275e61504a2d5816948237e516df' +readonly K3SLRH_EXPECTED_VALIDATOR_SHA256='6b869032fc325759ee25b219fad4ea6190a3389a561c496f0664a51b802f73e6' + +k3slrh_test_fail() { + printf 'TEST FAILURE: %s\n' "$*" >&2 + exit 1 +} + +# This RED gate intentionally precedes authority hashing, source scanning, +# sourcing, mktemp, command doubles, and every capability or live path. +[[ -f "$K3SLRH_PROOF_LIBRARY" && ! -L "$K3SLRH_PROOF_LIBRARY" && + -r "$K3SLRH_PROOF_LIBRARY" ]] || k3slrh_test_fail 'header-proof library missing' +[[ -f "$K3SLRH_WRAPPER" && ! -L "$K3SLRH_WRAPPER" && + -r "$K3SLRH_WRAPPER" ]] || k3slrh_test_fail 'header-proof wrapper missing' +[[ "${K3SLRH_FOCUS:-entrypoint}" == entrypoint ]] || + k3slrh_test_fail 'unsupported header-proof focus' + +k3slrh_sha_is_exact() { + local path="${1-}" expected="${2-}" record + (( $# == 2 )) || return 97 + record="$(/usr/bin/sha256sum -- "$path")" || return 1 + [[ "$record" == "${expected} ${path}" ]] +} + +for k3slrh_checkpoint in \ + "$K3SLRH_AUTHORITY_DESIGN:$K3SLRH_AUTHORITY_DESIGN_SHA256" \ + "$K3SLRH_AUTHORITY_PLAN:$K3SLRH_AUTHORITY_PLAN_SHA256" \ + "$K3SLRH_SHARED_LIBRARY:$K3SLRH_EXPECTED_SHARED_SHA256" \ + "$K3SLRH_A1_LIBRARY:$K3SLRH_EXPECTED_A1_SHA256" \ + "$K3SLRH_A1_HELPER:$K3SLRH_EXPECTED_A1_HELPER_SHA256" \ + "$K3SLRH_CONTRACT:$K3SLRH_EXPECTED_CONTRACT_SHA256" \ + "$K3SLRH_VALIDATOR:$K3SLRH_EXPECTED_VALIDATOR_SHA256"; do + k3slrh_checkpoint_path="${k3slrh_checkpoint%:*}" + k3slrh_checkpoint_hash="${k3slrh_checkpoint##*:}" + k3slrh_sha_is_exact "$k3slrh_checkpoint_path" "$k3slrh_checkpoint_hash" || + k3slrh_test_fail 'authority or frozen dependency drift' +done +unset k3slrh_checkpoint k3slrh_checkpoint_path k3slrh_checkpoint_hash + +k3slrh_library_static_is_safe() { + local source_path="${1-}" + (( $# == 1 )) || return 97 + [[ -f "$source_path" && ! -L "$source_path" ]] || return 1 + /usr/bin/awk ' + function trim(value) { + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + return value + } + function external_path(value) { + return value ~ /\/(usr\/)?(s?bin)\/[A-Za-z0-9._+:-]+/ + } + function sensitive_bare(value) { + return value ~ /(^|(&&|\|\||;)[[:space:]]*)(sudo|mount|umount|losetup|cryptsetup|keepassxc-cli|sqlite3|systemctl|timeout|k3s|findmnt|lsblk|blockdev|dmsetup)([[:space:]]|$)/ + } + { + raw = $0 + value = trim(raw) + if (value == "" || value ~ /^#/) next + if (value ~ /]+>/) bad = 1 + if (value ~ /_k3slr_collect_prepackage_snapshot|_k3slr_lifecycle_dispatch|_k3slr_prove_header_restore/) bad = 1 + if (in_function) { + if (raw == "}") { in_function = 0; function_name = ""; next } + if (value ~ /`/ || value ~ /(^|[;[:space:]])(eval|command|exec)[[:space:]]/) bad = 1 + if (sensitive_bare(value) && value !~ /_k3slrh_command[[:space:]]/) bad = 1 + if (external_path(value) && value !~ /_k3slrh_command[[:space:]]/ && + value !~ /_k3slr[a-z0-9_]+[[:space:]]/) bad = 1 + next + } + if (value ~ /^[A-Za-z_][A-Za-z0-9_]*\(\)[[:space:]]*\{$/) { + in_function = 1 + function_name = value + sub(/\(\)[[:space:]]*\{$/, "", function_name) + next + } + if (value ~ /^readonly[[:space:]]+[A-Za-z_][A-Za-z0-9_]*=(\047[^\047]*\047|[A-Za-z0-9_.:+\/-]+)$/) next + bad = 1 + } + END { if (in_function || bad) exit 1 } + ' "$source_path" +} + +k3slrh_wrapper_static_is_safe() { + local source_path="${1-}" + (( $# == 1 )) || return 97 + [[ -f "$source_path" && ! -L "$source_path" ]] || return 1 + /usr/bin/awk ' + { + if ($0 ~ /_k3slr_collect_prepackage_snapshot|_k3slr_lifecycle_dispatch|_k3slr_prove_header_restore/) bad = 1 + if ($0 ~ /(^|[[:space:]])(sudo|mount|umount|losetup|cryptsetup|keepassxc-cli|sqlite3)([[:space:]]|$)/) bad = 1 + if ($0 ~ /scripts\/lib\/k3s-local-recovery\.sh/) shared = NR + if ($0 ~ /scripts\/lib\/k3s-local-recovery-a1\.sh/) a1 = NR + if ($0 ~ /scripts\/lib\/k3s-local-recovery-header-proof\.sh/) proof = NR + if ($0 ~ /^if \[\[ "\$\{BASH_SOURCE\[0\]\}" == "\$0" \]\]; then$/) direct_guard++ + if ($0 ~ /_k3slrh_header_proof_main "\$@"/) direct_main++ + if ($0 ~ /_k3slrh_wrapper_initial_guard/) initial_guard++ + if ($0 ~ /unset -f[[:space:]]+_k3slrh_/) bad = 1 + if ($0 ~ /_k3slrh_wrapper_environment_is_clean/) inherited_guard = NR + } + END { + if (bad || shared == 0 || a1 == 0 || proof == 0 || + !(shared < a1 && a1 < proof) || direct_guard != 1 || + direct_main != 1 || initial_guard < 2 || inherited_guard == 0 || + inherited_guard >= shared) exit 1 + } + ' "$source_path" +} + +# No production source is evaluated until both conservative scans pass. +k3slrh_library_static_is_safe "$K3SLRH_PROOF_LIBRARY" || + k3slrh_test_fail 'header-proof library static preflight failed' +k3slrh_wrapper_static_is_safe "$K3SLRH_WRAPPER" || + k3slrh_test_fail 'header-proof wrapper static preflight failed' + +# shellcheck source=/dev/null +source "$K3SLRH_SHARED_LIBRARY" || k3slrh_test_fail 'shared library source failed' +# shellcheck source=/dev/null +source "$K3SLRH_A1_LIBRARY" || k3slrh_test_fail 'A1 library source failed' +# shellcheck source=/dev/null +source "$K3SLRH_PROOF_LIBRARY" || k3slrh_test_fail 'header-proof library source failed' +_k3slr_load_contract "$K3SLRH_CONTRACT" || k3slrh_test_fail 'strict contract load failed' + +k3slrh_require_function() { + local name="${1-}" + (( $# == 1 )) || return 97 + declare -F "$name" >/dev/null || k3slrh_test_fail "header-proof API missing: ${name}" +} + +for k3slrh_api in \ + _k3slrh_parse_cli \ + _k3slrh_derive_targets \ + _k3slrh_run_closed_validator \ + _k3slrh_k3s_health \ + _k3slrh_pre_mount_capability \ + _k3slrh_header_proof_main \ + _k3slrh_target_state_is_consistent; do + k3slrh_require_function "$k3slrh_api" +done +unset k3slrh_api + +k3slrh_assert_readonly_literal() { + local name="${1-}" expected="${2-}" declaration + (( $# == 2 )) || return 97 + declaration="$(declare -p "$name" 2>/dev/null)" || return 1 + [[ "$declaration" == 'declare -r '* || "$declaration" == 'declare -rx '* ]] || return 1 + [[ "${!name}" == "$expected" ]] +} + +k3slrh_assert_readonly_literal K3SLRH_PROOF_RELATIVE \ + 'HyeonworksRecovery/containers/k3s-recovery.header-proof.luks' || + k3slrh_test_fail 'proof relative name is not immutable' +k3slrh_assert_readonly_literal K3SLRH_PRE_KDBX_PREFIX \ + 'hyeonworks-recovery.header-proof-pre.' || + k3slrh_test_fail 'pre KDBX prefix is not immutable' +k3slrh_assert_readonly_literal K3SLRH_POST_KDBX_PREFIX \ + 'hyeonworks-recovery.header-proof-post.' || + k3slrh_test_fail 'post KDBX prefix is not immutable' +k3slrh_assert_readonly_literal K3SLRH_HEADER_BACKUP_NAME 'header-original.bin' || + k3slrh_test_fail 'header backup name is not immutable' +k3slrh_assert_readonly_literal K3SLRH_HEADER_EXPORT_NAME 'header-exported.bin' || + k3slrh_test_fail 'header export name is not immutable' +k3slrh_assert_readonly_literal K3SLRH_SQLITE_PACKAGE_VERSION '3.45.1-1ubuntu2.7' || + k3slrh_test_fail 'sqlite package version is not immutable' +k3slrh_assert_readonly_literal K3SLRH_SHARED_LIBRARY_SHA256 "$K3SLRH_EXPECTED_SHARED_SHA256" || + k3slrh_test_fail 'shared source checkpoint placeholder or drift' +k3slrh_assert_readonly_literal K3SLRH_A1_LIBRARY_SHA256 "$K3SLRH_EXPECTED_A1_SHA256" || + k3slrh_test_fail 'A1 source checkpoint placeholder or drift' +k3slrh_assert_readonly_literal K3SLRH_A1_HELPER_SHA256 "$K3SLRH_EXPECTED_A1_HELPER_SHA256" || + k3slrh_test_fail 'A1 helper checkpoint placeholder or drift' +k3slrh_assert_readonly_literal K3SLRH_CONTRACT_SHA256 "$K3SLRH_EXPECTED_CONTRACT_SHA256" || + k3slrh_test_fail 'contract checkpoint placeholder or drift' +k3slrh_assert_readonly_literal K3SLRH_VALIDATOR_SHA256 "$K3SLRH_EXPECTED_VALIDATOR_SHA256" || + k3slrh_test_fail 'validator checkpoint placeholder or drift' + +readonly K3SLRH_EXPECTED_UTC='20260809T120304Z' +readonly K3SLRH_RAW_IDENTIFIER_SENTINEL='RAW_IDENTIFIER_SENTINEL_DO_NOT_PRINT' +readonly K3SLRH_EXPECTED_CLOSED_RECORD=$'Recovery device: match\nRecovery state: closed\nLineage: match\nLatest bundle: not_checked\n' +readonly K3SLRH_EXPECTED_DRY_RUN=$'Header proof state: closed\nHeader proof capability: pass\nExecute-time checks: pending\nDRY RUN PASS: no changes\n' + +k3slrh_fixture_root="$(/usr/bin/mktemp -d /tmp/k3slrh-entrypoint-test.XXXXXXXX)" || + k3slrh_test_fail 'cannot allocate fixture root' +readonly k3slrh_fixture_root +readonly k3slrh_stdout="${k3slrh_fixture_root}/stdout" +readonly k3slrh_stderr="${k3slrh_fixture_root}/stderr" +readonly k3slrh_command_log="${k3slrh_fixture_root}/commands" +readonly k3slrh_mutation_log="${k3slrh_fixture_root}/mutations" +readonly k3slrh_validator_call_log="${k3slrh_fixture_root}/validator-calls" +readonly k3slrh_validator_hash_log="${k3slrh_fixture_root}/validator-hashes" +readonly k3slrh_sudo_validate_log="${k3slrh_fixture_root}/sudo-validates" +readonly k3slrh_readyz_log="${k3slrh_fixture_root}/readyz-calls" +readonly k3slrh_validator_file_log="${k3slrh_fixture_root}/validator-file-checks" +readonly k3slrh_validator_symlink_log="${k3slrh_fixture_root}/validator-symlink-checks" +readonly k3slrh_validator_stat_log="${k3slrh_fixture_root}/validator-stat-checks" +readonly k3slrh_binary_stat_log="${k3slrh_fixture_root}/binary-stat-checks" + +k3slrh_cleanup() { + case "$k3slrh_fixture_root" in + /tmp/k3slrh-entrypoint-test.[A-Za-z0-9]*) /usr/bin/rm -rf -- "$k3slrh_fixture_root" ;; + *) return 1 ;; + esac +} +trap k3slrh_cleanup EXIT + +k3slrh_assert_eq() { + local expected="${1-}" actual="${2-}" label="${3-}" + (( $# == 3 )) || k3slrh_test_fail 'invalid assert_eq invocation' + [[ "$actual" == "$expected" ]] || k3slrh_test_fail "$label" +} + +k3slrh_assert_rc() { + local expected="${1-}" label="${2-}" rc=0 + shift 2 || k3slrh_test_fail 'invalid assert_rc invocation' + if "$@"; then rc=0; else rc=$?; fi + [[ "$rc" == "$expected" ]] || k3slrh_test_fail "${label}: rc=${rc}, expected=${expected}" +} + +k3slrh_assert_succeeds() { + local label="${1-}" + shift || k3slrh_test_fail 'invalid assert_succeeds invocation' + "$@" || k3slrh_test_fail "$label" +} + +k3slrh_wrapper_clean_composition() { + local rc=0 + : >"$k3slrh_stdout" + : >"$k3slrh_stderr" + if /usr/bin/env -i \ + PATH=/usr/sbin:/usr/bin:/sbin:/bin LC_ALL=C \ + /usr/bin/bash --noprofile --norc -c ' + set -Eeuo pipefail + source "$1" + declare -F _k3slrh_header_proof_main >/dev/null + declare -F _k3slrh_wrapper_initial_guard >/dev/null + if _k3slrh_wrapper_initial_guard 0 hB; then exit 41; fi + if _k3slrh_wrapper_initial_guard 1000 hxB; then exit 42; fi + ' k3slrh-wrapper-probe "$K3SLRH_WRAPPER" \ + >"$k3slrh_stdout" 2>"$k3slrh_stderr"; then + rc=0 + else + rc=$? + fi + [[ "$rc" == 0 && ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" ]] +} + +k3slrh_wrapper_rejects_inherited_route() { + local rc=0 marker="${k3slrh_fixture_root}/inherited-route-marker" + : >"$k3slrh_stdout" + : >"$k3slrh_stderr" + if /usr/bin/env -i \ + PATH=/usr/sbin:/usr/bin:/sbin:/bin LC_ALL=C \ + /usr/bin/bash --noprofile --norc -c ' + set -Eeuo pipefail + K3SLRH_PROOF_RELATIVE=attacker-controlled + test_marker=$2 + _k3slrh_header_proof_main() { : >"$test_marker"; } + set +e + source "$1" >/dev/null 2>&1 + wrapper_rc=$? + set -e + [[ "$wrapper_rc" == 1 && ! -e "$test_marker" ]] + ' k3slrh-wrapper-probe "$K3SLRH_WRAPPER" "$marker" \ + >"$k3slrh_stdout" 2>"$k3slrh_stderr"; then + rc=0 + else + rc=$? + fi + [[ "$rc" == 0 && ! -e "$marker" && ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" ]] +} + +k3slrh_run_parse() { + local parsed='' rc=0 + if _k3slrh_parse_cli parsed "$@"; then rc=0; else rc=$?; fi + (( rc == 0 )) || return "$rc" + printf '%s' "$parsed" +} + +k3slrh_run_timestamp() ( + _k3slrh_derive_targets "$@" +) + +k3slrh_targets_are_exact() ( + _k3slrh_derive_targets "$K3SLRH_EXPECTED_UTC" || return 1 + [[ "${K3SLRH_RUN_UTC:-}" == "$K3SLRH_EXPECTED_UTC" && + "${K3SLRH_PRE_KDBX_RELATIVE:-}" == \ + "HyeonworksRecovery/vault/backups/${K3SLRH_PRE_KDBX_PREFIX}${K3SLRH_EXPECTED_UTC}.kdbx" && + "${K3SLRH_POST_KDBX_RELATIVE:-}" == \ + "HyeonworksRecovery/vault/backups/${K3SLRH_POST_KDBX_PREFIX}${K3SLRH_EXPECTED_UTC}.kdbx" ]] || return 1 + _k3slrh_target_state_is_consistent \ + "$K3SLRH_RUN_UTC" "$K3SLRH_PRE_KDBX_RELATIVE" "$K3SLRH_POST_KDBX_RELATIVE" +) + +k3slrh_run_targets_same_second_collision() ( + _k3slrh_derive_targets "$K3SLRH_EXPECTED_UTC" || return 1 + _k3slrh_derive_targets "$K3SLRH_EXPECTED_UTC" +) + +k3slrh_run_targets_pre_post_timestamp_drift() ( + local drifted_post + _k3slrh_derive_targets "$K3SLRH_EXPECTED_UTC" || return 1 + drifted_post="HyeonworksRecovery/vault/backups/${K3SLRH_POST_KDBX_PREFIX}20260809T120305Z.kdbx" + _k3slrh_target_state_is_consistent \ + "$K3SLRH_RUN_UTC" "$K3SLRH_PRE_KDBX_RELATIVE" "$drifted_post" +) + +k3slrh_model_case=valid +k3slrh_validator_count=0 +k3slrh_mutation_count=0 + +k3slrh_log_command() { + local argument + for argument in "$@"; do printf '%s\0' "$argument" >>"$k3slrh_command_log"; done + printf '\0' >>"$k3slrh_command_log" +} + +k3slrh_path_is_known_binary() { + local candidate + for candidate in \ + /usr/bin/awk /usr/bin/bash /usr/bin/chmod /usr/bin/chown /usr/bin/cmp \ + /usr/bin/date /usr/bin/dd /usr/bin/dpkg-query /usr/bin/env /usr/bin/find \ + /usr/bin/findmnt /usr/bin/grep /usr/bin/keepassxc-cli /usr/bin/lsblk \ + /usr/bin/mkdir /usr/bin/mktemp /usr/bin/mount /usr/bin/ntfsinfo \ + /usr/bin/od /usr/bin/readlink /usr/bin/rm /usr/bin/rmdir /usr/bin/sha256sum \ + /usr/bin/sqlite3 /usr/bin/stat /usr/bin/sudo /usr/bin/sync /usr/bin/systemctl \ + /usr/bin/test /usr/bin/timeout /usr/bin/umount /usr/bin/unlink /usr/bin/wc \ + /usr/local/bin/k3s /usr/sbin/blockdev /usr/sbin/cryptsetup /usr/sbin/dmsetup \ + /usr/sbin/losetup; do + [[ "${1-}" != "$candidate" ]] || return 0 + done + return 1 +} + +_k3slrh_command() { + local package='' path='' expected_backing='' hash_count=0 + k3slrh_log_command "$@" + case "${1-}" in + /usr/bin/sudo) + if (( $# == 3 )) && [[ "$2" == --non-interactive && "$3" == --validate ]]; then + printf x >>"$k3slrh_sudo_validate_log" + if [[ "$k3slrh_model_case" == uncached_sudo ]]; then + printf 'SUDO_PROMPT_SENTINEL\n' >&2 + return 1 + fi + return 0 + fi + if (( $# == 15 )) && [[ "$2" == --non-interactive && "$3" == -- && + "$4" == /usr/bin/env && "$5" == -i && + "$6" == 'PATH=/usr/sbin:/usr/bin:/sbin:/bin' && "$7" == LC_ALL=C && + "$8" == /usr/bin/timeout && "$9" == --signal=TERM && + "${10}" == --kill-after=1s && "${11}" == 30s && + "${12}" == /usr/local/bin/k3s && "${13}" == kubectl && + "${14}" == get && "${15}" == --raw=/readyz ]]; then + printf x >>"$k3slrh_readyz_log" + case "$k3slrh_model_case" in + readyz_extra) printf 'ok\nextra\n' ;; + readyz_missing_lf) printf 'ok' ;; + readyz_cr) printf 'ok\r\n' ;; + readyz_nul) printf 'ok\0\n' ;; + readyz_empty) : ;; + readyz_rc1) return 1 ;; + readyz_rc124) return 124 ;; + readyz_rc137) return 137 ;; + readyz_stderr) printf 'READYZ_STDERR_SENTINEL\n' >&2; printf 'ok\n' ;; + *) printf 'ok\n' ;; + esac + return 0 + fi + printf 'mutation:sudo\n' >>"$k3slrh_mutation_log" + return 97 + ;; + /usr/bin/systemctl) + [[ "$#" == 4 && "$2" == is-active && "$3" == --quiet && "$4" == k3s ]] || return 97 + [[ "$k3slrh_model_case" != k3s_inactive ]] || return 3 + ;; + /usr/bin/timeout) + # Direct non-root readyz is forbidden by the authority contract. + return 97 + ;; + /usr/bin/env) + [[ "$#" == 10 && "$2" == -i && "$3" == 'PATH=/usr/sbin:/usr/bin:/sbin:/bin' && + "$4" == LC_ALL=C && "$5" == /usr/bin/bash && "$6" == --noprofile && + "$7" == --norc && "$8" == -- && "$9" == "$K3SLRH_VALIDATOR" && + "${10}" == --expect-closed ]] || return 97 + k3slrh_validator_count=$((k3slrh_validator_count + 1)) + printf x >>"$k3slrh_validator_call_log" + case "$k3slrh_model_case" in + validator_cr) printf '%s\r\n' 'Recovery device: match' 'Recovery state: closed' 'Lineage: match' 'Latest bundle: not_checked' ;; + validator_nul) printf 'Recovery device: match\0Recovery state: closed\nLineage: match\nLatest bundle: not_checked\n' ;; + validator_missing_lf) printf '%s' "${K3SLRH_EXPECTED_CLOSED_RECORD%$'\n'}" ;; + validator_extra) printf '%sextra\n' "$K3SLRH_EXPECTED_CLOSED_RECORD" ;; + validator_raw_id) printf '%s%s\n' "$K3SLRH_EXPECTED_CLOSED_RECORD" "$K3SLRH_RAW_IDENTIFIER_SENTINEL" ;; + validator_rc1) return 1 ;; + validator_rc124) return 124 ;; + validator_rc137) return 137 ;; + validator_stderr) printf 'x' >&2; printf '%s' "$K3SLRH_EXPECTED_CLOSED_RECORD" ;; + *) printf '%s' "$K3SLRH_EXPECTED_CLOSED_RECORD" ;; + esac + ;; + /usr/bin/dpkg-query) + [[ "$#" == 4 && "$2" == --show && + "$3" == '--showformat=${Status}|${Version}\n' ]] || return 97 + package="${!#}" + case "$package" in + keepassxc) path='2.7.6+dfsg.1-1build3'; [[ "$k3slrh_model_case" != keepass_package_drift ]] || path='0.invalid' ;; + cryptsetup-bin) path='2:2.7.0-1ubuntu4.2'; [[ "$k3slrh_model_case" != cryptsetup_package_drift ]] || path='0.invalid' ;; + sqlite3) path='3.45.1-1ubuntu2.7'; [[ "$k3slrh_model_case" != sqlite_package_drift ]] || path='0.invalid' ;; + *) return 97 ;; + esac + printf 'install ok installed|%s\n' "$path" + ;; + /usr/bin/date) + [[ "$#" == 3 && "$2" == --utc && "$3" == +%Y%m%dT%H%M%SZ ]] || return 97 + printf '%s\n' "$K3SLRH_EXPECTED_UTC" + ;; + /usr/bin/findmnt) + [[ "$#" == 6 && "$2" == -rn && "$3" == -M && "$5" == -o && "$6" == TARGET ]] || return 97 + case "$4|$k3slrh_model_case" in + "$K3SLR_OUTER_MOUNT|outer_mount_collision"|"$K3SLR_INNER_MOUNT|inner_mount_collision") + printf '%s\n' "$4" + return 0 + ;; + "$K3SLR_OUTER_MOUNT|"*|"$K3SLR_INNER_MOUNT|"*) return 1 ;; + *) return 97 ;; + esac + ;; + /usr/sbin/losetup) + [[ "$#" == 5 && "$2" == --list && "$3" == --noheadings && + "$4" == --output && "$5" == NAME,BACK-FILE ]] || { + printf 'mutation:losetup\n' >>"$k3slrh_mutation_log" + return 97 + } + case "$k3slrh_model_case" in + main_loop_residue) + expected_backing="${K3SLR_OUTER_MOUNT}/${K3SLR_CONTAINER_RELATIVE}" + printf '/dev/loop91 %s\n' "$expected_backing" + ;; + proof_loop_residue) + expected_backing="${K3SLR_OUTER_MOUNT}/${K3SLRH_PROOF_RELATIVE}" + printf '/dev/loop92 %s\n' "$expected_backing" + ;; + esac + ;; + /usr/sbin/dmsetup) + return 97 + ;; + /usr/bin/find) + if (( $# == 8 )) && [[ "$2" == "$K3SLR_OUTER_MOUNT" && + "$3" == -mindepth && "$4" == 1 && "$5" == -maxdepth && + "$6" == 1 && "$7" == -print && "$8" == -quit ]]; then + [[ "$k3slrh_model_case" != outer_mountpoint_not_empty ]] || + printf '%s/unexpected\n' "$K3SLR_OUTER_MOUNT" + return 0 + fi + if (( $# == 10 )) && [[ "$2" == /run && "$3" == -mindepth && + "$4" == 1 && "$5" == -maxdepth && "$6" == 1 && "$7" == -name && + "$8" == 'hyeonworks-k3s-recovery.*' && "$9" == -print && + "${10}" == -quit ]]; then + [[ "$k3slrh_model_case" != tmpfs_residue ]] || + printf '/run/hyeonworks-k3s-recovery.ABCDEFGH\n' + return 0 + fi + return 97 + ;; + /usr/bin/test) + if (( $# == 3 )) && [[ "$2" == ! && "$3" == -L ]]; then return 97; fi + if (( $# == 4 )) && [[ "$2" == ! && "$3" == -e ]]; then + case "$4" in + "/dev/mapper/${K3SLR_MAPPING_NAME}") + [[ "$k3slrh_model_case" != main_mapping_residue ]] + return + ;; + "/dev/mapper/${K3SLR_PROOF_MAPPING_NAME}") + [[ "$k3slrh_model_case" != proof_mapping_residue ]] + return + ;; + *) return 97 ;; + esac + fi + if (( $# == 3 )); then + case "$2" in + -d) [[ "$3" == "$K3SLR_OUTER_MOUNT" ]] || return 97 ;; + -f|-x|-r) + case "$3" in + "$K3SLRH_SHARED_LIBRARY"|"$K3SLRH_A1_LIBRARY"|"$K3SLRH_A1_HELPER"|"$K3SLRH_CONTRACT"|"$K3SLRH_VALIDATOR") ;; + *) k3slrh_path_is_known_binary "$3" || return 97 ;; + esac + if [[ "$2" == -f && "$3" == "$K3SLRH_VALIDATOR" ]]; then + printf x >>"$k3slrh_validator_file_log" + fi + ;; + *) return 97 ;; + esac + return 0 + fi + if (( $# == 4 )) && [[ "$2" == ! && "$3" == -L ]]; then + case "$4" in + "/dev/mapper/${K3SLR_MAPPING_NAME}") + [[ "$k3slrh_model_case" != main_mapping_dangling ]] + return + ;; + "/dev/mapper/${K3SLR_PROOF_MAPPING_NAME}") + [[ "$k3slrh_model_case" != proof_mapping_dangling ]] + return + ;; + "$K3SLRH_VALIDATOR") printf x >>"$k3slrh_validator_symlink_log"; return 0 ;; + "$K3SLR_OUTER_MOUNT"|"$K3SLRH_SHARED_LIBRARY"|"$K3SLRH_A1_LIBRARY"|"$K3SLRH_A1_HELPER"|"$K3SLRH_CONTRACT") return 0 ;; + *) k3slrh_path_is_known_binary "$4" || return 97; return 0 ;; + esac + fi + return 97 + ;; + /usr/bin/stat) + path="${!#}" + if (( $# == 4 )) && [[ "$2" == "--format=%u|%a|%F" && "$3" == -- ]]; then + k3slrh_path_is_known_binary "$path" || return 97 + printf '%s\n' "$path" >>"$k3slrh_binary_stat_log" + if [[ "$k3slrh_model_case" == binary_untrusted && "$path" == /usr/bin/keepassxc-cli ]]; then + printf '0|775|regular file\n' + else + printf '0|755|regular file\n' + fi + return 0 + fi + if (( $# == 4 )) && [[ "$2" == "--format=%F|%u|%g|%a" && "$3" == -- ]]; then + case "$path" in + "$K3SLR_OUTER_MOUNT") printf 'directory|0|0|755\n' ;; + "$K3SLRH_SHARED_LIBRARY"|"$K3SLRH_A1_LIBRARY"|"$K3SLRH_A1_HELPER"|"$K3SLRH_CONTRACT"|"$K3SLRH_VALIDATOR") + [[ "$path" != "$K3SLRH_VALIDATOR" ]] || printf x >>"$k3slrh_validator_stat_log" + printf 'regular file|1000|1000|664\n' + ;; + *) return 97 ;; + esac + return 0 + fi + return 97 + ;; + /usr/bin/readlink) + [[ "$#" == 4 && "$2" == -e && "$3" == -- ]] || return 97 + path="$4" + case "$path" in + "$K3SLR_OUTER_MOUNT"|"$K3SLRH_SHARED_LIBRARY"|"$K3SLRH_A1_LIBRARY"|"$K3SLRH_A1_HELPER"|"$K3SLRH_CONTRACT"|"$K3SLRH_VALIDATOR") printf '%s\n' "$path" ;; + *) return 97 ;; + esac + ;; + /usr/bin/sha256sum) + [[ "$#" == 3 && "$2" == -- ]] || return 97 + path="${!#}" + case "$path" in + "$K3SLRH_SHARED_LIBRARY") printf '%s %s\n' "$K3SLRH_EXPECTED_SHARED_SHA256" "$path" ;; + "$K3SLRH_A1_LIBRARY") printf '%s %s\n' "$K3SLRH_EXPECTED_A1_SHA256" "$path" ;; + "$K3SLRH_A1_HELPER") printf '%s %s\n' "$K3SLRH_EXPECTED_A1_HELPER_SHA256" "$path" ;; + "$K3SLRH_CONTRACT") printf '%s %s\n' "$K3SLRH_EXPECTED_CONTRACT_SHA256" "$path" ;; + "$K3SLRH_VALIDATOR") + printf x >>"$k3slrh_validator_hash_log" + hash_count="$(/usr/bin/wc -c <"$k3slrh_validator_hash_log")" || return 97 + if [[ "$k3slrh_model_case" == validator_hash_pre_drift || + ( "$k3slrh_model_case" == validator_hash_post_drift && "$hash_count" -ge 2 ) ]]; then + printf '%064d %s\n' 0 "$path" + else + printf '%s %s\n' "$K3SLRH_EXPECTED_VALIDATOR_SHA256" "$path" + fi + ;; + *) return 97 ;; + esac + ;; + /usr/bin/mount|/usr/bin/umount|/usr/sbin/cryptsetup) + k3slrh_mutation_count=$((k3slrh_mutation_count + 1)) + printf 'mutation\n' >>"$k3slrh_mutation_log" + return 97 + ;; + *) return 97 ;; + esac +} + +_k3slr_command() { _k3slrh_command "$@"; } + +_k3slr_collect_prepackage_snapshot() { printf 'legacy-collector\n' >>"$k3slrh_mutation_log"; return 97; } +_k3slr_prove_header_restore() { printf 'legacy-marker\n' >>"$k3slrh_mutation_log"; return 97; } +_k3slr_lifecycle_dispatch() { printf 'prepare-dispatch\n' >>"$k3slrh_mutation_log"; return 97; } + +k3slrh_reset_model_logs() { + : >"$k3slrh_command_log" + : >"$k3slrh_mutation_log" + : >"$k3slrh_validator_call_log" + : >"$k3slrh_validator_hash_log" + : >"$k3slrh_sudo_validate_log" + : >"$k3slrh_readyz_log" + : >"$k3slrh_validator_file_log" + : >"$k3slrh_validator_symlink_log" + : >"$k3slrh_validator_stat_log" + : >"$k3slrh_binary_stat_log" +} + +k3slrh_capture_call() { + : >"$k3slrh_stdout" + : >"$k3slrh_stderr" + "$@" >"$k3slrh_stdout" 2>"$k3slrh_stderr" +} + +k3slrh_validator_case() { + local case_name="${1-}" expected_rc="${2-}" expected_calls="${3-}" + local expected_hashes="${4-}" rc=0 expected_physical=2 + (( $# == 4 )) || return 97 + k3slrh_reset_model_logs + k3slrh_model_case="$case_name" + [[ "$case_name" != validator_hash_pre_drift ]] || expected_physical=1 + if k3slrh_capture_call _k3slrh_run_closed_validator; then rc=0; else rc=$?; fi + [[ "$rc" == "$expected_rc" && + "$(/usr/bin/wc -c <"$k3slrh_validator_call_log")" == "$expected_calls" && + "$(/usr/bin/wc -c <"$k3slrh_validator_hash_log")" == "$expected_hashes" && + "$(/usr/bin/wc -c <"$k3slrh_validator_file_log")" == "$expected_physical" && + "$(/usr/bin/wc -c <"$k3slrh_validator_symlink_log")" == "$expected_physical" && + "$(/usr/bin/wc -c <"$k3slrh_validator_stat_log")" == "$expected_physical" ]] || return 1 + [[ ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" ]] || return 1 + ! /usr/bin/grep -aFq -- "$K3SLRH_RAW_IDENTIFIER_SENTINEL" "$k3slrh_stdout" "$k3slrh_stderr" +} + +k3slrh_health_case() { + local case_name="${1-}" expected_rc="${2-}" expected_readyz="${3-}" rc=0 + (( $# == 3 )) || return 97 + k3slrh_reset_model_logs + k3slrh_model_case="$case_name" + if k3slrh_capture_call _k3slrh_k3s_health; then rc=0; else rc=$?; fi + [[ "$rc" == "$expected_rc" && ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" && + "$(/usr/bin/wc -c <"$k3slrh_readyz_log")" == "$expected_readyz" ]] +} + +k3slrh_capability_case() { + local case_name="${1-}" expected_rc="${2-}" rc=0 + k3slrh_reset_model_logs + k3slrh_model_case="$case_name" + if k3slrh_capture_call _k3slrh_pre_mount_capability; then rc=0; else rc=$?; fi + [[ "$rc" == "$expected_rc" ]] || return 1 + if [[ "$expected_rc" == 0 ]]; then + [[ "$(<"$k3slrh_stdout")" == 'Header proof capability: pass' && ! -s "$k3slrh_stderr" ]] || return 1 + else + [[ ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" ]] || return 1 + fi + [[ ! -s "$k3slrh_mutation_log" ]] || return 1 + if [[ "$case_name" == uncached_sudo ]]; then + [[ "$(/usr/bin/wc -c <"$k3slrh_sudo_validate_log")" == 1 && + ! -s "$k3slrh_readyz_log" && ! -s "$k3slrh_validator_call_log" && + ! -s "$k3slrh_validator_hash_log" ]] + fi +} + +k3slrh_command_model_is_closed() { + local rc=0 k3slrh_bad_command + local -a k3slrh_bad_argv=() + k3slrh_reset_model_logs + for k3slrh_bad_command in \ + '/usr/bin/timeout --signal=TERM 30s /usr/local/bin/k3s kubectl get --raw=/readyz' \ + '/usr/bin/findmnt --target /fixture' \ + '/usr/sbin/losetup --find --show /fixture' \ + '/usr/bin/stat --format=%F -- /dev/loop0' \ + '/usr/bin/test -e /dev/mapper/k3s-recovery'; do + read -r -a k3slrh_bad_argv <<<"$k3slrh_bad_command" + if _k3slrh_command "${k3slrh_bad_argv[@]}" >/dev/null 2>&1; then rc=0; else rc=$?; fi + [[ "$rc" == 97 ]] || return 1 + done + [[ -s "$k3slrh_mutation_log" ]] +} + +k3slrh_invalid_main() { + local rc=0 + : >"$k3slrh_stdout" + : >"$k3slrh_stderr" + if _k3slrh_header_proof_main "$@" >"$k3slrh_stdout" 2>"$k3slrh_stderr"; then + rc=0 + else + rc=$? + fi + ! /usr/bin/grep -aFq -- "$K3SLRH_RAW_IDENTIFIER_SENTINEL" \ + "$k3slrh_stdout" "$k3slrh_stderr" || return 97 + return "$rc" +} + +k3slrh_execute_without_tty() ( + local rc=0 + _k3slrh_tty_capability() { return 1; } + _k3slr_require_execute_tty() { return 1; } + : >"$k3slrh_stdout" + : >"$k3slrh_stderr" + if _k3slrh_header_proof_main --execute >"$k3slrh_stdout" 2>"$k3slrh_stderr"; then + rc=0 + else + rc=$? + fi + [[ ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" ]] || return 97 + return "$rc" +) + +k3slrh_dry_run_closed() { + local rc=0 validator_calls validator_hashes sudo_validates readyz_calls + local validator_files validator_symlinks validator_stats binary_count k3slrh_required_binary + k3slrh_model_case=valid + k3slrh_validator_count=0 + k3slrh_mutation_count=0 + k3slrh_reset_model_logs + if k3slrh_capture_call _k3slrh_header_proof_main; then rc=0; else rc=$?; fi + validator_calls="$(/usr/bin/wc -c <"$k3slrh_validator_call_log")" || return 1 + validator_hashes="$(/usr/bin/wc -c <"$k3slrh_validator_hash_log")" || return 1 + sudo_validates="$(/usr/bin/wc -c <"$k3slrh_sudo_validate_log")" || return 1 + readyz_calls="$(/usr/bin/wc -c <"$k3slrh_readyz_log")" || return 1 + validator_files="$(/usr/bin/wc -c <"$k3slrh_validator_file_log")" || return 1 + validator_symlinks="$(/usr/bin/wc -c <"$k3slrh_validator_symlink_log")" || return 1 + validator_stats="$(/usr/bin/wc -c <"$k3slrh_validator_stat_log")" || return 1 + for k3slrh_required_binary in /usr/bin/keepassxc-cli /usr/sbin/cryptsetup /usr/bin/sqlite3; do + binary_count="$(/usr/bin/grep -Fxc -- "$k3slrh_required_binary" "$k3slrh_binary_stat_log")" || return 1 + [[ "$binary_count" == 1 ]] || return 1 + done + [[ "$rc" == 0 && "$(<"$k3slrh_stdout")"$'\n' == "$K3SLRH_EXPECTED_DRY_RUN" && + ! -s "$k3slrh_stderr" && ! -s "$k3slrh_mutation_log" && + "$validator_calls" == 1 && "$validator_hashes" == 2 && + "$validator_files" == 2 && "$validator_symlinks" == 2 && "$validator_stats" == 2 && + "$sudo_validates" == 1 && "$readyz_calls" == 1 && + "$k3slrh_mutation_count" == 0 ]] +} + +k3slrh_assert_eq dry-run "$(k3slrh_run_parse)" 'no argument must select dry-run' +k3slrh_assert_eq execute "$(k3slrh_run_parse --execute)" 'one execute must be accepted' +k3slrh_assert_rc 1 'duplicate parser execute must fail safely' k3slrh_run_parse --execute --execute +k3slrh_assert_rc 1 'caller parser path must fail safely' k3slrh_run_parse --container /fixture +k3slrh_assert_rc 2 'duplicate main execute must be usage rc2' k3slrh_invalid_main --execute --execute +k3slrh_assert_rc 2 'caller main path must be usage rc2' k3slrh_invalid_main --container /fixture +k3slrh_assert_succeeds 'actual clean wrapper composition and root/xtrace guards must pass' \ + k3slrh_wrapper_clean_composition +k3slrh_assert_succeeds 'actual wrapper must reject inherited route and constants' \ + k3slrh_wrapper_rejects_inherited_route +k3slrh_assert_rc 1 'execute without TTY must be safety rc1' k3slrh_execute_without_tty +k3slrh_assert_rc 1 'malformed timestamp must be safety rc1' k3slrh_run_timestamp malformed +k3slrh_assert_succeeds 'valid target names must derive exactly once' k3slrh_targets_are_exact +k3slrh_assert_rc 1 'same-second target reservation must be safety rc1' \ + k3slrh_run_targets_same_second_collision +k3slrh_assert_rc 1 'pre/post timestamp drift must be safety rc1' \ + k3slrh_run_targets_pre_post_timestamp_drift +k3slrh_assert_succeeds 'closed external command model must reject unknown argv' \ + k3slrh_command_model_is_closed + +for k3slrh_bad_health in k3s_inactive readyz_extra readyz_missing_lf readyz_cr \ + readyz_nul readyz_empty readyz_rc1 readyz_rc124 readyz_rc137 readyz_stderr; do + if [[ "$k3slrh_bad_health" == k3s_inactive ]]; then k3slrh_expected_readyz=0; else k3slrh_expected_readyz=1; fi + k3slrh_assert_succeeds "K3s health must reject ${k3slrh_bad_health}" \ + k3slrh_health_case "$k3slrh_bad_health" 1 "$k3slrh_expected_readyz" +done +unset k3slrh_bad_health k3slrh_expected_readyz +k3slrh_assert_succeeds 'exact cached-sudo readyz must pass silently' \ + k3slrh_health_case valid 0 1 + +for k3slrh_bad_validator in validator_cr validator_nul validator_missing_lf \ + validator_extra validator_raw_id validator_rc1 validator_rc124 validator_rc137 \ + validator_stderr; do + k3slrh_assert_succeeds "closed validator must reject ${k3slrh_bad_validator}" \ + k3slrh_validator_case "$k3slrh_bad_validator" 1 1 2 +done +unset k3slrh_bad_validator +k3slrh_assert_succeeds 'validator pre-call hash drift must stop before dispatch' \ + k3slrh_validator_case validator_hash_pre_drift 1 0 1 +k3slrh_assert_succeeds 'validator post-call hash drift must fail closed' \ + k3slrh_validator_case validator_hash_post_drift 1 1 2 +k3slrh_assert_succeeds 'exact pinned closed validator record must pass' \ + k3slrh_validator_case valid 0 1 2 + +for k3slrh_bad_capability in uncached_sudo k3s_inactive readyz_extra \ + keepass_package_drift cryptsetup_package_drift sqlite_package_drift \ + binary_untrusted \ + outer_mount_collision inner_mount_collision outer_mountpoint_not_empty tmpfs_residue \ + main_mapping_residue proof_mapping_residue main_mapping_dangling proof_mapping_dangling \ + main_loop_residue proof_loop_residue; do + k3slrh_assert_succeeds "capability must reject ${k3slrh_bad_capability}" \ + k3slrh_capability_case "$k3slrh_bad_capability" 1 +done +unset k3slrh_bad_capability +k3slrh_assert_succeeds 'closed dry-run must pass without mutation' k3slrh_dry_run_closed + +printf 'PASS: header-proof entrypoint and closed capability fixtures\n' diff --git a/scripts/validate/test-k3s-local-recovery-lsblk.sh b/scripts/validate/test-k3s-local-recovery-lsblk.sh new file mode 100644 index 0000000..5f485f3 --- /dev/null +++ b/scripts/validate/test-k3s-local-recovery-lsblk.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Regression test for whole-disk lsblk queries. Without --nodeps, lsblk emits +# the disk and all child partitions, while the validator requires one record. +set -Eeuo pipefail + +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly VALIDATOR_PATH="${REPOSITORY_ROOT}/scripts/validate/k3s-local-recovery.sh" + +# shellcheck source=/dev/null +source "$VALIDATOR_PATH" + +_k3slrv_exec() { + local scope="${1-}" + shift || return 1 + [[ "$scope" == user ]] || return 1 + + if (( $# == 8 )) && + [[ "$1" == /usr/bin/lsblk && "$2" == --noheadings && + "$3" == --nodeps && "$4" == --paths && "$5" == --output && + "$6" == MODEL && "$7" == -- && "$8" == /dev/fixture-disk ]]; then + printf '%s\n' FIXTURE_DISK_MODEL + else + # This mirrors lsblk against a whole disk when dependency traversal is + # not disabled: the disk row is followed by a child-partition row. + printf '%s\n%s\n' FIXTURE_DISK_MODEL FIXTURE_CHILD_ROW + fi +} + +actual='' +if ! actual="$(_k3slrv_lsblk MODEL /dev/fixture-disk)"; then + printf '%s\n' \ + 'TEST FAILURE: whole-disk lsblk query did not produce one record' >&2 + exit 1 +fi + +if [[ "$actual" != FIXTURE_DISK_MODEL ]]; then + printf '%s\n' \ + 'TEST FAILURE: whole-disk lsblk query returned an unexpected record' >&2 + exit 1 +fi + +printf '%s\n' 'PASS: recovery validator excludes lsblk child rows' diff --git a/scripts/validate/test-k3s-local-recovery.sh b/scripts/validate/test-k3s-local-recovery.sh new file mode 100644 index 0000000..fb94f72 --- /dev/null +++ b/scripts/validate/test-k3s-local-recovery.sh @@ -0,0 +1,8668 @@ +#!/usr/bin/env bash + +# Regression coverage for the source-safe local recovery hardware contract. +set -Eeuo pipefail + +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly LIBRARY_PATH="${REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery.sh" +readonly CONTRACT_PATH="${REPOSITORY_ROOT}/infrastructure/security/k3s/local-recovery.env" +readonly VALIDATOR_PATH="${REPOSITORY_ROOT}/scripts/validate/k3s-local-recovery.sh" +readonly FEASIBILITY_PATH="${REPOSITORY_ROOT}/scripts/validate/k3s-local-recovery-feasibility.sh" +readonly PREPARE_WRAPPER_PATH="${REPOSITORY_ROOT}/scripts/bootstrap/prepare-k3s-local-recovery.sh" +readonly OPEN_WRAPPER_PATH="${REPOSITORY_ROOT}/scripts/bootstrap/open-k3s-local-recovery.sh" +readonly CLOSE_WRAPPER_PATH="${REPOSITORY_ROOT}/scripts/bootstrap/close-k3s-local-recovery.sh" + +fail() { + printf 'TEST FAILURE: %s\n' "$*" >&2 + exit 1 +} + +assert_eq() { + local expected="$1" actual="$2" description="$3" + [[ "$actual" == "$expected" ]] || fail "$description (expected ${expected}, got ${actual})" +} + +assert_succeeds() { + "$@" || fail "expected success: $*" +} + +assert_fails() { + if "$@"; then + fail "expected failure: $*" + fi +} + +task5b_focus_selector_guard() { + local focus="${1-}" + (( $# == 1 )) || return 2 + case "$focus" in + ''|gate0|api_state|tty_seam|action1) return 0 ;; + *) + printf 'TEST FAILURE: unsupported TASK5B_FOCUS: %s\n' "$focus" >&2 + return 2 + ;; + esac +} + +# A nonempty Task 5B selector is closed before fixture creation, sourcing, or +# any Task 5A/older/full-suite assertion can run. +task5b_focus_selector_guard "${TASK5B_FOCUS:-}" || exit $? + +task4_review2_focus_selector_guard() { + local focus="${1-}" + (( $# == 1 )) || return 2 + case "$focus" in + ''|critical|capture|atomic|show_protected|feas_show_protected|feas_attachment_listing) return 0 ;; + *) + printf 'TEST FAILURE: unsupported TASK4_REVIEW2_FOCUS: %s\n' "$focus" >&2 + return 2 + ;; + esac +} + +# A Task 1 selector is closed before any optional full-suite branch can run. +task4_review2_focus_selector_guard "${TASK4_REVIEW2_FOCUS:-}" || exit $? + +fixture_root="$(mktemp -d "${TMPDIR:-/tmp}/k3s-local-recovery-test.XXXXXX")" +cleanup() { + case "$fixture_root" in + /tmp/k3s-local-recovery-test.*|"${TMPDIR:-/tmp}"/k3s-local-recovery-test.*) + rm -rf -- "$fixture_root" + ;; + *) + fail 'refusing to remove an unexpected fixture directory' + ;; + esac +} +trap cleanup EXIT + +write_fixture() { + local name="$1" destination + destination="${fixture_root}/${name}" + cp -- "$CONTRACT_PATH" "$destination" + case "$name" in + missing_key.env) + sed -i '/^K3SLR_OWNER_GID=/d' "$destination" + ;; + duplicate_key.env) + printf '%s\n' 'K3SLR_OWNER_GID=1000' >>"$destination" + ;; + unknown_key.env) + printf '%s\n' 'K3SLR_UNSUPPORTED=value' >>"$destination" + ;; + command_substitution.env) + printf 'K3SLR_OUTER_MOUNT=$(touch %s)\n' "${fixture_root}/evaluated" >"$destination.tmp" + sed '/^K3SLR_OUTER_MOUNT=/d' "$destination" >>"$destination.tmp" + mv -- "$destination.tmp" "$destination" + ;; + relative_device.env) + sed -i 's|^K3SLR_RECOVERY_DISK_BY_ID=.*|K3SLR_RECOVERY_DISK_BY_ID=dev/disk/by-id/wwn-0x500a0751e6aa6254|' "$destination" + ;; + parent_path.env) + sed -i 's|^K3SLR_DATABASE_RELATIVE=.*|K3SLR_DATABASE_RELATIVE=HyeonworksRecovery/..|' "$destination" + ;; + *) + fail "unknown fixture: $name" + ;; + esac + printf '%s\n' "$destination" +} + +load_fixture() { + _k3slr_load_contract "$(write_fixture "$1")" +} + +# The production change this catches is a missing or unsafe source-only contract +# library. The RED run before implementation asserted that source failed. +# shellcheck source=/dev/null +source "$LIBRARY_PATH" + +assert_succeeds _k3slr_load_contract "$CONTRACT_PATH" +assert_eq /dev/disk/by-id/wwn-0x500a0751e6aa6254 \ + "$( _k3slr_config_value "$CONTRACT_PATH" K3SLR_RECOVERY_DISK_BY_ID )" \ + 'approved recovery disk identity is available without sourcing config' +assert_eq 'K3s Recovery LUKS' "$K3SLR_KEEPASS_ENTRY" 'KeePass entry is library-owned' +assert_eq 'luks-header-backup.bin' "$K3SLR_KEEPASS_HEADER_ATTACHMENT" 'header attachment is library-owned' +assert_fails _k3slr_config_value "$CONTRACT_PATH" K3SLR_UNSUPPORTED + +assert_fails load_fixture missing_key.env +assert_fails load_fixture duplicate_key.env +assert_fails load_fixture unknown_key.env +assert_fails load_fixture command_substitution.env +[[ ! -e "${fixture_root}/evaluated" ]] || fail 'contract parser evaluated command substitution' +assert_fails load_fixture relative_device.env +assert_fails load_fixture parent_path.env +assert_eq 12884901891 "$( _k3slr_required_bytes 858993460 1 )" '25 percent padding and two phases' +assert_fails _k3slr_required_bytes -1 0 +assert_fails _k3slr_required_bytes 1 9223372036854775807 +assert_fails _k3slr_required_bytes 1844674407370955161 0 +assert_succeeds _k3slr_inner_capacity_fits 12884901891 1 858993460 +assert_fails _k3slr_inner_capacity_fits 12884901890 1 858993460 +assert_fails _k3slr_inner_capacity_fits 12884901891 1x 858993460 +assert_succeeds _k3slr_outer_capacity_fits 1000000000000 400000000000 34359738368 +assert_fails _k3slr_outer_capacity_fits 1000000000000 250000000000 34359738368 +assert_fails _k3slr_outer_capacity_fits 1000 1001 1 + +validator_device_evidence() { + local fixture="$1" + local recovery_partition=/dev/fixture-recovery-partition + local recovery_partition_parent=/dev/fixture-recovery-disk + local recovery_fs_uuid=4EA0196C0C5FA27E recovery_partuuid=4670aa9f-9045-4bce-930d-9e84dfec9f38 + local recovery_model=CT1000MX500SSD1 recovery_serial=2306E6AA6254 + local recovery_wwn=0x500a0751e6aa6254 k3s_major_minor=8:32 + local smart_health=PASSED smart_reallocated=0 smart_pending=0 smart_uncorrectable=0 ntfs_probe=pass + case "$fixture" in + wrong_by_id_target) recovery_partition_parent=/dev/fixture-wrong-disk ;; + wrong_fs_uuid) recovery_fs_uuid=AAAAAAAAAAAAAAAA ;; + wrong_partuuid) recovery_partuuid=aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa ;; + wrong_model) recovery_model=WRONGMODEL ;; + wrong_serial) recovery_serial=WRONGSERIAL ;; + wrong_wwn) recovery_wwn=0xaaaaaaaaaaaaaaaa ;; + same_parent_disk) k3s_major_minor=8:0 ;; + smart_health_failed) smart_health=FAILED ;; + smart_reallocated_nonzero) smart_reallocated=1 ;; + smart_pending_nonzero) smart_pending=1 ;; + smart_uncorrectable_nonzero) smart_uncorrectable=1 ;; + ntfs_dirty_or_hibernated) ntfs_probe=fail ;; + esac + cat <"$argv_log"; } + _k3slrv_collect_device_evidence device_ready >/dev/null + [[ "$(<"$argv_log")" == '/usr/bin/ntfs-3g.probe --readwrite /dev/fixture-recovery-partition' ]] +) + +probe_review1_blkid_argv() ( + local argv_log="${fixture_root}/review1-blkid.argv" + # shellcheck source=/dev/null + source "$VALIDATOR_PATH" + _k3slrv_one_line_root() { + printf '%s\n' "$*" >"$argv_log" + printf '%s\n' fixture + } + _k3slrv_blkid UUID /dev/fixture-recovery-partition >/dev/null + [[ "$(<"$argv_log")" == '/usr/sbin/blkid --output value --match-tag UUID -- /dev/fixture-recovery-partition' ]] +) + +probe_review1_ntfs3_effective_masks() ( + local device_raw state_raw + # shellcheck source=/dev/null + source "$VALIDATOR_PATH" + device_raw="$(validator_device_evidence open_exact_lineage)" + state_raw="$(validator_state_evidence open_exact_lineage open)" + state_raw="${state_raw/umask=077/dmask=0077,fmask=0077}" + _k3slrv_validate_state open "$state_raw" "$device_raw" +) + +probe_review1_initial_guard() ( + # shellcheck source=/dev/null + source "$VALIDATOR_PATH" + declare -F _k3slrv_initial_guard >/dev/null || return 1 + _k3slrv_initial_guard 1000 hB || return 1 + ! _k3slrv_initial_guard 0 hB +) + +probe_review1_bash_x_rejected() ( + local stdout_file="${fixture_root}/review1-xtrace.stdout" + local stderr_file="${fixture_root}/review1-xtrace.stderr" rc + set +e + /usr/bin/bash -x "$VALIDATOR_PATH" --help >"$stdout_file" 2>"$stderr_file" + rc=$? + set -e + [[ "$rc" -eq 1 && ! -s "$stdout_file" ]] || return 1 + [[ "$(<"$stderr_file")" == *'Recovery validation refused'* ]] || return 1 + [[ "$(<"$stderr_file")" != *'K3SLR_RECOVERY_FS_UUID'* ]] +) + +probe_review1_manifest_symlink_escape() ( + local root="${fixture_root}/review1-manifest-symlink" bundle outside hash + root="$(mktemp -d "${root}.XXXXXX")" + bundle="${root}/bundle" + outside="${root}/outside" + mkdir -p "$bundle/data" "$outside/parent" + chmod 0700 "$bundle" "$bundle/data" "$outside" "$outside/parent" + printf '%s\n' payload >"$bundle/data/payload" + printf '%s\n' escaped >"$outside/leaf" + printf '%s\n' escaped-parent >"$outside/parent/payload" + hash="$(/usr/bin/sha256sum "$bundle/data/payload")"; hash="${hash%% *}" + printf '%s %s\n' "$hash" './data/payload' >"$bundle/verification.manifest" + chmod 0600 "$bundle/verification.manifest" "$bundle/data/payload" "$outside/leaf" "$outside/parent/payload" + # shellcheck source=/dev/null + source "$VALIDATOR_PATH" + declare -F _k3slrv_verify_manifest_targets >/dev/null || return 1 + _k3slrv_root() { "$@"; } + _k3slrv_verify_manifest_targets "$bundle" "$bundle/verification.manifest" || return 1 + ln -s "$outside/leaf" "$bundle/leaf-link" + hash="$(/usr/bin/sha256sum "$outside/leaf")"; hash="${hash%% *}" + printf '%s %s\n' "$hash" './leaf-link' >"$bundle/verification.manifest" + ! _k3slrv_verify_manifest_targets "$bundle" "$bundle/verification.manifest" || return 1 + ln -s "$outside/parent" "$bundle/parent-link" + hash="$(/usr/bin/sha256sum "$outside/parent/payload")"; hash="${hash%% *}" + printf '%s %s\n' "$hash" './parent-link/payload' >"$bundle/verification.manifest" + ! _k3slrv_verify_manifest_targets "$bundle" "$bundle/verification.manifest" +) + +probe_review1_partition_mounted_elsewhere() ( + # shellcheck source=/dev/null + source "$VALIDATOR_PATH" + _k3slrv_one_line() { + case "${*: -1}" in + "$K3SLR_RECOVERY_PARTITION_BY_ID") printf '%s\n' /dev/fixture-recovery-partition ;; + "$K3SLR_RECOVERY_DISK_BY_ID") printf '%s\n' /dev/fixture-recovery-disk ;; + "$K3SLR_K3S_PARTITION_BY_ID") printf '%s\n' /dev/fixture-k3s-partition ;; + "$K3SLR_K3S_DISK_BY_ID") printf '%s\n' /dev/fixture-k3s-disk ;; + *) return 1 ;; + esac + } + _k3slrv_lsblk() { printf '%s\n' fixture; } + _k3slrv_blkid() { printf '%s\n' fixture; } + _k3slrv_smart_evidence() { + printf '%s\n' smart_health=PASSED smart_reallocated=0 smart_pending=0 smart_uncorrectable=0 + } + _k3slrv_findmnt_optional() { :; } + _k3slrv_findmnt_source_optional() { printf '%s\n' /mnt/fixture-unapproved; } + _k3slrv_root() { :; } + ! _k3slrv_collect_device_evidence device_ready >/dev/null +) + +probe_review1_open_lineage_snapshot() ( + local good mutation key current line candidate device_raw state_raw + # shellcheck source=/dev/null + source "$VALIDATOR_PATH" + declare -F _k3slrv_validate_open_snapshot >/dev/null || return 1 + good=$'container_chain_pinned=1\ncontainer_inode=4242\ncontainer_device=8:3\nloop_back_inode=4242\nloop_back_major_minor=8:3\nouter_major_minor=8:3\nloop_major_minor=7:0\nmapping_loop_major_minor=7:0\nmapping_major_minor=253:0\ninner_major_minor=253:0\nloop_offset=0\nloop_sizelimit=0\nsnapshot_stable=1' + _k3slrv_validate_open_snapshot "$good" || { printf 'review1 snapshot good failed\n' >&2; return 1; } + for mutation in \ + 'container_chain_pinned=0' \ + 'container_device=8:9' \ + 'loop_back_inode=9999' \ + 'loop_back_major_minor=8:9' \ + 'mapping_loop_major_minor=7:9' \ + 'mapping_major_minor=253:9' \ + 'loop_offset=1' \ + 'loop_sizelimit=1' \ + 'snapshot_stable=0'; do + key="${mutation%%=*}" + current='' + while IFS= read -r line; do [[ "$line" == "$key="* ]] && current="$line"; done <<<"$good" + [[ -n "$current" ]] || return 1 + candidate="${good/"$current"/"$mutation"}" + ! _k3slrv_validate_open_snapshot "$candidate" || { printf 'review1 snapshot mutation accepted: %s\n' "$mutation" >&2; return 1; } + done + device_raw="$(validator_device_evidence open_exact_lineage)" + state_raw="$(validator_state_evidence open_exact_lineage open)" + _k3slrv_validate_state open "$state_raw" "$device_raw" || { printf 'review1 state snapshot good failed\n' >&2; return 1; } + ! _k3slrv_validate_state open "${state_raw/snapshot_stable=1/snapshot_stable=0}" "$device_raw" || { + printf 'review1 state snapshot drift accepted\n' >&2 + return 1 + } +) + +probe_review1_predefined_parser_cannot_bypass() ( + local marker="${fixture_root}/review1-parser-attacked" stdout_file="${fixture_root}/review1-parser.stdout" + local stderr_file="${fixture_root}/review1-parser.stderr" rc + export -f validator_device_evidence validator_state_evidence + set +e + /usr/bin/bash -c ' + source "$1" + _k3slr_load_contract() { printf attacked >"$2"; return 1; } + _k3slrv_verify_trusted_binaries() { :; } + _k3slrv_require_cached_sudo() { :; } + _k3slrv_collect_device_evidence() { validator_device_evidence exact_device_ready; } + _k3slrv_collect_state_evidence() { validator_state_evidence closed_no_residue device_ready; } + k3slr_local_recovery_main --expect-device-ready + ' review1-parser "$VALIDATOR_PATH" "$marker" >"$stdout_file" 2>"$stderr_file" + rc=$? + set -e + [[ "$rc" -eq 0 && ! -e "$marker" && ! -s "$stderr_file" ]] || return 1 + [[ "$(<"$stdout_file")" == *'Recovery state: device_ready'* ]] +) + +probe_review2_exported_source_cannot_intercept() ( + local marker="${fixture_root}/review2-source-attacked" + local stdout_file="${fixture_root}/review2-source.stdout" + local stderr_file="${fixture_root}/review2-source.stderr" rc + export -f validator_device_evidence validator_state_evidence + source() { + printf '%s' attacked >"$K3SLR_SOURCE_ATTACK_MARKER" + return 1 + } + export -f source + set +e + K3SLR_SOURCE_ATTACK_MARKER="$marker" /usr/bin/bash -c ' + builtin source -- "$1" + _k3slrv_verify_trusted_binaries() { :; } + _k3slrv_require_cached_sudo() { :; } + _k3slrv_collect_device_evidence() { validator_device_evidence exact_device_ready; } + _k3slrv_collect_state_evidence() { validator_state_evidence closed_no_residue device_ready; } + k3slr_local_recovery_main --expect-device-ready + ' review2-source "$VALIDATOR_PATH" >"$stdout_file" 2>"$stderr_file" + rc=$? + set -e + [[ "$rc" -eq 0 && ! -e "$marker" && ! -s "$stderr_file" ]] || return 1 + [[ "$(<"$stdout_file")" == $'Recovery device: match\nRecovery state: device_ready\nLineage: match\nLatest bundle: not_checked' ]] +) + +validator_command_fake() { + local scope="$1" command="$2" argument field='' device='' tag='' format='' target='' mapping value occurrence + local stat_mode=600 stat_size=34359738368 stat_blocks=67108864 stat_device=8:3 + local smart_health=PASSED smart_reallocated=0 smart_pending=0 smart_uncorrectable=0 + shift 2 + { + printf '%s\t%s' "$scope" "$command" + for argument in "$@"; do printf '\t%s' "$argument"; done + printf '\n' + } >>"$K3SLR_FAKE_LOG" + case "$command" in + /usr/bin/test) + if [[ "${1-}" == -L ]]; then return 1; fi + return 0 + ;; + /usr/bin/stat) + while (( $# > 0 )); do + case "$1" in + --format=*) format="${1#--format=}"; shift ;; + --) shift; target="$1"; shift ;; + *) target="$1"; shift ;; + esac + done + case "$format|$target" in + '%u|%a|%F|'*) printf '%s\n' '0|755|regular file' ;; + '%d:%i|%F|/mnt/k3s-recovery-ssd') printf '%s\n' '2051:100|directory' ;; + '%d:%i|%F|/mnt/k3s-recovery-ssd/HyeonworksRecovery') printf '%s\n' '2051:101|directory' ;; + '%d:%i|%F|/mnt/k3s-recovery-ssd/HyeonworksRecovery/containers') printf '%s\n' '2051:102|directory' ;; + '%d:%i|%F|/mnt/k3s-recovery-ssd/HyeonworksRecovery/containers/k3s-recovery.luks') printf '%s\n' '2051:4242|regular file' ;; + '%F|%u|%g|%a|%s|%b|%i|/mnt/k3s-recovery-ssd/HyeonworksRecovery/containers/k3s-recovery.luks'|\ + '%F|%u|%g|%a|%s|%b|%i|%Hd:%Ld|/mnt/k3s-recovery-ssd/HyeonworksRecovery/containers/k3s-recovery.luks') + occurrence="$(/usr/bin/grep -Fc -- $'\t/usr/bin/stat\t--format=%F|%u|%g|%a|%s|%b|%i' "$K3SLR_FAKE_LOG")" + if (( occurrence >= 2 )); then + case "$K3SLR_FAKE_CASE" in + open_drift_container_mode) stat_mode=640 ;; + open_drift_container_size) stat_size=34359734272 ;; + open_drift_container_allocated) stat_blocks=67108856 ;; + esac + fi + if [[ "$format" == *'%Hd:%Ld' ]]; then + printf 'regular file|1000|1000|%s|%s|%s|4242|%s\n' \ + "$stat_mode" "$stat_size" "$stat_blocks" "$stat_device" + else + printf 'regular file|1000|1000|%s|%s|%s|4242\n' \ + "$stat_mode" "$stat_size" "$stat_blocks" + fi + ;; + '%u|%g|%a|%F|/srv/recovery/k3s') + occurrence="$(/usr/bin/grep -Fc -- $'\t/usr/bin/stat\t--format=%u|%g|%a|%F\t--\t/srv/recovery/k3s' "$K3SLR_FAKE_LOG")" + if [[ "$K3SLR_FAKE_CASE" == open_drift_inner_root && "$occurrence" -ge 2 ]]; then + printf '%s\n' '0|0|750|directory' + else + printf '%s\n' '0|0|700|directory' + fi + ;; + *) return 1 ;; + esac + ;; + /usr/bin/sudo) + [[ "$*" == '--non-interactive --validate' ]] + ;; + /usr/bin/readlink) + device="${*: -1}" + case "$device" in + /dev/disk/by-id/wwn-0x500a0751e6aa6254-part3) printf '%s\n' /dev/fixture-recovery-partition ;; + /dev/disk/by-id/wwn-0x500a0751e6aa6254) printf '%s\n' /dev/fixture-recovery-disk ;; + /dev/disk/by-id/wwn-0x500a07512df28bd1-part1) printf '%s\n' /dev/fixture-k3s-partition ;; + /dev/disk/by-id/wwn-0x500a07512df28bd1) printf '%s\n' /dev/fixture-k3s-disk ;; + *) printf '%s\n' "$device" ;; + esac + ;; + /usr/bin/lsblk) + while (( $# > 0 )); do + if [[ "$1" == --output ]]; then field="$2"; shift 2; else device="$1"; shift; fi + done + case "$field|$device" in + 'PKNAME|/dev/fixture-recovery-partition') + if [[ "$K3SLR_FAKE_CASE" == wrong_by_id_target ]]; then printf '%s\n' /dev/fixture-wrong-disk; else printf '%s\n' /dev/fixture-recovery-disk; fi + ;; + 'PKNAME|/dev/fixture-k3s-partition') printf '%s\n' /dev/fixture-k3s-disk ;; + 'MODEL|/dev/fixture-recovery-disk') + if [[ "$K3SLR_FAKE_CASE" == wrong_model ]]; then printf '%s\n' WRONGMODEL; else printf '%s\n' CT1000MX500SSD1; fi + ;; + 'SERIAL|/dev/fixture-recovery-disk') + if [[ "$K3SLR_FAKE_CASE" == wrong_serial ]]; then printf '%s\n' WRONGSERIAL; else printf '%s\n' 2306E6AA6254; fi + ;; + 'WWN|/dev/fixture-recovery-disk') + if [[ "$K3SLR_FAKE_CASE" == wrong_wwn ]]; then printf '%s\n' 0xaaaaaaaaaaaaaaaa; else printf '%s\n' 0x500a0751e6aa6254; fi + ;; + 'MAJ:MIN|/dev/fixture-recovery-disk') printf '%s\n' 8:0 ;; + 'MODEL|/dev/fixture-k3s-disk') printf '%s\n' CT250MX500SSD1 ;; + 'SERIAL|/dev/fixture-k3s-disk') printf '%s\n' 21132DF28BD1 ;; + 'WWN|/dev/fixture-k3s-disk') printf '%s\n' 0x500a07512df28bd1 ;; + 'MAJ:MIN|/dev/fixture-k3s-disk') + if [[ "$K3SLR_FAKE_CASE" == same_parent_disk ]]; then printf '%s\n' 8:0; else printf '%s\n' 8:32; fi + ;; + 'MAJ:MIN|/dev/fixture-loop') printf '%s\n' 7:0 ;; + 'MAJ:MIN|/dev/fixture-other-loop') printf '%s\n' 7:9 ;; + 'MAJ:MIN|/dev/mapper/k3s-recovery') printf '%s\n' 253:0 ;; + *) return 1 ;; + esac + ;; + /usr/sbin/blkid) + while (( $# > 0 )); do + if [[ "$1" == --match-tag ]]; then tag="$2"; shift 2; else device="$1"; shift; fi + done + case "$tag|$device" in + 'UUID|/dev/fixture-recovery-partition') + if [[ "$K3SLR_FAKE_CASE" == wrong_fs_uuid ]]; then printf '%s\n' AAAAAAAAAAAAAAAA; else printf '%s\n' 4EA0196C0C5FA27E; fi + ;; + 'PARTUUID|/dev/fixture-recovery-partition') + if [[ "$K3SLR_FAKE_CASE" == wrong_partuuid ]]; then printf '%s\n' aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa; else printf '%s\n' 4670aa9f-9045-4bce-930d-9e84dfec9f38; fi + ;; + 'TYPE|/dev/fixture-recovery-partition') printf '%s\n' ntfs ;; + 'UUID|/dev/fixture-k3s-partition') printf '%s\n' b86086ef-2b3c-4638-abcf-fc7f137dcb97 ;; + 'PARTUUID|/dev/fixture-k3s-partition') printf '%s\n' b081b955-d6bc-442c-ac0f-db76560a5245 ;; + 'TYPE|/dev/fixture-k3s-partition') printf '%s\n' ext4 ;; + 'TYPE|/dev/mapper/k3s-recovery') printf '%s\n' ext4 ;; + 'LABEL|/dev/mapper/k3s-recovery') + occurrence="$(/usr/bin/grep -Fc -- $'\t/usr/sbin/blkid\t--output\tvalue\t--match-tag\tLABEL\t--\t/dev/mapper/k3s-recovery' "$K3SLR_FAKE_LOG")" + if [[ "$K3SLR_FAKE_CASE" == open_wrong_inner_label || + ( "$K3SLR_FAKE_CASE" == open_drift_inner_label && "$occurrence" -ge 2 ) ]]; then + printf '%s\n' WRONG_LABEL + else + printf '%s\n' K3S_RECOVERY + fi + ;; + *) return 1 ;; + esac + ;; + /usr/sbin/smartctl) + case "$K3SLR_FAKE_CASE" in + smart_health_failed) smart_health=FAILED ;; + smart_reallocated_nonzero) smart_reallocated=1 ;; + smart_pending_nonzero) smart_pending=1 ;; + smart_uncorrectable_nonzero) smart_uncorrectable=1 ;; + esac + printf '%s\n' \ + "SMART overall-health self-assessment test result: ${smart_health}" \ + " 5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - ${smart_reallocated}" \ + "197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - ${smart_pending}" \ + "198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - ${smart_uncorrectable}" + ;; + /usr/bin/findmnt) + while (( $# > 0 )); do + case "$1" in + --mountpoint|--source) field="$1"; target="$2"; shift 2 ;; + *) shift ;; + esac + done + if [[ "$field" == --source ]]; then + [[ "$K3SLR_FAKE_CASE" == open_* ]] || return 1 + printf '%s\n' /mnt/k3s-recovery-ssd + elif [[ "$field" == --mountpoint && "$K3SLR_FAKE_CASE" == open_* ]]; then + case "$target" in + /mnt/k3s-recovery-ssd) + if [[ "$K3SLR_FAKE_CASE" == open_wrong_outer_source ]]; then device=/dev/fixture-other-partition; else device=/dev/fixture-recovery-partition; fi + if [[ "$K3SLR_FAKE_CASE" == open_wrong_mount_options ]]; then + value='rw,nodev,nosuid,uid=1000,gid=1000,dmask=0077,fmask=0077' + else + value='rw,nodev,nosuid,noexec,uid=1000,gid=1000,dmask=0077,fmask=0077' + fi + if [[ "$K3SLR_FAKE_CASE" == open_wrong_container_device ]]; then field=8:9; else field=8:3; fi + printf '%s %s %s %s %s\n' "$device" ntfs3 "$value" 41 "$field" + ;; + /srv/recovery/k3s) printf '%s\n' '/dev/mapper/k3s-recovery ext4 rw,nodev,nosuid,noexec 42 253:0' ;; + *) return 1 ;; + esac + else + return 1 + fi + ;; + /usr/bin/ntfs-3g.probe) + [[ "$#" -eq 2 && "$1" == --readwrite && "$2" == /dev/fixture-recovery-partition ]] || return 2 + [[ "$K3SLR_FAKE_CASE" != ntfs_dirty_or_hibernated ]] + ;; + /usr/sbin/losetup) + if [[ "$K3SLR_FAKE_CASE" == open_* ]]; then + if [[ "$K3SLR_FAKE_CASE" == open_wrong_container_device ]]; then field=8:9; else field=8:3; fi + printf '/dev/fixture-loop 4242 %s 7:0 0 0\n' "$field" + elif [[ "$K3SLR_FAKE_CASE" == unexpected_loop ]]; then + printf '%s\n' /dev/fixture-loop + fi + ;; + /usr/sbin/cryptsetup) + mapping="${*: -1}" + if [[ "$mapping" == k3s-recovery-proof ]]; then return 4; fi + if [[ "$K3SLR_FAKE_CASE" == open_* || "$K3SLR_FAKE_CASE" == unexpected_mapping ]]; then + if [[ "$K3SLR_FAKE_CASE" == open_plain_mapping ]]; then value=plain; else value=LUKS2; fi + if [[ "$K3SLR_FAKE_CASE" == open_wrong_backing_file ]]; then device=/dev/fixture-other-loop; else device=/dev/fixture-loop; fi + printf '%s\n' "/dev/mapper/k3s-recovery is active and is in use." " type: ${value}" " device: ${device}" + else + return 4 + fi + ;; + *) return 1 ;; + esac +} + +run_validator_command_fixture() { + local fixture="$1" stdout_file="$2" stderr_file="$3" log_file="$4" + shift 4 + : >"$log_file" + export -f validator_command_fake + K3SLR_FAKE_CASE="$fixture" K3SLR_FAKE_LOG="$log_file" /usr/bin/bash -c ' + source "$1"; shift + _k3slrv_exec() { validator_command_fake "$@"; } + k3slr_local_recovery_main "$@" + ' review1-command "$VALIDATOR_PATH" "$@" >"$stdout_file" 2>"$stderr_file" +} + +probe_review1_command_boundary_device_ready() ( + local stdout_file="${fixture_root}/review1-command.stdout" stderr_file="${fixture_root}/review1-command.stderr" + local log_file="${fixture_root}/review1-command.log" + run_validator_command_fixture exact_device_ready "$stdout_file" "$stderr_file" "$log_file" --expect-device-ready || return 1 + [[ ! -s "$stderr_file" && "$(<"$stdout_file")" == *'Recovery state: device_ready'* ]] || return 1 + /usr/bin/grep -Fqx $'root\t/usr/bin/ntfs-3g.probe\t--readwrite\t/dev/fixture-recovery-partition' "$log_file" || return 1 + /usr/bin/grep -Fqx $'root\t/usr/sbin/blkid\t--output\tvalue\t--match-tag\tUUID\t--\t/dev/fixture-recovery-partition' "$log_file" +) + +probe_review1_command_boundary_open() ( + local stdout_file="${fixture_root}/review1-command-open.stdout" stderr_file="${fixture_root}/review1-command-open.stderr" + local log_file="${fixture_root}/review1-command-open.log" + run_validator_command_fixture open_exact_lineage "$stdout_file" "$stderr_file" "$log_file" --expect-open || return 1 + [[ ! -s "$stderr_file" && "$(<"$stdout_file")" == *'Recovery state: open'* ]] || return 1 + /usr/bin/grep -Fqx $'root\t/usr/bin/findmnt\t--noheadings\t--raw\t--output\tSOURCE,FSTYPE,OPTIONS,ID,MAJ:MIN\t--mountpoint\t/mnt/k3s-recovery-ssd' "$log_file" || return 1 + /usr/bin/grep -Fqx $'root\t/usr/bin/findmnt\t--noheadings\t--raw\t--output\tTARGET\t--source\t/dev/fixture-recovery-partition' "$log_file" || return 1 + /usr/bin/grep -Fqx $'root\t/usr/sbin/losetup\t--list\t--associated\t/mnt/k3s-recovery-ssd/HyeonworksRecovery/containers/k3s-recovery.luks\t--noheadings\t--raw\t--output\tNAME,BACK-INO,BACK-MAJ:MIN,MAJ:MIN,OFFSET,SIZELIMIT' "$log_file" || return 1 + /usr/bin/grep -Fqx $'root\t/usr/sbin/cryptsetup\tstatus\t--\tk3s-recovery' "$log_file" || return 1 + /usr/bin/grep -Fqx $'root\t/usr/sbin/cryptsetup\tstatus\t--\tk3s-recovery-proof' "$log_file" +) + +prepare_review1_latest_bundle() { + local case_name="$1" root="$2" inner relative bundle outside identity payload_hash metadata_hash + inner="${root}/inner" + relative='k3s-secrets-encryption-20260801T000000Z/post' + bundle="${inner}/${relative}" + outside="${root}/outside-payload" + mkdir -p -- "$bundle" + printf '%s\n' 'encrypted payload fixture' >"${bundle}/payload.bin" + printf '%s\n' \ + 'schema=platform-k3s-bundle-v1' \ + 'bundle_id=01234567-89ab-4cde-8fab-0123456789ab' \ + 'phase=post' \ + 'k3s_version=v1.36.2+k3s1' \ + 'datastore=sqlite' \ + 'created_at_utc=2026-08-01T00:00:00Z' \ + 'secret_count=1' >"${bundle}/bundle.env" + case "$case_name" in + malformed) + sed -i 's/^schema=.*/schema=not-the-approved-schema/' "${bundle}/bundle.env" + ;; + duplicate) + printf '%s\n' 'phase=post' >>"${bundle}/bundle.env" + ;; + valid|symlink_escape|hash_mismatch) ;; + *) return 1 ;; + esac + payload_hash="$(/usr/bin/sha256sum -- "${bundle}/payload.bin")"; payload_hash="${payload_hash%% *}" + metadata_hash="$(/usr/bin/sha256sum -- "${bundle}/bundle.env")"; metadata_hash="${metadata_hash%% *}" + if [[ "$case_name" == symlink_escape ]]; then + printf '%s\n' 'escaped payload fixture' >"$outside" + ln -s -- "$outside" "${bundle}/escaped-link" + payload_hash="$(/usr/bin/sha256sum -- "$outside")"; payload_hash="${payload_hash%% *}" + printf '%s %s\n' "$payload_hash" './escaped-link' >"${bundle}/verification.manifest" + else + [[ "$case_name" != hash_mismatch ]] || payload_hash='0000000000000000000000000000000000000000000000000000000000000000' + printf '%s %s\n%s %s\n' \ + "$payload_hash" './payload.bin' "$metadata_hash" './bundle.env' >"${bundle}/verification.manifest" + fi + chmod 0700 -- "$bundle" + chmod 0600 -- "${bundle}/bundle.env" "${bundle}/verification.manifest" "${bundle}/payload.bin" + identity="$(/usr/bin/stat --format='%d:%i' -- "$bundle")" + printf '%s\n' \ + 'schema=k3slr-latest-post-bundle-v1' \ + "relative_path=${relative}" \ + "directory_identity=${identity}" >"${inner}/.latest-post-bundle.env" + chmod 0600 -- "${inner}/.latest-post-bundle.env" +} + +run_review1_latest_bundle_case() ( + local case_name="$1" expected_rc="$2" root inner rc + root="${fixture_root}/review1-latest-${case_name}" + mkdir -p -- "$root" + prepare_review1_latest_bundle "$case_name" "$root" || return 1 + inner="${root}/inner" + # shellcheck source=/dev/null + source "$VALIDATOR_PATH" + K3SLR_INNER_MOUNT="$inner" + _k3slrv_exec() { + local scope="$1" command="$2" argument mode_kind + shift 2 + [[ "$scope" == user || "$scope" == root ]] || return 1 + if [[ "$command" == /usr/bin/stat && " $* " == *" --format=%u:%g:%a:%F "* ]]; then + mode_kind="$($command "${@/--format=%u:%g:%a:%F/--format=%a:%F}")" || return 1 + printf '0:0:%s\n' "$mode_kind" + return 0 + fi + "$command" "$@" + } + set +e + _k3slrv_verify_latest_bundle + rc=$? + set -e + [[ "$rc" -eq "$expected_rc" ]] +) + +probe_review1_latest_bundle_actual() { + assert_succeeds run_review1_latest_bundle_case valid 0 + assert_succeeds run_review1_latest_bundle_case malformed 1 + assert_succeeds run_review1_latest_bundle_case duplicate 1 + assert_succeeds run_review1_latest_bundle_case symlink_escape 1 + assert_succeeds run_review1_latest_bundle_case hash_mismatch 1 +} + +run_review2_latest_main_case() ( + local fixture_name="$1" expected_rc="$2" bundle_case="$3" + local root="${fixture_root}/review2-latest-main-${fixture_name}" inner + local stdout_file="${fixture_root}/review2-latest-main-${fixture_name}.stdout" + local stderr_file="${fixture_root}/review2-latest-main-${fixture_name}.stderr" rc output + mkdir -p -- "$root" + prepare_review1_latest_bundle "$bundle_case" "$root" || return 1 + inner="${root}/inner" + export -f validator_device_evidence validator_state_evidence + set +e + K3SLR_TEST_INNER_MOUNT="$inner" /usr/bin/bash -c ' + builtin source -- "$1" + _k3slrv_exec() { + local scope="$1" command="$2" mode_kind + shift 2 + [[ "$scope" == user || "$scope" == root ]] || return 1 + if [[ "$command" == /usr/bin/stat && " $* " == *" --format=%u:%g:%a:%F "* ]]; then + mode_kind="$($command "${@/--format=%u:%g:%a:%F/--format=%a:%F}")" || return 1 + printf "0:0:%s\\n" "$mode_kind" + return 0 + fi + "$command" "$@" + } + _k3slrv_verify_trusted_binaries() { :; } + _k3slrv_require_cached_sudo() { K3SLR_INNER_MOUNT="$K3SLR_TEST_INNER_MOUNT"; } + _k3slrv_collect_device_evidence() { validator_device_evidence open_exact_lineage; } + _k3slrv_collect_state_evidence() { validator_state_evidence open_exact_lineage open; } + k3slr_local_recovery_main --expect-open --check-latest-bundle + ' review2-latest-main "$VALIDATOR_PATH" >"$stdout_file" 2>"$stderr_file" + rc=$? + set -e + if [[ "$rc" -ne "$expected_rc" ]]; then + printf 'task4 feasibility diagnostic: case=%s expected=%s actual=%s stderr=%s\n' \ + "$fixture_case" "$expected_rc" "$rc" "$(<"$stderr_file")" >&2 + /usr/bin/tail -n 30 "$TASK4_FEASIBILITY_COMMAND_LOG" >&2 + return 1 + fi + if (( expected_rc == 0 )); then + [[ ! -s "$stderr_file" ]] || return 1 + output="$(<"$stdout_file")" + [[ "$output" == $'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: verified' ]] || return 1 + else + [[ ! -s "$stdout_file" && "$(<"$stderr_file")" == 'Recovery validation failed' ]] || return 1 + fi + validator_outputs_are_sanitized "$stdout_file" "$stderr_file" +) + +probe_review2_latest_main_integration() { + assert_succeeds run_review2_latest_main_case latest_manifest_valid 0 valid + assert_succeeds run_review2_latest_main_case latest_manifest_invalid 1 hash_mismatch +} + +probe_review1_record_preserving_evidence() ( + local raw + local -A parsed=() + # shellcheck source=/dev/null + source "$VALIDATOR_PATH" + _k3slrv_exec() { printf 'fixture-value\n\n'; } + ! _k3slrv_one_line /usr/bin/readlink -f -- /fixture || return 1 + declare -F _k3slrv_capture >/dev/null || return 1 + trailing_collector() { printf 'alpha=one\n\n'; } + _k3slrv_capture raw trailing_collector || return 1 + ! _k3slrv_parse_evidence "$raw" parsed alpha || return 1 + ! _k3slrv_parse_evidence $'alpha=one\nalpha=two' parsed alpha || return 1 + ! _k3slrv_parse_evidence $'alpha=one\nunknown=two' parsed alpha || return 1 + ! _k3slrv_parse_evidence $'alpha=one\r' parsed alpha || return 1 + ! _k3slrv_parse_evidence $'alpha=one\ncontinued' parsed alpha +) + +probe_review1_leak_sentinels() ( + local stdout_file="${fixture_root}/review1-leak.stdout" stderr_file="${fixture_root}/review1-leak.stderr" + : >"$stdout_file" + : >"$stderr_file" + validator_outputs_are_sanitized "$stdout_file" "$stderr_file" || return 1 + printf '%s\n' '4670aa9f-9045-4bce-930d-9e84dfec9f38' >"$stdout_file" + ! validator_outputs_are_sanitized "$stdout_file" "$stderr_file" || return 1 + : >"$stdout_file" + printf '%s\n' '/srv/recovery/k3s/.latest-post-bundle.env' >"$stderr_file" + ! validator_outputs_are_sanitized "$stdout_file" "$stderr_file" +) + +assert_validator_case exact_device_ready 0 device_ready --expect-device-ready +assert_validator_case wrong_by_id_target 1 device_ready --expect-device-ready +assert_validator_case wrong_fs_uuid 1 device_ready --expect-device-ready +assert_validator_case wrong_partuuid 1 device_ready --expect-device-ready +assert_validator_case wrong_model 1 device_ready --expect-device-ready +assert_validator_case wrong_serial 1 device_ready --expect-device-ready +assert_validator_case wrong_wwn 1 device_ready --expect-device-ready +assert_validator_case same_parent_disk 1 device_ready --expect-device-ready +assert_validator_case smart_health_failed 1 device_ready --expect-device-ready +assert_validator_case smart_reallocated_nonzero 1 device_ready --expect-device-ready +assert_validator_case smart_pending_nonzero 1 device_ready --expect-device-ready +assert_validator_case smart_uncorrectable_nonzero 1 device_ready --expect-device-ready +assert_validator_case ntfs_dirty_or_hibernated 1 device_ready --expect-device-ready +assert_validator_case closed_no_residue 0 closed --expect-closed +assert_validator_case unexpected_loop 1 closed --expect-closed +assert_validator_case unexpected_mapping 1 closed --expect-closed +assert_validator_case open_exact_lineage 0 open --expect-open +assert_validator_case open_wrong_outer_source 1 open --expect-open +assert_validator_case open_plain_mapping 1 open --expect-open +assert_validator_case open_wrong_backing_file 1 open --expect-open +assert_validator_case open_wrong_inner_label 1 open --expect-open +assert_validator_case open_wrong_mount_options 1 open --expect-open +assert_validator_case open_wrong_container_device 1 open --expect-open +assert_validator_case open_drift_container_mode 1 open --expect-open +assert_validator_case open_drift_container_size 1 open --expect-open +assert_validator_case open_drift_container_allocated 1 open --expect-open +assert_validator_case open_drift_inner_label 1 open --expect-open +assert_validator_case open_drift_inner_root 1 open --expect-open +assert_validator_case exact_device_ready 2 device_ready +assert_validator_case exact_device_ready 2 device_ready --expect-open --expect-closed +assert_validator_case open_exact_lineage 2 open --expect-closed --check-latest-bundle + +if [[ -z "$review_focus" || "$review_focus" == 1 ]]; then + assert_succeeds probe_review1_ntfs_probe_argv +fi +if [[ -z "$review_focus" || "$review_focus" == 2 ]]; then + assert_succeeds probe_review1_blkid_argv +fi +if [[ -z "$review_focus" || "$review_focus" == 3 ]]; then + assert_succeeds probe_review1_ntfs3_effective_masks +fi +if [[ -z "$review_focus" || "$review_focus" == 4 ]]; then + assert_succeeds probe_review1_initial_guard +fi +if [[ -z "$review_focus" || "$review_focus" == 5 ]]; then + assert_succeeds probe_review1_bash_x_rejected +fi +if [[ -z "$review_focus" || "$review_focus" == 6 ]]; then + assert_succeeds probe_review1_manifest_symlink_escape +fi +if [[ -z "$review_focus" || "$review_focus" == 7 ]]; then + assert_succeeds probe_review1_partition_mounted_elsewhere +fi +if [[ -z "$review_focus" || "$review_focus" == 8 ]]; then + assert_succeeds probe_review1_open_lineage_snapshot +fi +if [[ -z "$review_focus" || "$review_focus" == 9 ]]; then + assert_succeeds probe_review1_predefined_parser_cannot_bypass +fi +if [[ -z "$review_focus" || "$review_focus" == r2_9 ]]; then + assert_succeeds probe_review2_exported_source_cannot_intercept +fi +if [[ -z "$review_focus" || "$review_focus" == r2_latest ]]; then + assert_succeeds probe_review2_latest_main_integration +fi +if [[ -z "$review_focus" || "$review_focus" == 10 ]]; then + assert_succeeds probe_review1_command_boundary_device_ready + assert_succeeds probe_review1_command_boundary_open + assert_succeeds probe_review1_latest_bundle_actual +fi +if [[ -z "$review_focus" || "$review_focus" == m1 ]]; then + assert_succeeds probe_review1_record_preserving_evidence +fi +if [[ -z "$review_focus" || "$review_focus" == m2 ]]; then + assert_succeeds probe_review1_leak_sentinels +fi + +task4_pipeline_contract() ( + declare -F _k3slr_luks_format_from_keepass >/dev/null || return 1 + declare -F _k3slr_luks_open_from_keepass >/dev/null || return 1 + declare -F _k3slr_password_pipe_preflight >/dev/null || return 1 + declare -F _k3slr_keepass_database_lineage_matches >/dev/null || return 1 +) + +task4_tty_and_secret_api_contract() ( + local stdout_file="${fixture_root}/task4-nontty.stdout" + local stderr_file="${fixture_root}/task4-nontty.stderr" + declare -F _k3slr_keepass_password_stdout >/dev/null || return 1 + ! _k3slr_keepass_password_stdout /nonexistent "$K3SLR_KEEPASS_ENTRY" \ + "$stdout_file" 2>"$stderr_file" || return 1 + [[ ! -s "$stdout_file" ]] || return 1 + declare -F _k3slr_keepass_password >/dev/null && return 1 + declare -F _k3slr_read_keepass_password >/dev/null && return 1 + if /usr/bin/rg -n \ + '(^|[[:space:]])(local|declare|typeset)[^#]*(secret|password|passphrase)|read[^#]*(secret|password|passphrase)|KEEPASS[^[:space:]]*PASSWORD=|LUKS[^[:space:]]*PASSWORD=' \ + "$LIBRARY_PATH" "$FEASIBILITY_PATH" >/dev/null 2>&1; then + return 1 + fi +) + +task4_feasibility_command_fake() { + local command="$1" operation path metadata process_identity='' + shift + if [[ ( "${TASK4_FEASIBILITY_CASE:-}" == signal_* || + "${TASK4_FEASIBILITY_CASE:-}" == anchor_* ) && + ! -e "${TASK4_FEASIBILITY_MAIN_PID_FILE:-/nonexistent}" ]]; then + printf '%s\n' "$BASHPID" >"$TASK4_FEASIBILITY_MAIN_PID_FILE" + fi + printf '%s %s\n' "$command" "$*" >>"$TASK4_FEASIBILITY_COMMAND_LOG" + case "$command" in + /usr/bin/test) + if [[ "${1-}" == -t ]]; then return 0; fi + path="${*: -1}" + if [[ "$path" == /tmp/k3slr-feasibility.* ]]; then /usr/bin/test "$@"; else return 0; fi + ;; + /usr/bin/stat) + path="${*: -1}" + if [[ "$path" == /tmp/k3slr-feasibility.* ]]; then + metadata="$(/usr/bin/stat "$@")" || return 1 + if [[ -e "${TASK4_ATOMIC_MUTATED_MARKER:-/nonexistent}" && + "$TASK4_FEASIBILITY_CASE" == atomic_wrong_owner && "$path" == */synthetic.kdbx ]]; then + printf '99999|%s\n' "${metadata#*|}" + elif [[ -e "${TASK4_ATOMIC_MUTATED_MARKER:-/nonexistent}" && + "$TASK4_FEASIBILITY_CASE" == atomic_parent_drift && "$path" == "$TASK4_FEASIBILITY_REQUESTED_DIR" && + ! -s "${TASK4_ATOMIC_PARENT_STAT_LOG:-/nonexistent}" ]]; then + printf x >"$TASK4_ATOMIC_PARENT_STAT_LOG" + printf '%s|99:99\n' "${metadata%|*}" + else + printf '%s\n' "$metadata" + fi + else + printf '0|755|regular file\n' + fi + ;; + /usr/bin/id|/usr/bin/mawk|/usr/bin/od|/usr/bin/ps|/usr/bin/sleep|/usr/bin/sha256sum|/usr/bin/cmp|/usr/bin/pkill) + if [[ -n "${TASK4_FEASIBILITY_ROLE_LOG:-}" && + ( "$command" == /usr/bin/mawk || "$command" == /usr/bin/sha256sum ) ]]; then + _k3slr_process_identity process_identity "$BASHPID" || return 1 + printf '%s|%s\n' "${command##*/}" "$process_identity" >>"$TASK4_FEASIBILITY_ROLE_LOG" + fi + "$command" "$@" + ;; + /usr/bin/readlink) + printf '%s\n' "${*: -1}" + ;; + /usr/bin/dpkg-query) + case "${*: -1}" in + keepassxc) + case "$TASK4_FEASIBILITY_CASE" in + package_missing_lf) printf 'ii |2.7.6+dfsg.1-1build3' ;; + package_double_lf) printf 'ii |2.7.6+dfsg.1-1build3\n\n' ;; + *) printf 'ii |2.7.6+dfsg.1-1build3\n' ;; + esac + ;; + cryptsetup-bin) printf 'ii |2:2.7.0-1ubuntu4.2\n' ;; + *) return 92 ;; + esac + ;; + /usr/bin/mktemp) + /usr/bin/mkdir -- "$TASK4_FEASIBILITY_REQUESTED_DIR" || return 1 + /usr/bin/chmod 0700 -- "$TASK4_FEASIBILITY_REQUESTED_DIR" || return 1 + printf '%s\n' "$TASK4_FEASIBILITY_REQUESTED_DIR" + ;; + /usr/bin/rm) + if [[ "$TASK4_FEASIBILITY_CASE" == cleanup_failure && "${1-}" == --recursive ]]; then + return 9 + fi + /usr/bin/rm "$@" + ;; + /usr/bin/sudo) + [[ "${1-}" == --non-interactive && "${2-}" == --validate ]] + ;; + /usr/bin/keepassxc-cli) + operation="$1" + if [[ -n "${TASK4_FEASIBILITY_ROLE_LOG:-}" ]]; then + _k3slr_process_identity process_identity "$BASHPID" || return 1 + printf 'keepass|%s\n' "$process_identity" >>"$TASK4_FEASIBILITY_ROLE_LOG" + fi + if [[ -n "${TASK4_CHILD_CMDLINE_LOG:-}" ]]; then + /usr/bin/tr '\0' '\n' <"/proc/${BASHPID}/cmdline" >>"$TASK4_CHILD_CMDLINE_LOG" + /usr/bin/tr '\0' '\n' <"/proc/${BASHPID}/environ" >>"$TASK4_CHILD_ENVIRON_LOG" + fi + if [[ "$TASK4_FEASIBILITY_CASE" == signal_* && "$operation" == db-create ]]; then + printf '%s\n' "$BASHPID" >"$TASK4_FEASIBILITY_CHILD_PID_FILE" + trap 'printf terminated >"$TASK4_FEASIBILITY_CHILD_TERM_FILE"; exit 143' TERM INT + while :; do /usr/bin/sleep 1; done + fi + if [[ "$TASK4_FEASIBILITY_CASE" == anchor_* && "$operation" == db-create ]]; then + printf '%s\n' "$BASHPID" >"$TASK4_FEASIBILITY_CHILD_PID_FILE" + trap '' TERM + printf ready >"$TASK4_FEASIBILITY_TERM_IGNORE_READY" + while :; do /usr/bin/sleep 1; done + fi + /usr/bin/mawk '{ next } END { exit 0 }' || return 1 + case "$operation" in + db-create) + path="${*: -1}" + ( set -o noclobber; printf 'synthetic-database\n' >"$path" ) || return 1 + /usr/bin/chmod 0600 -- "$path" + ;; + add) + path="${@: -2:1}" + printf 'synthetic-database-after-add\n' >"${path}.atomic" + if [[ "$TASK4_FEASIBILITY_CASE" == atomic_wrong_mode ]]; then + /usr/bin/chmod 0644 -- "${path}.atomic" + else + /usr/bin/chmod 0600 -- "${path}.atomic" + fi + /usr/bin/mv -- "${path}.atomic" "$path" + if [[ "$TASK4_FEASIBILITY_CASE" == atomic_symlink ]]; then + /usr/bin/unlink "$path" + /usr/bin/ln -s -- /dev/null "$path" + fi + if [[ -n "${TASK4_ATOMIC_MUTATED_MARKER:-}" ]]; then printf x >"$TASK4_ATOMIC_MUTATED_MARKER"; fi + ;; + show) + [[ "$TASK4_FEASIBILITY_CASE" != synthetic_failure ]] || return 7 + if (( $# == 6 )) && [[ "${2-}" == --show-protected && + "${3-}" == --attributes && "${4-}" == Password && + "${5-}" == */synthetic.kdbx && "${6-}" == "$K3SLR_KEEPASS_ENTRY" ]]; then + printf '%s\n' 'Zz9Yy8Xx7Ww6Vv5Uu4Tt3Ss2Rr1Qq0Pp8Oo7Nn6M' + elif (( $# == 7 )) && [[ "${2-}" == --quiet && + "${3-}" == --attributes && "${4-}" == Title && + "${5-}" == --show-attachments && "${6-}" == */synthetic.kdbx && + "${7-}" == "$K3SLR_KEEPASS_ENTRY" ]]; then + if [[ -f "${6}.attachment-state" ]]; then + printf 'K3s Recovery LUKS\n\nAttachments:\n fixture.bin (30.0 B)\n' + else + printf 'K3s Recovery LUKS\n\nNo attachments present.\n' + fi + else + return 91 + fi + ;; + attachment-import) + path="$3" + printf 'synthetic-database-after-attachment\n' >"${path}.atomic" + /usr/bin/chmod 0600 -- "${path}.atomic" + /usr/bin/mv -- "${path}.atomic" "$path" + : >"${path}.attachment-state" + ;; + attachment-export) + path="${*: -1}" + printf '%s\n' 'k3slr-attachment-roundtrip-v1' >"$path" + ;; + *) return 91 ;; + esac + ;; + *) return 90 ;; + esac +} + +run_task4_feasibility_fixture() ( + local fixture_case="$1" expected_rc="$2" requested_dir="$3" + local stdout_file="$4" stderr_file="$5" rc=0 + # shellcheck source=/dev/null + source "$FEASIBILITY_PATH" + export TASK4_FEASIBILITY_CASE="$fixture_case" + export TASK4_FEASIBILITY_REQUESTED_DIR="$requested_dir" + export TASK4_FEASIBILITY_COMMAND_LOG="${fixture_root}/task4-${fixture_case}.commands" + : >"$TASK4_FEASIBILITY_COMMAND_LOG" + _k3slr_command() { task4_feasibility_command_fake "$@"; } + set +e + k3slr_local_recovery_feasibility_main --execute "$stdout_file" 2>"$stderr_file" + rc=$? + set -e + if [[ "$rc" -ne "$expected_rc" ]]; then /usr/bin/tail -n 50 "$TASK4_FEASIBILITY_COMMAND_LOG" >&2; return 1; fi + if [[ "$fixture_case" == cleanup_failure ]]; then + [[ -d "$requested_dir" ]] || return 1 + /usr/bin/rm -rf -- "$requested_dir" + else + if [[ -e "$requested_dir" ]]; then printf 'task4 feasibility diagnostic: fixture remains\n' >&2; return 1; fi + fi + if /usr/bin/grep -Fq -- 'Zz9Yy8Xx7Ww6Vv5Uu4Tt3Ss2Rr1Qq0Pp8Oo7Nn6M' "$stdout_file" "$stderr_file"; then + printf 'task4 feasibility diagnostic: sentinel output leak\n' >&2 + return 1 + fi + if [[ "$fixture_case" == success ]]; then + /usr/bin/grep -Fx -- "/usr/bin/keepassxc-cli db-create --quiet --set-password ${requested_dir}/synthetic.kdbx" "$TASK4_FEASIBILITY_COMMAND_LOG" >/dev/null || return 1 + /usr/bin/grep -Fx -- "/usr/bin/keepassxc-cli add --quiet --generate --length 40 --lower --upper --numeric --every-group ${requested_dir}/synthetic.kdbx ${K3SLR_KEEPASS_ENTRY}" "$TASK4_FEASIBILITY_COMMAND_LOG" >/dev/null || return 1 + /usr/bin/grep -Fx -- "/usr/bin/keepassxc-cli show --show-protected --attributes Password ${requested_dir}/synthetic.kdbx ${K3SLR_KEEPASS_ENTRY}" "$TASK4_FEASIBILITY_COMMAND_LOG" >/dev/null || return 1 + [[ "$(/usr/bin/grep -Fxc -- "/usr/bin/keepassxc-cli show --quiet --attributes Title --show-attachments ${requested_dir}/synthetic.kdbx ${K3SLR_KEEPASS_ENTRY}" "$TASK4_FEASIBILITY_COMMAND_LOG")" -eq 2 ]] || return 1 + /usr/bin/grep -Fx -- "/usr/bin/keepassxc-cli attachment-import --quiet ${requested_dir}/synthetic.kdbx ${K3SLR_KEEPASS_ENTRY} fixture.bin ${requested_dir}/input.bin" "$TASK4_FEASIBILITY_COMMAND_LOG" >/dev/null || return 1 + /usr/bin/grep -E -- "^/usr/bin/keepassxc-cli attachment-export --quiet ${requested_dir}/synthetic\\.kdbx ${K3SLR_KEEPASS_ENTRY} fixture\\.bin /proc/[0-9]+/fd/[0-9]+$" "$TASK4_FEASIBILITY_COMMAND_LOG" >/dev/null || return 1 + if [[ "$(( $(/usr/bin/wc -l <"$stdout_file") ))" -ne 1 ]]; then + printf 'task4 diagnostic: success stdout line count mismatch\n' >&2 + return 1 + fi + if [[ "$(<"$stdout_file")" != 'Local recovery feasibility: pass' ]]; then + printf 'task4 diagnostic: success stdout classification mismatch\n' >&2 + return 1 + fi + if [[ -s "$stderr_file" ]]; then + printf 'task4 diagnostic: success stderr is nonempty\n' >&2 + return 1 + fi + elif [[ "$fixture_case" == synthetic_failure || "$fixture_case" == cleanup_failure ]]; then + [[ ! -s "$stdout_file" && "$(<"$stderr_file")" == 'Local recovery feasibility failed' ]] || return 1 + fi + return 0 +) + +task4_review1_blocking_child_is_reaped() ( + local signal_name expected_rc requested_dir stdout_file stderr_file child_pid index rc killer_pid + local role pid ppid pgid start_time recorded_roles + # shellcheck source=/dev/null + source "$FEASIBILITY_PATH" + _k3slr_lifecycle_boundary() { + local phase="$1" tracked_pid="${2-}" tracked_identity='' + if [[ "$phase" == feasibility-pid-published && -n "${TASK4_FEASIBILITY_ROLE_LOG:-}" ]]; then + _k3slr_process_identity tracked_identity "$tracked_pid" || return 1 + printf 'supervisor|%s\n' "$tracked_identity" >>"$TASK4_FEASIBILITY_ROLE_LOG" + fi + } + for signal_name in TERM INT; do + requested_dir="/tmp/k3slr-feasibility.signal${signal_name}${BASHPID}${RANDOM}" + stdout_file="${fixture_root}/task4-signal-${signal_name}.stdout" + stderr_file="${fixture_root}/task4-signal-${signal_name}.stderr" + export TASK4_FEASIBILITY_CASE="signal_${signal_name}" + export TASK4_FEASIBILITY_REQUESTED_DIR="$requested_dir" + export TASK4_FEASIBILITY_COMMAND_LOG="${fixture_root}/task4-signal-${signal_name}.commands" + export TASK4_FEASIBILITY_CHILD_PID_FILE="${fixture_root}/task4-signal-${signal_name}.pid" + export TASK4_FEASIBILITY_CHILD_TERM_FILE="${fixture_root}/task4-signal-${signal_name}.terminated" + export TASK4_FEASIBILITY_MAIN_PID_FILE="${fixture_root}/task4-signal-${signal_name}.main-pid" + export TASK4_FEASIBILITY_ROLE_LOG="${fixture_root}/task4-signal-${signal_name}.roles" + : >"$TASK4_FEASIBILITY_COMMAND_LOG" + : >"$TASK4_FEASIBILITY_ROLE_LOG" + _k3slr_command() { task4_feasibility_command_fake "$@"; } + ( + for ((index=0; index<100; index++)); do + [[ -s "$TASK4_FEASIBILITY_CHILD_PID_FILE" && -s "$TASK4_FEASIBILITY_MAIN_PID_FILE" ]] && break + /usr/bin/sleep 0.05 + done + [[ -s "$TASK4_FEASIBILITY_CHILD_PID_FILE" && -s "$TASK4_FEASIBILITY_MAIN_PID_FILE" ]] || exit 1 + kill -"$signal_name" "$(<"$TASK4_FEASIBILITY_MAIN_PID_FILE")" + ) & + killer_pid=$! + set +e + k3slr_local_recovery_feasibility_main --execute "$stdout_file" 2>"$stderr_file" + rc=$? + wait "$killer_pid" + set -e + child_pid="$(<"$TASK4_FEASIBILITY_CHILD_PID_FILE")" + if [[ "$signal_name" == TERM ]]; then expected_rc=143; else expected_rc=130; fi + if [[ "$rc" -ne "$expected_rc" || ! -e "$TASK4_FEASIBILITY_CHILD_TERM_FILE" || -e "$requested_dir" ]] || + kill -0 "$child_pid" 2>/dev/null; then + printf 'task4 signal diagnostic: signal=%s rc=%s expected=%s term=%s fixture=%s live=%s\n' \ + "$signal_name" "$rc" "$expected_rc" "$([[ -e "$TASK4_FEASIBILITY_CHILD_TERM_FILE" ]] && printf yes || printf no)" \ + "$([[ -e "$requested_dir" ]] && printf yes || printf no)" "$([[ -e /proc/$child_pid ]] && printf yes || printf no)" >&2 + return 1 + fi + recorded_roles=0 + while IFS='|' read -r role pid start_time ppid pgid; do + [[ "$pid" =~ ^[1-9][0-9]*$ && "$ppid" =~ ^[0-9]+$ && "$pgid" =~ ^[1-9][0-9]*$ && + "$start_time" =~ ^[1-9][0-9]*$ ]] || return 1 + ! kill -0 "$pid" 2>/dev/null || return 1 + recorded_roles=$((recorded_roles + 1)) + done <"$TASK4_FEASIBILITY_ROLE_LOG" + (( recorded_roles >= 3 )) || return 1 + done +) + +task4_feasibility_cleanup_and_leak_contract() ( + local case_name expected_rc output error temporary uid + export TASK4_CHILD_CMDLINE_LOG="${fixture_root}/task4-child.cmdline" + export TASK4_CHILD_ENVIRON_LOG="${fixture_root}/task4-child.environ" + : >"$TASK4_CHILD_CMDLINE_LOG" + : >"$TASK4_CHILD_ENVIRON_LOG" + [[ -f "$FEASIBILITY_PATH" && ! -L "$FEASIBILITY_PATH" ]] || return 1 + for case_name in success synthetic_failure cleanup_failure; do + case "$case_name" in + success) expected_rc=0 ;; + synthetic_failure|cleanup_failure) expected_rc=1 ;; + esac + temporary="/tmp/k3slr-feasibility.task4${BASHPID}${RANDOM}" + output="${fixture_root}/task4-${case_name}.stdout" + error="${fixture_root}/task4-${case_name}.stderr" + assert_succeeds run_task4_feasibility_fixture "$case_name" "$expected_rc" "$temporary" "$output" "$error" + done + uid="$(/usr/bin/id -u)" + if /usr/bin/find /tmp /run -xdev -user "$uid" \ + -name '*Zz9Yy8Xx7Ww6Vv5Uu4Tt3Ss2Rr1Qq0Pp8Oo7Nn6M*' \ + -print -quit 2>/dev/null | /usr/bin/grep -q .; then + return 1 + fi + if /usr/bin/find /tmp /run -xdev -user "$uid" -type f -readable \ + ! -path "${fixture_root}/*" \ + -exec /usr/bin/grep -IlF -- 'Zz9Yy8Xx7Ww6Vv5Uu4Tt3Ss2Rr1Qq0Pp8Oo7Nn6M' '{}' \; \ + -print -quit 2>/dev/null | /usr/bin/grep -q .; then + return 1 + fi +) + +task4_review1_noclobber_lifecycle() ( + local collision fixture_dir target + # shellcheck source=/dev/null + source "$FEASIBILITY_PATH" + export TASK4_FEASIBILITY_CASE=success + export TASK4_FEASIBILITY_COMMAND_LOG="${fixture_root}/task4-noclobber.commands" + _k3slr_command() { task4_feasibility_command_fake "$@"; } + for collision in preexisting symlink; do + fixture_dir="/tmp/k3slr-feasibility.noclobber${collision}${BASHPID}${RANDOM}" + /usr/bin/mkdir -m 0700 -- "$fixture_dir" + : >"$TASK4_FEASIBILITY_COMMAND_LOG" + if [[ "$collision" == preexisting ]]; then + target="${fixture_dir}/input.bin" + printf 'owned\n' >"$target" + else + target="${fixture_dir}/synthetic.kdbx" + /usr/bin/ln -s -- /dev/null "$target" + fi + K3SLRF_ACTIVE_CHILD_PID='' + assert_fails _k3slrf_run_synthetic "$fixture_dir" + [[ -e "$target" || -L "$target" ]] || return 1 + ! /usr/bin/grep -Fq -- '/usr/bin/keepassxc-cli ' "$TASK4_FEASIBILITY_COMMAND_LOG" || return 1 + /usr/bin/rm -rf -- "$fixture_dir" + done +) + +task4_review2_atomic_save_transition() ( + local requested_dir="/tmp/k3slr-feasibility.atomic${BASHPID}${RANDOM}" atomic_case + local stdout_file="${fixture_root}/task4-review2-atomic.stdout" stderr_file="${fixture_root}/task4-review2-atomic.stderr" + export TASK4_ATOMIC_MUTATED_MARKER="${fixture_root}/task4-review2-atomic-mutated" + export TASK4_ATOMIC_PARENT_STAT_LOG="${fixture_root}/task4-review2-parent-stat" + /usr/bin/unlink "$TASK4_ATOMIC_MUTATED_MARKER" 2>/dev/null || true + : >"$TASK4_ATOMIC_PARENT_STAT_LOG" + assert_succeeds run_task4_feasibility_fixture success 0 "$requested_dir" "$stdout_file" "$stderr_file" + for atomic_case in atomic_symlink atomic_wrong_owner atomic_wrong_mode atomic_parent_drift; do + /usr/bin/unlink "$TASK4_ATOMIC_MUTATED_MARKER" 2>/dev/null || true + : >"$TASK4_ATOMIC_PARENT_STAT_LOG" + requested_dir="/tmp/k3slr-feasibility.atomicfault${BASHPID}${RANDOM}" + stdout_file="${fixture_root}/task4-review2-${atomic_case}.stdout" + stderr_file="${fixture_root}/task4-review2-${atomic_case}.stderr" + assert_succeeds run_task4_feasibility_fixture "$atomic_case" 1 "$requested_dir" "$stdout_file" "$stderr_file" + done +) + +task4_review1_raw_package_record() ( + local record_case + # shellcheck source=/dev/null + source "$FEASIBILITY_PATH" + export TASK4_FEASIBILITY_COMMAND_LOG="${fixture_root}/task4-package.commands" + : >"$TASK4_FEASIBILITY_COMMAND_LOG" + _k3slr_command() { task4_feasibility_command_fake "$@"; } + TASK4_FEASIBILITY_CASE=success + assert_succeeds _k3slrf_package_version_is_exact keepassxc '2.7.6+dfsg.1-1build3' + for record_case in package_missing_lf package_double_lf; do + TASK4_FEASIBILITY_CASE="$record_case" + assert_fails _k3slrf_package_version_is_exact keepassxc '2.7.6+dfsg.1-1build3' + done +) + +task4_review1_command_fake() { + local command="$1" format path role pgid caller_pgid early_signal role_identity='' start_time='' + shift + printf '%s %s\n' "$command" "$*" >>"$TASK4_REVIEW1_COMMAND_LOG" + case "$command" in + /usr/bin/test) + if [[ "${1-}" == -t && "${TASK4_REVIEW3_SIGNAL_CASE:-}" == 1 && + ! -e "${TASK4_REVIEW3_CALLER_PID_FILE:-/nonexistent}" ]]; then + printf '%s\n' "$BASHPID" >"$TASK4_REVIEW3_CALLER_PID_FILE" + caller_pgid="$(/usr/bin/ps -o pgid= -p "$BASHPID")" || return 1 + caller_pgid="${caller_pgid//[[:space:]]/}" + printf '%s\n' "$caller_pgid" >"$TASK4_REVIEW3_CALLER_PGID_FILE" + fi + if [[ "${1-}" == -t && "${TASK4_REVIEW2_REQUIRE_REAL_TTY:-}" == 1 ]]; then + printf 'tty-check pid=%s result=%s\n' "$BASHPID" "$([[ -t 0 ]] && printf tty || printf pipe)" >>"$TASK4_REVIEW1_COMMAND_LOG" + /usr/bin/test -t 0 + return + fi + if [[ "${1-}" == -t && "${TASK4_REVIEW1_VALIDATION_CASE:-}" == non_tty ]]; then return 1; fi + if [[ "${1-}" == '!' && "${2-}" == -L && "${3-}" == "${K3SLR_OUTER_MOUNT}/${K3SLR_ROOT_RELATIVE}/vault" && + "${TASK4_REVIEW1_VALIDATION_CASE:-}" == parent_symlink ]]; then return 1; fi + return 0 + ;; + /usr/bin/id) + [[ "$*" == '-u' ]] || return 97 + printf '1000\n' + ;; + /usr/bin/readlink) + printf '%s\n' "${*: -1}" + ;; + /usr/bin/stat) + format="$1" + path="${*: -1}" + case "$format" in + --format=%u\|%a\|%F) printf '0|755|regular file\n' ;; + --format=%F) printf 'block special file\n' ;; + --format=%d:%i\|%F\|%u\|%a) + case "$path" in + "$K3SLR_OUTER_MOUNT") printf '8:10|directory|1000|700\n' ;; + "${K3SLR_OUTER_MOUNT}/${K3SLR_ROOT_RELATIVE}") printf '8:11|directory|1000|700\n' ;; + "${K3SLR_OUTER_MOUNT}/${K3SLR_ROOT_RELATIVE}/vault") printf '8:12|directory|1000|700\n' ;; + "${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}") + if [[ "${TASK4_REVIEW1_VALIDATION_CASE:-}" == post_drift ]]; then + printf 'x\n' >>"$TASK4_REVIEW1_DB_STAT_LOG" + if [[ "$(/usr/bin/wc -l <"$TASK4_REVIEW1_DB_STAT_LOG")" -gt 1 ]]; then + printf '8:99|regular file|1000|600\n' + else + printf '8:13|regular file|1000|600\n' + fi + else + printf '8:13|regular file|1000|600\n' + fi + ;; + *) return 96 ;; + esac + ;; + *) return 95 ;; + esac + ;; + /usr/bin/keepassxc-cli) + if [[ -n "${TASK4_REVIEW4_IDENTITY_ROLE_LOG:-}" ]]; then + _k3slr_process_identity role_identity "$BASHPID" || return 1 + printf 'keepass|%s\n' "$role_identity" >>"$TASK4_REVIEW4_IDENTITY_ROLE_LOG" + /usr/bin/sleep 0.05 + fi + if [[ -n "${TASK4_CHILD_CMDLINE_LOG:-}" ]]; then + /usr/bin/tr '\0' '\n' <"/proc/${BASHPID}/cmdline" >>"$TASK4_CHILD_CMDLINE_LOG" + /usr/bin/tr '\0' '\n' <"/proc/${BASHPID}/environ" >>"$TASK4_CHILD_ENVIRON_LOG" + fi + printf '%s\n' "$*" >>"$TASK4_REVIEW1_PRODUCER_LOG" + if [[ "${TASK4_REVIEW2_REQUIRE_REAL_TTY:-}" == 1 ]]; then + /usr/bin/test -t 0 || return 88 + fi + if [[ "$TASK4_REVIEW1_PRODUCER_CASE" == blocking_signal ]]; then + exec /usr/bin/bash -c ' + pgid="$(/usr/bin/ps -o pgid= -p "$$")" || exit 1 + pgid="${pgid//[[:space:]]/}" + IFS= read -r stat_record <"/proc/$$/stat" || exit 1 + stat_tail="${stat_record##*) }" + read -r -a stat_fields <<<"$stat_tail" + start_time="${stat_fields[19]}" + printf "keepass %s %s %s %s\n" "$$" "$PPID" "$pgid" "$start_time" >>"$TASK4_REVIEW3_ROLE_LOG" + trap "exit 143" TERM + trap "exit 130" INT + while :; do :; done + ' + fi + if [[ "$TASK4_REVIEW1_PRODUCER_CASE" == term_ignoring_descendant ]]; then + exec /usr/bin/bash -c ' + trap "" TERM + printf ready >"$TASK4_REVIEW5_TERM_IGNORE_READY" + while :; do :; done + ' + fi + case "$TASK4_REVIEW1_PRODUCER_CASE" in + partial_failure) printf '%s' partial; return 7 ;; + zero_failure) return 8 ;; + zero_success) return 0 ;; + missing_lf) printf '%s' 'Aa0Bb1Cc2Dd3Ee4Ff5Gg6Hh7Ii8Jj9Kk0Ll1Mm2N' ;; + double_lf) printf '%s\n\n' 'Aa0Bb1Cc2Dd3Ee4Ff5Gg6Hh7Ii8Jj9Kk0Ll1Mm2N' ;; + invalid_class) printf '%s\n' 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' ;; + malformed_length) printf '%s\n' 'Aa0Bb1Cc2Dd3Ee4Ff5Gg6Hh7Ii8Jj9Kk0Ll1Mm' ;; + oversized_success) /usr/bin/mawk 'BEGIN { for (i=0; i<200000; i++) printf "X"; printf "\n" }' ;; + success) printf '%s\n' 'Aa0Bb1Cc2Dd3Ee4Ff5Gg6Hh7Ii8Jj9Kk0Ll1Mm2N' ;; + *) return 94 ;; + esac + ;; + /usr/bin/sudo) + if [[ "${1-}" == --non-interactive && "${2-}" == --validate ]]; then return 0; fi + printf 'dispatch\n' >>"$TASK4_REVIEW1_CONSUMER_LOG" + /usr/bin/mawk 'END { exit 0 }' + ;; + /usr/bin/pkill) + /usr/bin/pkill "$@" + ;; + /usr/bin/ps) + if [[ "${TASK4_REVIEW3_EARLY_SIGNAL:-}" =~ ^(INT|TERM)$ && "${1-} ${2-}" == '-o stat=' ]]; then + early_signal="$TASK4_REVIEW3_EARLY_SIGNAL" + TASK4_REVIEW3_EARLY_SIGNAL='' + printf '%s\n' "${*: -1}" >"$TASK4_REVIEW3_SUPERVISOR_PID_FILE" + /usr/bin/ps -o pgid= -p "${*: -1}" | /usr/bin/tr -d '[:space:]' >"$TASK4_REVIEW3_SUPERVISOR_PGID_FILE" + kill -"$early_signal" "$(<"$TASK4_REVIEW3_CALLER_PID_FILE")" + fi + /usr/bin/ps "$@" + ;; + /usr/bin/sleep) + /usr/bin/sleep "$@" + ;; + /usr/bin/od|/usr/bin/mawk|/usr/bin/dd) + if [[ -n "${TASK4_REVIEW4_IDENTITY_ROLE_LOG:-}" && "$command" != /usr/bin/dd ]]; then + role="${command##*/}" + _k3slr_process_identity role_identity "$BASHPID" || return 1 + printf '%s|%s\n' "$role" "$role_identity" >>"$TASK4_REVIEW4_IDENTITY_ROLE_LOG" + fi + if [[ "$TASK4_REVIEW1_PRODUCER_CASE" == blocking_signal && "$command" != /usr/bin/dd ]]; then + role="${command##*/}" + pgid="$(/usr/bin/ps -o pgid= -p "$BASHPID")" || return 1 + pgid="${pgid//[[:space:]]/}" + _k3slr_process_identity role_identity "$BASHPID" || return 1 + IFS='|' read -r _ start_time _ _ <<<"$role_identity" + printf '%s %s %s %s %s\n' "$role" "$BASHPID" "$PPID" "$pgid" "$start_time" >>"$TASK4_REVIEW3_ROLE_LOG" + exec "$command" "$@" + fi + "$command" "$@" + ;; + *) return 93 ;; + esac +} + +task4_review1_producer_must_finish_before_consumer() ( + local database="${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" + export TASK4_REVIEW1_CONSUMER_LOG="${fixture_root}/task4-review1-consumer.log" + export TASK4_REVIEW1_PRODUCER_LOG="${fixture_root}/task4-review1-producer.log" + export TASK4_REVIEW1_COMMAND_LOG="${fixture_root}/task4-review1-command.log" + export TASK4_REVIEW1_DB_STAT_LOG="${fixture_root}/task4-review1-db-stat.log" + export TASK4_CHILD_CMDLINE_LOG="${fixture_root}/task4-child.cmdline" + export TASK4_CHILD_ENVIRON_LOG="${fixture_root}/task4-child.environ" + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_COMMAND_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + : >"$TASK4_CHILD_CMDLINE_LOG" + : >"$TASK4_CHILD_ENVIRON_LOG" + _k3slr_command() { task4_review1_command_fake "$@"; } + + TASK4_REVIEW1_PRODUCER_CASE=partial_failure + export TASK4_REVIEW1_PRODUCER_CASE + assert_fails _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" + [[ ! -s "$TASK4_REVIEW1_CONSUMER_LOG" ]] || return 1 + + TASK4_REVIEW1_PRODUCER_CASE=zero_failure + export TASK4_REVIEW1_PRODUCER_CASE + assert_fails _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" + [[ ! -s "$TASK4_REVIEW1_CONSUMER_LOG" ]] || return 1 + + TASK4_REVIEW1_PRODUCER_CASE=success + export TASK4_REVIEW1_PRODUCER_CASE + if ! _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME"; then + printf 'task4 critical diagnostic: producer=%s consumer=%s\n' \ + "$(/usr/bin/wc -l <"$TASK4_REVIEW1_PRODUCER_LOG")" \ + "$(/usr/bin/wc -l <"$TASK4_REVIEW1_CONSUMER_LOG")" >&2 + /usr/bin/tail -n 20 "$TASK4_REVIEW1_COMMAND_LOG" >&2 + return 1 + fi + [[ "$(/usr/bin/wc -l <"$TASK4_REVIEW1_CONSUMER_LOG")" -eq 1 ]] || return 1 + [[ "$(/usr/bin/tail -n 1 "$TASK4_REVIEW1_PRODUCER_LOG")" == \ + "show --show-protected --attributes Password ${database} ${K3SLR_KEEPASS_ENTRY}" ]] +) + +task4_review3_cleanup_recorded_roles() { + local role pid ppid pgid start_time expected_identity current_identity caller_pgid cleanup_rc=0 + [[ -f "$1" ]] || return 0 + caller_pgid="$(/usr/bin/ps -o pgid= -p "$BASHPID")" || return 1 + caller_pgid="${caller_pgid//[[:space:]]/}" + while read -r role pid ppid pgid start_time; do + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || continue + expected_identity="${pid}|${start_time}|${ppid}|${pgid}" + if _k3slr_process_identity current_identity "$pid" 2>/dev/null && + [[ "$current_identity" == "$expected_identity" && "$pgid" != "$caller_pgid" ]]; then + builtin kill -TERM "$pid" 2>/dev/null || cleanup_rc=1 + fi + done <"$1" + /usr/bin/sleep 0.1 + while read -r role pid ppid pgid start_time; do + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || continue + expected_identity="${pid}|${start_time}|${ppid}|${pgid}" + if _k3slr_process_identity current_identity "$pid" 2>/dev/null && + [[ "$current_identity" == "$expected_identity" && "$pgid" != "$caller_pgid" ]]; then + builtin kill -KILL "$pid" 2>/dev/null || cleanup_rc=1 + fi + done <"$1" + return "$cleanup_rc" +} + +task4_review3_nested_pipeline_signal_cleanup() ( + local database="${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" + local signal_name expected_rc iteration prefix killer_pid rc started_ns ended_ns elapsed_ms + local role pid ppid pgid start_time leaked dedicated_pgid caller_pgid + export TASK4_REVIEW2_REQUIRE_REAL_TTY=1 + export TASK4_REVIEW3_SIGNAL_CASE=1 + TASK4_REVIEW1_PRODUCER_CASE=blocking_signal + export TASK4_REVIEW1_PRODUCER_CASE + _k3slr_command() { task4_review1_command_fake "$@"; } + for signal_name in INT TERM; do + if [[ "$signal_name" == INT ]]; then expected_rc=130; else expected_rc=143; fi + for iteration in 1 2; do + prefix="${fixture_root}/task4-review3-${signal_name}-${iteration}" + export TASK4_REVIEW1_CONSUMER_LOG="${prefix}.consumer" + export TASK4_REVIEW1_PRODUCER_LOG="${prefix}.producer" + export TASK4_REVIEW1_COMMAND_LOG="${prefix}.commands" + export TASK4_REVIEW1_DB_STAT_LOG="${prefix}.db-stat" + export TASK4_REVIEW3_CALLER_PID_FILE="${prefix}.caller-pid" + export TASK4_REVIEW3_CALLER_PGID_FILE="${prefix}.caller-pgid" + export TASK4_REVIEW3_ROLE_LOG="${prefix}.roles" + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_COMMAND_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + : >"$TASK4_REVIEW3_ROLE_LOG" + /usr/bin/unlink "$TASK4_REVIEW3_CALLER_PID_FILE" 2>/dev/null || true + /usr/bin/unlink "$TASK4_REVIEW3_CALLER_PGID_FILE" 2>/dev/null || true + ( + for ((probe=0; probe<200; probe++)); do + if [[ -s "$TASK4_REVIEW3_CALLER_PID_FILE" && + "$(/usr/bin/wc -l <"$TASK4_REVIEW3_ROLE_LOG")" -ge 3 ]]; then break; fi + /usr/bin/sleep 0.01 + done + [[ -s "$TASK4_REVIEW3_CALLER_PID_FILE" && + "$(/usr/bin/wc -l <"$TASK4_REVIEW3_ROLE_LOG")" -ge 3 ]] || exit 1 + kill -"$signal_name" "$(<"$TASK4_REVIEW3_CALLER_PID_FILE")" + ) & + killer_pid=$! + started_ns="$(/usr/bin/date +%s%N)" + set +e + _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" /dev/null; then leaked=$((leaked + 1)); fi + done <"$TASK4_REVIEW3_ROLE_LOG" + if [[ "$rc" -ne "$expected_rc" || "$elapsed_ms" -ge 3000 || + -s "$TASK4_REVIEW1_CONSUMER_LOG" || "$leaked" -ne 0 || + -z "$dedicated_pgid" || "$dedicated_pgid" == "$caller_pgid" ]]; then + printf 'task4 review3 signal diagnostic: signal=%s iteration=%s rc=%s elapsed_ms=%s leaked=%s\n' \ + "$signal_name" "$iteration" "$rc" "$elapsed_ms" "$leaked" >&2 + /usr/bin/sed -n '1,20p' "$TASK4_REVIEW3_ROLE_LOG" >&2 + /usr/bin/tail -n 40 "$TASK4_REVIEW1_COMMAND_LOG" >&2 + while read -r role pid ppid pgid start_time; do + /usr/bin/ps -o pid=,ppid=,pgid=,stat=,args= -p "$pid" >&2 || true + done <"$TASK4_REVIEW3_ROLE_LOG" + task4_review3_cleanup_recorded_roles "$TASK4_REVIEW3_ROLE_LOG" + return 1 + fi + done + done +) + +task4_review3_launch_window_signal_cleanup() ( + local database="${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" + local signal_name expected_rc prefix rc started_ns ended_ns elapsed_ms supervisor_pid supervisor_pgid + local supervisor_identity current_identity caller_pgid cleanup_rc=0 + export TASK4_REVIEW2_REQUIRE_REAL_TTY=1 + export TASK4_REVIEW3_SIGNAL_CASE=1 + TASK4_REVIEW1_PRODUCER_CASE=blocking_signal + export TASK4_REVIEW1_PRODUCER_CASE + _k3slr_command() { task4_review1_command_fake "$@"; } + _k3slr_lifecycle_boundary() { + local phase="$1" recorded_pgid='' recorded_identity='' + if [[ "$phase" == pid-published && "${TASK4_REVIEW3_EARLY_SIGNAL:-}" =~ ^(INT|TERM)$ ]]; then + printf '%s\n' "$producer_pid" >"$TASK4_REVIEW3_SUPERVISOR_PID_FILE" + recorded_pgid="$(/usr/bin/ps -o pgid= -p "$producer_pid")" || return 1 + recorded_pgid="${recorded_pgid//[[:space:]]/}" + printf '%s\n' "$recorded_pgid" >"$TASK4_REVIEW3_SUPERVISOR_PGID_FILE" + _k3slr_process_identity recorded_identity "$producer_pid" || return 1 + printf '%s\n' "$recorded_identity" >"$TASK4_REVIEW3_SUPERVISOR_IDENTITY_FILE" + kill -"$TASK4_REVIEW3_EARLY_SIGNAL" "$(<"$TASK4_REVIEW3_CALLER_PID_FILE")" + TASK4_REVIEW3_EARLY_SIGNAL='' + fi + } + for signal_name in INT TERM; do + if [[ "$signal_name" == INT ]]; then expected_rc=130; else expected_rc=143; fi + prefix="${fixture_root}/task4-review3-early-${signal_name}" + export TASK4_REVIEW1_CONSUMER_LOG="${prefix}.consumer" + export TASK4_REVIEW1_PRODUCER_LOG="${prefix}.producer" + export TASK4_REVIEW1_COMMAND_LOG="${prefix}.commands" + export TASK4_REVIEW1_DB_STAT_LOG="${prefix}.db-stat" + export TASK4_REVIEW3_CALLER_PID_FILE="${prefix}.caller-pid" + export TASK4_REVIEW3_CALLER_PGID_FILE="${prefix}.caller-pgid" + export TASK4_REVIEW3_ROLE_LOG="${prefix}.roles" + export TASK4_REVIEW3_SUPERVISOR_PID_FILE="${prefix}.supervisor-pid" + export TASK4_REVIEW3_SUPERVISOR_PGID_FILE="${prefix}.supervisor-pgid" + export TASK4_REVIEW3_SUPERVISOR_IDENTITY_FILE="${prefix}.supervisor-identity" + export TASK4_REVIEW3_EARLY_SIGNAL="$signal_name" + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_COMMAND_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + : >"$TASK4_REVIEW3_ROLE_LOG" + /usr/bin/unlink "$TASK4_REVIEW3_CALLER_PID_FILE" 2>/dev/null || true + /usr/bin/unlink "$TASK4_REVIEW3_CALLER_PGID_FILE" 2>/dev/null || true + started_ns="$(/usr/bin/date +%s%N)" + set +e + _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" /dev/null; then + printf 'task4 review3 early diagnostic: signal=%s rc=%s elapsed_ms=%s supervisor=%s pgid=%s\n' \ + "$signal_name" "$rc" "$elapsed_ms" "$supervisor_pid" "$supervisor_pgid" >&2 + caller_pgid="$(/usr/bin/ps -o pgid= -p "$BASHPID")" || cleanup_rc=1 + caller_pgid="${caller_pgid//[[:space:]]/}" + if _k3slr_process_identity current_identity "$supervisor_pid" 2>/dev/null && + [[ "$current_identity" == "$supervisor_identity" && "$supervisor_pgid" != "$caller_pgid" ]]; then + builtin kill -KILL "$supervisor_pid" 2>/dev/null || cleanup_rc=1 + fi + (( cleanup_rc == 0 )) || printf 'task4 review3 early cleanup failed\n' >&2 + return 1 + fi + done +) + +task4_review4_stale_cached_group_boundary() ( + local child_a unrelated_pid unrelated_pgid unrelated_identity='' caller_pgid supplied_identity='' + local current_identity='' current_pgid='' boundary_rc=0 killed=0 cleanup_rc=0 wait_rc + /usr/bin/setsid /usr/bin/sleep 30 & + unrelated_pid=$! + /usr/bin/sleep 0.05 + _k3slr_process_identity unrelated_identity "$unrelated_pid" || return 1 + unrelated_pgid="$(/usr/bin/ps -o pgid= -p "$unrelated_pid")" || return 1 + unrelated_pgid="${unrelated_pgid//[[:space:]]/}" + caller_pgid="$(/usr/bin/ps -o pgid= -p "$BASHPID")" || return 1 + caller_pgid="${caller_pgid//[[:space:]]/}" + [[ "$unrelated_pgid" =~ ^[1-9][0-9]*$ && "$unrelated_pgid" != "$caller_pgid" ]] || return 1 + /usr/bin/sleep 30 & + child_a=$! + _k3slr_process_identity supplied_identity "$child_a" || return 1 + set +e + _k3slr_terminate_and_reap "$child_a" "$unrelated_pgid" "$supplied_identity" + boundary_rc=$? + set -e + if ! kill -0 "$unrelated_pid" 2>/dev/null; then killed=1; fi + if kill -0 "$unrelated_pid" 2>/dev/null; then + _k3slr_process_identity current_identity "$unrelated_pid" 2>/dev/null || current_identity='' + current_pgid="$(/usr/bin/ps -o pgid= -p "$unrelated_pid" 2>/dev/null || true)" + current_pgid="${current_pgid//[[:space:]]/}" + caller_pgid="$(/usr/bin/ps -o pgid= -p "$BASHPID" 2>/dev/null || true)" + caller_pgid="${caller_pgid//[[:space:]]/}" + if [[ "$current_identity" == "$unrelated_identity" && "$current_pgid" == "$unrelated_pgid" && + "$caller_pgid" =~ ^[1-9][0-9]*$ && "$current_pgid" != "$caller_pgid" ]]; then + builtin kill -TERM -- "-${unrelated_pgid}" 2>/dev/null || cleanup_rc=1 + fi + fi + if wait "$unrelated_pid" 2>/dev/null; then wait_rc=0; else wait_rc=$?; fi + (( wait_rc != 127 )) || cleanup_rc=1 + if _k3slr_process_identity current_identity "$child_a" 2>/dev/null && + [[ "$current_identity" == "$supplied_identity" ]]; then + builtin kill -TERM "$child_a" 2>/dev/null || cleanup_rc=1 + fi + if wait "$child_a" 2>/dev/null; then wait_rc=0; else wait_rc=$?; fi + (( wait_rc != 127 )) || cleanup_rc=1 + (( cleanup_rc == 0 )) || return 1 + if [[ "$killed" -ne 0 ]]; then + printf 'task4 review4 stale group RED: child=%s supplied_group=%s unrelated=killed boundary_rc=%s\n' \ + "$child_a" "$unrelated_pgid" "$boundary_rc" >&2 + return 1 + fi +) + +task4_review4_direct_fallback_boundaries() ( + local mode child_pid child_identity='' child_group='' child_state='' attempt original_group_function + for mode in caller_group query_failure; do + ( kill -STOP "$BASHPID"; printf 'unexpected child continuation\n' >&2 ) & + child_pid=$! + child_identity='' + for ((attempt=0; attempt<200; attempt++)); do + if _k3slr_process_record child_identity child_state "$child_pid" && [[ "$child_state" == T ]]; then break; fi + /usr/bin/sleep 0.01 + done + [[ -n "$child_identity" && "$child_state" == T ]] || return 1 + IFS='|' read -r _ _ _ child_group <<<"$child_identity" + if [[ "$mode" == query_failure ]]; then + original_group_function="$(declare -f _k3slr_process_group_for_pid)" + _k3slr_process_group_for_pid() { return 1; } + fi + _k3slr_terminate_and_reap "$child_pid" "$child_group" "$child_identity" || return 1 + if [[ "$mode" == query_failure ]]; then eval "$original_group_function"; fi + if kill -0 "$child_pid" 2>/dev/null; then return 1; fi + done +) + +task4_review4_term_ignoring_group_escalates() ( + local supervisor_pid supervisor_pgid='' supervisor_identity='' descendant_pid='' state='' attempt + local descendant_file="${fixture_root}/task4-review4-term-ignoring-descendant" + : >"$descendant_file" + set -m + ( + kill -STOP "$BASHPID" + trap '' TERM + ( trap '' TERM; while :; do :; done ) & + printf '%s\n' "$!" >"$descendant_file" + wait + ) & + supervisor_pid=$! + _k3slr_wait_for_stopped_group supervisor_pgid supervisor_identity "$supervisor_pid" || { + set +m + return 1 + } + set +m + kill -CONT "$supervisor_pid" || return 1 + for ((attempt=0; attempt<200; attempt++)); do + [[ -s "$descendant_file" ]] && break + /usr/bin/sleep 0.01 + done + descendant_pid="$(<"$descendant_file")" + [[ "$descendant_pid" =~ ^[1-9][0-9]*$ ]] || return 1 + _k3slr_terminate_and_reap "$supervisor_pid" "$supervisor_pgid" "$supervisor_identity" || return 1 + ! kill -0 "$supervisor_pid" 2>/dev/null || return 1 + ! kill -0 "$descendant_pid" 2>/dev/null || return 1 + ! _k3slr_group_exists "$supervisor_pgid" +) + +task4_review4_wait_clear_is_atomic() ( + local database="${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" + local rc role pid ppid pgid start_time sentinel_pid sentinel_identity='' current_identity='' + local seen_supervisor=0 seen_process_sub=0 seen_keepass=0 seen_od=0 seen_mawk=0 + export TASK4_REVIEW2_REQUIRE_REAL_TTY=1 TASK4_REVIEW3_SIGNAL_CASE=1 + export TASK4_REVIEW1_PRODUCER_CASE=success + export TASK4_REVIEW1_CONSUMER_LOG="${fixture_root}/task4-review4-wait-clear.consumer" + export TASK4_REVIEW1_PRODUCER_LOG="${fixture_root}/task4-review4-wait-clear.producer" + export TASK4_REVIEW1_COMMAND_LOG="${fixture_root}/task4-review4-wait-clear.commands" + export TASK4_REVIEW1_DB_STAT_LOG="${fixture_root}/task4-review4-wait-clear.db-stat" + export TASK4_REVIEW3_CALLER_PID_FILE="${fixture_root}/task4-review4-wait-clear.caller" + export TASK4_REVIEW3_CALLER_PGID_FILE="${fixture_root}/task4-review4-wait-clear.caller-pgid" + export TASK4_REVIEW4_IDENTITY_ROLE_LOG="${fixture_root}/task4-review4-wait-clear.roles" + export TASK4_REVIEW4_NEGATIVE_KILL_LOG="${fixture_root}/task4-review4-wait-clear.negative-kill" + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_COMMAND_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + : >"$TASK4_REVIEW4_IDENTITY_ROLE_LOG" + : >"$TASK4_REVIEW4_NEGATIVE_KILL_LOG" + _k3slr_command() { task4_review1_command_fake "$@"; } + _k3slr_lifecycle_boundary() { + local phase="$1" boundary_pid="${2-}" boundary_identity='' + case "$phase" in + pid-published) + _k3slr_process_identity boundary_identity "$boundary_pid" || return 1 + printf 'supervisor|%s\n' "$boundary_identity" >>"$TASK4_REVIEW4_IDENTITY_ROLE_LOG" + ;; + validator-stage-published) + _k3slr_process_identity boundary_identity "$boundary_pid" || return 1 + printf 'process_sub|%s\n' "$boundary_identity" >>"$TASK4_REVIEW4_IDENTITY_ROLE_LOG" + ;; + wait-reaped-before-clear) + builtin kill -TERM "$(<"$TASK4_REVIEW3_CALLER_PID_FILE")" + ;; + esac + } + kill() { + local argument + for argument in "$@"; do + if [[ "$argument" =~ ^-[1-9][0-9]*$ ]]; then printf '%s\n' "$argument" >>"$TASK4_REVIEW4_NEGATIVE_KILL_LOG"; fi + done + builtin kill "$@" + } + /usr/bin/setsid /usr/bin/sleep 30 & + sentinel_pid=$! + /usr/bin/sleep 0.02 + _k3slr_process_identity sentinel_identity "$sentinel_pid" || return 1 + set +e + _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" /dev/null || return 1 + done <"$TASK4_REVIEW4_IDENTITY_ROLE_LOG" + (( seen_supervisor && seen_process_sub && seen_keepass && seen_od && seen_mawk )) || return 1 + _k3slr_process_identity current_identity "$sentinel_pid" || return 1 + [[ "$current_identity" == "$sentinel_identity" ]] || return 1 + builtin kill -TERM "$sentinel_pid" + wait "$sentinel_pid" 2>/dev/null || true +) + +task4_review4_signal_phase_matrix() ( + local database="${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" + local signal_name expected_rc injection_phase phase rc prefix supervisor_pid='' supervisor_pgid='' + local supervisor_identity='' sentinel_pid sentinel_identity='' current_identity='' role pid ppid pgid + local sentinel_start_time='' sentinel_parent='' sentinel_pgid='' fixture_cleanup_rc=0 + local -a phases=(coproc-launch-before pid-published stop-query-complete cont-before cont-after \ + wait-reaped-before-clear pre-consumer) + export TASK4_REVIEW2_REQUIRE_REAL_TTY=1 TASK4_REVIEW3_SIGNAL_CASE=1 + /usr/bin/setsid /usr/bin/sleep 60 & + sentinel_pid=$! + /usr/bin/sleep 0.02 + _k3slr_process_identity sentinel_identity "$sentinel_pid" || return 1 + IFS='|' read -r sentinel_pid sentinel_start_time sentinel_parent sentinel_pgid <<<"$sentinel_identity" + _task4_review4_phase_sentinel_cleanup() { + if _k3slr_process_identity current_identity "$sentinel_pid" 2>/dev/null && + [[ "$current_identity" == "$sentinel_identity" ]]; then + _k3slr_signal_pid TERM "$sentinel_pid" "$sentinel_identity" "$sentinel_parent" \ + "$sentinel_pgid" 2>/dev/null || return 1 + if _k3slr_wait_child "$sentinel_pid"; then :; else [[ "$?" -ne 127 ]] || return 1; fi + fi + } + _task4_review4_phase_exit_cleanup() { + local original_status="$1" cleanup_status=0 + trap - EXIT + if [[ -n "$prefix" ]]; then + _task4_review5_phase_fixture_cleanup "$prefix" || cleanup_status=1 + fi + _task4_review4_phase_sentinel_cleanup || cleanup_status=1 + (( cleanup_status == 0 )) || exit 1 + exit "$original_status" + } + trap '_task4_review4_phase_exit_cleanup "$?"' EXIT + _k3slr_command() { task4_review1_command_fake "$@"; } + _k3slr_lifecycle_boundary() { + local boundary_phase="$1" boundary_pid="${2-}" recorded_pgid='' + if [[ "$boundary_phase" == pid-published ]]; then + printf '%s\n' "$boundary_pid" >"${prefix}.supervisor-pid" + _k3slr_process_identity supervisor_identity "$boundary_pid" || return 1 + printf '%s\n' "$supervisor_identity" >"${prefix}.supervisor-identity" + recorded_pgid="$(/usr/bin/ps -o pgid= -p "$boundary_pid")" || return 1 + recorded_pgid="${recorded_pgid//[[:space:]]/}" + printf '%s\n' "$recorded_pgid" >"${prefix}.supervisor-pgid" + fi + if [[ "$boundary_phase" == "$injection_phase" ]]; then + printf 'injected\n' >"${prefix}.injected" + builtin kill -"$signal_name" "$(<"$TASK4_REVIEW3_CALLER_PID_FILE")" + fi + } + for signal_name in INT TERM; do + if [[ "$signal_name" == INT ]]; then expected_rc=130; else expected_rc=143; fi + for phase in "${phases[@]}"; do + injection_phase="$phase" + prefix="${fixture_root}/task4-review4-phase-${signal_name}-${phase}" + export TASK4_REVIEW1_CONSUMER_LOG="${prefix}.consumer" + export TASK4_REVIEW1_PRODUCER_LOG="${prefix}.producer" + export TASK4_REVIEW1_COMMAND_LOG="${prefix}.commands" + export TASK4_REVIEW1_DB_STAT_LOG="${prefix}.db-stat" + export TASK4_REVIEW3_CALLER_PID_FILE="${prefix}.caller-pid" + export TASK4_REVIEW3_CALLER_PGID_FILE="${prefix}.caller-pgid" + export TASK4_REVIEW3_ROLE_LOG="${prefix}.roles" + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_COMMAND_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + : >"$TASK4_REVIEW3_ROLE_LOG" + /usr/bin/unlink "${prefix}.injected" "${prefix}.supervisor-pid" \ + "${prefix}.supervisor-pgid" "${prefix}.supervisor-identity" 2>/dev/null || true + if [[ "$phase" == wait-reaped-before-clear || "$phase" == pre-consumer ]]; then + TASK4_REVIEW1_PRODUCER_CASE=success + else + TASK4_REVIEW1_PRODUCER_CASE=blocking_signal + fi + export TASK4_REVIEW1_PRODUCER_CASE + set +e + _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" &2 + return 1 + fi + if [[ "$phase" == coproc-launch-before || "$phase" == pid-published || + "$phase" == stop-query-complete || "$phase" == cont-before ]] && + [[ -s "$TASK4_REVIEW3_ROLE_LOG" ]]; then + printf 'task4 review5 pre-CONT worker dispatch: signal=%s phase=%s\n' \ + "$signal_name" "$phase" >&2 + return 1 + fi + if [[ -s "${prefix}.supervisor-pid" ]]; then + supervisor_pid="$(<"${prefix}.supervisor-pid")" + supervisor_pgid="$(<"${prefix}.supervisor-pgid")" + if kill -0 "$supervisor_pid" 2>/dev/null || _k3slr_group_exists "$supervisor_pgid"; then + printf 'task4 review4 phase leak: signal=%s phase=%s supervisor=%s pgid=%s\n' \ + "$signal_name" "$phase" "$supervisor_pid" "$supervisor_pgid" >&2 + return 1 + fi + fi + while read -r role pid ppid pgid start_time; do + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || continue + ! kill -0 "$pid" 2>/dev/null || return 1 + done <"$TASK4_REVIEW3_ROLE_LOG" + _k3slr_process_identity current_identity "$sentinel_pid" || return 1 + [[ "$current_identity" == "$sentinel_identity" ]] || return 1 + done + done + return 0 +) + +task4_review5_exact_proc_record_contract() ( + local raw_file="${fixture_root}/task4-review5-proc-stat" raw_record='' identity='' state='' live_identity='' + local valid='123 (comm with ) and ) delimiter) T 0 456 456 0 -1 4194560 1 2 3 4 5 6 7 8 9 10 11 12 18446744073709551615' + printf '%s\n' "$valid" >"$raw_file" + _k3slr_read_exact_record raw_record "$raw_file" || return 1 + [[ "$raw_record" == "$valid" ]] || return 1 + _k3slr_parse_proc_stat_record identity state 123 "$raw_record" || return 1 + [[ "$identity" == '123|18446744073709551615|0|456' && "$state" == T ]] || return 1 + _k3slr_parse_proc_stat_record identity state 123 \ + '123 (lower stopped) t 7 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 999999999999999999999999' || return 1 + [[ "$identity" == '123|999999999999999999999999|7|456' && "$state" == t ]] || return 1 + _k3slr_parse_proc_stat_record identity state 123 \ + '123 (idle) I 7 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' || return 1 + _k3slr_parse_proc_stat_record identity state 123 \ + '123 (paging) W 7 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' || return 1 + _k3slr_parse_proc_stat_record identity state 123 \ + '123 (consecutive ))) close) R 8 457 457 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 14' || return 1 + [[ "$identity" == '123|14|8|457' && "$state" == R ]] || return 1 + _k3slr_parse_proc_stat_record identity state 123 \ + '123 (decoy ) R 91 777 777 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 15 still comm) S 7 458 458 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 16' || return 1 + [[ "$identity" == '123|16|7|458' && "$state" == S ]] || return 1 + _k3slr_parse_proc_stat_record identity state 123 \ + '123 (comm ending in )) T 6 459 459 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 17' || return 1 + [[ "$identity" == '123|17|6|459' && "$state" == T ]] || return 1 + _k3slr_process_identity live_identity "$BASHPID" || return 1 + [[ "$live_identity" == "${BASHPID}|"* ]] || return 1 + + local raw_line=sentinel resolved_record=sentinel queried_identity=sentinel + local raw_proc_record=sentinel parsed_state=sentinel collision_identity=sentinel + printf '%s\n' "$valid" >"$raw_file" + _k3slr_read_exact_record raw_line "$raw_file" || return 1 + [[ "$raw_line" == "$valid" ]] || return 1 + _k3slr_proc_stat_record resolved_record "$BASHPID" || return 1 + [[ "$resolved_record" == "${BASHPID} "* ]] || return 1 + _k3slr_process_identity queried_identity "$BASHPID" || return 1 + [[ "$queried_identity" == "${BASHPID}|"* ]] || return 1 + _k3slr_process_record raw_proc_record parsed_state "$BASHPID" || return 1 + [[ "$raw_proc_record" == "${BASHPID}|"* && "$parsed_state" != sentinel ]] || return 1 + local identity_destination=sentinel process_state=sentinel collision_state=sentinel collision_identity=sentinel + _k3slr_parse_proc_stat_record identity_destination collision_state 123 "$valid" || return 1 + [[ "$identity_destination" == '123|18446744073709551615|0|456' && "$collision_state" == T ]] || return 1 + _k3slr_parse_proc_stat_record collision_identity process_state 123 "$valid" || return 1 + [[ "$collision_identity" == '123|18446744073709551615|0|456' && "$process_state" == T ]] || return 1 + local destination_name=sentinel resolved_group=sentinel child_pid=sentinel + local identity_destination=sentinel state_destination=sentinel group_destination=sentinel + _k3slr_capture_one_line destination_name /usr/bin/bash -c 'printf "capture-matrix\n"' || return 1 + [[ "$destination_name" == capture-matrix ]] || return 1 + _k3slr_process_group_for_pid resolved_group "$BASHPID" || return 1 + [[ "$resolved_group" =~ ^[1-9][0-9]*$ ]] || return 1 + _k3slr_proc_stat_record child_pid "$BASHPID" || return 1 + [[ "$child_pid" == "${BASHPID} "* ]] || return 1 + _k3slr_process_identity destination_name "$BASHPID" || return 1 + [[ "$destination_name" == "${BASHPID}|"* ]] || return 1 + _k3slr_process_record identity_destination state_destination "$BASHPID" || return 1 + [[ "$identity_destination" == "${BASHPID}|"* && "$state_destination" != sentinel ]] || return 1 + ! _k3slr_parse_proc_stat_record identity identity 123 "$valid" || return 1 + ! _k3slr_process_record identity identity "$BASHPID" || return 1 + ! _k3slr_wait_for_stopped_group identity identity "$BASHPID" || return 1 + ! _k3slr_parse_proc_stat_record _k3slr_parse_state state 123 "$valid" || return 1 + + printf '%s' "$valid" >"$raw_file" + ! _k3slr_read_exact_record raw_record "$raw_file" || return 1 + printf '%s\n\n' "$valid" >"$raw_file" + ! _k3slr_read_exact_record raw_record "$raw_file" || return 1 + printf '%s\r\n' "$valid" >"$raw_file" + ! _k3slr_read_exact_record raw_record "$raw_file" || return 1 + + local malformed + for malformed in \ + $'123 (cr\rcomm) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + $'123 (embedded\nrecord) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + $'123\t(tab prefix) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + $'123 (tab delimiter)\tT 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + '123 (double-space prefix) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + '123(no-space prefix) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + '123 (double-space delimiter) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + '123 (no-space delimiter)T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + '123 (decoy ) R 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13 real end) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 14' \ + '123 (tail close) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 ) 13' \ + '124 (wrong pid) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + '0123 (leading pid) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + '123 (leading ppid) T 00 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + '123 (leading pgid) T 0 0456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + '123 (leading start) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 00' \ + '123 (zero start) T 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 0' \ + '123 (bad state) ? 0 456 456 0 -1 1 1 2 3 4 5 6 7 8 9 10 11 12 13' \ + '123 (short) T 0 456'; do + identity=identity-sentinel + state=state-sentinel + ! _k3slr_parse_proc_stat_record identity state 123 "$malformed" || return 1 + [[ "$identity" == identity-sentinel && "$state" == state-sentinel ]] || return 1 + done +) + +task4_review5_cleanup_exact_recorded_roles() { + local log_file="$1" role pid start_time ppid pgid expected_identity current_identity caller_group + _k3slr_process_group_for_pid caller_group "$BASHPID" || return 1 + while IFS='|' read -r role pid start_time ppid pgid; do + [[ "$pid" =~ ^[1-9][0-9]*$ && "$pgid" =~ ^[1-9][0-9]*$ ]] || continue + expected_identity="${pid}|${start_time}|${ppid}|${pgid}" + if _k3slr_process_identity current_identity "$pid" 2>/dev/null && + [[ "$current_identity" == "$expected_identity" && "$pgid" != "$caller_group" ]]; then + builtin kill -KILL "$pid" 2>/dev/null || return 1 + fi + done <"$log_file" +} + +_task4_review5_phase_fixture_cleanup() { + local prefix="$1" supervisor_pid='' supervisor_pgid='' supervisor_identity='' + local current_identity='' observed_group='' caller_group='' identity_pid='' start_time='' + local parent_pid='' identity_group='' role pid ppid pgid expected_identity signal_name wait_rc + local cleanup_rc=0 + if [[ -s "${prefix}.supervisor-pid" && -s "${prefix}.supervisor-pgid" && + -s "${prefix}.supervisor-identity" ]]; then + supervisor_pid="$(<"${prefix}.supervisor-pid")" + supervisor_pgid="$(<"${prefix}.supervisor-pgid")" + supervisor_identity="$(<"${prefix}.supervisor-identity")" + IFS='|' read -r identity_pid start_time parent_pid identity_group <<<"$supervisor_identity" + if _k3slr_process_identity current_identity "$supervisor_pid" 2>/dev/null && + [[ "$current_identity" == "$supervisor_identity" && "$identity_pid" == "$supervisor_pid" && + "$identity_group" == "$supervisor_pgid" ]]; then + _k3slr_process_group_for_pid observed_group "$supervisor_pid" 2>/dev/null || observed_group='' + _k3slr_process_group_for_pid caller_group "$BASHPID" 2>/dev/null || caller_group='' + if [[ "$observed_group" == "$supervisor_pgid" && "$observed_group" != "$caller_group" ]]; then + _k3slr_terminate_and_reap "$supervisor_pid" "$supervisor_pgid" \ + "$supervisor_identity" 2>/dev/null || : + fi + fi + if _k3slr_process_identity current_identity "$supervisor_pid" 2>/dev/null && + [[ "$current_identity" == "$supervisor_identity" ]]; then + _k3slr_process_group_for_pid observed_group "$supervisor_pid" 2>/dev/null || observed_group='' + _k3slr_process_group_for_pid caller_group "$BASHPID" 2>/dev/null || caller_group='' + if [[ "$observed_group" == "$supervisor_pgid" && "$observed_group" != "$caller_group" ]]; then + _k3slr_signal_pid KILL "$supervisor_pid" "$supervisor_identity" "$parent_pid" \ + "$supervisor_pgid" 2>/dev/null || : + if _k3slr_wait_child "$supervisor_pid"; then wait_rc=0; else wait_rc=$?; fi + (( wait_rc != 127 )) || cleanup_rc=1 + fi + fi + fi + if [[ -f "${prefix}.roles" ]]; then + for signal_name in TERM KILL; do + while read -r role pid ppid pgid start_time; do + [[ "$pid" =~ ^[1-9][0-9]*$ && "$ppid" =~ ^[0-9]+$ && + "$pgid" =~ ^[1-9][0-9]*$ && "$start_time" =~ ^[1-9][0-9]*$ ]] || continue + expected_identity="${pid}|${start_time}|${ppid}|${pgid}" + if _k3slr_process_identity current_identity "$pid" 2>/dev/null && + [[ "$current_identity" == "$expected_identity" ]]; then + _k3slr_process_group_for_pid observed_group "$pid" 2>/dev/null || observed_group='' + _k3slr_process_group_for_pid caller_group "$BASHPID" 2>/dev/null || caller_group='' + if [[ "$observed_group" == "$pgid" && "$observed_group" != "$caller_group" ]]; then + _k3slr_signal_pid "$signal_name" "$pid" "$expected_identity" "$ppid" "$pgid" \ + 2>/dev/null || : + fi + fi + done <"${prefix}.roles" + [[ "$signal_name" == TERM ]] && /usr/bin/sleep 0.1 + done + while read -r role pid ppid pgid start_time; do + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || continue + expected_identity="${pid}|${start_time}|${ppid}|${pgid}" + if _k3slr_process_identity current_identity "$pid" 2>/dev/null && + [[ "$current_identity" == "$expected_identity" ]]; then + cleanup_rc=1 + fi + done <"${prefix}.roles" + fi + if [[ -n "$supervisor_pid" ]] && + _k3slr_process_identity current_identity "$supervisor_pid" 2>/dev/null && + [[ "$current_identity" == "$supervisor_identity" ]]; then + cleanup_rc=1 + fi + return "$cleanup_rc" +} + +task4_review5_phase_fixture_failure_cleanup_contract() ( + local prefix="${fixture_root}/task4-review5-phase-fixture-cleanup" + local supervisor_pid supervisor_identity='' supervisor_pgid='' current_identity='' + /usr/bin/setsid /usr/bin/bash -c 'trap "" TERM; while :; do /usr/bin/sleep 1; done' & + supervisor_pid=$! + /usr/bin/sleep 0.02 + _k3slr_process_identity supervisor_identity "$supervisor_pid" || return 1 + _k3slr_process_group_for_pid supervisor_pgid "$supervisor_pid" || return 1 + printf '%s\n' "$supervisor_pid" >"${prefix}.supervisor-pid" + printf '%s\n' "$supervisor_pgid" >"${prefix}.supervisor-pgid" + printf '%s\n' "$supervisor_identity" >"${prefix}.supervisor-identity" + : >"${prefix}.roles" + _task4_review5_phase_fixture_cleanup "$prefix" || { + if _k3slr_process_identity current_identity "$supervisor_pid" 2>/dev/null && + [[ "$current_identity" == "$supervisor_identity" ]]; then + _k3slr_signal_pid KILL "$supervisor_pid" "$supervisor_identity" "$BASHPID" \ + "$supervisor_pgid" 2>/dev/null || return 1 + _k3slr_wait_child "$supervisor_pid" || : + fi + return 1 + } + ! kill -0 "$supervisor_pid" 2>/dev/null && ! _k3slr_group_exists "$supervisor_pgid" +) + +task4_review5_default_supervisor_anchor() ( + local database="${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" + local prefix="${fixture_root}/task4-review5-anchor" killer_pid rc role pid ppid pgid start_time + local leaked=0 cleanup_rc=0 supervisor_pgid='' + export TASK4_REVIEW2_REQUIRE_REAL_TTY=1 TASK4_REVIEW3_SIGNAL_CASE=1 + export TASK4_REVIEW1_PRODUCER_CASE=term_ignoring_descendant + export TASK4_REVIEW1_CONSUMER_LOG="${prefix}.consumer" + export TASK4_REVIEW1_PRODUCER_LOG="${prefix}.producer" + export TASK4_REVIEW1_COMMAND_LOG="${prefix}.commands" + export TASK4_REVIEW1_DB_STAT_LOG="${prefix}.db-stat" + export TASK4_REVIEW3_CALLER_PID_FILE="${prefix}.caller-pid" + export TASK4_REVIEW3_CALLER_PGID_FILE="${prefix}.caller-pgid" + export TASK4_REVIEW4_IDENTITY_ROLE_LOG="${prefix}.roles" + export TASK4_REVIEW5_TERM_IGNORE_READY="${prefix}.ready" + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_COMMAND_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + : >"$TASK4_REVIEW4_IDENTITY_ROLE_LOG" + _k3slr_command() { task4_review1_command_fake "$@"; } + _k3slr_lifecycle_boundary() { + local phase="$1" boundary_pid="${2-}" identity='' + if [[ "$phase" == pid-published ]]; then + _k3slr_process_identity identity "$boundary_pid" || return 1 + printf 'supervisor|%s\n' "$identity" >>"$TASK4_REVIEW4_IDENTITY_ROLE_LOG" + supervisor_pgid="$(/usr/bin/ps -o pgid= -p "$boundary_pid")" || return 1 + supervisor_pgid="${supervisor_pgid//[[:space:]]/}" + printf '%s\n' "$supervisor_pgid" >"${prefix}.pgid" + fi + } + ( + for ((probe=0; probe<300; probe++)); do + if [[ -s "$TASK4_REVIEW3_CALLER_PID_FILE" && -s "$TASK4_REVIEW5_TERM_IGNORE_READY" ]]; then break; fi + /usr/bin/sleep 0.01 + done + [[ -s "$TASK4_REVIEW3_CALLER_PID_FILE" && -s "$TASK4_REVIEW5_TERM_IGNORE_READY" ]] || exit 1 + builtin kill -TERM "$(<"$TASK4_REVIEW3_CALLER_PID_FILE")" + ) & + killer_pid=$! + set +e + _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" /dev/null; then leaked=$((leaked + 1)); fi + done <"$TASK4_REVIEW4_IDENTITY_ROLE_LOG" + if (( leaked > 0 )); then + task4_review5_cleanup_exact_recorded_roles "$TASK4_REVIEW4_IDENTITY_ROLE_LOG" || cleanup_rc=$? + fi + [[ "$cleanup_rc" -eq 0 ]] || return 1 + if [[ "$rc" -ne 143 || "$leaked" -ne 0 || -s "$TASK4_REVIEW1_CONSUMER_LOG" ]] || + _k3slr_group_exists "$supervisor_pgid"; then + printf 'task4 review5 anchor RED: rc=%s leaked=%s pgid_live=%s\n' \ + "$rc" "$leaked" "$(_k3slr_group_exists "$supervisor_pgid" && printf yes || printf no)" >&2 + return 1 + fi +) + +task4_review5_feasibility_default_supervisor_anchor() ( + local prefix="${fixture_root}/task4-review5-feas-anchor" requested_dir + local killer_pid rc supervisor_pgid='' role pid start_time ppid pgid leaked=0 cleanup_rc=0 + source "$FEASIBILITY_PATH" + requested_dir="/tmp/k3slr-feasibility.anchor${BASHPID}${RANDOM}" + export TASK4_FEASIBILITY_CASE=anchor_TERM + export TASK4_FEASIBILITY_REQUESTED_DIR="$requested_dir" + export TASK4_FEASIBILITY_COMMAND_LOG="${prefix}.commands" + export TASK4_FEASIBILITY_MAIN_PID_FILE="${prefix}.main" + export TASK4_FEASIBILITY_CHILD_PID_FILE="${prefix}.child" + export TASK4_FEASIBILITY_ROLE_LOG="${prefix}.roles" + export TASK4_FEASIBILITY_TERM_IGNORE_READY="${prefix}.ready" + : >"$TASK4_FEASIBILITY_COMMAND_LOG" + : >"$TASK4_FEASIBILITY_ROLE_LOG" + _k3slr_command() { task4_feasibility_command_fake "$@"; } + _k3slr_lifecycle_boundary() { + local phase="$1" tracked_pid="${2-}" tracked_identity='' + if [[ "$phase" == feasibility-pid-published ]]; then + _k3slr_process_identity tracked_identity "$tracked_pid" || return 1 + printf 'supervisor|%s\n' "$tracked_identity" >>"$TASK4_FEASIBILITY_ROLE_LOG" + supervisor_pgid="$(/usr/bin/ps -o pgid= -p "$tracked_pid")" || return 1 + supervisor_pgid="${supervisor_pgid//[[:space:]]/}" + printf '%s\n' "$supervisor_pgid" >"${prefix}.pgid" + fi + } + ( + local probe + for ((probe=0; probe<300; probe++)); do + if [[ -s "$TASK4_FEASIBILITY_MAIN_PID_FILE" && + -s "$TASK4_FEASIBILITY_TERM_IGNORE_READY" ]]; then break; fi + /usr/bin/sleep 0.01 + done + [[ -s "$TASK4_FEASIBILITY_MAIN_PID_FILE" && + -s "$TASK4_FEASIBILITY_TERM_IGNORE_READY" ]] || exit 1 + builtin kill -TERM "$(<"$TASK4_FEASIBILITY_MAIN_PID_FILE")" + ) & + killer_pid=$! + set +e + k3slr_local_recovery_feasibility_main --execute /dev/null 2>"${prefix}.stderr" + rc=$? + wait "$killer_pid" + set -e + supervisor_pgid="$(<"${prefix}.pgid")" + while IFS='|' read -r role pid start_time ppid pgid; do + if kill -0 "$pid" 2>/dev/null; then leaked=$((leaked + 1)); fi + done <"$TASK4_FEASIBILITY_ROLE_LOG" + if (( leaked > 0 )); then + task4_review5_cleanup_exact_recorded_roles "$TASK4_FEASIBILITY_ROLE_LOG" || cleanup_rc=$? + fi + [[ "$cleanup_rc" -eq 0 ]] || return 1 + if [[ "$rc" -ne 143 || "$leaked" -ne 0 || -e "$requested_dir" ]] || + _k3slr_group_exists "$supervisor_pgid"; then + printf 'task4 review5 feasibility anchor RED: rc=%s leaked=%s fixture=%s pgid_live=%s\n' \ + "$rc" "$leaked" "$([[ -e "$requested_dir" ]] && printf yes || printf no)" \ + "$(_k3slr_group_exists "$supervisor_pgid" && printf yes || printf no)" >&2 + return 1 + fi +) + +task4_review5_published_unpinned_cleanup() ( + local database="${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" + local mode prefix rc supervisor_pid supervisor_pgid recorded_identity current_identity caller_pgid + local definition leaked cleanup_rc raw='' state='' + definition="$(declare -f _k3slr_proc_stat_record)" + eval "${definition/_k3slr_proc_stat_record/_task4_review5_original_proc_stat_record}" + export TASK4_REVIEW2_REQUIRE_REAL_TTY=1 TASK4_REVIEW3_SIGNAL_CASE=1 + export TASK4_REVIEW1_PRODUCER_CASE=blocking_signal + _k3slr_command() { task4_review1_command_fake "$@"; } + for mode in missing short malformed persistent; do + cleanup_rc=0 + prefix="${fixture_root}/task4-review5-unpinned-${mode}" + export TASK4_REVIEW1_CONSUMER_LOG="${prefix}.consumer" + export TASK4_REVIEW1_PRODUCER_LOG="${prefix}.producer" + export TASK4_REVIEW1_COMMAND_LOG="${prefix}.commands" + export TASK4_REVIEW1_DB_STAT_LOG="${prefix}.db-stat" + export TASK4_REVIEW3_CALLER_PID_FILE="${prefix}.caller-pid" + export TASK4_REVIEW3_CALLER_PGID_FILE="${prefix}.caller-pgid" + export TASK4_REVIEW3_ROLE_LOG="${prefix}.roles" + export TASK4_REVIEW5_UNPINNED_MODE="$mode" + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_COMMAND_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + : >"$TASK4_REVIEW3_ROLE_LOG" + : >"${prefix}.pid-signals" + : >"${prefix}.group-signals" + : >"${prefix}.waits" + TASK4_REVIEW5_UNPINNED_PID='' + _k3slr_lifecycle_boundary() { + local phase="$1" boundary_pid="${2-}" raw='' pinned='' state='' + if [[ "$phase" == pid-published ]]; then + _task4_review5_original_proc_stat_record raw "$boundary_pid" || return 1 + _k3slr_parse_proc_stat_record pinned state "$boundary_pid" "$raw" || return 1 + printf '%s\n' "$boundary_pid" >"${prefix}.pid" + printf '%s\n' "$pinned" >"${prefix}.identity" + /usr/bin/ps -o pgid= -p "$boundary_pid" | /usr/bin/tr -d '[:space:]' >"${prefix}.pgid" + TASK4_REVIEW5_UNPINNED_PID="$boundary_pid" + fi + } + _k3slr_proc_stat_record() { + local destination_name="$1" queried_pid="$2" + if [[ -n "$TASK4_REVIEW5_UNPINNED_PID" && "$queried_pid" == "$TASK4_REVIEW5_UNPINNED_PID" ]]; then + case "$TASK4_REVIEW5_UNPINNED_MODE" in + missing|persistent) return 1 ;; + short) printf -v "$destination_name" '%s' "${queried_pid} (short) T 1" ;; + malformed) printf -v "$destination_name" '%s' "${queried_pid} (bad) ? 01 02" ;; + esac + return 0 + fi + _task4_review5_original_proc_stat_record "$destination_name" "$queried_pid" + } + _k3slr_signal_published_pid() { + printf '%s %s\n' "$1" "$2" >>"${prefix}.pid-signals" + builtin kill -KILL "$2" + } + _k3slr_signal_group() { + printf '%s %s\n' "$1" "$3" >>"${prefix}.group-signals" + return 99 + } + _k3slr_wait_child() { + printf '%s\n' "$1" >>"${prefix}.waits" + wait "$1" >/dev/null 2>&1 + } + set +e + _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" /dev/null || _k3slr_group_exists "$supervisor_pgid"; then leaked=1; fi + if (( leaked )); then + caller_pgid="$(/usr/bin/ps -o pgid= -p "$BASHPID")" + caller_pgid="${caller_pgid//[[:space:]]/}" + if _task4_review5_original_proc_stat_record raw "$supervisor_pid" 2>/dev/null && + _k3slr_parse_proc_stat_record current_identity state "$supervisor_pid" "$raw" && + [[ "$current_identity" == "$recorded_identity" && "$supervisor_pgid" != "$caller_pgid" ]]; then + builtin kill -KILL "$supervisor_pid" 2>/dev/null || cleanup_rc=$? + else + cleanup_rc=1 + fi + fi + [[ "${cleanup_rc:-0}" -eq 0 ]] || return 1 + if [[ "$rc" -ne 1 || "$leaked" -ne 0 || -s "$TASK4_REVIEW1_CONSUMER_LOG" || + -s "$TASK4_REVIEW3_ROLE_LOG" || -s "${prefix}.group-signals" || + "$(/usr/bin/wc -l <"${prefix}.pid-signals")" -ne 1 || + "$(/usr/bin/wc -l <"${prefix}.waits")" -ne 1 ]]; then + printf 'task4 review5 unpinned diagnostic: mode=%s rc=%s leaked=%s pid_signals=%s group_signals=%s waits=%s\n' \ + "$mode" "$rc" "$leaked" "$(/usr/bin/wc -l <"${prefix}.pid-signals")" \ + "$(/usr/bin/wc -l <"${prefix}.group-signals")" "$(/usr/bin/wc -l <"${prefix}.waits")" >&2 + return 1 + fi + done +) + +task4_review5_pending_priority() ( + local database="${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" + local prefix="${fixture_root}/task4-review5-priority-password" rc requested_dir + export TASK4_REVIEW2_REQUIRE_REAL_TTY=1 TASK4_REVIEW3_SIGNAL_CASE=1 + export TASK4_REVIEW1_PRODUCER_CASE=blocking_signal + export TASK4_REVIEW1_CONSUMER_LOG="${prefix}.consumer" + export TASK4_REVIEW1_PRODUCER_LOG="${prefix}.producer" + export TASK4_REVIEW1_COMMAND_LOG="${prefix}.commands" + export TASK4_REVIEW1_DB_STAT_LOG="${prefix}.db-stat" + export TASK4_REVIEW3_CALLER_PID_FILE="${prefix}.caller-pid" + export TASK4_REVIEW3_CALLER_PGID_FILE="${prefix}.caller-pgid" + export TASK4_REVIEW3_ROLE_LOG="${prefix}.roles" + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_COMMAND_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + : >"$TASK4_REVIEW3_ROLE_LOG" + _k3slr_command() { task4_review1_command_fake "$@"; } + _k3slr_lifecycle_boundary() { + if [[ "$1" == coproc-launch-before ]]; then + builtin kill -TERM "$(<"$TASK4_REVIEW3_CALLER_PID_FILE")" + builtin kill -INT "$(<"$TASK4_REVIEW3_CALLER_PID_FILE")" + fi + } + set +e + _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" &2 + return 1 + fi + + source "$FEASIBILITY_PATH" + requested_dir="/tmp/k3slr-feasibility.priority${BASHPID}${RANDOM}" + export TASK4_FEASIBILITY_CASE=signal_TERM + export TASK4_FEASIBILITY_REQUESTED_DIR="$requested_dir" + export TASK4_FEASIBILITY_COMMAND_LOG="${prefix}.feasibility-commands" + export TASK4_FEASIBILITY_MAIN_PID_FILE="${prefix}.feasibility-main" + : >"$TASK4_FEASIBILITY_COMMAND_LOG" + _k3slr_command() { task4_feasibility_command_fake "$@"; } + _k3slr_lifecycle_boundary() { + if [[ "$1" == feasibility-launch-before ]]; then + builtin kill -TERM "$(<"$TASK4_FEASIBILITY_MAIN_PID_FILE")" + builtin kill -INT "$(<"$TASK4_FEASIBILITY_MAIN_PID_FILE")" + fi + } + set +e + k3slr_local_recovery_feasibility_main --execute /dev/null 2>"${prefix}.feasibility-stderr" + rc=$? + set -e + if [[ "$rc" -ne 143 || -e "$requested_dir" ]]; then + printf 'task4 review5 feasibility priority RED: rc=%s fixture=%s\n' "$rc" \ + "$([[ -e "$requested_dir" ]] && printf yes || printf no)" >&2 + return 1 + fi +) + +task4_review5_feasibility_cleanup_failure_is_not_signal_success() ( + local prefix="${fixture_root}/task4-review5-feas-cleanup-failure" requested_dir + local rc child_pid definition raw='' fail_pid='' + source "$FEASIBILITY_PATH" + requested_dir="/tmp/k3slr-feasibility.pendingfail${BASHPID}${RANDOM}" + export TASK4_FEASIBILITY_CASE=signal_TERM + export TASK4_FEASIBILITY_REQUESTED_DIR="$requested_dir" + export TASK4_FEASIBILITY_COMMAND_LOG="${prefix}.commands" + export TASK4_FEASIBILITY_MAIN_PID_FILE="${prefix}.main" + : >"$TASK4_FEASIBILITY_COMMAND_LOG" + : >"${prefix}.signal-count" + definition="$(declare -f _k3slr_proc_stat_record)" + eval "${definition/_k3slr_proc_stat_record/_task4_review5_feas_original_proc_stat_record}" + _k3slr_command() { task4_feasibility_command_fake "$@"; } + _k3slr_lifecycle_boundary() { + if [[ "$1" == feasibility-pid-published ]]; then + fail_pid="$2" + printf '%s\n' "$2" >"${prefix}.pid" + builtin kill -TERM "$(<"$TASK4_FEASIBILITY_MAIN_PID_FILE")" + fi + } + _k3slr_proc_stat_record() { + if [[ -n "$fail_pid" && "$2" == "$fail_pid" ]]; then return 1; fi + _task4_review5_feas_original_proc_stat_record "$@" + } + _k3slr_signal_published_pid() { + printf 'call\n' >>"${prefix}.signal-count" + if [[ "$(/usr/bin/wc -l <"${prefix}.signal-count")" -eq 1 ]]; then return 1; fi + builtin kill -KILL "$2" + } + set +e + k3slr_local_recovery_feasibility_main --execute /dev/null 2>"${prefix}.stderr" + rc=$? + set -e + child_pid="$(<"${prefix}.pid")" + if [[ "$rc" -ne 1 || "$(/usr/bin/wc -l <"${prefix}.signal-count")" -ne 2 || + -e "$requested_dir" ]] || kill -0 "$child_pid" 2>/dev/null; then + printf 'task4 review5 feasibility cleanup diagnostic: rc=%s calls=%s child_live=%s fixture=%s\n' \ + "$rc" "$(/usr/bin/wc -l <"${prefix}.signal-count")" \ + "$([[ -e /proc/$child_pid ]] && printf yes || printf no)" \ + "$([[ -e "$requested_dir" ]] && printf yes || printf no)" >&2 + return 1 + fi +) + +task4_review2_tty_and_bounded_validated_pipe() ( + local database="${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" producer_case + export TASK4_REVIEW1_CONSUMER_LOG="${fixture_root}/task4-review2-consumer.log" + export TASK4_REVIEW1_PRODUCER_LOG="${fixture_root}/task4-review2-producer.log" + export TASK4_REVIEW1_COMMAND_LOG="${fixture_root}/task4-review2-command.log" + export TASK4_REVIEW1_DB_STAT_LOG="${fixture_root}/task4-review2-db-stat.log" + export TASK4_REVIEW2_REQUIRE_REAL_TTY=1 + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_COMMAND_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + _k3slr_command() { task4_review1_command_fake "$@"; } + + TASK4_REVIEW1_PRODUCER_CASE=success + export TASK4_REVIEW1_PRODUCER_CASE + if ! _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" &2 + /usr/bin/tail -n 30 "$TASK4_REVIEW1_COMMAND_LOG" >&2 + return 1 + fi + [[ "$(/usr/bin/wc -l <"$TASK4_REVIEW1_CONSUMER_LOG")" -eq 1 ]] || return 1 + /usr/bin/grep -q 'result=tty' "$TASK4_REVIEW1_COMMAND_LOG" || return 1 + + : >"$TASK4_REVIEW1_CONSUMER_LOG" + for producer_case in zero_success missing_lf double_lf invalid_class malformed_length oversized_success; do + TASK4_REVIEW1_PRODUCER_CASE="$producer_case" + export TASK4_REVIEW1_PRODUCER_CASE + assert_fails _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" "$command_log" + : >"$stdout_file" + : >"$stderr_file" + # shellcheck source=/dev/null + source "$LIBRARY_PATH" + _k3slr_command() { + printf '%s\0' "$@" >>"$command_log" + case "$#:$1:${2-}:${3-}:${4-}:${5-}" in + '3:/usr/bin/test:-t:0::') return 0 ;; + '7:/usr/bin/keepassxc-cli:show:--show-protected:--attributes:Password') + [[ "${6-}" == /fixture/recovery.kdbx && "${7-}" == 'K3s Recovery LUKS' ]] || return 97 + printf '%s\n' "$secret_sentinel" + ;; + '11:/usr/bin/sudo:--non-interactive:--:/usr/sbin/cryptsetup:open') + [[ "${6-}" == --readonly && "${7-}" == --type && "${8-}" == luks2 && + "${9-}" == --key-file=- && "${10-}" == /dev/loop23 && + "${11-}" == k3s-recovery-proof ]] || return 97 + IFS= read -r _ || return 1 + ;; + *) return 97 ;; + esac + } + _k3slr_keepass_password_stdout /fixture/recovery.kdbx 'K3s Recovery LUKS' \ + >/dev/null 2>"$stderr_file" || rc=$? + if (( rc != 0 )); then + mapfile -d '' -t task1_red_argv <"$command_log" + [[ " ${task1_red_argv[*]} " == *' /usr/bin/keepassxc-cli show --quiet '* && + " ${task1_red_argv[*]} " != *' --show-protected '* ]] || + fail 'producer RED did not reach the legacy KeePass argv missing --show-protected' + fi + assert_eq 0 "$rc" 'protected KeePass producer uses the protected Password argv' + rc=0 + _k3slr_cryptsetup_open_readonly_stdin /dev/loop23 k3s-recovery-proof \ + < <(printf '%s\n' "$secret_sentinel") >"$stdout_file" 2>>"$stderr_file" || rc=$? + assert_eq 0 "$rc" 'read-only LUKS consumer succeeds' + export TASK4_REVIEW1_CONSUMER_LOG="${fixture_root}/task1-readonly-pipe.consumer" + export TASK4_REVIEW1_PRODUCER_LOG="${fixture_root}/task1-readonly-pipe.producer" + export TASK4_REVIEW1_COMMAND_LOG="${fixture_root}/task1-readonly-pipe.commands" + export TASK4_REVIEW1_DB_STAT_LOG="${fixture_root}/task1-readonly-pipe.db-stat" + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_COMMAND_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + TASK4_REVIEW1_PRODUCER_CASE=success + TASK4_REVIEW1_VALIDATION_CASE='' + export TASK4_REVIEW1_PRODUCER_CASE TASK4_REVIEW1_VALIDATION_CASE + _k3slr_command() { task4_review1_command_fake "$@"; } + assert_fails _k3slr_luks_open_readonly_from_keepass /fixture/recovery.kdbx \ + 'K3s Recovery LUKS' /dev/loop23 + assert_fails _k3slr_luks_open_readonly_from_keepass /fixture/recovery.kdbx \ + 'K3s Recovery LUKS' /dev/loop23 k3s-recovery-proof extra + rc=0 + _k3slr_luks_open_readonly_from_keepass \ + "${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" 'K3s Recovery LUKS' \ + /dev/loop7 "$K3SLR_PROOF_MAPPING_NAME" "$command_log" + : >"$trace_file" + # shellcheck source=/dev/null + source "$FEASIBILITY_PATH" + _k3slrf_require_interactive_stdin() { return 0; } + _k3slrf_verify_prerequisites() { return 0; } + _k3slrf_require_cached_sudo() { return 0; } + _k3slrf_create_fixture_dir() { + printf -v "$1" '%s' "$case_root/database" + printf -v "$2" '%s' fixture-directory + printf -v "$3" '%s' true + mkdir -p -- "$case_root/database" + } + _k3slrf_cleanup_fixture_dir() { [[ "$1" == "$case_root/database" ]]; } + _k3slrf_directory_identity() { printf -v "$1" '%s' fixture-directory; } + _k3slrf_directory_identity_matches() { [[ "$2" == fixture-directory ]]; } + _k3slrf_file_identity() { [[ -f "$2" ]] && printf -v "$1" '%s' "file:$2"; } + _k3slrf_file_identity_matches() { [[ -f "$1" && "$2" == "file:$1" ]]; } + _k3slrf_run_tracked() { + ( "$@" ) & + local child=$! + wait "$child" + } + _k3slrf_validate_generated_password_stdin() { + local received='' + IFS= read -r received + [[ "$received" == "$secret_sentinel" ]] + } + _k3slr_command() { + local database='' payload='' + printf '%s\0' "$@" >>"$command_log" + case "$1:${2-}" in + '/usr/bin/sha256sum:') + (( $# == 1 )) || return 97 + printf 'master-hash\n' >>"$trace_file" + while IFS= read -r _; do :; done + printf '%064d -\n' 0 + ;; + '/usr/bin/mawk:'*) + (( $# == 2 )) && [[ "${2-}" == '{ print $1 }' ]] || return 97 + printf 'master-field\n' >>"$trace_file" + while IFS= read -r _; do :; done + printf '%s\n' fixture-master + ;; + '/usr/bin/stat:--format=%s') + (( $# == 4 )) && [[ "${3-}" == -- && + "${4-}" == "${case_root}/database/keepass-listing."*.stdout ]] || return 97 + /usr/bin/stat --format='%s' -- "${4}" + ;; + '/usr/bin/rm:--') + (( $# == 4 )) && [[ "${3-}" == "${case_root}/database/keepass-listing."*.stdout && + "${4-}" == "${case_root}/database/keepass-listing."*.stderr ]] || return 97 + /usr/bin/rm -- "${3}" "${4}" + ;; + '/usr/bin/keepassxc-cli:db-create') + (( $# == 5 )) && [[ "${3-}" == --quiet && "${4-}" == --set-password && "${5-}" == */synthetic.kdbx ]] || return 97 + while IFS= read -r _; do :; done + : >"${5}" + printf 'db-create\n' >>"$trace_file" + ;; + '/usr/bin/keepassxc-cli:add') + (( $# == 12 )) && [[ "${3-}" == --quiet && "${4-}" == --generate && "${5-}" == --length && + "${6-}" == 40 && "${7-}" == --lower && "${8-}" == --upper && + "${9-}" == --numeric && "${10-}" == --every-group && + "${11-}" == */synthetic.kdbx && "${12-}" == 'K3s Recovery LUKS' ]] || return 97 + while IFS= read -r _; do :; done + printf '%s\n' entry >"${11}" + printf 'entry-add\n' >>"$trace_file" + ;; + '/usr/bin/keepassxc-cli:show') + if (( $# == 7 )); then + [[ "${4-}" == --attributes && "${5-}" == Password && + "${6-}" == */synthetic.kdbx && "${7-}" == 'K3s Recovery LUKS' ]] || return 97 + if [[ "${3-}" == --show-protected ]]; then + printf 'password-show-protected\n' >>"$trace_file" + while IFS= read -r _; do :; done + printf '%s\n' "$secret_sentinel" + elif [[ "${3-}" == --quiet ]]; then + printf 'password-show-legacy\n' >>"$trace_file" + return 97 + else + return 97 + fi + elif (( $# == 8 )); then + [[ "${3-}" == --quiet && "${4-}" == --attributes && "${5-}" == Title && + "${6-}" == --show-attachments && "${7-}" == */synthetic.kdbx && + "${8-}" == 'K3s Recovery LUKS' ]] || return 97 + while IFS= read -r _; do :; done + if [[ -f "${7}.attachment-state" ]]; then + printf 'K3s Recovery LUKS\n\nAttachments:\n fixture.bin (30.0 B)\n' + else + printf 'K3s Recovery LUKS\n\nNo attachments present.\n' + fi + else + return 97 + fi + ;; + '/usr/bin/keepassxc-cli:attachment-import') + (( $# == 7 )) && [[ "${3-}" == --quiet && "${4-}" == */synthetic.kdbx && + "${5-}" == 'K3s Recovery LUKS' && "${6-}" == fixture.bin && "${7-}" == */input.bin ]] || return 97 + while IFS= read -r _; do :; done + printf '%s\n' attached >"${4}.attachment-state" + IFS= read -r payload <"${7}" || return 1 + printf '%s\n' "$payload" >"${4}.attachment-data" + ;; + '/usr/bin/keepassxc-cli:attachment-export') + (( $# == 7 )) && [[ "${3-}" == --quiet && "${4-}" == */synthetic.kdbx && + "${5-}" == 'K3s Recovery LUKS' && "${6-}" == fixture.bin && "${7-}" == /proc/*/fd/* ]] || return 97 + while IFS= read -r _; do :; done + IFS= read -r payload <"${4}.attachment-data" || return 1 + printf '%s\n' "$payload" >"${7}" + ;; + '/usr/bin/cmp:--silent') (( $# == 5 )) && [[ "${3-}" == -- && "${4-}" == */input.bin && "${5-}" == */output.bin ]] || return 97 ;; + *) return 97 ;; + esac + } + k3slr_local_recovery_feasibility_main --execute >"$stdout_file" 2>"$stderr_file" || rc=$? + if (( rc != 0 )); then + local task1_feas_index=0 task1_feas_legacy_seen=false task1_feas_protected_seen=false + mapfile -d '' -t task1_feas_red_argv <"$command_log" + for (( task1_feas_index=0; task1_feas_index+6<${#task1_feas_red_argv[@]}; task1_feas_index++ )); do + if [[ "${task1_feas_red_argv[task1_feas_index]}" == /usr/bin/keepassxc-cli && + "${task1_feas_red_argv[task1_feas_index + 1]}" == show && + "${task1_feas_red_argv[task1_feas_index + 2]}" == --show-protected && + "${task1_feas_red_argv[task1_feas_index + 3]}" == --attributes && + "${task1_feas_red_argv[task1_feas_index + 4]}" == Password && + "${task1_feas_red_argv[task1_feas_index + 6]}" == 'K3s Recovery LUKS' ]]; then + task1_feas_protected_seen=true + break + elif [[ "${task1_feas_red_argv[task1_feas_index]}" == /usr/bin/keepassxc-cli && + "${task1_feas_red_argv[task1_feas_index + 1]}" == show && + "${task1_feas_red_argv[task1_feas_index + 2]}" == --quiet && + "${task1_feas_red_argv[task1_feas_index + 3]}" == --attributes && + "${task1_feas_red_argv[task1_feas_index + 4]}" == Password && + "${task1_feas_red_argv[task1_feas_index + 6]}" == 'K3s Recovery LUKS' ]]; then + task1_feas_legacy_seen=true + break + fi + done + if "$task1_feas_protected_seen"; then + fail "feasibility failed after the protected password stage (stages: $(/usr/bin/tr '\n' ',' <"$trace_file"))" + fi + "$task1_feas_legacy_seen" || fail "feasibility RED did not reach the exact legacy password show argv (stages: $(/usr/bin/tr '\n' ',' <"$trace_file"))" + /usr/bin/grep -Fqx password-show-legacy "$trace_file" || + fail 'feasibility RED did not persist the legacy password stage' + fi + assert_eq 0 "$rc" 'feasibility password check uses the protected Password argv' + ! /usr/bin/grep -Fq -- "$secret_sentinel" "$stdout_file" "$stderr_file" "$command_log" || + fail 'feasibility fixture leaked the password sentinel' + mapfile -d '' -t task1_feas_argv <"$command_log" + [[ " ${task1_feas_argv[*]} " == *' --show-protected '* ]] || + fail 'feasibility did not request the protected Password field' +) + +# The raw-NUL record is emitted only by the exact modeled show-attachments +# command. It is never placed in a Bash variable or command substitution by +# this fixture; feasibility must capture and reject it at its real boundary. +task4_review2_feasibility_attachment_listing() ( + local case_root="${fixture_root}/task1-feas-listing" + local command_log="${case_root}/commands.nul" + local trace_file="${case_root}/trace" + local stdout_file="${case_root}/stdout" + local stderr_file="${case_root}/stderr" + local listing_case=raw_nul rc=0 state='' + mkdir -p -- "$case_root" + : >"$command_log" + : >"$trace_file" + # shellcheck source=/dev/null + source "$FEASIBILITY_PATH" + _k3slr_parse_keepass_attachment_listing() { + (( $# == 3 )) || return 97 + case "$1" in + $'K3s Recovery LUKS\n\nNo attachments present.\n') + printf -v "$3" '%s' absent + printf 'parser:absent\n' >>"$trace_file" + ;; + $'K3s Recovery LUKS\n\nAttachments:\n fixture.bin (30.0 B)\n') + printf -v "$3" '%s' present + printf 'parser:present\n' >>"$trace_file" + ;; + *) + printf 'parser:rejected\n' >>"$trace_file" + return 1 + ;; + esac + } + _k3slrf_require_interactive_stdin() { return 0; } + _k3slrf_verify_prerequisites() { return 0; } + _k3slrf_require_cached_sudo() { return 0; } + _k3slrf_create_fixture_dir() { + local new_dir="${case_root}/database-${RANDOM}" + mkdir -p -- "$new_dir" + printf -v "$1" '%s' "$new_dir" + printf -v "$2" '%s' fixture-directory + printf -v "$3" '%s' true + } + _k3slrf_cleanup_fixture_dir() { [[ "$1" == "${case_root}"/database-* ]]; } + _k3slrf_directory_identity() { printf -v "$1" '%s' fixture-directory; } + _k3slrf_directory_identity_matches() { [[ "$2" == fixture-directory ]]; } + _k3slrf_file_identity() { [[ -f "$2" ]] && printf -v "$1" '%s' "file:$2"; } + _k3slrf_file_identity_matches() { [[ -f "$1" && "$2" == "file:$1" ]]; } + _k3slrf_run_tracked() { + ( "$@" ) & + local child=$! + wait "$child" + } + _k3slrf_validate_generated_password_stdin() { IFS= read -r _; } + _k3slr_command() { + local payload='' + printf '%s\0' "$@" >>"$command_log" + case "$1:${2-}" in + '/usr/bin/sha256sum:') + (( $# == 1 )) || return 97 + while IFS= read -r _; do :; done + printf '%064d -\n' 0 + ;; + '/usr/bin/mawk:'*) + (( $# == 2 )) && [[ "${2-}" == '{ print $1 }' ]] || return 97 + while IFS= read -r _; do :; done + printf '%s\n' fixture-master + ;; + '/usr/bin/stat:--format=%s') + (( $# == 4 )) && [[ "${3-}" == -- && + "${4-}" == "${case_root}/database-"*/keepass-listing.*.stdout ]] || return 97 + /usr/bin/stat --format='%s' -- "${4}" + ;; + '/usr/bin/rm:--') + (( $# == 4 )) && [[ "${3-}" == "${case_root}/database-"*/keepass-listing.*.stdout && + "${4-}" == "${case_root}/database-"*/keepass-listing.*.stderr ]] || return 97 + /usr/bin/rm -- "${3}" "${4}" + ;; + '/usr/bin/keepassxc-cli:db-create') + (( $# == 5 )) && [[ "${3-}" == --quiet && "${4-}" == --set-password && "${5-}" == */synthetic.kdbx ]] || return 97 + while IFS= read -r _; do :; done + : >"${5}" + ;; + '/usr/bin/keepassxc-cli:add') + (( $# == 12 )) && [[ "${3-}" == --quiet && "${4-}" == --generate && "${5-}" == --length && + "${6-}" == 40 && "${7-}" == --lower && "${8-}" == --upper && + "${9-}" == --numeric && "${10-}" == --every-group && "${11-}" == */synthetic.kdbx && + "${12-}" == 'K3s Recovery LUKS' ]] || return 97 + while IFS= read -r _; do :; done + : >"${11}.entry" + ;; + '/usr/bin/keepassxc-cli:show') + if (( $# == 7 )); then + [[ ( "${3-}" == --quiet || "${3-}" == --show-protected ) && "${4-}" == --attributes && + "${5-}" == Password && "${6-}" == */synthetic.kdbx && + "${7-}" == 'K3s Recovery LUKS' ]] || return 97 + while IFS= read -r _; do :; done + printf '%s\n' 'Aa0Bb1Cc2Dd3Ee4Ff5Gg6Hh7Ii8Jj9Kk0Ll1Mm2N' + elif (( $# == 8 )); then + [[ "${3-}" == --quiet && "${4-}" == --attributes && "${5-}" == Title && + "${6-}" == --show-attachments && "${7-}" == */synthetic.kdbx && + "${8-}" == 'K3s Recovery LUKS' ]] || return 97 + while IFS= read -r _; do :; done + case "$listing_case" in + raw_nul) + printf 'listing:raw-nul\n' >>"$trace_file" + printf 'K3s Recovery LUKS\n\nAttachments:\n fixture.bin\0 (30.0 B)\n' + ;; + absent) + if [[ -f "${7}.attachment-state" ]]; then + printf 'listing:present\n' >>"$trace_file" + printf 'K3s Recovery LUKS\n\nAttachments:\n fixture.bin (30.0 B)\n' + else + printf 'listing:absent\n' >>"$trace_file" + printf 'K3s Recovery LUKS\n\nNo attachments present.\n' + fi + ;; + present) + printf 'listing:present\n' >>"$trace_file" + printf 'K3s Recovery LUKS\n\nAttachments:\n fixture.bin (30.0 B)\n' + ;; + size_8192) + printf 'listing:size-8192\n' >>"$trace_file" + printf 'K3s Recovery LUKS\n\n%08172d\n' 0 + ;; + size_8193) + printf 'listing:size-8193\n' >>"$trace_file" + printf 'K3s Recovery LUKS\n\n%08173d\n' 0 + ;; + *) return 97 ;; + esac + else + return 97 + fi + ;; + '/usr/bin/keepassxc-cli:attachment-import') + (( $# == 7 )) && [[ "${3-}" == --quiet && "${4-}" == */synthetic.kdbx && + "${5-}" == 'K3s Recovery LUKS' && "${6-}" == fixture.bin && "${7-}" == */input.bin ]] || return 97 + while IFS= read -r _; do :; done + IFS= read -r payload <"${7}" || return 1 + printf '%s\n' "$payload" >"${4}.attachment-data" + : >"${4}.attachment-state" + printf 'import\n' >>"$trace_file" + ;; + '/usr/bin/keepassxc-cli:attachment-export') + (( $# == 7 )) && [[ "${3-}" == --quiet && "${4-}" == */synthetic.kdbx && + "${5-}" == 'K3s Recovery LUKS' && "${6-}" == fixture.bin && "${7-}" == /proc/*/fd/* ]] || return 97 + while IFS= read -r _; do :; done + IFS= read -r payload <"${4}.attachment-data" || return 1 + printf '%s\n' "$payload" >"${7}" + printf 'export\n' >>"$trace_file" + ;; + '/usr/bin/cmp:--silent') (( $# == 5 )) && [[ "${3-}" == -- && "${4-}" == */input.bin && "${5-}" == */output.bin ]] || return 97 ;; + *) return 97 ;; + esac + } + set +e + k3slr_local_recovery_feasibility_main --execute >"$stdout_file" 2>"$stderr_file" + rc=$? + set -e + assert_eq 1 "$(/usr/bin/grep -Fxc 'listing:raw-nul' "$trace_file" || :)" \ + 'raw-NUL path dispatches exactly one attachment listing' + assert_eq 0 "$(/usr/bin/grep -Fc 'parser:' "$trace_file" || :)" \ + 'raw-NUL path reaches no parser' + assert_eq 0 "$(/usr/bin/grep -Fxc import "$trace_file" || :)" \ + 'raw-NUL path reaches no import' + assert_eq 0 "$(/usr/bin/grep -Fxc export "$trace_file" || :)" \ + 'raw-NUL path reaches no export' + ! /usr/bin/grep -Fiq -- 'null byte' "$stderr_file" || + fail 'raw-NUL capture emitted a Bash null-byte warning' + (( rc != 0 )) || fail 'raw-NUL attachment listing was accepted before bounded capture rejected it' + for listing_case in size_8192 size_8193; do + : >"$trace_file" + set +e + k3slr_local_recovery_feasibility_main --execute >>"$stdout_file" 2>>"$stderr_file" + rc=$? + set -e + (( rc != 0 )) || fail "$listing_case attachment listing was accepted" + if [[ "$listing_case" == size_8192 ]]; then + assert_eq $'listing:size-8192\nparser:rejected' "$(<"$trace_file")" \ + '8192-byte listing reaches parser and is rejected only by grammar' + else + assert_eq 'listing:size-8193' "$(<"$trace_file")" \ + '8193-byte listing is rejected by the capture size gate before parser' + fi + done + : >"$trace_file" + listing_case=absent + assert_succeeds k3slr_local_recovery_feasibility_main --execute >>"$stdout_file" 2>>"$stderr_file" + assert_eq $'listing:absent\nparser:absent\nimport\nlisting:present\nparser:present\nexport' \ + "$(<"$trace_file")" 'normal feasibility trace is absent to present with exact parser ordering' + ! /usr/bin/grep -Fq -- 'Aa0Bb1Cc2Dd3Ee4Ff5Gg6Hh7Ii8Jj9Kk0Ll1Mm2N' \ + "$stdout_file" "$stderr_file" "$command_log" || fail 'attachment fixture leaked password sentinel' +) + +task4_review2_attachment_parser_contract() ( + local state=unchanged record='' + # shellcheck source=/dev/null + source "$LIBRARY_PATH" + assert_succeeds _k3slr_parse_keepass_attachment_listing \ + $'K3s Recovery LUKS\n\nNo attachments present.\n' fixture.bin state + assert_eq absent "$state" 'canonical KeePassXC absent record parses' + assert_succeeds _k3slr_parse_keepass_attachment_listing \ + $'K3s Recovery LUKS\n\nAttachments:\n fixture.bin (30.0 B)\n' fixture.bin state + assert_eq present "$state" 'canonical KeePassXC present record parses' + for record in \ + $'K3s Recovery LUKS\n\nAttachments:\n archive.bin (1.0 KiB)\n' \ + $'K3s Recovery LUKS\n\nAttachments:\n archive.bin (2.5 MiB)\n' \ + $'K3s Recovery LUKS\n\nAttachments:\n archive.bin (3.0 GiB)\n'; do + state=unchanged + assert_succeeds _k3slr_parse_keepass_attachment_listing "$record" archive.bin state + assert_eq present "$state" 'parser accepts exact non-fixture attachment and IEC size unit' + state=unchanged + assert_fails _k3slr_parse_keepass_attachment_listing "$record" fixture.bin state + assert_eq unchanged "$state" 'non-target attachment leaves output unchanged' + done + for record in '' $'K3s Recovery LUKS\n\nAttachments:\n fixture.bin (30.0 B)\n fixture.bin (30.0 B)\n' \ + $'K3s Recovery LUKS\r\n\r\nNo attachments present.\r\n' \ + $'Title: K3s Recovery LUKS\n\nAttachments:\n fixture.bin (30.0 B)\n' \ + $'K3s Recovery LUKS\n\nAttachments:\n fixture.bin.old (30.0 B)\n' \ + $'K3s Recovery LUKS\n\nNo attachments present.\nextra\n' \ + $'K3s Recovery LUKS\n\nNo attachments present.'; do + state=unchanged + assert_fails _k3slr_parse_keepass_attachment_listing "$record" fixture.bin state + assert_eq unchanged "$state" 'malformed attachment record leaves output state unchanged' + done + printf -v record 'K3s Recovery LUKS\n\n%08172d\n' 0 + assert_eq 8192 "${#record}" '8192-byte parser record is final-LF framed' + state=unchanged + assert_fails _k3slr_parse_keepass_attachment_listing "$record" fixture.bin state + assert_eq unchanged "$state" '8192-byte grammar failure leaves output unchanged' + printf -v record 'K3s Recovery LUKS\n\n%08173d\n' 0 + assert_eq 8193 "${#record}" '8193-byte parser record is final-LF framed' + state=unchanged + assert_fails _k3slr_parse_keepass_attachment_listing "$record" fixture.bin state + assert_eq unchanged "$state" 'oversize parser record leaves output unchanged' + assert_fails _k3slr_parse_keepass_attachment_listing \ + $'K3s Recovery LUKS\n\nNo attachments present.\n' fixture.bin unsafe-name + state=unchanged + assert_fails _k3slr_parse_keepass_attachment_listing \ + $'K3s Recovery LUKS\n\nNo attachments present.\n' fixture.bin _k3slr_attachment_state + assert_eq unchanged "$state" 'reserved production-local output name cannot mutate caller state' + record=$'K3s Recovery LUKS\n\nNo attachments present.\n' + assert_fails _k3slr_parse_keepass_attachment_listing \ + "$record" fixture.bin _k3slr_attachment_record + assert_eq $'K3s Recovery LUKS\n\nNo attachments present.\n' "$record" \ + 'reserved record-local output name leaves the caller record unchanged' +) + +task4_review1_two_sentinel_leak_contract() ( + local first='Aa0Bb1Cc2Dd3Ee4Ff5Gg6Hh7Ii8Jj9Kk0Ll1Mm2N' + local second='Zz9Yy8Xx7Ww6Vv5Uu4Tt3Ss2Rr1Qq0Pp8Oo7Nn6M' + local value uid sink + [[ "$first" != "$second" && "${#first}" -eq 40 && "${#second}" -eq 40 ]] || return 1 + for value in "$first" "$second"; do + [[ "$value" =~ [a-z] && "$value" =~ [A-Z] && "$value" =~ [0-9] ]] || return 1 + for sink in "${fixture_root}/task4-child.cmdline" "${fixture_root}/task4-child.environ" \ + "${fixture_root}"/task4-*.stdout "${fixture_root}"/task4-*.stderr; do + [[ -e "$sink" ]] || continue + ! /usr/bin/grep -Fq -- "$value" "$sink" || return 1 + done + uid="$(/usr/bin/id -u)" + ! /usr/bin/find /tmp /run -xdev -user "$uid" -name "*${value}*" -print -quit 2>/dev/null | + /usr/bin/grep -q . || return 1 + ! /usr/bin/find /tmp /run -xdev -user "$uid" -type f -readable \ + ! -path "${fixture_root}/*" -exec /usr/bin/grep -IlF -- "$value" '{}' + \ + 2>/dev/null | /usr/bin/grep -q . || return 1 + done +) + +task4_review1_preflight_and_lineage_gate() ( + local database="${K3SLR_OUTER_MOUNT}/${K3SLR_DATABASE_RELATIVE}" validation_case + export TASK4_REVIEW1_CONSUMER_LOG="${fixture_root}/task4-review1-gate-consumer.log" + export TASK4_REVIEW1_PRODUCER_LOG="${fixture_root}/task4-review1-gate-producer.log" + export TASK4_REVIEW1_COMMAND_LOG="${fixture_root}/task4-review1-gate-command.log" + export TASK4_REVIEW1_DB_STAT_LOG="${fixture_root}/task4-review1-gate-db-stat.log" + _k3slr_command() { task4_review1_command_fake "$@"; } + TASK4_REVIEW1_PRODUCER_CASE=success + export TASK4_REVIEW1_PRODUCER_CASE + for validation_case in non_tty parent_symlink; do + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_COMMAND_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + TASK4_REVIEW1_VALIDATION_CASE="$validation_case" + export TASK4_REVIEW1_VALIDATION_CASE + assert_fails _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" + [[ ! -s "$TASK4_REVIEW1_PRODUCER_LOG" && ! -s "$TASK4_REVIEW1_CONSUMER_LOG" ]] || return 1 + done + : >"$TASK4_REVIEW1_CONSUMER_LOG" + : >"$TASK4_REVIEW1_PRODUCER_LOG" + : >"$TASK4_REVIEW1_DB_STAT_LOG" + TASK4_REVIEW1_VALIDATION_CASE=post_drift + export TASK4_REVIEW1_VALIDATION_CASE + assert_fails _k3slr_luks_open_from_keepass "$database" "$K3SLR_KEEPASS_ENTRY" /dev/loop7 "$K3SLR_MAPPING_NAME" + [[ "$(/usr/bin/wc -l <"$TASK4_REVIEW1_PRODUCER_LOG")" -eq 1 && ! -s "$TASK4_REVIEW1_CONSUMER_LOG" ]] +) + +task4_review1_exact_constants_become_readonly() ( + local assignment + /usr/bin/bash -c ' + K3SLR_KEEPASS_ENTRY="K3s Recovery LUKS" + K3SLR_KEEPASS_HEADER_ATTACHMENT="luks-header-backup.bin" + _K3SLR_INT64_MAX=9223372036854775807 + _K3SLR_REQUIRED_MINIMUM_FREE_BYTES=10737418240 + source "$1" + ' task4-readonly "$LIBRARY_PATH" + for assignment in 'K3SLR_KEEPASS_ENTRY=changed' \ + 'K3SLR_KEEPASS_HEADER_ATTACHMENT=changed' '_K3SLR_INT64_MAX=1' \ + '_K3SLR_REQUIRED_MINIMUM_FREE_BYTES=1'; do + ! /usr/bin/bash -c ' + K3SLR_KEEPASS_ENTRY="K3s Recovery LUKS" + K3SLR_KEEPASS_HEADER_ATTACHMENT="luks-header-backup.bin" + _K3SLR_INT64_MAX=9223372036854775807 + _K3SLR_REQUIRED_MINIMUM_FREE_BYTES=10737418240 + source "$1" + eval "$2" + ' task4-readonly "$LIBRARY_PATH" "$assignment" 2>/dev/null || return 1 + done +) + +task4_review1_generated_password_exact_lf() ( + # shellcheck source=/dev/null + source "$FEASIBILITY_PATH" + if ! printf '%s\n' 'Aa0Bb1Cc2Dd3Ee4Ff5Gg6Hh7Ii8Jj9Kk0Ll1Mm2N' | + _k3slrf_validate_generated_password_stdin; then printf 'exact-lf valid rejected\n' >&2; return 1; fi + if printf '%s' 'Aa0Bb1Cc2Dd3Ee4Ff5Gg6Hh7Ii8Jj9Kk0Ll1Mm2N' | + _k3slrf_validate_generated_password_stdin; then printf 'exact-lf missing accepted\n' >&2; return 1; fi + if printf '%s\n\n' 'Aa0Bb1Cc2Dd3Ee4Ff5Gg6Hh7Ii8Jj9Kk0Ll1Mm2N' | + _k3slrf_validate_generated_password_stdin; then printf 'exact-lf double accepted\n' >&2; return 1; fi +) + +task5a_wrapper_and_guard_contract() ( + local wrapper main_name mode output status trace_output + + for wrapper in "$PREPARE_WRAPPER_PATH" "$OPEN_WRAPPER_PATH" "$CLOSE_WRAPPER_PATH"; do + [[ -f "$wrapper" && ! -L "$wrapper" ]] || { + printf 'missing Task 5A lifecycle wrapper: %s\n' "$wrapper" >&2 + return 1 + } + assert_succeeds /usr/bin/bash -n "$wrapper" + done + + for mode in prepare open close; do + case "$mode" in + prepare) wrapper="$PREPARE_WRAPPER_PATH" ;; + open) wrapper="$OPEN_WRAPPER_PATH" ;; + close) wrapper="$CLOSE_WRAPPER_PATH" ;; + esac + main_name="_k3slr_${mode}_main" + + output="$({ + PATH=/tmp/untrusted-path + LC_ALL=POSIX + umask 022 + set -- --execute + # shellcheck source=/dev/null + source "$wrapper" + declare -F "$main_name" >/dev/null || return 1 + declare -F _k3slr_wrapper_initial_guard >/dev/null || return 1 + _k3slr_wrapper_initial_guard 0 '' >/dev/null 2>&1 && return 1 + _k3slr_wrapper_initial_guard 1000 x >/dev/null 2>&1 && return 1 + _k3slr_wrapper_initial_guard 1000 '' || return 1 + printf '%s|%s|' "$PATH" "$LC_ALL" + umask + } 2>/dev/null)" || return 1 + assert_eq '/usr/sbin:/usr/bin:/sbin:/bin|C|0077' "$output" \ + "$mode wrapper establishes its early source-safe process guard" + + trace_output="${fixture_root}/task5a-${mode}-xtrace.stderr" + if /usr/bin/bash -x "$wrapper" 2>"$trace_output"; then + printf '%s wrapper accepted xtrace\n' "$mode" >&2 + return 1 + fi + ! /usr/bin/grep -Fq -- 'scripts/lib/k3s-local-recovery.sh' "$trace_output" || { + printf '%s wrapper discovered/sourced the repository before xtrace refusal\n' "$mode" >&2 + return 1 + } + + output="$({ + # shellcheck source=/dev/null + source "$wrapper" + _k3slr_lifecycle_main() { + (( $# == 2 )) || return 1 + printf '%s|%s\n' "$1" "$2" + } + "$main_name" + } 2>/dev/null)" || return 1 + assert_eq "${mode}|dry-run" "$output" "$mode no-argument parser selects dry-run" + + for bad_args in '--help' '-h' '--execute --execute' '--unknown' 'positional'; do + # This deliberate word split enumerates fixed test arguments, never input. + # shellcheck disable=SC2086 + if output="$({ source "$wrapper"; "$main_name" $bad_args; } 2>&1)"; then + printf '%s wrapper accepted invalid argv: %s\n' "$mode" "$bad_args" >&2 + return 1 + else + status=$? + fi + [[ "$status" == 2 && "$output" == *'Usage:'* ]] || { + printf '%s wrapper invalid argv did not produce usage exit 2: %s\n' "$mode" "$bad_args" >&2 + return 1 + } + done + done + + assert_fails _k3slr_require_execute_tty +) + +task5a_prepare_provider_capability_gate() ( + local log="${fixture_root}/task5a-prepare-capability.log" + + _k3slr_prove_header_restore() { :; } + export -f _k3slr_prove_header_restore + /usr/bin/bash -c ' + source "$1" + ! declare -F _k3slr_prove_header_restore >/dev/null + ' task5a-provider-cleanup "$PREPARE_WRAPPER_PATH" || { + printf 'prepare wrapper retained an inherited/exported provider\n' >&2 + return 1 + } + unset -f _k3slr_prove_header_restore + + # shellcheck source=/dev/null + source "$PREPARE_WRAPPER_PATH" + _k3slr_load_contract() { + (( $# == 1 )) || return 1 + printf 'contract\n' >>"$log" + } + _k3slr_require_execute_tty() { + printf 'tty\n' >>"$log" + return 1 + } + _k3slr_lifecycle_step() { + printf 'step:%s\n' "$1" >>"$log" + return 1 + } + _k3slr_lifecycle_dispatch() { + printf 'dispatch\n' >>"$log" + return 1 + } + _k3slr_command() { + printf 'unexpected-command:%s\n' "$*" >>"$log" + return 97 + } + + assert_fails _k3slr_prepare_main --execute + assert_eq contract "$(<"$log")" \ + 'prepare provider absence refuses after contract and before every later boundary' +) + +task5a_open_close_capability_gate() ( + local mode wrapper main_name state missing_binary log output expected line + local -a trusted_binaries=( + /usr/bin/keepassxc-cli + /usr/sbin/cryptsetup + /usr/sbin/losetup + /usr/sbin/blkid + /usr/sbin/smartctl + /usr/bin/mount + /usr/bin/umount + /usr/bin/findmnt + /usr/bin/lsblk + /usr/bin/readlink + /usr/bin/stat + /usr/bin/ntfsinfo + /usr/bin/ntfs-3g.probe + /usr/bin/fuser + /usr/bin/sync + /usr/bin/cmp + /usr/bin/cat + /usr/bin/sha256sum + /usr/bin/mawk + /usr/bin/od + /usr/bin/bash + /usr/bin/du + /usr/bin/test + /usr/bin/dd + /usr/bin/id + /usr/bin/ps + /usr/bin/sleep + /usr/bin/mkdir + /usr/bin/dpkg-query + /usr/bin/mktemp + /usr/bin/rm + /usr/bin/rmdir + /usr/bin/sudo + ) + + for mode in open close; do + case "$mode" in + open) wrapper="$OPEN_WRAPPER_PATH" ;; + close) wrapper="$CLOSE_WRAPPER_PATH" ;; + esac + main_name="_k3slr_${mode}_main" + log="${fixture_root}/task5a-${mode}-capability.log" + : >"$log" + + # shellcheck source=/dev/null + source "$wrapper" + _k3slr_load_contract() { + (( $# == 1 )) || return 1 + printf 'contract\n' >>"$log" + } + _k3slr_query_package_state() { + local destination_name="$1" package="$2" expected_version="$3" result + printf 'package:%s:%s\n' "$package" "$expected_version" >>"$log" + case "$package" in + keepassxc) result="${TASK5A_KEEPASS_STATE:-installed}" ;; + cryptsetup-bin) result="${TASK5A_CRYPTSETUP_STATE:-installed}" ;; + *) return 1 ;; + esac + printf -v "$destination_name" '%s' "$result" + } + _k3slr_trusted_root_executable() { + printf 'binary:%s\n' "$1" >>"$log" + [[ -z "${TASK5A_MISSING_BINARY:-}" || "$1" != "$TASK5A_MISSING_BINARY" ]] + } + _k3slr_require_execute_tty() { + printf 'tty\n' >>"$log" + } + _k3slr_lifecycle_dispatch() { + printf 'dispatch:%s:%s\n' "$1" "$2" >>"$log" + } + _k3slr_command() { + printf 'unexpected-command:%s\n' "$*" >>"$log" + return 97 + } + + TASK5A_KEEPASS_STATE=installed + TASK5A_CRYPTSETUP_STATE=installed + TASK5A_MISSING_BINARY='' + assert_succeeds "$main_name" --execute + expected=$'contract\npackage:keepassxc:2.7.6+dfsg.1-1build3\npackage:cryptsetup-bin:2:2.7.0-1ubuntu4.2' + for line in "${trusted_binaries[@]}"; do + expected+=$'\n'binary:"$line" + done + expected+=$'\ntty\ndispatch:'"$mode"':_k3slr_prove_header_restore' + assert_eq "$expected" "$(<"$log")" \ + "$mode gates packages and every trusted binary before TTY/dispatch" + + for state in absent mixed wrong malformed; do + : >"$log" + case "$state" in + absent) + TASK5A_KEEPASS_STATE=absent + TASK5A_CRYPTSETUP_STATE=absent + ;; + mixed) + TASK5A_KEEPASS_STATE=installed + TASK5A_CRYPTSETUP_STATE=absent + ;; + wrong) + TASK5A_KEEPASS_STATE=wrong-version + TASK5A_CRYPTSETUP_STATE=installed + ;; + malformed) + TASK5A_KEEPASS_STATE=malformed + TASK5A_CRYPTSETUP_STATE=installed + ;; + esac + assert_fails "$main_name" --execute + output="$(<"$log")" + expected=$'contract\npackage:keepassxc:2.7.6+dfsg.1-1build3\npackage:cryptsetup-bin:2:2.7.0-1ubuntu4.2' + assert_eq "$expected" "$output" "$mode $state package state stops before binary/TTY/dispatch" + done + + : >"$log" + TASK5A_KEEPASS_STATE=installed + TASK5A_CRYPTSETUP_STATE=installed + TASK5A_MISSING_BINARY=/usr/bin/stat + assert_fails "$main_name" --execute + output="$(<"$log")" + [[ "$output" == *$'binary:/usr/bin/stat' && "$output" != *$'\ntty' && + "$output" != *$'\ndispatch:' && "$output" != *'unexpected-command:'* ]] || { + printf '%s missing binary did not stop at the capability boundary\n' "$mode" >&2 + return 1 + } + done +) + +task5a_lifecycle_dispatch_contract() ( + local mode step failure_step log expected actual + local -a prepare_steps=( + sudo + context + device-validator + estimate-phase + capacity-preflight + confirm-prepare + install-packages + feasibility + revalidate-device + mount-outer + revalidate-capacity + create-layout + create-kdbx + create-container + validate-allocation + create-loop + luks-format + luks-open + mkfs-ext4 + mount-inner + write-marker-metadata + header-proof + close-inner + close-mapping + detach-loop + unmount-outer + closed-validator + ) + local -a open_steps=( + sudo + context + confirm-open + 'device-validator(--expect-device-ready)' + revalidate-context-device + mount-outer + validate-layout-kdbx-container + validate-allocation-and-ntfs-attributes + validate-outer-metadata + collision-check + create-loop + luks-open + mount-inner + validate-inner-metadata-marker-capacity + 'open-validator(--expect-open)' + ) + local -a close_steps=( + sudo + context + confirm-close + 'open-validator(--expect-open)' + validate-metadata-and-full-lineage + busy-process-query-pass-1 + busy-process-query-pass-2 + sync-inner-filesystem + unmount-inner + close-main-mapping + detach-original-loop + unmount-outer + 'closed-validator(--expect-closed)' + ) + + log="${fixture_root}/task5a-dispatch.log" + _k3slr_lifecycle_step() { + (( $# == 1 )) || return 1 + printf 'hook:%s\n' "$1" >>"$log" + } + _k3slr_lifecycle_action() { + case "$1" in + prepare) + (( $# == 3 )) || return 1 + [[ "$3" == _k3slr_prepare_invocation ]] || return 1 + ;; + open|close) + (( $# == 2 )) || return 1 + ;; + *) return 1 ;; + esac + printf 'action:%s\n' "$2" >>"$log" + [[ -z "${failure_step:-}" || "$2" != "$failure_step" ]] + } + task5a_fixture_header_provider() { + return 1 + } + + for mode in prepare open close; do + local -a steps=() + case "$mode" in + prepare) steps=("${prepare_steps[@]}") ;; + open) steps=("${open_steps[@]}") ;; + close) steps=("${close_steps[@]}") ;; + esac + : >"$log" + failure_step='' + assert_succeeds _k3slr_lifecycle_dispatch "$mode" task5a_fixture_header_provider + expected='' + for step in "${steps[@]}"; do + [[ -z "$expected" ]] || expected+=$'\n' + expected+="hook:${step}"$'\n'"action:${step}" + done + actual="$(<"$log")" + assert_eq "$expected" "$actual" "$mode emits the independent exact lifecycle order" + + case "$mode" in + prepare) failure_step=create-loop ;; + open) failure_step=collision-check ;; + close) failure_step=unmount-inner ;; + esac + : >"$log" + assert_fails _k3slr_lifecycle_dispatch "$mode" task5a_fixture_header_provider + expected='' + for step in "${steps[@]}"; do + [[ -z "$expected" ]] || expected+=$'\n' + expected+="hook:${step}"$'\n'"action:${step}" + [[ "$step" == "$failure_step" ]] && break + done + actual="$(<"$log")" + assert_eq "$expected" "$actual" \ + "$mode failure includes its own hook/action and dispatches no later action" + done +) + +task5a_production_actions_fail_closed() ( + local lifecycle step command_calls=0 action_rc=0 probe_rc=0 record='' + local -a prepare_unavailable=( + context estimate-phase capacity-preflight confirm-prepare install-packages + feasibility revalidate-device mount-outer revalidate-capacity create-layout + create-kdbx create-container validate-allocation create-loop luks-format luks-open + mkfs-ext4 mount-inner write-marker-metadata header-proof close-inner close-mapping + detach-loop unmount-outer closed-validator + ) + local -a open_unavailable=( + sudo context confirm-open 'device-validator(--expect-device-ready)' + revalidate-context-device mount-outer validate-layout-kdbx-container + validate-allocation-and-ntfs-attributes validate-outer-metadata collision-check + create-loop luks-open mount-inner validate-inner-metadata-marker-capacity + 'open-validator(--expect-open)' + ) + local -a close_unavailable=( + sudo context confirm-close 'open-validator(--expect-open)' + validate-metadata-and-full-lineage busy-process-query-pass-1 + busy-process-query-pass-2 sync-inner-filesystem unmount-inner close-main-mapping + detach-original-loop unmount-outer 'closed-validator(--expect-closed)' + ) + _k3slr_command() { + (( command_calls += 1 )) + return 97 + } + task5b_unavailable_probe_payload() { + local lifecycle="${1-}" step="${2-}" action_rc=0 command_calls=0 + (( $# == 2 )) || return 1 + if [[ "$lifecycle" == prepare ]]; then + _k3slr_lifecycle_action prepare "$step" _k3slr_prepare_invocation || + action_rc=$? + else + _k3slr_lifecycle_action "$lifecycle" "$step" || action_rc=$? + fi + task5b_production_probe_emit_ok unavailable "$action_rc" "$command_calls" + } + task5b_production_probe_adversary_matrix || return 1 + task5b_production_probe_static_graph_is_closed unavailable || return 1 + for lifecycle in prepare open close; do + local -a actions=() + case "$lifecycle" in + prepare) actions=("${prepare_unavailable[@]}") ;; + open) actions=("${open_unavailable[@]}") ;; + close) actions=("${close_unavailable[@]}") ;; + esac + for step in "${actions[@]}"; do + record='' + probe_rc=0 + record="$( + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm unavailable || return 1 + task5b_unavailable_probe_payload "$lifecycle" "$step" + )" || probe_rc=$? + assert_eq 0 "$probe_rc" \ + "historical unavailable probe completes safely: $lifecycle/$step" + assert_eq 'TASK5B_PROBE_OK|unavailable|1|0' "$record" \ + "historical unavailable action is fail-closed before commands: $lifecycle/$step" + done + done +) + +task5a_unowned_capture_cleanup_is_rejected() ( + local description="$1" capture_directory capture_record + local rm_rc=0 rmdir_rc=0 records_preserved=0 directory_preserved=0 + capture_directory="$(/usr/bin/mktemp --directory --tmpdir=/tmp \ + k3slr-capture.XXXXXXXX)" || return 1 + [[ "$capture_directory" =~ ^/tmp/k3slr-capture\.[A-Za-z0-9]{8}$ && + -d "$capture_directory" && ! -L "$capture_directory" ]] || return 1 + : >"${capture_directory}/stdout" + : >"${capture_directory}/stderr" + + _k3slr_command /usr/bin/rm -- "${capture_directory}/stdout" \ + "${capture_directory}/stderr" >/dev/null 2>&1 || rm_rc=$? + if [[ -f "${capture_directory}/stdout" && ! -L "${capture_directory}/stdout" && + -f "${capture_directory}/stderr" && ! -L "${capture_directory}/stderr" ]]; then + records_preserved=1 + fi + for capture_record in "${capture_directory}/stdout" "${capture_directory}/stderr"; do + if [[ -f "$capture_record" && ! -L "$capture_record" ]]; then + /usr/bin/rm -- "$capture_record" || return 1 + elif [[ -e "$capture_record" || -L "$capture_record" ]]; then + return 1 + fi + done + + _k3slr_command /usr/bin/rmdir -- "$capture_directory" \ + >/dev/null 2>&1 || rmdir_rc=$? + if [[ -d "$capture_directory" && ! -L "$capture_directory" ]]; then + directory_preserved=1 + /usr/bin/rmdir -- "$capture_directory" || return 1 + elif [[ -e "$capture_directory" || -L "$capture_directory" ]]; then + return 1 + fi + + assert_eq 97 "$rm_rc" "$description rejects unowned record deletion before execution" + assert_eq 1 "$records_preserved" "$description preserves unowned capture records" + assert_eq 97 "$rmdir_rc" "$description rejects unowned directory deletion before execution" + assert_eq 1 "$directory_preserved" "$description preserves the unowned capture directory" +) + +task5a_capture_path_is_safe() { + (( $# == 1 )) && [[ "$1" =~ ^/tmp/k3slr-capture\.[A-Za-z0-9]{8}$ ]] +} + +task5a_capture_directory_identity() { + local capture_directory="${1-}" identity + (( $# == 1 )) && task5a_capture_path_is_safe "$capture_directory" && + [[ -d "$capture_directory" && ! -L "$capture_directory" ]] || return 97 + identity="$(/usr/bin/stat --format='%d:%i' -- "$capture_directory")" || return 97 + [[ "$identity" =~ ^[0-9]+:[0-9]+$ ]] || return 97 + printf '%s\n' "$identity" +} + +task5a_capture_log_read_exact() { + local log="${1-}" destination_name="${2-}" size line probe + local row_count=0 + (( $# == 2 )) && [[ -f "$log" && ! -L "$log" ]] && + [[ "$destination_name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || return 97 + local -n destination="$destination_name" + destination=() + size="$(/usr/bin/stat --format='%s' -- "$log")" || return 97 + [[ "$size" =~ ^[0-9]+$ ]] && (( size <= 1048576 )) || return 97 + if IFS= read -r -d '' probe <"$log"; then + return 97 + fi + while :; do + line='' + if IFS= read -r line; then + row_count=$((row_count + 1)) + (( row_count <= 4096 && ${#line} > 0 && ${#line} <= 512 )) || return 97 + [[ "$line" != *$'\r'* ]] || return 97 + destination+=("$line") + else + [[ -z "$line" ]] || return 97 + break + fi + done <"$log" +} + +task5a_capture_ownership_log_require() { + local ownership_log="${1-}" capture_directory="${2-}" expected_phase="${3-}" + local row event path identity active_path='' active_identity='' active_phase='' + local target_phase=absent target_identity='' + local -a rows=() + local -A phase_by_path=() identity_by_path=() + (( $# == 3 )) || return 97 + case "$expected_phase" in + complete) [[ -z "$capture_directory" ]] || return 97 ;; + absent|created|records-removed|removed) + task5a_capture_path_is_safe "$capture_directory" || return 97 + ;; + *) return 97 ;; + esac + task5a_capture_log_read_exact "$ownership_log" rows || return 97 + for row in "${rows[@]}"; do + [[ "$row" =~ ^(created|records-removed|removed)\|(/tmp/k3slr-capture\.[A-Za-z0-9]{8})\|([0-9]+:[0-9]+)$ ]] || return 97 + event="${BASH_REMATCH[1]}" + path="${BASH_REMATCH[2]}" + identity="${BASH_REMATCH[3]}" + case "$event" in + created) + [[ -z "$active_path" && -z "${phase_by_path[$path]+present}" ]] || return 97 + active_path="$path" + active_identity="$identity" + active_phase=created + phase_by_path["$path"]=created + identity_by_path["$path"]="$identity" + ;; + records-removed) + [[ "$active_path" == "$path" && "$active_phase" == created && + "$active_identity" == "$identity" ]] || return 97 + active_phase=records-removed + phase_by_path["$path"]=records-removed + ;; + removed) + [[ "$active_path" == "$path" && "$active_phase" == records-removed && + "$active_identity" == "$identity" ]] || return 97 + phase_by_path["$path"]=removed + active_path='' + active_identity='' + active_phase='' + ;; + esac + done + if [[ -n "$capture_directory" && -n "${phase_by_path[$capture_directory]+present}" ]]; then + target_phase="${phase_by_path[$capture_directory]}" + target_identity="${identity_by_path[$capture_directory]}" + fi + case "$expected_phase" in + complete) [[ -z "$active_path" ]] ;; + absent) [[ "$target_phase" == absent && -z "$active_path" ]] ;; + created|records-removed) + [[ "$target_phase" == "$expected_phase" && "$active_path" == "$capture_directory" ]] + ;; + removed) [[ "$target_phase" == removed && -z "$active_path" ]] ;; + esac || return 97 + [[ "$expected_phase" == complete || "$expected_phase" == absent ]] || + printf '%s\n' "$target_identity" +} + +task5a_capture_event_log_require() { + local event_log="${1-}" capture_directory="${2-}" expected_phase="${3-}" + local row event path command_id active_path='' active_phase='' target_phase=absent + local command_count=0 + local -a rows=() + local -A phase_by_path=() + (( $# == 3 )) || return 97 + case "$expected_phase" in + complete) [[ -z "$capture_directory" ]] || return 97 ;; + absent|created|records-removed|removed) + task5a_capture_path_is_safe "$capture_directory" || return 97 + ;; + *) return 97 ;; + esac + task5a_capture_log_read_exact "$event_log" rows || return 97 + for row in "${rows[@]}"; do + if [[ "$row" =~ ^(mktemp|rm|rmdir)\|(/tmp/k3slr-capture\.[A-Za-z0-9]{8})$ ]]; then + event="${BASH_REMATCH[1]}" + path="${BASH_REMATCH[2]}" + case "$event" in + mktemp) + [[ -z "$active_path" && -z "${phase_by_path[$path]+present}" ]] || return 97 + active_path="$path" + active_phase=created + command_count=0 + phase_by_path["$path"]=created + ;; + rm) + [[ "$active_path" == "$path" && "$active_phase" == created ]] || return 97 + active_phase=records-removed + phase_by_path["$path"]=records-removed + ;; + rmdir) + [[ "$active_path" == "$path" && "$active_phase" == records-removed ]] || return 97 + phase_by_path["$path"]=removed + active_path='' + active_phase='' + command_count=0 + ;; + esac + elif [[ "$row" =~ ^command\|([A-Za-z0-9][A-Za-z0-9._:-]{0,63})$ ]]; then + command_id="${BASH_REMATCH[1]}" + [[ -n "$command_id" && -n "$active_path" && "$active_phase" == created && + "$command_count" == 0 ]] || return 97 + command_count=1 + else + return 97 + fi + done + if [[ -n "$capture_directory" && -n "${phase_by_path[$capture_directory]+present}" ]]; then + target_phase="${phase_by_path[$capture_directory]}" + fi + case "$expected_phase" in + complete) [[ -z "$active_path" ]] ;; + absent) [[ "$target_phase" == absent && -z "$active_path" ]] ;; + created|records-removed) + [[ "$target_phase" == "$expected_phase" && "$active_path" == "$capture_directory" ]] + ;; + removed) [[ "$target_phase" == removed && -z "$active_path" ]] ;; + esac || return 97 +} + +task5a_capture_ledgers_require_coherent() { + local ownership_log="${1-}" event_log="${2-}" capture_directory="${3-}" + local expected_phase="${4-}" ownership_identity row phase path entry_index + local -a ownership_rows=() event_rows=() + local -a ownership_entries=() event_entries=() + (( $# == 4 )) || return 97 + + ownership_identity="$(task5a_capture_ownership_log_require \ + "$ownership_log" "$capture_directory" "$expected_phase")" || return 97 + task5a_capture_event_log_require \ + "$event_log" "$capture_directory" "$expected_phase" || return 97 + + task5a_capture_log_read_exact "$ownership_log" ownership_rows || return 97 + task5a_capture_log_read_exact "$event_log" event_rows || return 97 + for row in "${ownership_rows[@]}"; do + [[ "$row" =~ ^(created|records-removed|removed)\|(/tmp/k3slr-capture\.[A-Za-z0-9]{8})\|[0-9]+:[0-9]+$ ]] || return 97 + ownership_entries+=("${BASH_REMATCH[1]}|${BASH_REMATCH[2]}") + done + for row in "${event_rows[@]}"; do + if [[ "$row" =~ ^(mktemp|rm|rmdir)\|(/tmp/k3slr-capture\.[A-Za-z0-9]{8})$ ]]; then + case "${BASH_REMATCH[1]}" in + mktemp) phase=created ;; + rm) phase=records-removed ;; + rmdir) phase=removed ;; + *) return 97 ;; + esac + path="${BASH_REMATCH[2]}" + event_entries+=("${phase}|${path}") + elif [[ "$row" =~ ^command\|[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$ ]]; then + continue + else + return 97 + fi + done + (( ${#ownership_entries[@]} == ${#event_entries[@]} )) || return 97 + for (( entry_index=0; entry_index<${#ownership_entries[@]}; entry_index++ )); do + [[ "${ownership_entries[entry_index]}" == "${event_entries[entry_index]}" ]] || + return 97 + done + case "$expected_phase" in + created|records-removed|removed) printf '%s\n' "$ownership_identity" ;; + esac +} + +task5a_capture_ledgers_allow_registration() { + local ownership_log="${1-}" event_log="${2-}" + (( $# == 2 )) || return 97 + task5a_capture_ledgers_require_coherent \ + "$ownership_log" "$event_log" '' complete +} + +task5a_capture_ledger_register() { + local ownership_log="${1-}" event_log="${2-}" capture_directory="${3-}" + local current_identity + (( $# == 3 )) && task5a_capture_path_is_safe "$capture_directory" || return 97 + task5a_capture_ledgers_require_coherent \ + "$ownership_log" "$event_log" "$capture_directory" absent || return 97 + current_identity="$(task5a_capture_directory_identity "$capture_directory")" || return 97 + printf 'created|%s|%s\n' "$capture_directory" "$current_identity" >>"$ownership_log" || + return 97 + printf 'mktemp|%s\n' "$capture_directory" >>"$event_log" || return 97 + task5a_capture_ledgers_require_coherent \ + "$ownership_log" "$event_log" "$capture_directory" created >/dev/null || return 97 +} + +task5a_capture_ledger_require() { + task5a_capture_ownership_log_require "$@" +} + +task5a_capture_event_phase_is() { + task5a_capture_event_log_require "$@" +} + +task5a_remove_owned_capture_records() { + local ownership_log="${1-}" event_log="${2-}" capture_directory pinned_identity + local current_identity + shift 2 || return 97 + (( $# == 4 )) && [[ "$1" == /usr/bin/rm && "$2" == -- && + "$3" =~ ^/tmp/k3slr-capture\.[A-Za-z0-9]{8}/stdout$ ]] || return 97 + capture_directory="${3%/stdout}" + [[ "$4" == "${capture_directory}/stderr" && -d "$capture_directory" && + ! -L "$capture_directory" && -f "$3" && ! -L "$3" && + -f "$4" && ! -L "$4" ]] || return 97 + pinned_identity="$(task5a_capture_ledgers_require_coherent \ + "$ownership_log" "$event_log" "$capture_directory" created)" || return 97 + current_identity="$(task5a_capture_directory_identity "$capture_directory")" || return 97 + [[ "$current_identity" == "$pinned_identity" ]] || return 97 + "$@" || return $? + [[ ! -e "$3" && ! -L "$3" && ! -e "$4" && ! -L "$4" ]] || return 1 + printf 'records-removed|%s|%s\n' "$capture_directory" "$pinned_identity" \ + >>"$ownership_log" || return 97 + printf 'rm|%s\n' "$capture_directory" >>"$event_log" || return 97 + task5a_capture_ledgers_require_coherent "$ownership_log" "$event_log" \ + "$capture_directory" records-removed >/dev/null || return 97 +} + +task5a_remove_owned_capture_directory() { + local ownership_log="${1-}" event_log="${2-}" capture_directory pinned_identity + local current_identity + shift 2 || return 97 + (( $# == 3 )) && [[ "$1" == /usr/bin/rmdir && "$2" == -- ]] || return 97 + capture_directory="$3" + task5a_capture_path_is_safe "$capture_directory" && + [[ -d "$capture_directory" && ! -L "$capture_directory" ]] || return 97 + pinned_identity="$(task5a_capture_ledgers_require_coherent \ + "$ownership_log" "$event_log" "$capture_directory" records-removed)" || return 97 + current_identity="$(task5a_capture_directory_identity "$capture_directory")" || return 97 + [[ "$current_identity" == "$pinned_identity" ]] || return 97 + "$@" || return $? + [[ ! -e "$capture_directory" && ! -L "$capture_directory" ]] || return 1 + printf 'removed|%s|%s\n' "$capture_directory" "$pinned_identity" \ + >>"$ownership_log" || return 97 + printf 'rmdir|%s\n' "$capture_directory" >>"$event_log" || return 97 + task5a_capture_ledgers_require_coherent "$ownership_log" "$event_log" \ + "$capture_directory" removed >/dev/null || return 97 +} + +task5a_cleanup_capture_probe_path() { + local capture_directory="${1-}" capture_record + (( $# == 1 )) && task5a_capture_path_is_safe "$capture_directory" || return 1 + if [[ ! -e "$capture_directory" && ! -L "$capture_directory" ]]; then + return 0 + fi + [[ -d "$capture_directory" && ! -L "$capture_directory" ]] || return 1 + for capture_record in "${capture_directory}/stdout" "${capture_directory}/stderr"; do + if [[ -f "$capture_record" && ! -L "$capture_record" ]]; then + /usr/bin/rm -- "$capture_record" || return 1 + elif [[ -e "$capture_record" || -L "$capture_record" ]]; then + return 1 + fi + done + /usr/bin/rmdir -- "$capture_directory" +} + +task5a_capture_cleanup_phase_and_identity_is_enforced() ( + local description="$1" early_path repeat_path drift_path replacement_path + local early_rmdir_rc=0 early_rm_rc=0 early_finish_rmdir_rc=0 + local repeat_rm_rc=0 repeat_second_rm_rc=0 repeat_rmdir_rc=0 + local reuse_rm_rc=0 reuse_rmdir_rc=0 drift_rm_rc=0 drift_rmdir_rc=0 + local early_preserved=0 early_completed=0 repeat_records_preserved=0 repeat_removed=0 + local reuse_records_preserved=0 reuse_directory_preserved=0 + local drift_records_preserved=0 drift_directory_preserved=0 + local drift_identity replacement_identity + + early_path="$(_k3slr_command /usr/bin/mktemp --directory --tmpdir=/tmp \ + k3slr-capture.XXXXXXXX)" || return 1 + task5a_capture_path_is_safe "$early_path" && + [[ -d "$early_path" && ! -L "$early_path" ]] || return 1 + _k3slr_command /usr/bin/rmdir -- "$early_path" \ + >/dev/null 2>&1 || early_rmdir_rc=$? + [[ -d "$early_path" && ! -L "$early_path" ]] && early_preserved=1 + : >"${early_path}/stdout" + : >"${early_path}/stderr" + _k3slr_command /usr/bin/rm -- "${early_path}/stdout" \ + "${early_path}/stderr" >/dev/null 2>&1 || early_rm_rc=$? + _k3slr_command /usr/bin/rmdir -- "$early_path" \ + >/dev/null 2>&1 || early_finish_rmdir_rc=$? + [[ ! -e "$early_path" && ! -L "$early_path" ]] && early_completed=1 + task5a_cleanup_capture_probe_path "$early_path" || return 1 + + repeat_path="$(_k3slr_command /usr/bin/mktemp --directory --tmpdir=/tmp \ + k3slr-capture.XXXXXXXX)" || return 1 + task5a_capture_path_is_safe "$repeat_path" && + [[ -d "$repeat_path" && ! -L "$repeat_path" ]] || return 1 + : >"${repeat_path}/stdout" + : >"${repeat_path}/stderr" + _k3slr_command /usr/bin/rm -- "${repeat_path}/stdout" \ + "${repeat_path}/stderr" >/dev/null 2>&1 || repeat_rm_rc=$? + : >"${repeat_path}/stdout" + : >"${repeat_path}/stderr" + _k3slr_command /usr/bin/rm -- "${repeat_path}/stdout" \ + "${repeat_path}/stderr" >/dev/null 2>&1 || repeat_second_rm_rc=$? + if [[ -f "${repeat_path}/stdout" && ! -L "${repeat_path}/stdout" && + -f "${repeat_path}/stderr" && ! -L "${repeat_path}/stderr" ]]; then + repeat_records_preserved=1 + fi + /usr/bin/rm -- "${repeat_path}/stdout" "${repeat_path}/stderr" || return 1 + _k3slr_command /usr/bin/rmdir -- "$repeat_path" \ + >/dev/null 2>&1 || repeat_rmdir_rc=$? + if [[ ! -e "$repeat_path" && ! -L "$repeat_path" ]]; then + repeat_removed=1 + /usr/bin/mkdir -- "$repeat_path" || return 1 + fi + [[ -d "$repeat_path" && ! -L "$repeat_path" ]] || return 1 + : >"${repeat_path}/stdout" + : >"${repeat_path}/stderr" + _k3slr_command /usr/bin/rm -- "${repeat_path}/stdout" \ + "${repeat_path}/stderr" >/dev/null 2>&1 || reuse_rm_rc=$? + if [[ -f "${repeat_path}/stdout" && ! -L "${repeat_path}/stdout" && + -f "${repeat_path}/stderr" && ! -L "${repeat_path}/stderr" ]]; then + reuse_records_preserved=1 + fi + /usr/bin/rm -- "${repeat_path}/stdout" "${repeat_path}/stderr" || return 1 + _k3slr_command /usr/bin/rmdir -- "$repeat_path" \ + >/dev/null 2>&1 || reuse_rmdir_rc=$? + [[ -d "$repeat_path" && ! -L "$repeat_path" ]] && reuse_directory_preserved=1 + task5a_cleanup_capture_probe_path "$repeat_path" || return 1 + + drift_path="$(_k3slr_command /usr/bin/mktemp --directory --tmpdir=/tmp \ + k3slr-capture.XXXXXXXX)" || return 1 + replacement_path="$(/usr/bin/mktemp --directory --tmpdir=/tmp \ + k3slr-capture.XXXXXXXX)" || return 1 + task5a_capture_path_is_safe "$drift_path" && + task5a_capture_path_is_safe "$replacement_path" || return 1 + drift_identity="$(task5a_capture_directory_identity "$drift_path")" || return 1 + replacement_identity="$(task5a_capture_directory_identity "$replacement_path")" || return 1 + [[ "$drift_identity" != "$replacement_identity" ]] || return 1 + /usr/bin/rmdir -- "$drift_path" || return 1 + /usr/bin/mv -- "$replacement_path" "$drift_path" || return 1 + [[ -d "$drift_path" && ! -L "$drift_path" ]] || return 1 + : >"${drift_path}/stdout" + : >"${drift_path}/stderr" + _k3slr_command /usr/bin/rm -- "${drift_path}/stdout" \ + "${drift_path}/stderr" >/dev/null 2>&1 || drift_rm_rc=$? + if [[ -f "${drift_path}/stdout" && ! -L "${drift_path}/stdout" && + -f "${drift_path}/stderr" && ! -L "${drift_path}/stderr" ]]; then + drift_records_preserved=1 + fi + /usr/bin/rm -- "${drift_path}/stdout" "${drift_path}/stderr" || return 1 + _k3slr_command /usr/bin/rmdir -- "$drift_path" \ + >/dev/null 2>&1 || drift_rmdir_rc=$? + [[ -d "$drift_path" && ! -L "$drift_path" ]] && drift_directory_preserved=1 + task5a_cleanup_capture_probe_path "$drift_path" || return 1 + + assert_eq 97 "$early_rmdir_rc" "$description rejects rmdir before record removal" + assert_eq 1 "$early_preserved" "$description preserves an early-rmdir owned directory" + assert_eq 0 "$early_rm_rc" "$description completes record removal after early rmdir rejection" + assert_eq 0 "$early_finish_rmdir_rc" \ + "$description completes rmdir after early rmdir rejection" + assert_eq 1 "$early_completed" "$description completes the early-rmdir capture lifecycle" + assert_eq 0 "$repeat_rm_rc" "$description permits the first owned record removal" + assert_eq 97 "$repeat_second_rm_rc" "$description rejects repeated owned record removal" + assert_eq 1 "$repeat_records_preserved" "$description preserves records on repeated rm" + assert_eq 0 "$repeat_rmdir_rc" "$description permits rmdir after one successful rm" + assert_eq 1 "$repeat_removed" "$description removes a completed owned directory" + assert_eq 97 "$reuse_rm_rc" "$description rejects completed-path reuse for rm" + assert_eq 1 "$reuse_records_preserved" "$description preserves reused-path records" + assert_eq 97 "$reuse_rmdir_rc" "$description rejects completed-path reuse for rmdir" + assert_eq 1 "$reuse_directory_preserved" "$description preserves a reused directory" + assert_eq 97 "$drift_rm_rc" "$description rejects identity drift for rm" + assert_eq 1 "$drift_records_preserved" "$description preserves identity-drifted records" + assert_eq 97 "$drift_rmdir_rc" "$description rejects identity drift for rmdir" + assert_eq 1 "$drift_directory_preserved" "$description preserves an identity-drifted directory" +) + +task5a_global_corrupt_capture_ledgers_are_rejected() ( + local ownership_log="$1" event_log="$2" description="$3" target_path foreign_path + local rm_rc=0 records_preserved=0 + local -a foreign_paths=( + /tmp/k3slr-capture.R5A00001 + /tmp/k3slr-capture.R5A00002 + /tmp/k3slr-capture.R5A00003 + /tmp/k3slr-capture.R5A00004 + ) + target_path="$(_k3slr_command /usr/bin/mktemp --directory --tmpdir=/tmp \ + k3slr-capture.XXXXXXXX)" || return 1 + task5a_capture_path_is_safe "$target_path" && + [[ -d "$target_path" && ! -L "$target_path" ]] || return 1 + for foreign_path in "${foreign_paths[@]}"; do + [[ "$target_path" != "$foreign_path" ]] || return 1 + done + : >"${target_path}/stdout" + : >"${target_path}/stderr" + + printf '%s\n' \ + 'removed|/tmp/k3slr-capture.R5A00001|101:101' \ + 'created|/tmp/k3slr-capture.R5A00002|102:102' \ + 'created|/tmp/k3slr-capture.R5A00002|102:102' \ + 'created|/tmp/k3slr-capture.R5A00003|103:103' \ + 'records-removed|/tmp/k3slr-capture.R5A00003|104:104' \ + >>"$ownership_log" + printf '%s' 'created|/tmp/k3slr-capture.R5A00004|105:105' >>"$ownership_log" + printf '%s\n' \ + 'rm|/tmp/k3slr-capture.R5A00001' \ + 'mktemp|/tmp/k3slr-capture.R5A00002' \ + 'mktemp|/tmp/k3slr-capture.R5A00002' \ + 'malformed-event-row' \ + >>"$event_log" + printf '%s' 'command|unterminated-probe' >>"$event_log" + + _k3slr_command /usr/bin/rm -- "${target_path}/stdout" \ + "${target_path}/stderr" >/dev/null 2>&1 || rm_rc=$? + if [[ -f "${target_path}/stdout" && ! -L "${target_path}/stdout" && + -f "${target_path}/stderr" && ! -L "${target_path}/stderr" ]]; then + records_preserved=1 + fi + task5a_cleanup_capture_probe_path "$target_path" || return 1 + + assert_eq 97 "$rm_rc" "$description rejects globally corrupt ledgers before deletion" + assert_eq 1 "$records_preserved" "$description preserves records for global ledger corruption" +) + +task5a_cross_ledger_coherence_is_enforced() ( + local ownership_log="$1" event_log="$2" path_log="$3" description="$4" + local registration_output='' registration_rc=0 registration_orphan=0 + local rm_target rmdir_target target_identity rm_rc=0 rmdir_rc=0 + local rm_preserved=0 rmdir_preserved=0 path_index + local -a registration_paths_before=() registration_paths_after=() + local -a path_log_before=() path_log_after=() + + : >"$ownership_log" + : >"$event_log" + : >"$path_log" + printf '%s\n' \ + 'created|/tmp/k3slr-capture.R6OWN001|601:701' \ + 'records-removed|/tmp/k3slr-capture.R6OWN001|601:701' \ + 'removed|/tmp/k3slr-capture.R6OWN001|601:701' >"$ownership_log" + printf '%s\n' \ + 'mktemp|/tmp/k3slr-capture.R6EVT001' \ + 'rm|/tmp/k3slr-capture.R6EVT001' \ + 'rmdir|/tmp/k3slr-capture.R6EVT001' >"$event_log" + shopt -s nullglob + registration_paths_before=(/tmp/k3slr-capture.????????) + mapfile -t path_log_before <"$path_log" + registration_output="$(_k3slr_command /usr/bin/mktemp --directory \ + --tmpdir=/tmp k3slr-capture.XXXXXXXX)" || registration_rc=$? + registration_paths_after=(/tmp/k3slr-capture.????????) + mapfile -t path_log_after <"$path_log" + if (( ${#registration_paths_before[@]} != ${#registration_paths_after[@]} )); then + registration_orphan=1 + else + for (( path_index=0; path_index<${#registration_paths_before[@]}; path_index++ )); do + if [[ "${registration_paths_before[path_index]}" != \ + "${registration_paths_after[path_index]}" ]]; then + registration_orphan=1 + fi + done + fi + if [[ -n "$registration_output" ]]; then + task5a_capture_path_is_safe "$registration_output" || return 1 + task5a_cleanup_capture_probe_path "$registration_output" || return 1 + fi + + rm_target="$(/usr/bin/mktemp --directory --tmpdir=/tmp \ + k3slr-capture.XXXXXXXX)" || return 1 + target_identity="$(task5a_capture_directory_identity "$rm_target")" || return 1 + : >"${rm_target}/stdout" + : >"${rm_target}/stderr" + printf '%s\n' \ + 'created|/tmp/k3slr-capture.R6OWN002|602:702' \ + 'records-removed|/tmp/k3slr-capture.R6OWN002|602:702' \ + 'removed|/tmp/k3slr-capture.R6OWN002|602:702' \ + "created|${rm_target}|${target_identity}" >"$ownership_log" + printf '%s\n' \ + 'mktemp|/tmp/k3slr-capture.R6EVT002' \ + 'rm|/tmp/k3slr-capture.R6EVT002' \ + 'rmdir|/tmp/k3slr-capture.R6EVT002' \ + "mktemp|${rm_target}" >"$event_log" + _k3slr_command /usr/bin/rm -- "${rm_target}/stdout" \ + "${rm_target}/stderr" >/dev/null 2>&1 || rm_rc=$? + if [[ -f "${rm_target}/stdout" && ! -L "${rm_target}/stdout" && + -f "${rm_target}/stderr" && ! -L "${rm_target}/stderr" ]]; then + rm_preserved=1 + fi + task5a_cleanup_capture_probe_path "$rm_target" || return 1 + + rmdir_target="$(/usr/bin/mktemp --directory --tmpdir=/tmp \ + k3slr-capture.XXXXXXXX)" || return 1 + target_identity="$(task5a_capture_directory_identity "$rmdir_target")" || return 1 + printf '%s\n' \ + 'created|/tmp/k3slr-capture.R6OWN003|603:703' \ + 'records-removed|/tmp/k3slr-capture.R6OWN003|603:703' \ + 'removed|/tmp/k3slr-capture.R6OWN003|603:703' \ + "created|${rmdir_target}|${target_identity}" \ + "records-removed|${rmdir_target}|${target_identity}" >"$ownership_log" + printf '%s\n' \ + 'mktemp|/tmp/k3slr-capture.R6EVT003' \ + 'rm|/tmp/k3slr-capture.R6EVT003' \ + 'rmdir|/tmp/k3slr-capture.R6EVT003' \ + "mktemp|${rmdir_target}" \ + "rm|${rmdir_target}" >"$event_log" + _k3slr_command /usr/bin/rmdir -- "$rmdir_target" \ + >/dev/null 2>&1 || rmdir_rc=$? + [[ -d "$rmdir_target" && ! -L "$rmdir_target" ]] && rmdir_preserved=1 + task5a_cleanup_capture_probe_path "$rmdir_target" || return 1 + + assert_eq 97 "$registration_rc" \ + "$description rejects registration for individually complete mismatched ledgers" + assert_eq '' "$registration_output" \ + "$description emits no mktemp output for mismatched completed histories" + assert_eq 0 "$registration_orphan" \ + "$description creates no real mktemp directory for mismatched completed histories" + assert_eq "${#path_log_before[@]}" "${#path_log_after[@]}" \ + "$description does not register a path for mismatched completed histories" + assert_eq 97 "$rm_rc" \ + "$description rejects rm for mismatched history plus the same active target" + assert_eq 1 "$rm_preserved" \ + "$description preserves records for mismatched history plus the same active target" + assert_eq 97 "$rmdir_rc" \ + "$description rejects rmdir for mismatched history plus the same active target" + assert_eq 1 "$rmdir_preserved" \ + "$description preserves the directory for mismatched history plus the same active target" +) + +task5a_command_append_failure_is_rejected() ( + local append_target="$1" ownership_log="$2" event_log="$3" path_log="$4" + local description="$5" capture_path original_mode output='' command_rc=0 + local target_preserved=0 row_index + local -a append_before=() append_after=() + local -a ownership_before=() ownership_after=() + local -a event_before=() event_after=() + local -a paths_before=() paths_after=() + shift 5 || return 1 + (( $# > 0 )) && [[ -f "$append_target" && ! -L "$append_target" ]] || return 1 + original_mode="$(/usr/bin/stat --format='%a' -- "$append_target")" || return 1 + [[ "$original_mode" =~ ^[0-7]{3,4}$ ]] || return 1 + + capture_path="$(_k3slr_command /usr/bin/mktemp --directory --tmpdir=/tmp \ + k3slr-capture.XXXXXXXX)" || return 1 + task5a_capture_path_is_safe "$capture_path" && + [[ -d "$capture_path" && ! -L "$capture_path" ]] || return 1 + task5a_append_failure_cleanup() { + /usr/bin/chmod "$original_mode" -- "$append_target" >/dev/null 2>&1 || : + task5a_cleanup_capture_probe_path "$capture_path" >/dev/null 2>&1 || : + } + trap task5a_append_failure_cleanup EXIT + + mapfile -t append_before <"$append_target" + mapfile -t ownership_before <"$ownership_log" + mapfile -t event_before <"$event_log" + mapfile -t paths_before <"$path_log" + /usr/bin/chmod 0400 -- "$append_target" || return 1 + output="$(_k3slr_command "$@" 2>/dev/null)" || command_rc=$? + /usr/bin/chmod "$original_mode" -- "$append_target" || return 1 + mapfile -t append_after <"$append_target" + mapfile -t ownership_after <"$ownership_log" + mapfile -t event_after <"$event_log" + mapfile -t paths_after <"$path_log" + [[ -d "$capture_path" && ! -L "$capture_path" ]] && target_preserved=1 + task5a_cleanup_capture_probe_path "$capture_path" || return 1 + trap - EXIT + + assert_eq 97 "$command_rc" "$description returns rc 97 when its required audit append fails" + assert_eq '' "$output" "$description does not enter the recognized command branch" + assert_eq 1 "$target_preserved" "$description preserves the active capture target" + assert_eq "${#append_before[@]}" "${#append_after[@]}" \ + "$description preserves the append target row count" + for (( row_index=0; row_index<${#append_before[@]}; row_index++ )); do + assert_eq "${append_before[row_index]}" "${append_after[row_index]}" \ + "$description preserves every append target row" + done + assert_eq "${#ownership_before[@]}" "${#ownership_after[@]}" \ + "$description preserves the ownership ledger row count" + for (( row_index=0; row_index<${#ownership_before[@]}; row_index++ )); do + assert_eq "${ownership_before[row_index]}" "${ownership_after[row_index]}" \ + "$description preserves every ownership ledger row" + done + assert_eq "${#event_before[@]}" "${#event_after[@]}" \ + "$description appends no command event" + for (( row_index=0; row_index<${#event_before[@]}; row_index++ )); do + assert_eq "${event_before[row_index]}" "${event_after[row_index]}" \ + "$description preserves every event ledger row" + done + assert_eq "${#paths_before[@]}" "${#paths_after[@]}" \ + "$description preserves the path ledger row count" + for (( row_index=0; row_index<${#paths_before[@]}; row_index++ )); do + assert_eq "${paths_before[row_index]}" "${paths_after[row_index]}" \ + "$description preserves every path ledger row" + done +) + +task5a_capture_global_parser_contract() ( + local ownership_log="${fixture_root}/task5a-parser-ownership.log" + local event_log="${fixture_root}/task5a-parser-events.log" + local target=/tmp/k3slr-capture.R5P00001 + declare -F task5a_capture_ownership_log_require >/dev/null || return 1 + declare -F task5a_capture_event_log_require >/dev/null || return 1 + declare -F task5a_capture_ledgers_require_coherent >/dev/null || return 1 + + printf '%s\n' "created|${target}|201:301" >"$ownership_log" + printf '%s\n' "mktemp|${target}" 'command|parser-probe' >"$event_log" + assert_succeeds task5a_capture_ownership_log_require \ + "$ownership_log" "$target" created >/dev/null + assert_succeeds task5a_capture_event_log_require "$event_log" "$target" created + + printf 'created|%s|201:301\0\n' "$target" >"$ownership_log" + assert_fails task5a_capture_ownership_log_require "$ownership_log" "$target" created + printf 'created|%s|201:301\r\n' "$target" >"$ownership_log" + assert_fails task5a_capture_ownership_log_require "$ownership_log" "$target" created + printf '\n' >"$ownership_log" + assert_fails task5a_capture_ownership_log_require "$ownership_log" "$target" created + printf 'created|%s|201:301|extra\n' "$target" >"$ownership_log" + assert_fails task5a_capture_ownership_log_require "$ownership_log" "$target" created + printf 'malformed-ownership-row\n' >"$ownership_log" + assert_fails task5a_capture_ownership_log_require "$ownership_log" "$target" created + printf 'created|%s|201:301' "$target" >"$ownership_log" + assert_fails task5a_capture_ownership_log_require "$ownership_log" "$target" created + printf '%s\n' \ + 'removed|/tmp/k3slr-capture.R5P00002|202:302' \ + "created|${target}|201:301" >"$ownership_log" + assert_fails task5a_capture_ownership_log_require "$ownership_log" "$target" created + printf '%s\n' \ + 'created|/tmp/k3slr-capture.R5P00002|202:302' \ + 'created|/tmp/k3slr-capture.R5P00002|202:302' \ + "created|${target}|201:301" >"$ownership_log" + assert_fails task5a_capture_ownership_log_require "$ownership_log" "$target" created + printf '%s\n' \ + 'created|/tmp/k3slr-capture.R5P00002|202:302' \ + 'records-removed|/tmp/k3slr-capture.R5P00002|203:303' \ + "created|${target}|201:301" >"$ownership_log" + assert_fails task5a_capture_ownership_log_require "$ownership_log" "$target" created + printf '%s\n' \ + 'created|/tmp/k3slr-capture.R5P00002|202:302' \ + "created|${target}|201:301" >"$ownership_log" + assert_fails task5a_capture_ownership_log_require "$ownership_log" "$target" created + + printf 'mktemp|%s\0\n' "$target" >"$event_log" + assert_fails task5a_capture_event_log_require "$event_log" "$target" created + printf 'mktemp|%s\r\n' "$target" >"$event_log" + assert_fails task5a_capture_event_log_require "$event_log" "$target" created + printf '\n' >"$event_log" + assert_fails task5a_capture_event_log_require "$event_log" "$target" created + printf 'mktemp|%s|extra\n' "$target" >"$event_log" + assert_fails task5a_capture_event_log_require "$event_log" "$target" created + printf 'malformed-event-row\n' >"$event_log" + assert_fails task5a_capture_event_log_require "$event_log" "$target" created + printf 'mktemp|%s' "$target" >"$event_log" + assert_fails task5a_capture_event_log_require "$event_log" "$target" created + printf '%s\n' \ + 'rm|/tmp/k3slr-capture.R5P00002' \ + "mktemp|${target}" >"$event_log" + assert_fails task5a_capture_event_log_require "$event_log" "$target" created + printf '%s\n' \ + 'mktemp|/tmp/k3slr-capture.R5P00002' \ + 'mktemp|/tmp/k3slr-capture.R5P00002' \ + "mktemp|${target}" >"$event_log" + assert_fails task5a_capture_event_log_require "$event_log" "$target" created + printf '%s\n' 'command|orphan-command' "mktemp|${target}" >"$event_log" + assert_fails task5a_capture_event_log_require "$event_log" "$target" created + + printf '%s\n' \ + 'created|/tmp/k3slr-capture.R6ORD001|611:711' \ + 'records-removed|/tmp/k3slr-capture.R6ORD001|611:711' \ + 'removed|/tmp/k3slr-capture.R6ORD001|611:711' \ + 'created|/tmp/k3slr-capture.R6ORD002|612:712' \ + 'records-removed|/tmp/k3slr-capture.R6ORD002|612:712' \ + 'removed|/tmp/k3slr-capture.R6ORD002|612:712' >"$ownership_log" + printf '%s\n' \ + 'mktemp|/tmp/k3slr-capture.R6ORD002' \ + 'rm|/tmp/k3slr-capture.R6ORD002' \ + 'rmdir|/tmp/k3slr-capture.R6ORD002' \ + 'mktemp|/tmp/k3slr-capture.R6ORD001' \ + 'rm|/tmp/k3slr-capture.R6ORD001' \ + 'rmdir|/tmp/k3slr-capture.R6ORD001' >"$event_log" + assert_fails task5a_capture_ledgers_require_coherent \ + "$ownership_log" "$event_log" '' complete + + printf '%s\n' \ + 'created|/tmp/k3slr-capture.R6PHS001|613:713' \ + 'records-removed|/tmp/k3slr-capture.R6PHS001|613:713' \ + 'removed|/tmp/k3slr-capture.R6PHS001|613:713' \ + "created|${target}|201:301" >"$ownership_log" + printf '%s\n' \ + 'mktemp|/tmp/k3slr-capture.R6PHS001' \ + 'rm|/tmp/k3slr-capture.R6PHS001' \ + 'rmdir|/tmp/k3slr-capture.R6PHS001' \ + "mktemp|${target}" \ + "rm|${target}" >"$event_log" + assert_fails task5a_capture_ledgers_require_coherent \ + "$ownership_log" "$event_log" "$target" created + + printf '%s\n' \ + 'created|/tmp/k3slr-capture.R6MAT001|614:714' \ + 'records-removed|/tmp/k3slr-capture.R6MAT001|614:714' \ + 'removed|/tmp/k3slr-capture.R6MAT001|614:714' >"$ownership_log" + printf '%s\n' \ + 'mktemp|/tmp/k3slr-capture.R6MAT001' \ + 'rm|/tmp/k3slr-capture.R6MAT001' \ + 'rmdir|/tmp/k3slr-capture.R6MAT001' >"$event_log" + assert_succeeds task5a_capture_ledgers_require_coherent \ + "$ownership_log" "$event_log" '' complete + printf '%s\n' \ + 'mktemp|/tmp/k3slr-capture.R6MAT001' \ + 'command|already-validated' \ + 'rm|/tmp/k3slr-capture.R6MAT001' \ + 'rmdir|/tmp/k3slr-capture.R6MAT001' >"$event_log" + assert_succeeds task5a_capture_ledgers_require_coherent \ + "$ownership_log" "$event_log" '' complete +) + +task5a_corrupt_capture_registration_is_rejected() ( + local ownership_log="$1" event_log="$2" description="$3" + local created_output='' registration_rc=0 + : >"$ownership_log" + : >"$event_log" + printf '%s\n' 'created|/tmp/k3slr-capture.R5R00001|401:501' >"$ownership_log" + printf '%s\n' 'mktemp|/tmp/k3slr-capture.R5R00001' >"$event_log" + created_output="$(_k3slr_command /usr/bin/mktemp --directory --tmpdir=/tmp \ + k3slr-capture.XXXXXXXX)" || registration_rc=$? + if [[ -n "$created_output" ]]; then + task5a_capture_path_is_safe "$created_output" && + [[ -d "$created_output" && ! -L "$created_output" ]] || return 1 + task5a_cleanup_capture_probe_path "$created_output" || return 1 + fi + assert_eq 97 "$registration_rc" \ + "$description rejects registration with a foreign incomplete capture" + assert_eq '' "$created_output" "$description creates no directory for rejected registration" +) + +task5a_assert_capture_ownership_complete() { + local ownership_log="$1" event_log="$2" path_log="$3" description="$4" + local capture_path pinned_identity + local -a capture_paths=() ownership_rows=() + mapfile -t capture_paths <"$path_log" + mapfile -t ownership_rows <"$ownership_log" + assert_eq "$(( ${#capture_paths[@]} * 3 ))" "${#ownership_rows[@]}" \ + "$description records exactly three ownership phases per capture" + for capture_path in "${capture_paths[@]}"; do + pinned_identity="$(task5a_capture_ledger_require \ + "$ownership_log" "$capture_path" removed)" || return 1 + [[ "$pinned_identity" =~ ^[0-9]+:[0-9]+$ ]] || return 1 + task5a_capture_event_log_require "$event_log" "$capture_path" removed || return 1 + done + task5a_capture_ownership_log_require "$ownership_log" '' complete || return 1 + task5a_capture_event_log_require "$event_log" '' complete || return 1 +} + +task5a_exact_record_and_pure_parser_contract() ( + local value state total free used phase marker_hash package_output package_error package_rc + local valid_hash=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + local valid_metadata valid_metadata_hex valid_marker valid_marker_hex invalid_hex expected + local context_case='' package_case='' invalid_rc=0 + local event_log="${fixture_root}/task5a-record-events.log" + local path_log="${fixture_root}/task5a-record-paths.log" + local ownership_log="${fixture_root}/task5a-record-ownership.log" + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + + task5a_record_argv_equals() { + local expected_count="${1-}" argument_index + local -a compared_arguments=() + shift || return 1 + [[ "$expected_count" =~ ^[1-9][0-9]*$ ]] || return 1 + compared_arguments=("$@") + (( ${#compared_arguments[@]} == 2 * expected_count )) || return 1 + for (( argument_index=0; argument_index>"$path_log" + printf '%s\n' "$created" + return 0 + ;; + /usr/bin/rm) + task5a_remove_owned_capture_records "$ownership_log" "$event_log" "$@" + return + ;; + /usr/bin/rmdir) + task5a_remove_owned_capture_directory "$ownership_log" "$event_log" "$@" + return + ;; + /usr/local/bin/k3s) + task5a_record_argv_equals 4 /usr/local/bin/k3s kubectl config \ + current-context "$@" || return 97 + command_id=current-context + ;; + /usr/bin/dpkg-query) + if task5a_record_argv_equals 4 /usr/bin/dpkg-query --show \ + '--showformat=${Status}|${Version}\n' keepassxc "$@"; then + command_id=package-keepassxc + elif task5a_record_argv_equals 4 /usr/bin/dpkg-query --show \ + '--showformat=${Status}|${Version}\n' cryptsetup-bin "$@"; then + command_id=package-cryptsetup-bin + else + return 97 + fi + ;; + *) return 97 ;; + esac + + printf 'command|%s\n' "$command_id" >>"$event_log" || return 97 + case "$command_id" in + current-context) + case "$context_case" in + success) printf 'default\n' ;; + missing_lf) printf default ;; + stderr) printf 'default\n'; printf 'warning\n' >&2 ;; + status) printf 'default\n'; return 1 ;; + nul) printf 'default\0\n' ;; + *) return 97 ;; + esac + ;; + package-keepassxc|package-cryptsetup-bin) + printf '%s' "$package_output" + printf '%s' "$package_error" >&2 + return "$package_rc" + ;; + esac + } + + task5a_fixture_ascii_hex() { + local input="$1" index character encoded + for (( index=0; index<${#input}; index++ )); do + character="${input:index:1}" + printf -v encoded '%02x' "'$character" + printf '%s' "$encoded" + done + } + valid_metadata=$'schema=k3slr-runtime-v1\nluks_uuid=12345678-1234-4abc-8def-1234567890ab\ncontainer_bytes=34359738368\ninner_label=K3S_RECOVERY\nmarker_sha256=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\n' + valid_metadata_hex="$(task5a_fixture_ascii_hex "$valid_metadata")"$'\n' + valid_marker=$'12345678-1234-4abc-8def-1234567890ab\n' + valid_marker_hex="$(task5a_fixture_ascii_hex "$valid_marker")"$'\n' + + assert_succeeds _k3slr_parse_current_context_record $'default\n' value + assert_eq default "$value" 'exact current context parses' + for value in default $'default\n\n' $'default\r\n' $'other\n' ''; do + assert_fails _k3slr_parse_current_context_record "$value" value + done + + assert_succeeds task5a_unowned_capture_cleanup_is_rejected 'record capture fake' + assert_succeeds task5a_corrupt_capture_registration_is_rejected \ + "$ownership_log" "$event_log" 'record capture fake' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + assert_succeeds task5a_cross_ledger_coherence_is_enforced \ + "$ownership_log" "$event_log" "$path_log" 'record capture fake' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + assert_succeeds task5a_capture_global_parser_contract + assert_succeeds task5a_global_corrupt_capture_ledgers_are_rejected \ + "$ownership_log" "$event_log" 'record capture fake' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + assert_succeeds task5a_capture_cleanup_phase_and_identity_is_enforced \ + 'record capture fake' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + context_case=success + assert_succeeds task5a_command_append_failure_is_rejected \ + "$event_log" "$ownership_log" "$event_log" "$path_log" \ + 'record fake command-event append failure' \ + /usr/local/bin/k3s kubectl config current-context + context_case='' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + _k3slr_command /fixture/unknown >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'record fake rejects unknown argv with rc 97' + invalid_rc=0 + _k3slr_command /usr/local/bin/k3s 'kubectl config' current-context \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" \ + 'context fake rejects flattened text with wrong argv grouping' + invalid_rc=0 + _k3slr_command /usr/local/bin/k3s kubectl config current-context unexpected \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'context fake rejects an extra argv element' + invalid_rc=0 + _k3slr_command /usr/local/bin/k3s kubectl config \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'context fake rejects a missing argv element' + invalid_rc=0 + _k3slr_command /usr/local/bin/k3s kubectl config get-contexts \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'context fake rejects unknown same-executable argv' + invalid_rc=0 + _k3slr_command /usr/bin/dpkg-query --show \ + '--showformat=${Status}|${Version}\n keepassxc' \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" \ + 'package fake rejects flattened text with wrong argv grouping' + invalid_rc=0 + _k3slr_command /usr/bin/dpkg-query --show \ + '--showformat=${Status}|${Version}\n' keepassxc unexpected \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'package fake rejects an extra argv element' + invalid_rc=0 + _k3slr_command /usr/bin/dpkg-query --show \ + '--showformat=${Status}|${Version}\n' \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'package fake rejects a missing argv element' + invalid_rc=0 + _k3slr_command /usr/bin/dpkg-query --list keepassxc \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'package fake rejects unknown same-executable argv' + [[ ! -s "$event_log" && ! -s "$path_log" ]] || return 1 + + context_case=success + assert_succeeds _k3slr_current_context value + assert_eq default "$value" 'current context query uses exact argv and exact record' + context_case=missing_lf + assert_fails _k3slr_current_context value + context_case=stderr + assert_fails _k3slr_current_context value + context_case=status + assert_fails _k3slr_current_context value + context_case=nul + assert_fails _k3slr_current_context value + + assert_succeeds _k3slr_parse_package_query_record state keepassxc \ + '2.7.6+dfsg.1-1build3' 0 $'install ok installed|2.7.6+dfsg.1-1build3\n' '' + assert_eq installed "$state" 'exact installed package record parses' + assert_succeeds _k3slr_parse_package_query_record state cryptsetup-bin \ + '2:2.7.0-1ubuntu4.2' 1 '' $'dpkg-query: no packages found matching cryptsetup-bin\n' + assert_eq absent "$state" 'exact absent package record parses' + assert_succeeds _k3slr_parse_package_query_record state keepassxc \ + '2.7.6+dfsg.1-1build3' 0 $'install ok installed|9.9.9\n' '' + assert_eq wrong-version "$state" 'wrong installed package version is classified' + assert_succeeds _k3slr_parse_package_query_record state keepassxc \ + '2.7.6+dfsg.1-1build3' 0 $'deinstall ok config-files|2.7.6+dfsg.1-1build3\n' '' + assert_eq malformed "$state" 'wrong dpkg status is malformed' + assert_succeeds _k3slr_parse_package_query_record state keepassxc \ + '2.7.6+dfsg.1-1build3' 1 'unexpected' $'dpkg-query: no packages found matching keepassxc\n' + assert_eq malformed "$state" 'absent record with stdout is malformed' + assert_succeeds _k3slr_parse_package_query_record state keepassxc \ + '2.7.6+dfsg.1-1build3' 0 $'install ok installed|2.7.6+dfsg.1-1build3\n' $'warning\n' + assert_eq malformed "$state" 'installed record with stderr is malformed' + assert_succeeds _k3slr_parse_package_query_record state keepassxc \ + '2.7.6+dfsg.1-1build3' 2 '' '' + assert_eq malformed "$state" 'unexpected dpkg exit is malformed' + assert_fails _k3slr_parse_package_query_record state other-package 1.0 0 $'install ok installed|1.0\n' '' + + package_output=$'install ok installed|2.7.6+dfsg.1-1build3\n' + package_error='' + package_rc=0 + assert_succeeds _k3slr_query_package_state state keepassxc '2.7.6+dfsg.1-1build3' + assert_eq installed "$state" 'package query uses exact argv and parses its record' + package_output='' + package_error=$'dpkg-query: no packages found matching cryptsetup-bin\n' + package_rc=1 + assert_succeeds _k3slr_query_package_state state cryptsetup-bin '2:2.7.0-1ubuntu4.2' + assert_eq absent "$state" 'package query preserves exact absent stderr record' + package_output='install ok installed|2.7.6+dfsg.1-1build3' + package_error='' + package_rc=0 + assert_succeeds _k3slr_query_package_state state keepassxc '2.7.6+dfsg.1-1build3' + assert_eq malformed "$state" 'package query preserves missing-LF framing for rejection' + package_output=$'install ok installed|2.7.6+dfsg.1-1build3\n' + package_error=$'warning\n' + package_rc=0 + assert_succeeds _k3slr_query_package_state state keepassxc '2.7.6+dfsg.1-1build3' + assert_eq malformed "$state" 'package query preserves unexpected stderr for rejection' + + expected="$(printf '%s\n' \ + current-context \ + current-context \ + current-context \ + current-context \ + current-context \ + package-keepassxc \ + package-cryptsetup-bin \ + package-keepassxc \ + package-keepassxc)" + assert_succeeds task5a_assert_record_capture_lineage "$expected" \ + 'context and package composition' + + assert_succeeds _k3slr_parse_phase_bytes_record $'phase_bytes=4294967296\n' phase + assert_eq 4294967296 "$phase" 'exact estimator record parses' + for value in 'phase_bytes=1' $'phase_bytes=01\n' $'phase_bytes=-1\n' \ + $'phase_bytes=1\nextra=2\n' $'bytes=1\n' $'phase_bytes=9223372036854775808\n'; do + assert_fails _k3slr_parse_phase_bytes_record "$value" phase + done + assert_succeeds _k3slr_max_phase_bytes phase 7 11 9 + assert_eq 11 "$phase" 'phase maximum never decreases after a lower observation' + assert_succeeds _k3slr_max_phase_bytes phase 11 11 15 + assert_eq 15 "$phase" 'phase maximum advances for a larger third observation' + assert_fails _k3slr_max_phase_bytes phase 1 bad 3 + + assert_succeeds _k3slr_parse_statfs_record $'4096|1000|250\n' total free used + assert_eq 4096000 "$total" 'statfs total bytes use fundamental block size' + assert_eq 1024000 "$free" 'statfs free bytes use blocks available to user' + assert_eq 3072000 "$used" 'statfs conservative used bytes exclude unavailable blocks' + for value in '4096|1000|250' $'4096|1000|1001\n' $'0|1|1\n' \ + $'4096|01|1\n' $'9223372036854775807|2|1\n' $'4096|1|1\nextra\n'; do + assert_fails _k3slr_parse_statfs_record "$value" total free used + done + assert_fails _k3slr_parse_statfs_record $'1|1|1\n' total total used + + K3SLR_MINIMUM_FREE_BYTES=10737418240 + assert_succeeds _k3slr_conservative_inner_capacity_fits \ + 34359738368 2147483648 + assert_fails _k3slr_conservative_inner_capacity_fits \ + 34359738368 4294967296 + assert_fails _k3slr_conservative_inner_capacity_fits 0 1 + + K3SLR_CONTAINER_SIZE_BYTES=34359738368 + K3SLR_INNER_LABEL=K3S_RECOVERY + ! declare -F _k3slr_parse_runtime_metadata >/dev/null || \ + fail 'raw runtime metadata/UUID shell-variable parser API is forbidden' + ! declare -F _k3slr_marker_record_is_valid >/dev/null || \ + fail 'raw 37-byte marker shell-variable parser API is forbidden' + assert_succeeds _k3slr_parse_runtime_metadata_hex "$valid_metadata_hex" marker_hash + assert_eq "$valid_hash" "$marker_hash" 'runtime metadata returns marker hash' + assert_succeeds _k3slr_marker_hex_is_valid "$valid_marker_hex" + for value in \ + "${valid_metadata%$'\n'}" \ + "${valid_metadata/luks_uuid=/luks-uuid=}" \ + "${valid_metadata/container_bytes=34359738368/container_bytes=34359738369}" \ + "${valid_metadata/inner_label=K3S_RECOVERY/inner_label=OTHER}" \ + "${valid_metadata/marker_sha256=/marker_sha256=A}" \ + $'schema=k3slr-runtime-v1\r\nluks_uuid=12345678-1234-4abc-8def-1234567890ab\ncontainer_bytes=34359738368\ninner_label=K3S_RECOVERY\nmarker_sha256=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\n'; do + invalid_hex="$(task5a_fixture_ascii_hex "$value")"$'\n' + assert_fails _k3slr_parse_runtime_metadata_hex "$invalid_hex" marker_hash + done + for value in $'12345678-1234-3abc-8def-1234567890ab\n' \ + $'12345678-1234-4abc-7def-1234567890ab\n' \ + '12345678-1234-4abc-8def-1234567890ab'; do + invalid_hex="$(task5a_fixture_ascii_hex "$value")"$'\n' + assert_fails _k3slr_marker_hex_is_valid "$invalid_hex" + done +) + +task5a_ntfs_capacity_and_prepackage_seams() ( + local value valid_text valid_hex invalid_hex od_record oversized snapshot snapshot_after + local total free identity output expected_identity calls=0 + + task5a_fixture_ascii_hex() { + local input="$1" index character encoded + for (( index=0; index<${#input}; index++ )); do + character="${input:index:1}" + printf -v encoded '%02x' "'$character" + printf '%s' "$encoded" + done + } + task5a_fixture_prepackage_snapshot() { + printf '%s\n' \ + 'canonical_partition=/dev/sda3' \ + 'partition_major_minor=8:3' \ + 'canonical_parent_disk=/dev/sda' \ + 'disk_major_minor=8:0' \ + "filesystem_uuid=${K3SLR_RECOVERY_FS_UUID}" \ + "partuuid=${K3SLR_RECOVERY_PARTUUID}" \ + 'filesystem_type=ntfs' \ + "model=${K3SLR_RECOVERY_MODEL}" \ + "serial=${K3SLR_RECOVERY_SERIAL}" \ + "wwn=${K3SLR_RECOVERY_WWN}" \ + 'canonical_k3s_partition=/dev/sdb2' \ + 'k3s_partition_major_minor=8:34' \ + 'canonical_k3s_disk=/dev/sdb' \ + 'k3s_disk_major_minor=8:32' \ + "k3s_filesystem_uuid=${K3SLR_K3S_FS_UUID}" \ + "k3s_partuuid=${K3SLR_K3S_PARTUUID}" \ + 'k3s_filesystem_type=ext4' \ + "k3s_model=${K3SLR_K3S_MODEL}" \ + "k3s_serial=${K3SLR_K3S_SERIAL}" \ + "k3s_wwn=${K3SLR_K3S_WWN}" \ + 'smart_health=PASSED' \ + 'smart_reallocated=0' \ + 'smart_pending=0' \ + 'smart_uncorrectable=0' \ + 'outer_mount=absent' \ + 'inner_mount=absent' \ + 'source_mounts=absent' \ + 'loops=absent' \ + 'mapper_directory=physical-root-safe' \ + 'mapping_path=absent' \ + 'proof_mapping_path=absent' \ + 'mapping_names=absent' \ + 'ntfs_probe=pass' + } + + od_record=$' 20 30 61 0a\n' + assert_succeeds _k3slr_normalize_od_hex_record "$od_record" value + assert_eq $'2030610a\n' "$value" 'od tokens normalize to one bounded lowercase hex record' + for od_record in $' 20 0A 0a\n' $' 20 gg 0a\n' ' 20 0a' $'\n'; do + assert_fails _k3slr_normalize_od_hex_record "$od_record" value + done + oversized='00 ' + for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16; do + oversized+="$oversized" + done + oversized+='00 ' + oversized="${oversized:0:196611}"$'\n' + assert_fails _k3slr_normalize_od_hex_record "$oversized" value + + valid_text=$'Volume Information\n\tCluster Size: 4096\n Volume Size in Clusters: 1000000\nAllocated clusters 250000 (25.0%)\nFree Clusters: 750000 (75.0%)\n' + valid_hex="$(task5a_fixture_ascii_hex "$valid_text")"$'\n' + assert_succeeds _k3slr_parse_unmounted_ntfs_capacity_hex "$valid_hex" total free + assert_eq 4096000000 "$total" 'NTFS total bytes are cluster size times volume clusters' + assert_eq 3072000000 "$free" 'NTFS free bytes are cluster size times free clusters' + output="$(_k3slr_parse_unmounted_ntfs_capacity_hex "$valid_hex" total free)" || return 1 + assert_eq '' "$output" 'NTFS parser never logs normalized or decoded evidence' + + for value in \ + "${valid_text%$'\n'}" \ + "${valid_text/Cluster Size: 4096/Cluster Size: 04096}" \ + $'Volume Information\n\tCluster\tSize: 4096\nVolume Size in Clusters: 1000000\nAllocated clusters 250000 (25.0%)\nFree Clusters: 750000 (75.0%)\n' \ + $'Volume Information\n\tCluster Size:\t4096\nVolume Size in Clusters: 1000000\nAllocated clusters 250000 (25.0%)\nFree Clusters: 750000 (75.0%)\n' \ + "${valid_text/Allocated clusters 250000/Allocated clusters 250001}" \ + "${valid_text/25.0%/25.1%}" \ + "${valid_text/Free Clusters:/Free clusters:}" \ + "${valid_text/Free Clusters: 750000 (75.0%)/Free Clusters: 750000 (75.0%) suffix}" \ + "${valid_text}Free Clusters: 750000 (75.0%)" \ + "${valid_text}Cluster Size: 4096 suffix"$'\n' \ + $'Cluster Size: 9223372036854775807\nVolume Size in Clusters: 2\nAllocated clusters 1 (50.0%)\nFree Clusters: 1 (50.0%)\n'; do + invalid_hex="$(task5a_fixture_ascii_hex "$value")"$'\n' + assert_fails _k3slr_parse_unmounted_ntfs_capacity_hex "$invalid_hex" total free + done + invalid_hex="${valid_hex%$'\n'}" + invalid_hex="${invalid_hex:0:12}00${invalid_hex:12}"$'\n' + assert_fails _k3slr_parse_unmounted_ntfs_capacity_hex "$invalid_hex" total free + invalid_hex="${valid_hex%$'\n'}" + invalid_hex="${invalid_hex:0:12}0d${invalid_hex:12}"$'\n' + assert_fails _k3slr_parse_unmounted_ntfs_capacity_hex "$invalid_hex" total free + invalid_hex="${valid_hex^^}" + assert_fails _k3slr_parse_unmounted_ntfs_capacity_hex "$invalid_hex" total free + assert_fails _k3slr_parse_unmounted_ntfs_capacity_hex "${valid_hex%?}f" total free + + snapshot="$(task5a_fixture_prepackage_snapshot)"$'\n' + assert_succeeds _k3slr_validate_prepackage_snapshot "$snapshot" + assert_fails _k3slr_parse_prepackage_snapshot_fields "$snapshot" 'bad-name' + assert_fails _k3slr_parse_prepackage_snapshot_fields "$snapshot" _k3slr_snapshot_alias + for snapshot_after in \ + "${snapshot/smart_pending=0/smart_pending=1}" \ + "${snapshot/ntfs_probe=pass/ntfs_probe=fail}" \ + "${snapshot/loops=absent/loops=present}" \ + "${snapshot/disk_major_minor=8:0/disk_major_minor=8:32}" \ + "${snapshot/filesystem_uuid=${K3SLR_RECOVERY_FS_UUID}/filesystem_uuid=AAAAAAAAAAAAAAAA}" \ + "${snapshot}mapping_names=absent"; do + assert_fails _k3slr_validate_prepackage_snapshot "$snapshot_after" + done + + _k3slr_collect_prepackage_snapshot() { + local destination_name="$1" selected="$snapshot" + calls=$((calls + 1)) + if (( calls == 2 )) && [[ -n "${TASK5A_SECOND_SNAPSHOT:-}" ]]; then + selected="$TASK5A_SECOND_SNAPSHOT" + fi + printf -v "$destination_name" '%s' "$selected" + } + _k3slr_capture_ntfsinfo_hex() { + local destination_name="$1" partition="$2" + [[ "$partition" == /dev/sda3 ]] || return 1 + printf -v "$destination_name" '%s' "$valid_hex" + } + calls=0 + TASK5A_SECOND_SNAPSHOT='' + assert_succeeds _k3slr_prepackage_device_preflight + assert_eq 2 "$calls" 'pre-package device preflight pins byte-equal before/after snapshots' + + calls=0 + TASK5A_SECOND_SNAPSHOT="${snapshot/serial=${K3SLR_RECOVERY_SERIAL}/serial=DRIFTED}" + assert_fails _k3slr_prepackage_device_preflight + assert_eq 2 "$calls" 'pre-package drift is observed before refusal' + + calls=0 + TASK5A_SECOND_SNAPSHOT='' + assert_succeeds _k3slr_collect_unmounted_ntfs_capacity value identity + assert_eq "$valid_hex" "$value" 'capacity collector returns only normalized hex' + expected_identity="/dev/sda3|8:3|/dev/sda|8:0|${K3SLR_RECOVERY_FS_UUID}|${K3SLR_RECOVERY_PARTUUID}|ntfs|${K3SLR_RECOVERY_MODEL}|${K3SLR_RECOVERY_SERIAL}|${K3SLR_RECOVERY_WWN}|8:32" + assert_eq "$expected_identity" "$identity" 'capacity collector returns the strict pinned identity tuple' + assert_eq 2 "$calls" 'capacity collector repeats the full snapshot around ntfsinfo' + + calls=0 + TASK5A_SECOND_SNAPSHOT="${snapshot/partition_major_minor=8:3/partition_major_minor=8:4}" + value=unchanged + identity=unchanged + assert_fails _k3slr_collect_unmounted_ntfs_capacity value identity + assert_eq unchanged "$value" 'drift failure does not publish normalized hex' + assert_eq unchanged "$identity" 'drift failure does not publish identity' +) + +task5a_exact_process_capture_contract() ( + local capture_case=success stdout stderr status invalid_rc=0 expected + local event_log="${fixture_root}/task5a-capture-events.log" + local path_log="${fixture_root}/task5a-capture-paths.log" + local ownership_log="${fixture_root}/task5a-capture-ownership.log" + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + + task5a_capture_argv_equals() { + local expected_count="${1-}" argument_index + local -a compared_arguments=() + shift || return 1 + [[ "$expected_count" =~ ^[1-9][0-9]*$ ]] || return 1 + compared_arguments=("$@") + (( ${#compared_arguments[@]} == 2 * expected_count )) || return 1 + for (( argument_index=0; argument_index>"$path_log" + printf '%s\n' "$created" + return 0 + ;; + /usr/bin/rm) + task5a_remove_owned_capture_records "$ownership_log" "$event_log" "$@" + return + ;; + /usr/bin/rmdir) + task5a_remove_owned_capture_directory "$ownership_log" "$event_log" "$@" + return + ;; + /fixture/capture) + task5a_capture_argv_equals 3 /fixture/capture --mode emit "$@" || return 97 + printf 'command|fixture-capture\n' >>"$event_log" || return 97 + case "$capture_case" in + success) + printf 'stdout line\n' + printf 'stderr line\n' >&2 + return 7 + ;; + nul) printf 'unsafe\0record\n' ;; + *) return 97 ;; + esac + ;; + *) return 97 ;; + esac + } + + assert_succeeds task5a_unowned_capture_cleanup_is_rejected \ + 'generic process capture fake' + assert_succeeds task5a_capture_cleanup_phase_and_identity_is_enforced \ + 'generic process capture fake' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + assert_succeeds task5a_cross_ledger_coherence_is_enforced \ + "$ownership_log" "$event_log" "$path_log" 'generic process capture fake' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + capture_case=success + assert_succeeds task5a_command_append_failure_is_rejected \ + "$event_log" "$ownership_log" "$event_log" "$path_log" \ + 'generic fake command-event append failure' \ + /fixture/capture --mode emit + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + _k3slr_command /fixture/unknown >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'process-capture fake rejects unknown argv with rc 97' + invalid_rc=0 + _k3slr_command /fixture/capture '--mode emit' \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" \ + 'process-capture fake rejects flattened text with wrong argv grouping' + invalid_rc=0 + _k3slr_command /fixture/capture --mode emit unexpected \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" \ + 'process-capture fake rejects the known executable with an extra argv element' + invalid_rc=0 + _k3slr_command /fixture/capture --mode >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" \ + 'process-capture fake rejects the known executable with a missing argv element' + invalid_rc=0 + _k3slr_command /fixture/capture --mode discard \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" \ + 'process-capture fake rejects unknown same-executable argv' + [[ ! -s "$event_log" && ! -s "$path_log" ]] || return 1 + + assert_succeeds _k3slr_capture_process stdout stderr status \ + /fixture/capture --mode emit + assert_eq $'stdout line\n' "$stdout" 'process capture preserves stdout terminal LF' + assert_eq $'stderr line\n' "$stderr" 'process capture preserves stderr terminal LF' + assert_eq 7 "$status" 'process capture preserves nonzero exit status' + + capture_case=nul + assert_fails _k3slr_capture_process stdout stderr status \ + /fixture/capture --mode emit + expected="$(printf '%s\n' fixture-capture fixture-capture)" + assert_succeeds task5a_assert_process_capture_lineage "$expected" \ + 'success and NUL-rejection process capture' + assert_fails _k3slr_capture_process stdout stdout status \ + /fixture/capture --mode emit + assert_succeeds task5a_assert_process_capture_lineage "$expected" \ + 'duplicate-destination rejection leaves no capture residue' +) + +task5a_prepackage_system_command_matrix() ( + local log="${fixture_root}/task5a-prepackage-system.log" + local event_log="${fixture_root}/task5a-prepackage-system-events.log" + local path_log="${fixture_root}/task5a-prepackage-system-paths.log" + local ownership_log="${fixture_root}/task5a-prepackage-system-ownership.log" + local cryptsetup_log="${fixture_root}/task5a-prepackage-system-cryptsetup.log" + local snapshot system_case expected unknown_rc=0 wrong_group_rc=0 extra_arg_rc=0 missing_arg_rc=0 + : >"$log" + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + : >"$cryptsetup_log" + + task5a_argv_equals() { + local expected_count="${1-}" argument_index + local -a compared_arguments=() + shift || return 1 + [[ "$expected_count" =~ ^[1-9][0-9]*$ ]] || return 1 + compared_arguments=("$@") + (( ${#compared_arguments[@]} == 2 * expected_count )) || return 1 + for (( argument_index=0; argument_index>"$cryptsetup_log" + ;; + esac + done + case "${1-}" in + /usr/bin/mktemp) + (( $# == 4 )) && [[ "$2" == --directory && "$3" == --tmpdir=/tmp && + "$4" == k3slr-capture.XXXXXXXX ]] || return 97 + task5a_capture_ledgers_require_coherent \ + "$ownership_log" "$event_log" '' complete || return 97 + created="$("$@")" || return 1 + [[ "$created" =~ ^/tmp/k3slr-capture\.[A-Za-z0-9]{8}$ && + -d "$created" && ! -L "$created" ]] || return 97 + if ! task5a_capture_ledger_register \ + "$ownership_log" "$event_log" "$created"; then + task5a_cleanup_capture_probe_path "$created" || return 1 + return 97 + fi + printf '%s\n' "$created" >>"$path_log" + printf '%s\n' "$created" + return 0 + ;; + /usr/bin/rm) + task5a_remove_owned_capture_records "$ownership_log" "$event_log" "$@" + return + ;; + /usr/bin/rmdir) + task5a_remove_owned_capture_directory "$ownership_log" "$event_log" "$@" + return + ;; + esac + + case "${1-}" in + /usr/bin/readlink) + if task5a_argv_equals 4 \ + /usr/bin/readlink -f -- "$K3SLR_RECOVERY_PARTITION_BY_ID" "$@"; then + command_id=readlink-recovery-partition + elif task5a_argv_equals 4 \ + /usr/bin/readlink -f -- "$K3SLR_RECOVERY_DISK_BY_ID" "$@"; then + command_id=readlink-recovery-disk + elif task5a_argv_equals 4 \ + /usr/bin/readlink -f -- "$K3SLR_K3S_PARTITION_BY_ID" "$@"; then + command_id=readlink-k3s-partition + elif task5a_argv_equals 4 \ + /usr/bin/readlink -f -- "$K3SLR_K3S_DISK_BY_ID" "$@"; then + command_id=readlink-k3s-disk + else + : + fi + ;; + /usr/bin/lsblk) + if task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output PKNAME -- /dev/sda3 "$@"; then + command_id=lsblk-recovery-partition-parent + elif task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output MAJ:MIN -- /dev/sda3 "$@"; then + command_id=lsblk-recovery-partition-majmin + elif task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output MODEL -- /dev/sda "$@"; then + command_id=lsblk-recovery-model + elif task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output SERIAL -- /dev/sda "$@"; then + command_id=lsblk-recovery-serial + elif task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output WWN -- /dev/sda "$@"; then + command_id=lsblk-recovery-wwn + elif task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output MAJ:MIN -- /dev/sda "$@"; then + command_id=lsblk-recovery-disk-majmin + elif task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output PKNAME -- /dev/sdb2 "$@"; then + command_id=lsblk-k3s-partition-parent + elif task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output MAJ:MIN -- /dev/sdb2 "$@"; then + command_id=lsblk-k3s-partition-majmin + elif task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output MODEL -- /dev/sdb "$@"; then + command_id=lsblk-k3s-model + elif task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output SERIAL -- /dev/sdb "$@"; then + command_id=lsblk-k3s-serial + elif task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output WWN -- /dev/sdb "$@"; then + command_id=lsblk-k3s-wwn + elif task5a_argv_equals 7 \ + /usr/bin/lsblk --noheadings --paths --output MAJ:MIN -- /dev/sdb "$@"; then + command_id=lsblk-k3s-disk-majmin + elif task5a_argv_equals 5 \ + /usr/bin/lsblk --noheadings --raw --output NAME,TYPE "$@"; then + command_id=lsblk-all-names-types + else + : + fi + ;; + /usr/bin/sudo) + if task5a_argv_equals 10 \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value \ + --match-tag UUID -- /dev/sda3 "$@"; then + command_id=blkid-recovery-uuid + elif task5a_argv_equals 10 \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value \ + --match-tag PARTUUID -- /dev/sda3 "$@"; then + command_id=blkid-recovery-partuuid + elif task5a_argv_equals 10 \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value \ + --match-tag TYPE -- /dev/sda3 "$@"; then + command_id=blkid-recovery-type + elif task5a_argv_equals 10 \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value \ + --match-tag UUID -- /dev/sdb2 "$@"; then + command_id=blkid-k3s-uuid + elif task5a_argv_equals 10 \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value \ + --match-tag PARTUUID -- /dev/sdb2 "$@"; then + command_id=blkid-k3s-partuuid + elif task5a_argv_equals 10 \ + /usr/bin/sudo --non-interactive -- /usr/sbin/blkid --output value \ + --match-tag TYPE -- /dev/sdb2 "$@"; then + command_id=blkid-k3s-type + elif task5a_argv_equals 7 \ + /usr/bin/sudo --non-interactive -- /usr/sbin/smartctl -H -A /dev/sda "$@"; then + command_id=smartctl-recovery-health-attributes + elif task5a_argv_equals 10 \ + /usr/bin/sudo --non-interactive -- /usr/bin/findmnt --noheadings --raw \ + --output SOURCE,FSTYPE,OPTIONS,ID,MAJ:MIN --mountpoint \ + /mnt/k3s-recovery-ssd "$@"; then + command_id=findmnt-outer-mountpoint + elif task5a_argv_equals 10 \ + /usr/bin/sudo --non-interactive -- /usr/bin/findmnt --noheadings --raw \ + --output SOURCE,FSTYPE,OPTIONS,ID,MAJ:MIN --mountpoint \ + /srv/recovery/k3s "$@"; then + command_id=findmnt-inner-mountpoint + elif task5a_argv_equals 10 \ + /usr/bin/sudo --non-interactive -- /usr/bin/findmnt --noheadings --raw \ + --output TARGET --source /dev/sda3 "$@"; then + command_id=findmnt-recovery-source + elif task5a_argv_equals 9 \ + /usr/bin/sudo --non-interactive -- /usr/sbin/losetup --list \ + --noheadings --raw --output \ + NAME,BACK-FILE,BACK-INO,BACK-MAJ:MIN,MAJ:MIN,OFFSET,SIZELIMIT "$@"; then + command_id=losetup-all + elif task5a_argv_equals 6 \ + /usr/bin/sudo --non-interactive -- /usr/bin/ntfs-3g.probe \ + --readwrite /dev/sda3 "$@"; then + command_id=ntfs-probe-readwrite + else + : + fi + ;; + /usr/bin/stat) + if task5a_argv_equals 4 /usr/bin/stat '--format=%F|%u|%g|%a' -- \ + /dev/mapper "$@"; then + command_id=stat-mapper-directory + fi + ;; + /usr/bin/test) + if task5a_argv_equals 4 /usr/bin/test ! -L /dev/mapper "$@"; then + command_id=test-mapper-directory-not-symlink + elif task5a_argv_equals 4 \ + /usr/bin/test ! -e /dev/mapper/k3s-recovery "$@"; then + command_id=test-main-mapping-absent + elif task5a_argv_equals 4 \ + /usr/bin/test ! -L /dev/mapper/k3s-recovery "$@"; then + command_id=test-main-mapping-not-symlink + elif task5a_argv_equals 4 \ + /usr/bin/test ! -e /dev/mapper/k3s-recovery-proof "$@"; then + command_id=test-proof-mapping-absent + elif task5a_argv_equals 4 \ + /usr/bin/test ! -L /dev/mapper/k3s-recovery-proof "$@"; then + command_id=test-proof-mapping-not-symlink + else + : + fi + ;; + *) : ;; + esac + + printf '%s\n' "$command_id" >>"$log" || return 97 + printf 'command|%s\n' "$command_id" >>"$event_log" || return 97 + [[ "$command_id" != rejected-command ]] || return 97 + case "$command_id" in + readlink-recovery-partition) + case "${system_case:-}" in + capture_stdout_no_lf) printf '/dev/sda3' ;; + capture_stderr) printf '/dev/sda3\n'; printf 'unexpected stderr\n' >&2 ;; + capture_status) printf '/dev/sda3\n'; return 7 ;; + *) printf '/dev/sda3\n' ;; + esac + ;; + readlink-recovery-disk) + if [[ "${system_case:-}" == wrong_identity ]]; then printf '/dev/sdz\n'; else printf '/dev/sda\n'; fi + ;; + readlink-k3s-partition) printf '/dev/sdb2\n' ;; + readlink-k3s-disk) printf '/dev/sdb\n' ;; + lsblk-recovery-partition-parent) printf '/dev/sda\n' ;; + lsblk-recovery-partition-majmin) printf '8:3\n' ;; + lsblk-recovery-model) printf '%s\n' "$K3SLR_RECOVERY_MODEL" ;; + lsblk-recovery-serial) printf '%s\n' "$K3SLR_RECOVERY_SERIAL" ;; + lsblk-recovery-wwn) printf '%s\n' "$K3SLR_RECOVERY_WWN" ;; + lsblk-recovery-disk-majmin) printf '8:0\n' ;; + lsblk-k3s-partition-parent) printf '/dev/sdb\n' ;; + lsblk-k3s-partition-majmin) printf '8:34\n' ;; + lsblk-k3s-model) printf '%s\n' "$K3SLR_K3S_MODEL" ;; + lsblk-k3s-serial) printf '%s\n' "$K3SLR_K3S_SERIAL" ;; + lsblk-k3s-wwn) printf '%s\n' "$K3SLR_K3S_WWN" ;; + lsblk-k3s-disk-majmin) printf '8:32\n' ;; + blkid-recovery-uuid) printf '%s\n' "$K3SLR_RECOVERY_FS_UUID" ;; + blkid-recovery-partuuid) printf '%s\n' "$K3SLR_RECOVERY_PARTUUID" ;; + blkid-recovery-type) printf 'ntfs\n' ;; + blkid-k3s-uuid) printf '%s\n' "$K3SLR_K3S_FS_UUID" ;; + blkid-k3s-partuuid) printf '%s\n' "$K3SLR_K3S_PARTUUID" ;; + blkid-k3s-type) printf 'ext4\n' ;; + smartctl-recovery-health-attributes) + if [[ "${system_case:-}" == smart ]]; then + printf 'SMART overall-health self-assessment test result: PASSED\n 5 Reallocated_Sector_Ct 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1\n197 Current_Pending_Sector 0\n198 Offline_Uncorrectable 0\n' + else + printf 'SMART overall-health self-assessment test result: PASSED\n 5 Reallocated_Sector_Ct 0\n197 Current_Pending_Sector 0\n198 Offline_Uncorrectable 0\n' + fi + ;; + findmnt-outer-mountpoint) + if [[ "${system_case:-}" == mounted ]]; then printf '/dev/sda3 ntfs3 rw 1 8:3\n'; else return 1; fi + ;; + findmnt-inner-mountpoint|findmnt-recovery-source) return 1 ;; + losetup-all) + if [[ "${system_case:-}" == loop ]]; then printf '/dev/loop7 /mnt/file 42 8:3 7:7 0 0\n'; fi + ;; + stat-mapper-directory) printf 'directory|0|0|755\n' ;; + test-mapper-directory-not-symlink) + if [[ "${system_case:-}" == mapper_dir_symlink ]]; then return 1; fi + ;; + test-main-mapping-absent|test-main-mapping-not-symlink|\ + test-proof-mapping-absent|test-proof-mapping-not-symlink) ;; + lsblk-all-names-types) + if [[ "${system_case:-}" == mapping ]]; then printf 'sda disk\nk3s-recovery crypt\n'; else printf 'sda disk\nsda3 part\nsdb disk\nsdb2 part\n'; fi + ;; + ntfs-probe-readwrite) + if [[ "${system_case:-}" == probe ]]; then return 1; fi + ;; + esac + return 0 + } + + assert_succeeds task5a_unowned_capture_cleanup_is_rejected \ + 'pre-package system capture fake' + assert_succeeds task5a_capture_cleanup_phase_and_identity_is_enforced \ + 'pre-package system capture fake' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + assert_succeeds task5a_cross_ledger_coherence_is_enforced \ + "$ownership_log" "$event_log" "$path_log" 'pre-package system capture fake' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + : >"$log" + system_case='' + assert_succeeds task5a_command_append_failure_is_rejected \ + "$log" "$ownership_log" "$event_log" "$path_log" \ + 'pre-package fake command-log append failure' \ + /usr/bin/readlink -f -- "$K3SLR_RECOVERY_PARTITION_BY_ID" + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + : >"$log" + assert_succeeds task5a_command_append_failure_is_rejected \ + "$event_log" "$ownership_log" "$event_log" "$path_log" \ + 'pre-package fake command-event append failure' \ + /usr/bin/readlink -f -- "$K3SLR_RECOVERY_PARTITION_BY_ID" + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + : >"$log" + _k3slr_command /fixture/unknown >/dev/null 2>&1 || unknown_rc=$? + assert_eq 97 "$unknown_rc" 'pre-package fake rejects unknown argv with rc 97' + _k3slr_command /usr/bin/readlink \ + "-f -- ${K3SLR_RECOVERY_PARTITION_BY_ID}" >/dev/null 2>&1 || wrong_group_rc=$? + assert_eq 97 "$wrong_group_rc" \ + 'pre-package fake rejects same executable and flattened text with wrong argv grouping' + _k3slr_command /usr/bin/readlink -f -- \ + "$K3SLR_RECOVERY_PARTITION_BY_ID" unexpected >/dev/null 2>&1 || extra_arg_rc=$? + assert_eq 97 "$extra_arg_rc" \ + 'pre-package fake rejects same executable with an extra argv element' + _k3slr_command /usr/bin/readlink -f -- >/dev/null 2>&1 || missing_arg_rc=$? + assert_eq 97 "$missing_arg_rc" \ + 'pre-package fake rejects same executable with a missing argv element' + : >"$log" + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + : >"$cryptsetup_log" + + system_case='' + assert_succeeds _k3slr_collect_prepackage_snapshot snapshot + assert_succeeds _k3slr_validate_prepackage_snapshot "$snapshot" + expected="$(printf '%s\n' \ + readlink-recovery-partition \ + readlink-recovery-disk \ + readlink-k3s-partition \ + readlink-k3s-disk \ + lsblk-recovery-partition-parent \ + lsblk-recovery-partition-majmin \ + lsblk-recovery-model \ + lsblk-recovery-serial \ + lsblk-recovery-wwn \ + lsblk-recovery-disk-majmin \ + lsblk-k3s-partition-parent \ + lsblk-k3s-partition-majmin \ + lsblk-k3s-model \ + lsblk-k3s-serial \ + lsblk-k3s-wwn \ + lsblk-k3s-disk-majmin \ + blkid-recovery-uuid \ + blkid-recovery-partuuid \ + blkid-recovery-type \ + blkid-k3s-uuid \ + blkid-k3s-partuuid \ + blkid-k3s-type \ + smartctl-recovery-health-attributes \ + findmnt-outer-mountpoint \ + findmnt-inner-mountpoint \ + findmnt-recovery-source \ + losetup-all \ + stat-mapper-directory \ + test-mapper-directory-not-symlink \ + test-main-mapping-absent \ + test-main-mapping-not-symlink \ + test-proof-mapping-absent \ + test-proof-mapping-not-symlink \ + lsblk-all-names-types \ + ntfs-probe-readwrite)" + assert_eq "$expected" "$(<"$log")" \ + 'pre-package collector uses the independently hard-coded canonical command order' + assert_succeeds task5a_assert_capture_lineage_and_cleanup "$expected" \ + 'successful pre-package composition' + [[ ! -s "$cryptsetup_log" ]] || return 1 + + for system_case in capture_stdout_no_lf capture_stderr capture_status \ + wrong_identity smart mounted loop mapping mapper_dir_symlink probe; do + : >"$log" + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + : >"$cryptsetup_log" + assert_fails _k3slr_collect_prepackage_snapshot snapshot + assert_succeeds task5a_assert_capture_lineage_and_cleanup "$(<"$log")" \ + "failing pre-package composition (${system_case})" + [[ ! -s "$cryptsetup_log" ]] || return 1 + done +) + +task5a_ntfsinfo_pipeline_contract() ( + local hex expected_hex pipeline_case='' invalid_rc=0 expected_mawk + local event_log="${fixture_root}/task5a-ntfs-pipeline-events.log" + local path_log="${fixture_root}/task5a-ntfs-pipeline-paths.log" + local ownership_log="${fixture_root}/task5a-ntfs-pipeline-ownership.log" + local stage_log="${fixture_root}/task5a-ntfs-pipeline-stages.log" + local sudo_status_log="${fixture_root}/task5a-ntfs-pipeline-sudo-status.log" + local od_status_log="${fixture_root}/task5a-ntfs-pipeline-od-status.log" + local mawk_status_log="${fixture_root}/task5a-ntfs-pipeline-mawk-status.log" + + task5a_ntfs_argv_equals() { + local expected_count="${1-}" argument_index + local -a compared_arguments=() + shift || return 1 + [[ "$expected_count" =~ ^[1-9][0-9]*$ ]] || return 1 + compared_arguments=("$@") + (( ${#compared_arguments[@]} == 2 * expected_count )) || return 1 + for (( argument_index=0; argument_index"$stage_log" + : >"$sudo_status_log" + : >"$od_status_log" + : >"$mawk_status_log" + } + + task5a_assert_ntfs_stage_run() { + local description="$1" expected_sudo="$2" expected_od="$3" expected_mawk_status="$4" + local stage sudo_count=0 od_count=0 mawk_count=0 + local -a stages=() sudo_statuses=() od_statuses=() mawk_statuses=() + mapfile -t stages <"$stage_log" + mapfile -t sudo_statuses <"$sudo_status_log" + mapfile -t od_statuses <"$od_status_log" + mapfile -t mawk_statuses <"$mawk_status_log" + for stage in "${stages[@]}"; do + case "$stage" in + sudo) sudo_count=$((sudo_count + 1)) ;; + od) od_count=$((od_count + 1)) ;; + mawk) mawk_count=$((mawk_count + 1)) ;; + *) return 1 ;; + esac + done + assert_eq 3 "${#stages[@]}" "$description runs exactly three pipeline stages" + assert_eq 1 "$sudo_count" "$description runs the producer exactly once" + assert_eq 1 "$od_count" "$description runs od exactly once" + assert_eq 1 "$mawk_count" "$description runs mawk exactly once" + assert_eq 1 "${#sudo_statuses[@]}" "$description records one producer status" + assert_eq 1 "${#od_statuses[@]}" "$description records one od status" + assert_eq 1 "${#mawk_statuses[@]}" "$description records one mawk status" + assert_eq "$expected_sudo" "${sudo_statuses[0]}" "$description preserves producer status" + assert_eq "$expected_od" "${od_statuses[0]}" "$description preserves od status" + assert_eq "$expected_mawk_status" "${mawk_statuses[0]}" \ + "$description preserves mawk status" + } + + task5a_assert_ntfs_capture_lineage() { + local expected_count="$1" description="$2" + local event_index capture_path + local -a events=() capture_paths=() + mapfile -t events <"$event_log" + mapfile -t capture_paths <"$path_log" + assert_eq "$expected_count" "${#capture_paths[@]}" \ + "$description creates one capture directory per pipeline" + assert_eq "$((expected_count * 3))" "${#events[@]}" \ + "$description records complete capture cleanup groups" + for (( event_index=0; event_index 65536) {\n bad = 1\n exit\n }\n hex = hex $i\n }\n}\nEND {\n if (bad || bytes == 0) exit 1\n print hex\n}' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + task5a_reset_ntfs_stage_logs + + _k3slr_command() { + local created capture_directory command_rc=0 + case "${1-}" in + /usr/bin/mktemp) + task5a_ntfs_argv_equals 4 /usr/bin/mktemp --directory --tmpdir=/tmp \ + k3slr-capture.XXXXXXXX "$@" || return 97 + task5a_capture_ledgers_require_coherent \ + "$ownership_log" "$event_log" '' complete || return 97 + created="$("$@")" || return 1 + [[ "$created" =~ ^/tmp/k3slr-capture\.[A-Za-z0-9]{8}$ && + -d "$created" && ! -L "$created" ]] || return 97 + if ! task5a_capture_ledger_register \ + "$ownership_log" "$event_log" "$created"; then + task5a_cleanup_capture_probe_path "$created" || return 1 + return 97 + fi + printf '%s\n' "$created" >>"$path_log" + printf '%s\n' "$created" + return 0 + ;; + /usr/bin/rm) + task5a_remove_owned_capture_records "$ownership_log" "$event_log" "$@" + return + ;; + /usr/bin/rmdir) + task5a_remove_owned_capture_directory "$ownership_log" "$event_log" "$@" + return + ;; + /usr/bin/sudo) + task5a_ntfs_argv_equals 7 /usr/bin/sudo --non-interactive -- \ + /usr/bin/ntfsinfo --mft --notime /dev/sda3 "$@" || return 97 + printf 'sudo\n' >>"$stage_log" + case "$pipeline_case" in + boundary_65536) + if /usr/bin/head --bytes=65536 /dev/zero; then command_rc=0; else command_rc=$?; fi + ;; + boundary_65537) + if /usr/bin/head --bytes=65537 /dev/zero; then command_rc=0; else command_rc=$?; fi + ;; + *) + printf 'Cluster Size: 4096\nVolume Size in Clusters: 1000000\nAllocated clusters 250000 (25.0%%)\nFree Clusters: 750000 (75.0%%)\n' + command_rc=0 + ;; + esac + if [[ "$pipeline_case" == producer_stderr ]]; then + printf 'unexpected stderr\n' >&2 + elif [[ "$pipeline_case" == producer_failure ]]; then + command_rc=7 + fi + printf '%s\n' "$command_rc" >>"$sudo_status_log" + return "$command_rc" + ;; + /usr/bin/od) + task5a_ntfs_argv_equals 4 /usr/bin/od -An -v -tx1 "$@" || return 97 + printf 'od\n' >>"$stage_log" + if "$@"; then command_rc=0; else command_rc=$?; fi + [[ "$pipeline_case" != od_failure || "$command_rc" != 0 ]] || command_rc=8 + printf '%s\n' "$command_rc" >>"$od_status_log" + return "$command_rc" + ;; + /usr/bin/mawk) + task5a_ntfs_argv_equals 2 /usr/bin/mawk "$expected_mawk" "$@" || return 97 + printf 'mawk\n' >>"$stage_log" + if "$@"; then command_rc=0; else command_rc=$?; fi + [[ "$pipeline_case" != mawk_failure || "$command_rc" != 0 ]] || command_rc=9 + printf '%s\n' "$command_rc" >>"$mawk_status_log" + return "$command_rc" + ;; + *) return 97 ;; + esac + } + + assert_succeeds task5a_unowned_capture_cleanup_is_rejected \ + 'ntfsinfo pipeline capture fake' + assert_succeeds task5a_capture_cleanup_phase_and_identity_is_enforced \ + 'ntfsinfo pipeline capture fake' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + assert_succeeds task5a_cross_ledger_coherence_is_enforced \ + "$ownership_log" "$event_log" "$path_log" 'ntfsinfo pipeline capture fake' + : >"$event_log" + : >"$path_log" + : >"$ownership_log" + _k3slr_command /fixture/unknown >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'ntfsinfo fake rejects unknown argv with rc 97' + invalid_rc=0 + _k3slr_command /usr/bin/sudo \ + '--non-interactive -- /usr/bin/ntfsinfo --mft --notime /dev/sda3' \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" \ + 'ntfsinfo fake rejects grouped sudo arguments that flatten to the expected text' + invalid_rc=0 + _k3slr_command /usr/bin/sudo --non-interactive -- /usr/bin/ntfsinfo --mft \ + --notime /dev/sda3 unexpected >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'ntfsinfo fake rejects an extra sudo argv element' + invalid_rc=0 + _k3slr_command /usr/bin/sudo --non-interactive -- /usr/bin/ntfsinfo --mft \ + --notime >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'ntfsinfo fake rejects a missing sudo argv element' + invalid_rc=0 + _k3slr_command /usr/bin/sudo --version >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'ntfsinfo fake rejects unknown same-sudo argv' + invalid_rc=0 + _k3slr_command /usr/bin/od '-An -v -tx1' >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'ntfsinfo fake rejects grouped od argv' + invalid_rc=0 + _k3slr_command /usr/bin/od -An -v -tx1 unexpected \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'ntfsinfo fake rejects an extra od argv element' + invalid_rc=0 + _k3slr_command /usr/bin/od -An -v >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'ntfsinfo fake rejects a missing od argv element' + invalid_rc=0 + _k3slr_command /usr/bin/od --version >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'ntfsinfo fake rejects unknown same-od argv' + invalid_rc=0 + _k3slr_command /usr/bin/mawk "$expected_mawk" unexpected \ + >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'ntfsinfo fake rejects an extra mawk argv element' + invalid_rc=0 + _k3slr_command /usr/bin/mawk >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'ntfsinfo fake rejects a missing mawk argv element' + invalid_rc=0 + _k3slr_command /usr/bin/mawk '{ print }' >/dev/null 2>&1 || invalid_rc=$? + assert_eq 97 "$invalid_rc" 'ntfsinfo fake rejects unknown same-mawk argv' + [[ ! -s "$event_log" && ! -s "$path_log" && ! -s "$stage_log" ]] || return 1 + + pipeline_case='' + assert_succeeds _k3slr_capture_ntfsinfo_hex hex /dev/sda3 + expected_hex="$(task5a_fixture_ascii_hex $'Cluster Size: 4096\nVolume Size in Clusters: 1000000\nAllocated clusters 250000 (25.0%)\nFree Clusters: 750000 (75.0%)\n')"$'\n' + assert_eq "$expected_hex" "$hex" 'ntfsinfo pipeline returns only normalized bounded hex' + assert_succeeds task5a_assert_ntfs_stage_run 'successful ntfsinfo pipeline' 0 0 0 + + for pipeline_case in producer_failure od_failure mawk_failure producer_stderr; do + task5a_reset_ntfs_stage_logs + assert_fails _k3slr_capture_ntfsinfo_hex hex /dev/sda3 + case "$pipeline_case" in + producer_failure) + assert_succeeds task5a_assert_ntfs_stage_run \ + 'producer-only failure pipeline' 7 0 0 + ;; + od_failure) + assert_succeeds task5a_assert_ntfs_stage_run \ + 'od-only failure pipeline' 0 8 0 + ;; + mawk_failure) + assert_succeeds task5a_assert_ntfs_stage_run \ + 'mawk-only failure pipeline' 0 0 9 + ;; + producer_stderr) + assert_succeeds task5a_assert_ntfs_stage_run \ + 'stderr-rejection pipeline' 0 0 0 + ;; + esac + done + + pipeline_case=boundary_65536 + task5a_reset_ntfs_stage_logs + assert_succeeds _k3slr_capture_ntfsinfo_hex hex /dev/sda3 + assert_eq 131073 "${#hex}" 'ntfsinfo pipeline accepts exactly 65,536 bytes plus output LF' + [[ "$hex" =~ ^0+$'\n'$ ]] || return 1 + assert_succeeds task5a_assert_ntfs_stage_run '65,536-byte pipeline boundary' 0 0 0 + + pipeline_case=boundary_65537 + task5a_reset_ntfs_stage_logs + assert_fails _k3slr_capture_ntfsinfo_hex hex /dev/sda3 + assert_succeeds task5a_assert_ntfs_stage_run '65,537-byte pipeline rejection' 0 0 1 + + assert_succeeds task5a_assert_ntfs_capture_lineage 7 \ + 'success, status, stderr, and byte-boundary pipelines' +) + +task5a_dry_run_and_prepackage_action_binding() ( + local mode wrapper main_name log output action_rc=0 probe_rc=0 record='' + local command_calls=0 + for mode in prepare open close; do + case "$mode" in + prepare) wrapper="$PREPARE_WRAPPER_PATH" ;; + open) wrapper="$OPEN_WRAPPER_PATH" ;; + close) wrapper="$CLOSE_WRAPPER_PATH" ;; + esac + main_name="_k3slr_${mode}_main" + log="${fixture_root}/task5a-${mode}-dry-run.log" + : >"$log" + # shellcheck source=/dev/null + source "$wrapper" + _k3slr_load_contract() { + printf 'contract\n' >>"$log" + } + _k3slr_prepackage_device_preflight() { + printf 'prepackage\n' >>"$log" + } + _k3slr_require_execute_tty() { + printf 'unexpected-tty\n' >>"$log" + return 97 + } + _k3slr_command() { + (( command_calls += 1 )) + printf 'unexpected-command:%s\n' "$*" >>"$log" + return 97 + } + output="$("$main_name")" || return 1 + [[ "$output" == "DRY RUN: ${mode} lifecycle; no changes made." ]] || return 1 + assert_eq $'contract\nprepackage' "$(<"$log")" \ + "$mode dry-run uses only the non-interactive pre-package evidence seam" + + : >"$log" + _k3slr_prepackage_device_preflight() { + printf 'prepackage\n' >>"$log" + return 1 + } + assert_fails "$main_name" + assert_eq $'contract\nprepackage' "$(<"$log")" \ + "$mode dry-run fails closed when cached read-only evidence is unavailable" + done + + log="${fixture_root}/task5a-device-validator-binding.log" + : >"$log" + _k3slr_prepackage_device_preflight() { + printf 'prepackage\n' >>"$log" + } + _k3slr_command() { + (( command_calls += 1 )) + return 97 + } + task5b_legacy_binding_probe_payload() { + local binding_case="${1-}" action_rc=0 command_calls=0 + (( $# == 1 )) || return 1 + case "$binding_case" in + prepare-device) + _k3slr_lifecycle_action prepare device-validator \ + _k3slr_prepare_invocation || action_rc=$? + ;; + prepare-sudo) + _k3slr_lifecycle_action prepare sudo _k3slr_prepare_invocation || + action_rc=$? + ;; + open-device) + _k3slr_lifecycle_action open 'device-validator(--expect-device-ready)' || + action_rc=$? + ;; + *) return 1 ;; + esac + task5b_production_probe_emit_ok legacy_binding "$action_rc" "$command_calls" + } + task5b_production_probe_adversary_matrix || return 1 + task5b_production_probe_static_graph_is_closed legacy_binding || return 1 + record="$( + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm legacy_binding || return 1 + task5b_legacy_binding_probe_payload prepare-device + )" || probe_rc=$? + assert_eq 0 "$probe_rc" 'prepare device-validator binding probe completes safely' + assert_eq 'TASK5B_PROBE_OK|legacy_binding|1|0' "$record" \ + 'prepare device-validator binding is fail-closed before commands' + assert_eq '' "$(<"$log")" \ + 'prepare action integration remains fail-closed until its later checkpoint' + record='' + probe_rc=0 + record="$( + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm legacy_binding || return 1 + task5b_legacy_binding_probe_payload prepare-sudo + )" || probe_rc=$? + assert_eq 0 "$probe_rc" 'prepare sudo binding probe completes safely' + assert_eq 'TASK5B_PROBE_OK|legacy_binding|97|1' "$record" \ + 'prepare sudo binding reaches only the modeled command failure' + record='' + probe_rc=0 + record="$( + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm legacy_binding || return 1 + task5b_legacy_binding_probe_payload open-device + )" || probe_rc=$? + assert_eq 0 "$probe_rc" 'open device-validator binding probe completes safely' + assert_eq 'TASK5B_PROBE_OK|legacy_binding|1|0' "$record" \ + 'open device-validator binding is fail-closed before commands' +) + +# The production break this catches is a Task 5B fake command boundary that +# accepts a hazardous near-match or reaches a real executable. Expected argv +# below are independent test literals; none are derived from production arrays. +task5b_gate0_route_equals() { + local route_id="${1-}" argument_index + local -a actual_arguments=() expected_arguments=() + shift || return 1 + actual_arguments=("$@") + case "$route_id" in + sudo-validate) + expected_arguments=(/usr/bin/sudo --validate) + ;; + sudo-cached-validate) + expected_arguments=(/usr/bin/sudo --non-interactive --validate) + ;; + apt-update) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/apt-get update) + ;; + apt-install) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/apt-get install --yes + keepassxc=2.7.6+dfsg.1-1build3 cryptsetup-bin=2:2.7.0-1ubuntu4.2) + ;; + feasibility-execute) + expected_arguments=(/usr/bin/bash + /home/donghyeon/workspace/platform/scripts/validate/k3s-local-recovery-feasibility.sh + --execute) + ;; + outer-scaffold-mkdir) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/mkdir + --mode=0700 -- /mnt/k3s-recovery-ssd) + ;; + outer-mount) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/mount -t ntfs3 + -o rw,uid=1000,gid=1000,dmask=0077,fmask=0177,nodev,nosuid,noexec + /dev/disk/by-id/wwn-0x500a0751e6aa6254-part3 /mnt/k3s-recovery-ssd) + ;; + layout-root-mkdir) + expected_arguments=(/usr/bin/mkdir --mode=0700 -- + /mnt/k3s-recovery-ssd/HyeonworksRecovery) + ;; + layout-vault-mkdir) + expected_arguments=(/usr/bin/mkdir --mode=0700 -- + /mnt/k3s-recovery-ssd/HyeonworksRecovery/vault) + ;; + layout-backups-mkdir) + expected_arguments=(/usr/bin/mkdir --mode=0700 -- + /mnt/k3s-recovery-ssd/HyeonworksRecovery/vault/backups) + ;; + layout-containers-mkdir) + expected_arguments=(/usr/bin/mkdir --mode=0700 -- + /mnt/k3s-recovery-ssd/HyeonworksRecovery/containers) + ;; + probe-mktemp) + expected_arguments=(/usr/bin/mktemp + --tmpdir=/mnt/k3s-recovery-ssd/HyeonworksRecovery + .k3slr-hardlink-probe.XXXXXXXX) + ;; + probe-link) + expected_arguments=(/usr/bin/ln -- + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-hardlink-probe.FIXTURE1 + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-hardlink-probe.FIXTURE1.link) + ;; + probe-unlink-link) + expected_arguments=(/usr/bin/unlink -- + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-hardlink-probe.FIXTURE1.link) + ;; + probe-unlink-source) + expected_arguments=(/usr/bin/unlink -- + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-hardlink-probe.FIXTURE1) + ;; + probe-sync) + expected_arguments=(/usr/bin/sync --file-system + /mnt/k3s-recovery-ssd/HyeonworksRecovery) + ;; + keepass-db-create) + expected_arguments=(/usr/bin/keepassxc-cli db-create --quiet --set-password + /mnt/k3s-recovery-ssd/HyeonworksRecovery/vault/hyeonworks-recovery.kdbx) + ;; + keepass-add) + expected_arguments=(/usr/bin/keepassxc-cli add --quiet --generate --length 40 + --lower --upper --numeric --every-group + /mnt/k3s-recovery-ssd/HyeonworksRecovery/vault/hyeonworks-recovery.kdbx + 'K3s Recovery LUKS') + ;; + container-dd) + expected_arguments=(/usr/bin/dd if=/dev/zero + of=/mnt/k3s-recovery-ssd/HyeonworksRecovery/containers/k3s-recovery.luks + bs=16M count=2048 conv=excl,fsync status=progress) + ;; + outer-umount) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/umount -- + /mnt/k3s-recovery-ssd) + ;; + loop-create) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/sbin/losetup + --find --show --nooverlap -- + /mnt/k3s-recovery-ssd/HyeonworksRecovery/containers/k3s-recovery.luks) + ;; + luks-format) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/sbin/cryptsetup + luksFormat --batch-mode --type luks2 --key-file=- /dev/loop37) + ;; + luks-open) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/sbin/cryptsetup + open --type luks2 --key-file=- /dev/loop37 k3s-recovery) + ;; + mkfs-ext4) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/sbin/mkfs.ext4 + -F -L K3S_RECOVERY /dev/mapper/k3s-recovery) + ;; + inner-parent-mkdir) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/mkdir + --mode=0700 -- /srv/recovery) + ;; + inner-mountpoint-mkdir) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/mkdir + --mode=0700 -- /srv/recovery/k3s) + ;; + inner-mount) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/mount -t ext4 + -o rw,nodev,nosuid,noexec /dev/mapper/k3s-recovery /srv/recovery/k3s) + ;; + inner-chown) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/chown + root:root /srv/recovery/k3s) + ;; + inner-chmod) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/chmod + 0700 /srv/recovery/k3s) + ;; + marker-mktemp) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/mktemp + --tmpdir=/srv/recovery/k3s .k3slr-marker.tmp.XXXXXXXX) + ;; + marker-dd) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/dd + of=/srv/recovery/k3s/.k3slr-marker.tmp.FIXTURE1 bs=37 count=1 + iflag=fullblock conv=notrunc,fsync status=none) + ;; + outer-metadata-mktemp) + expected_arguments=(/usr/bin/mktemp + --tmpdir=/mnt/k3s-recovery-ssd/HyeonworksRecovery + .k3slr-volume.tmp.XXXXXXXX) + ;; + inner-metadata-mktemp) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/mktemp + --tmpdir=/srv/recovery/k3s .k3slr-volume.tmp.XXXXXXXX) + ;; + outer-metadata-dd) + expected_arguments=(/usr/bin/dd + of=/mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-volume.tmp.FIXTURE1 + bs=4096 count=1 iflag=fullblock conv=notrunc,fsync status=none) + ;; + inner-metadata-dd) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/dd + if=/mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-volume.tmp.FIXTURE1 + of=/srv/recovery/k3s/.k3slr-volume.tmp.FIXTURE1 bs=4096 count=1 + iflag=fullblock conv=notrunc,fsync status=none) + ;; + marker-stage-sync) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/sync + --file-system /srv/recovery/k3s/.k3slr-marker.tmp.FIXTURE1) + ;; + marker-publish) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/ln -- + /srv/recovery/k3s/.k3slr-marker.tmp.FIXTURE1 + /srv/recovery/k3s/.recovery-marker) + ;; + outer-stage-sync) + expected_arguments=(/usr/bin/sync --file-system + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-volume.tmp.FIXTURE1) + ;; + inner-stage-sync) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/sync + --file-system /srv/recovery/k3s/.k3slr-volume.tmp.FIXTURE1) + ;; + outer-metadata-publish) + expected_arguments=(/usr/bin/ln -- + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-volume.tmp.FIXTURE1 + /mnt/k3s-recovery-ssd/HyeonworksRecovery/volume.env) + ;; + inner-metadata-publish) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/ln -- + /srv/recovery/k3s/.k3slr-volume.tmp.FIXTURE1 + /srv/recovery/k3s/.recovery-volume.env) + ;; + marker-stage-unlink) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/unlink -- + /srv/recovery/k3s/.k3slr-marker.tmp.FIXTURE1) + ;; + outer-stage-unlink) + expected_arguments=(/usr/bin/unlink -- + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-volume.tmp.FIXTURE1) + ;; + inner-stage-unlink) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/unlink -- + /srv/recovery/k3s/.k3slr-volume.tmp.FIXTURE1) + ;; + inner-parent-sync) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/sync + --file-system /srv/recovery/k3s) + ;; + inner-umount) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/bin/umount -- + /srv/recovery/k3s) + ;; + mapping-close) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/sbin/cryptsetup + close k3s-recovery) + ;; + loop-detach) + expected_arguments=(/usr/bin/sudo --non-interactive -- /usr/sbin/losetup + --detach /dev/loop37) + ;; + *) return 1 ;; + esac + (( ${#actual_arguments[@]} == ${#expected_arguments[@]} )) || return 1 + for (( argument_index=0; argument_index<${#expected_arguments[@]}; argument_index++ )); do + [[ "${actual_arguments[argument_index]}" == "${expected_arguments[argument_index]}" ]] || + return 1 + done +} + +task5b_gate0_fake_command() { + local route_id + local -a route_ids=( + sudo-validate sudo-cached-validate apt-update apt-install feasibility-execute + outer-scaffold-mkdir outer-mount layout-root-mkdir layout-vault-mkdir + layout-backups-mkdir layout-containers-mkdir probe-mktemp probe-link + probe-unlink-link probe-unlink-source probe-sync keepass-db-create keepass-add + container-dd outer-umount loop-create luks-format luks-open mkfs-ext4 + inner-parent-mkdir inner-mountpoint-mkdir inner-mount inner-chown inner-chmod + marker-mktemp marker-dd outer-metadata-mktemp inner-metadata-mktemp + outer-metadata-dd inner-metadata-dd marker-stage-sync marker-publish + outer-stage-sync inner-stage-sync outer-metadata-publish inner-metadata-publish + marker-stage-unlink outer-stage-unlink inner-stage-unlink inner-parent-sync + inner-umount mapping-close loop-detach + ) + for route_id in "${route_ids[@]}"; do + if task5b_gate0_route_equals "$route_id" "$@"; then + (( TASK5B_GATE0_MODEL_DISPATCH_COUNT += 1 )) + (( TASK5B_GATE0_MODEL_EFFECT_COUNT += 1 )) + TASK5B_GATE0_LAST_ROUTE="$route_id" + TASK5B_GATE0_LAST_OUTPUT="modeled:${route_id}" + return 0 + fi + done + return 97 +} + +readonly -a TASK5B_GATE0_ACTIVE_GRAPH_FUNCTIONS=( + _k3slr_command assert_eq fail task5b_gate0_active_matrix + task5b_gate0_assert_rejected task5b_gate0_assert_route + task5b_gate0_fake_command task5b_gate0_reset_route_record + task5b_gate0_route_equals task5b_gate0_assignment_only_status + task5b_gate0_real_exec_fuse +) + +task5b_gate0_assignment_only_status() { + local candidate="${1-}" lhs rhs character next_character quote='' mode=scalar + local index=0 length depth=0 escaped=0 word_ended=0 append_assignment=0 + (( $# == 1 )) || return 1 + candidate="${candidate#"${candidate%%[![:space:]]*}"}" + candidate="${candidate%"${candidate##*[![:space:]]}"}" + [[ "$candidate" == *=* ]] || return 1 + lhs="${candidate%%=*}" + rhs="${candidate#*=}" + if [[ "$lhs" == *+ ]]; then + lhs="${lhs%+}" + append_assignment=1 + fi + [[ "$lhs" =~ ^[a-zA-Z_][a-zA-Z0-9_]*(\[([a-zA-Z0-9_]+|[a-zA-Z0-9_]+[[:space:]]*[-+][[:space:]]*[0-9]+)\])?$ ]] || + return 1 + length="${#rhs}" + (( length > 0 )) || return 0 + (( append_assignment == 0 )) || [[ "${rhs:0:1}" == '(' ]] || return 1 + if [[ "${rhs:0:1}" == '(' ]]; then + mode=compound + depth=1 + index=1 + fi + while (( index < length )); do + character="${rhs:index:1}" + next_character='' + (( index + 1 >= length )) || next_character="${rhs:index + 1:1}" + if (( escaped == 1 )); then + escaped=0 + (( index += 1 )) + continue + fi + case "$quote" in + single) + [[ "$character" != "'" ]] || quote='' + (( index += 1 )) + continue + ;; + double) + case "$character" in + '\\') escaped=1 ;; + '"') quote='' ;; + '`') return 1 ;; + '$') + [[ "$next_character" != '(' ]] || return 1 + ;; + esac + (( index += 1 )) + continue + ;; + esac + if (( depth > 0 )) && [[ "$mode" == arithmetic ]]; then + case "$character" in + '`') return 1 ;; + '$') + [[ "$next_character" != '(' ]] || return 1 + ;; + '(') (( depth += 1 )) ;; + ')') (( depth -= 1 )) ;; + esac + (( index += 1 )) + continue + fi + case "$character" in + "'") quote=single ;; + '"') quote=double ;; + '\\') escaped=1 ;; + '`'|';'|'&'|'|'|'<'|'>') return 1 ;; + '$') + if [[ "${rhs:index:3}" == '$((' ]]; then + mode=arithmetic + depth=2 + (( index += 3 )) + continue + fi + [[ "$next_character" != '(' ]] || return 1 + ;; + '(') + [[ "$mode" == compound ]] || return 1 + (( depth += 1 )) + ;; + ')') + [[ "$mode" == compound ]] || return 1 + (( depth -= 1 )) + (( depth >= 0 )) || return 1 + if (( depth == 0 )); then + (( index += 1 )) + while (( index < length )); do + [[ "${rhs:index:1}" =~ [[:space:]] ]] || return 1 + (( index += 1 )) + done + return 0 + fi + ;; + $'\n'|$'\r') + [[ "$mode" == compound ]] || return 1 + ;; + [[:space:]]) + if [[ "$mode" != compound ]]; then + word_ended=1 + fi + ;; + *) + (( word_ended == 0 )) || return 1 + ;; + esac + (( index += 1 )) + done + [[ -z "$quote" && "$escaped" == 0 ]] || return 1 + if [[ "$mode" == compound && "$depth" -gt 0 ]]; then + return 2 + fi + (( depth == 0 )) +} + +task5b_gate0_real_exec_fuse() { + local candidate="${BASH_COMMAND-}" execution_word='' assignment_rc=0 + candidate="${candidate#"${candidate%%[![:space:]]*}"}" + while [[ "$candidate" =~ ^(if|elif|while|until|then|\!|time)[[:space:]]+(.+)$ ]]; do + candidate="${BASH_REMATCH[2]}" + done + case "$candidate" in + ''|'{'|'}'|'[['*|'(('*|for\ *|case\ *|do|done|esac) return 0 ;; + esac + if [[ "$candidate" =~ ^[a-zA-Z_][a-zA-Z0-9_]*(\[[^]]+\])?\+?= ]]; then + task5b_gate0_assignment_only_status "$candidate" || assignment_rc=$? + (( assignment_rc == 0 )) && return 0 + (( TASK5B_GATE0_ACTUAL_EXEC_COUNT += 1 )) + if [[ "${TASK5B_GATE0_FUSE_PROBE_SILENT:-0}" != 1 ]]; then + printf 'Task 5B Gate 0 blocked assignment-prefixed execution\n' >&9 + fi + exit 96 + fi + read -r execution_word _ <<<"$candidate" + execution_word="${execution_word%;}" + case "$execution_word" in + \$*|\"\$*|\'\$*|/*|\"/*|\'/*|command|builtin|exec|eval|source|.) + (( TASK5B_GATE0_ACTUAL_EXEC_COUNT += 1 )) + if [[ "${TASK5B_GATE0_FUSE_PROBE_SILENT:-0}" != 1 ]]; then + printf 'Task 5B Gate 0 blocked dynamic/external execution word: %s\n' \ + "$execution_word" >&9 + fi + exit 96 + ;; + esac + case "$execution_word" in + _k3slr_command|assert_eq|fail|task5b_gate0_active_matrix|\ + task5b_gate0_assert_rejected|task5b_gate0_assert_route|\ + task5b_gate0_fake_command|task5b_gate0_reset_route_record|\ + task5b_gate0_route_equals|exit|local|shift|return|printf|read|:) + return 0 + ;; + esac + (( TASK5B_GATE0_ACTUAL_EXEC_COUNT += 1 )) + if [[ "${TASK5B_GATE0_FUSE_PROBE_SILENT:-0}" != 1 ]]; then + printf 'Task 5B Gate 0 blocked non-builtin execution word: %s\n' \ + "$execution_word" >&9 + fi + exit 96 +} + +task5b_gate0_adversary_direct() { + /fixture/task5b-unlisted-direct +} + +task5b_gate0_adversary_indented() { + "$@" +} + +task5b_gate0_adversary_control_word() { + if command "$@"; then + : + fi +} + +task5b_gate0_adversary_array() { + local -a task5b_runner=(/fixture/task5b-unlisted-array) + "${task5b_runner[@]}" +} + +task5b_gate0_adversary_variable() { + local task5b_runner=/fixture/task5b-unlisted-variable + "$task5b_runner" +} + +task5b_gate0_adversary_shell_c() { + /fixture/unlisted/bash -c ':' +} + +task5b_gate0_adversary_helper_root() { + task5b_gate0_adversary_helper_leaf +} + +task5b_gate0_adversary_helper_leaf() { + /fixture/task5b-unlisted-helper +} + +task5b_gate0_adversary_trap_disable() { + trap - DEBUG + set +T + shopt -u extdebug +} + +task5b_gate0_adversary_assignment_bare() { + TASK5B_ASSIGNMENT_PREFIX= trap - DEBUG +} + +task5b_gate0_adversary_assignment_simple() { + TASK5B_ASSIGNMENT_PREFIX=1 /usr/bin/printf TASK5B_ASSIGNMENT_PREFIX_CANARY +} + +task5b_gate0_adversary_assignment_quoted() { + TASK5B_ASSIGNMENT_PREFIX="a b" trap - DEBUG +} + +task5b_gate0_adversary_assignment_indexed() { + TASK5B_ASSIGNMENT_PREFIX[0]=1 trap - DEBUG +} + +task5b_gate0_static_safe_sample() { + local value='' + task5b_gate0_route_equals sudo-validate /usr/bin/sudo --validate || return 1 + value=modeled + [[ "$value" == modeled ]] +} + +task5b_gate0_static_assignment_prefixes_are_closed() { + local body="${1-}" line candidate='' compound_candidate='' + local assignment_rc=0 + (( $# == 1 )) || return 1 + while IFS= read -r line; do + candidate="${line#"${line%%[![:space:]]*}"}" + candidate="${candidate%"${candidate##*[![:space:]]}"}" + while [[ "$candidate" =~ ^(if|elif|while|until|then|\!|time)[[:space:]]+(.+)$ ]]; do + candidate="${BASH_REMATCH[2]}" + done + [[ "$candidate" != *';' ]] || candidate="${candidate%;}" + if [[ -n "$compound_candidate" ]]; then + compound_candidate+=$'\n'"$candidate" + assignment_rc=0 + task5b_gate0_assignment_only_status "$compound_candidate" || + assignment_rc=$? + case "$assignment_rc" in + 0) compound_candidate='' ;; + 2) ;; + *) return 1 ;; + esac + continue + fi + [[ "$candidate" =~ ^[a-zA-Z_][a-zA-Z0-9_]*(\[[^]]+\])?\+?= ]] || + continue + assignment_rc=0 + task5b_gate0_assignment_only_status "$candidate" || assignment_rc=$? + case "$assignment_rc" in + 0) ;; + 2) compound_candidate="$candidate" ;; + *) return 1 ;; + esac + done <<<"$body" + if [[ -n "$compound_candidate" ]]; then + return 1 + fi +} + +task5b_gate0_static_execution_positions_are_closed() { + local body="${1-}" + (( $# == 1 )) || return 1 + [[ -n "$body" ]] || return 1 + if /usr/bin/grep -En \ + '^[[:space:]]*((if|elif|while|until|then|!|time)[[:space:]]+)*("?/?[^[:space:]]*/(ba|z|k|da)?sh|bash|sh)[[:space:]].*(-c|--command)([[:space:]]|$)|^[[:space:]]*((if|elif|while|until|then|!|time)[[:space:]]+)*("?\$|\$\{|/|"/|command([[:space:]]|$)|builtin[[:space:]]+command([[:space:]]|$)|exec([[:space:]]|$)|eval([[:space:]]|$)|source([[:space:]]|$)|\.[[:space:]])|\$\*|\$\{[^}]+\[\*\][^}]*\}|`|\$\(|^[[:space:]]*(trap([[:space:]]|$)|set[[:space:]]+\+T([[:space:]]|$)|shopt[[:space:]].*extdebug)' \ + <<<"$body" >/dev/null; then + return 1 + fi +} + +task5b_gate0_static_body_is_closed() { + local body="${1-}" + (( $# == 1 )) || return 1 + task5b_gate0_static_assignment_prefixes_are_closed "$body" || return 1 + task5b_gate0_static_execution_positions_are_closed "$body" +} + +task5b_gate0_definition_hash() { + local destination_name="${1-}" function_name="${2-}" body='' hash_record='' + (( $# == 2 )) || return 1 + [[ "$destination_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1 + body="$(declare -f "$function_name")" || return 1 + hash_record="$(printf '%s\n' "$body" | /usr/bin/sha256sum)" || return 1 + [[ "$hash_record" =~ ^([0-9a-f]{64})[[:space:]]+-$ ]] || return 1 + printf -v "$destination_name" '%s' "${BASH_REMATCH[1]}" +} + +task5b_gate0_normalize_static_body_data_lines() { + local body="${1-}" destination_name="${2-}" function_name="${3-}" + local line normalized='' previous_continues=0 + (( $# == 3 )) || return 1 + [[ "$destination_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1 + while IFS= read -r line; do + if (( previous_continues == 1 )); then + normalized+=' task5b_static_argument ' + elif [[ "$function_name" == task5b_gate0_route_equals && + "$line" =~ ^[[:space:]]+/ ]]; then + normalized+=' task5b_static_array_value ' + fi + normalized+="$line"$'\n' + previous_continues=0 + [[ "$line" != *'\' ]] || previous_continues=1 + done <<<"$body" + if [[ "$function_name" == fail ]]; then + normalized="${normalized//\"\$*\"/task5b_static_message}" + fi + normalized="${normalized//\$\(\(/task5b_static_arithmetic_(}" + printf -v "$destination_name" '%s' "$normalized" +} + +task5b_gate0_fuse_body_is_closed() { + local body="${1-}" exact_allowlist + (( $# == 1 )) || return 1 + [[ -n "$body" ]] || return 1 + [[ "$body" != *'trap - DEBUG'* && "$body" != *'set +T'* && + "$body" != *'shopt -u extdebug'* ]] || return 1 + exact_allowlist='_k3slr_command | assert_eq | fail | task5b_gate0_active_matrix | task5b_gate0_assert_rejected | task5b_gate0_assert_route | task5b_gate0_fake_command | task5b_gate0_reset_route_record | task5b_gate0_route_equals | exit | local | shift | return | printf | read | :)' + [[ "$body" == *"$exact_allowlist"* ]] || return 1 + [[ "$body" != *'task5b_gate0_dynamic_external_helper'* ]] || return 1 + [[ "$body" != *'*=*)'* ]] || return 1 +} + +task5b_gate0_assignment_parser_body_is_closed() { + local body="${1-}" + (( $# == 1 )) || return 1 + [[ -n "$body" ]] || return 1 + if /usr/bin/grep -En \ + '^[[:space:]]*(trap([[:space:]]|$)|set[[:space:]]+\+T([[:space:]]|$)|shopt[[:space:]].*extdebug|/|"/|command([[:space:]]|$)|builtin[[:space:]]+command([[:space:]]|$)|exec([[:space:]]|$)|eval([[:space:]]|$)|source([[:space:]]|$)|\.[[:space:]])' \ + <<<"$body" >/dev/null; then + return 1 + fi +} + +task5b_gate0_active_graph_is_closed() { + local function_name body='' normalized_body='' + for function_name in "${TASK5B_GATE0_ACTIVE_GRAPH_FUNCTIONS[@]}"; do + body="$(declare -f "$function_name")" || return 1 + if [[ "$function_name" == task5b_gate0_real_exec_fuse ]]; then + task5b_gate0_fuse_body_is_closed "$body" || { + printf 'Task 5B Gate 0 static fuse scan rejected: %s\n' \ + "$function_name" >&2 + return 1 + } + continue + fi + if [[ "$function_name" == task5b_gate0_assignment_only_status ]]; then + task5b_gate0_assignment_parser_body_is_closed "$body" || { + printf 'Task 5B Gate 0 assignment parser scan rejected: %s\n' \ + "$function_name" >&2 + return 1 + } + continue + fi + task5b_gate0_normalize_static_body_data_lines "$body" normalized_body \ + "$function_name" || return 1 + task5b_gate0_static_assignment_prefixes_are_closed "$body" || { + printf 'Task 5B Gate 0 assignment-prefix scan rejected: %s\n' \ + "$function_name" >&2 + return 1 + } + task5b_gate0_static_execution_positions_are_closed "$normalized_body" || { + printf 'Task 5B Gate 0 active graph scan rejected: %s\n' \ + "$function_name" >&2 + return 1 + } + done +} + +task5b_gate0_active_graph_hash() { + local destination_name="${1-}" function_name body='' graph_record='' + (( $# == 1 )) || return 1 + [[ "$destination_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1 + for function_name in "${TASK5B_GATE0_ACTIVE_GRAPH_FUNCTIONS[@]}"; do + body="$(declare -f "$function_name")" || return 1 + graph_record+="${function_name}"$'\n'"${body}"$'\n' + done + body="$(printf '%s' "$graph_record" | /usr/bin/sha256sum)" || return 1 + [[ "$body" =~ ^([0-9a-f]{64})[[:space:]]+-$ ]] || return 1 + printf -v "$destination_name" '%s' "${BASH_REMATCH[1]}" +} + +task5b_gate0_static_adversarial_contract() { + local body='' + body="$(declare -f task5b_gate0_static_safe_sample)" || return 1 + assert_succeeds task5b_gate0_static_body_is_closed "$body" + for adversary in task5b_gate0_adversary_direct task5b_gate0_adversary_indented \ + task5b_gate0_adversary_control_word task5b_gate0_adversary_array \ + task5b_gate0_adversary_variable task5b_gate0_adversary_shell_c \ + task5b_gate0_adversary_trap_disable \ + task5b_gate0_adversary_assignment_bare \ + task5b_gate0_adversary_assignment_simple \ + task5b_gate0_adversary_assignment_quoted \ + task5b_gate0_adversary_assignment_indexed; do + body="$(declare -f "$adversary")" || return 1 + assert_fails task5b_gate0_static_body_is_closed "$body" + done + body="$(declare -f task5b_gate0_adversary_helper_root)"$'\n'\ +"$(declare -f task5b_gate0_adversary_helper_leaf)" || return 1 + assert_fails task5b_gate0_static_body_is_closed "$body" +} + +task5b_gate0_full_graph_static_adversarial_contract() ( + local scan_rc=0 + task5b_gate0_assert_rejected() { + TASK5B_ASSIGNMENT_PREFIX=1 trap - DEBUG + /usr/bin/printf 'TASK5B_FULL_GRAPH_SCAN_CANARY' + } + task5b_gate0_active_graph_is_closed 2>/dev/null || scan_rc=$? + assert_eq 1 "$scan_rc" \ + 'actual full-graph scanner rejects a mutated allowlisted helper' +) + +task5b_gate0_dynamic_external_helper() { + /usr/bin/printf 'TASK5B_REAL_EXEC_CANARY' +} + +task5b_gate0_dynamic_fuse_contract() { + local probe_rc=0 probe_output='' + probe_output="$( + TASK5B_GATE0_ACTUAL_EXEC_COUNT=0 + TASK5B_GATE0_FUSE_PROBE_SILENT=1 + shopt -s extdebug + set -T + trap 'task5b_gate0_real_exec_fuse' DEBUG + /usr/bin/true + )" || probe_rc=$? + assert_eq 96 "$probe_rc" 'generic fuse blocks an unlisted absolute executable' + assert_eq '' "$probe_output" 'unlisted absolute executable produces no output' + + probe_rc=0 + probe_output="$( + TASK5B_GATE0_ACTUAL_EXEC_COUNT=0 + TASK5B_GATE0_FUSE_PROBE_SILENT=1 + shopt -s extdebug + set -T + trap 'task5b_gate0_real_exec_fuse' DEBUG + task5b_gate0_dynamic_external_helper + )" || probe_rc=$? + assert_eq 96 "$probe_rc" 'generic fuse blocks helper-mediated external execution' + assert_eq '' "$probe_output" 'helper-mediated external canary never executes' + + probe_rc=0 + probe_output="$( + TASK5B_GATE0_ACTUAL_EXEC_COUNT=0 + TASK5B_GATE0_FUSE_PROBE_SILENT=1 + task5b_gate0_assert_rejected() { + trap - DEBUG + /usr/bin/printf 'TASK5B_TRAP_DISABLE_CANARY' + } + shopt -s extdebug + set -T + trap 'task5b_gate0_real_exec_fuse' DEBUG + task5b_gate0_assert_rejected + )" || probe_rc=$? + assert_eq 96 "$probe_rc" 'generic fuse blocks trap removal in an allowlisted helper' + assert_eq '' "$probe_output" 'trap-removal helper canary never executes' + + probe_rc=0 + probe_output="$( + TASK5B_GATE0_ACTUAL_EXEC_COUNT=0 + TASK5B_GATE0_FUSE_PROBE_SILENT=1 + shopt -s extdebug + set -T + trap 'task5b_gate0_real_exec_fuse' DEBUG + TASK5B_ASSIGNMENT_PREFIX= trap - DEBUG + /usr/bin/printf 'TASK5B_BARE_ASSIGNMENT_CANARY' + )" || probe_rc=$? + assert_eq 96 "$probe_rc" 'generic fuse blocks a bare assignment prefix' + assert_eq '' "$probe_output" 'bare assignment prefix leaves trap and canary closed' + + probe_rc=0 + probe_output="$( + TASK5B_GATE0_ACTUAL_EXEC_COUNT=0 + TASK5B_GATE0_FUSE_PROBE_SILENT=1 + shopt -s extdebug + set -T + trap 'task5b_gate0_real_exec_fuse' DEBUG + TASK5B_ASSIGNMENT_PREFIX=1 /usr/bin/printf 'TASK5B_SIMPLE_ASSIGNMENT_CANARY' + )" || probe_rc=$? + assert_eq 96 "$probe_rc" 'generic fuse blocks a simple assignment prefix' + assert_eq '' "$probe_output" 'simple assignment prefix external effect stays zero' + + probe_rc=0 + probe_output="$( + TASK5B_GATE0_ACTUAL_EXEC_COUNT=0 + TASK5B_GATE0_FUSE_PROBE_SILENT=1 + shopt -s extdebug + set -T + trap 'task5b_gate0_real_exec_fuse' DEBUG + TASK5B_ASSIGNMENT_PREFIX="a b" trap - DEBUG + /usr/bin/printf 'TASK5B_QUOTED_ASSIGNMENT_CANARY' + )" || probe_rc=$? + assert_eq 96 "$probe_rc" 'generic fuse blocks a quoted assignment prefix' + assert_eq '' "$probe_output" 'quoted assignment prefix leaves trap and canary closed' + + probe_rc=0 + probe_output="$( + TASK5B_GATE0_ACTUAL_EXEC_COUNT=0 + TASK5B_GATE0_FUSE_PROBE_SILENT=1 + shopt -s extdebug + set -T + trap 'task5b_gate0_real_exec_fuse' DEBUG + TASK5B_ASSIGNMENT_PREFIX[0]=1 trap - DEBUG + /usr/bin/printf 'TASK5B_INDEXED_ASSIGNMENT_CANARY' + )" || probe_rc=$? + assert_eq 96 "$probe_rc" 'generic fuse blocks an indexed assignment prefix' + assert_eq '' "$probe_output" 'indexed assignment prefix leaves trap and canary closed' +} + +task5b_gate0_reset_route_record() { + TASK5B_GATE0_MODEL_DISPATCH_COUNT=0 + TASK5B_GATE0_MODEL_EFFECT_COUNT=0 + TASK5B_GATE0_ACTUAL_EXEC_COUNT=0 + TASK5B_GATE0_LAST_ROUTE='' + TASK5B_GATE0_LAST_OUTPUT='' + : >"$TASK5B_GATE0_STDOUT_PATH" + : >"$TASK5B_GATE0_STDERR_PATH" +} + +task5b_gate0_assert_rejected() { + local description="$1" rejected_rc=0 + shift + task5b_gate0_reset_route_record + _k3slr_command "$@" >"$TASK5B_GATE0_STDOUT_PATH" \ + 2>"$TASK5B_GATE0_STDERR_PATH" || rejected_rc=$? + assert_eq 97 "$rejected_rc" "$description returns exact rc 97" + assert_eq 0 "$TASK5B_GATE0_MODEL_DISPATCH_COUNT" "$description has model dispatch 0" + assert_eq 0 "$TASK5B_GATE0_MODEL_EFFECT_COUNT" "$description has model effect 0" + assert_eq 0 "$TASK5B_GATE0_ACTUAL_EXEC_COUNT" "$description has actual exec 0" + [[ ! -s "$TASK5B_GATE0_STDOUT_PATH" && ! -s "$TASK5B_GATE0_STDERR_PATH" ]] || + fail "$description emitted stdout or stderr" +} + +task5b_gate0_assert_route() { + local route_id="$1" route_rc=0 last_index swap + local -a exact_arguments=() near_arguments=() + shift + exact_arguments=("$@") + + task5b_gate0_reset_route_record + _k3slr_command "${exact_arguments[@]}" >"$TASK5B_GATE0_STDOUT_PATH" \ + 2>"$TASK5B_GATE0_STDERR_PATH" || route_rc=$? + assert_eq 0 "$route_rc" "$route_id accepts its exact argv" + assert_eq 1 "$TASK5B_GATE0_MODEL_DISPATCH_COUNT" "$route_id dispatches exactly once" + assert_eq 1 "$TASK5B_GATE0_MODEL_EFFECT_COUNT" "$route_id records exactly one model effect" + assert_eq 0 "$TASK5B_GATE0_ACTUAL_EXEC_COUNT" "$route_id has actual exec 0" + assert_eq "$route_id" "$TASK5B_GATE0_LAST_ROUTE" "$route_id records its route" + assert_eq "modeled:${route_id}" "$TASK5B_GATE0_LAST_OUTPUT" "$route_id records output" + [[ ! -s "$TASK5B_GATE0_STDOUT_PATH" && ! -s "$TASK5B_GATE0_STDERR_PATH" ]] || + fail "$route_id exact route emitted stdout or stderr" + + near_arguments=("${exact_arguments[@]}" task5b-extra) + task5b_gate0_assert_rejected "$route_id extra-argument near-match" \ + "${near_arguments[@]}" + + last_index=$(( ${#exact_arguments[@]} - 1 )) + near_arguments=("${exact_arguments[@]:0:last_index}") + task5b_gate0_assert_rejected "$route_id missing-argument near-match" \ + "${near_arguments[@]}" + + near_arguments=("${exact_arguments[@]}") + near_arguments[last_index]=task5b-wrong-value + task5b_gate0_assert_rejected "$route_id wrong-value near-match" \ + "${near_arguments[@]}" + + if (( ${#exact_arguments[@]} >= 3 )); then + near_arguments=("${exact_arguments[@]}") + swap="${near_arguments[last_index - 1]}" + near_arguments[last_index - 1]="${near_arguments[last_index]}" + near_arguments[last_index]="$swap" + task5b_gate0_assert_rejected "$route_id wrong-order near-match" \ + "${near_arguments[@]}" + + near_arguments=("${exact_arguments[@]:0:last_index - 1}") + near_arguments+=("${exact_arguments[last_index - 1]} ${exact_arguments[last_index]}") + task5b_gate0_assert_rejected "$route_id wrong-grouping near-match" \ + "${near_arguments[@]}" + fi +} + +task5b_gate0_static_fake_scan() { + local fake_body route_body catchall_body fake_hash='' route_hash='' catchall_hash='' + local active_graph_hash='' + fake_body="$(declare -f task5b_gate0_fake_command)" || return 1 + route_body="$(declare -f task5b_gate0_route_equals)" || return 1 + catchall_body="$(declare -f _k3slr_command)" || return 1 + task5b_gate0_static_body_is_closed "$fake_body" || return 1 + task5b_gate0_static_body_is_closed "$catchall_body" || return 1 + task5b_gate0_definition_hash fake_hash task5b_gate0_fake_command || return 1 + task5b_gate0_definition_hash route_hash task5b_gate0_route_equals || return 1 + task5b_gate0_definition_hash catchall_hash _k3slr_command || return 1 + task5b_gate0_active_graph_is_closed || return 1 + task5b_gate0_full_graph_static_adversarial_contract || return 1 + task5b_gate0_active_graph_hash active_graph_hash || return 1 + [[ "$fake_hash" == ef2cfc5af6fff12713b20f8734fd28114312d695a7df7d478f9bddd155398d09 ]] || + return 1 + [[ "$route_hash" == 53f6a8cdb6726626a1db2136d7c38e4f2c52c4eaaeb3671506fb504475586a19 ]] || + return 1 + [[ "$catchall_hash" == 76448cb3a82bf60c7904b7d6217762b2769c65069a5e54f086b960d5a63ca3ce ]] || + return 1 + [[ "$active_graph_hash" == 5b3cbe49958bfabe8556d2ec60cdfaf2fdcf586e41953e2681b1720399332293 ]] || + return 1 + [[ "$fake_body" == *'return 0'* && "$fake_body" == *'return 97'* ]] || return 1 + [[ "$fake_body" != *'task5b_gate0_real_exec_fuse'* ]] || return 1 + task5b_gate0_static_adversarial_contract +} + +task5b_gate0_contract() ( + local gate0_stdout="${fixture_root}/task5b-gate0.stdout" + local gate0_stderr="${fixture_root}/task5b-gate0.stderr" + local matrix_rc=0 + + # Foreground source-loaded fixture: production code is loaded, then only its + # lowest command seam is replaced. No child/background fixture is started. + # shellcheck source=/dev/null + source "$LIBRARY_PATH" + _k3slr_command() { task5b_gate0_fake_command "$@"; } + exec 9>&2 + + TASK5B_GATE0_STDOUT_PATH="$gate0_stdout" + TASK5B_GATE0_STDERR_PATH="$gate0_stderr" + TASK5B_GATE0_MODEL_DISPATCH_COUNT=0 + TASK5B_GATE0_MODEL_EFFECT_COUNT=0 + TASK5B_GATE0_ACTUAL_EXEC_COUNT=0 + TASK5B_GATE0_LAST_ROUTE='' + TASK5B_GATE0_LAST_OUTPUT='' + : >"$TASK5B_GATE0_STDOUT_PATH" + : >"$TASK5B_GATE0_STDERR_PATH" + + task5b_gate0_active_matrix() { + task5b_gate0_assert_route sudo-validate /usr/bin/sudo --validate + task5b_gate0_assert_route sudo-cached-validate \ + /usr/bin/sudo --non-interactive --validate + task5b_gate0_assert_route apt-update \ + /usr/bin/sudo --non-interactive -- /usr/bin/apt-get update + task5b_gate0_assert_route apt-install \ + /usr/bin/sudo --non-interactive -- /usr/bin/apt-get install --yes \ + keepassxc=2.7.6+dfsg.1-1build3 cryptsetup-bin=2:2.7.0-1ubuntu4.2 + task5b_gate0_assert_route feasibility-execute /usr/bin/bash \ + /home/donghyeon/workspace/platform/scripts/validate/k3s-local-recovery-feasibility.sh \ + --execute + task5b_gate0_assert_route outer-scaffold-mkdir \ + /usr/bin/sudo --non-interactive -- /usr/bin/mkdir --mode=0700 -- \ + /mnt/k3s-recovery-ssd + task5b_gate0_assert_route outer-mount \ + /usr/bin/sudo --non-interactive -- /usr/bin/mount -t ntfs3 -o \ + rw,uid=1000,gid=1000,dmask=0077,fmask=0177,nodev,nosuid,noexec \ + /dev/disk/by-id/wwn-0x500a0751e6aa6254-part3 /mnt/k3s-recovery-ssd + task5b_gate0_assert_route layout-root-mkdir /usr/bin/mkdir --mode=0700 -- \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery + task5b_gate0_assert_route layout-vault-mkdir /usr/bin/mkdir --mode=0700 -- \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/vault + task5b_gate0_assert_route layout-backups-mkdir /usr/bin/mkdir --mode=0700 -- \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/vault/backups + task5b_gate0_assert_route layout-containers-mkdir /usr/bin/mkdir --mode=0700 -- \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/containers + task5b_gate0_assert_route probe-mktemp /usr/bin/mktemp \ + --tmpdir=/mnt/k3s-recovery-ssd/HyeonworksRecovery \ + .k3slr-hardlink-probe.XXXXXXXX + task5b_gate0_assert_route probe-link /usr/bin/ln -- \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-hardlink-probe.FIXTURE1 \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-hardlink-probe.FIXTURE1.link + task5b_gate0_assert_route probe-unlink-link /usr/bin/unlink -- \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-hardlink-probe.FIXTURE1.link + task5b_gate0_assert_route probe-unlink-source /usr/bin/unlink -- \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-hardlink-probe.FIXTURE1 + task5b_gate0_assert_route probe-sync /usr/bin/sync --file-system \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery + task5b_gate0_assert_route keepass-db-create /usr/bin/keepassxc-cli db-create \ + --quiet --set-password \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/vault/hyeonworks-recovery.kdbx + task5b_gate0_assert_route keepass-add /usr/bin/keepassxc-cli add --quiet \ + --generate --length 40 --lower --upper --numeric --every-group \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/vault/hyeonworks-recovery.kdbx \ + 'K3s Recovery LUKS' + task5b_gate0_assert_route container-dd /usr/bin/dd if=/dev/zero \ + of=/mnt/k3s-recovery-ssd/HyeonworksRecovery/containers/k3s-recovery.luks \ + bs=16M count=2048 conv=excl,fsync status=progress + task5b_gate0_assert_route outer-umount /usr/bin/sudo --non-interactive -- \ + /usr/bin/umount -- /mnt/k3s-recovery-ssd + task5b_gate0_assert_route loop-create /usr/bin/sudo --non-interactive -- \ + /usr/sbin/losetup --find --show --nooverlap -- \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/containers/k3s-recovery.luks + task5b_gate0_assert_route luks-format /usr/bin/sudo --non-interactive -- \ + /usr/sbin/cryptsetup luksFormat --batch-mode --type luks2 --key-file=- \ + /dev/loop37 + task5b_gate0_assert_route luks-open /usr/bin/sudo --non-interactive -- \ + /usr/sbin/cryptsetup open --type luks2 --key-file=- /dev/loop37 k3s-recovery + task5b_gate0_assert_route mkfs-ext4 /usr/bin/sudo --non-interactive -- \ + /usr/sbin/mkfs.ext4 -F -L K3S_RECOVERY /dev/mapper/k3s-recovery + task5b_gate0_assert_route inner-parent-mkdir /usr/bin/sudo --non-interactive -- \ + /usr/bin/mkdir --mode=0700 -- /srv/recovery + task5b_gate0_assert_route inner-mountpoint-mkdir \ + /usr/bin/sudo --non-interactive -- /usr/bin/mkdir --mode=0700 -- \ + /srv/recovery/k3s + task5b_gate0_assert_route inner-mount /usr/bin/sudo --non-interactive -- \ + /usr/bin/mount -t ext4 -o rw,nodev,nosuid,noexec \ + /dev/mapper/k3s-recovery /srv/recovery/k3s + task5b_gate0_assert_route inner-chown /usr/bin/sudo --non-interactive -- \ + /usr/bin/chown root:root /srv/recovery/k3s + task5b_gate0_assert_route inner-chmod /usr/bin/sudo --non-interactive -- \ + /usr/bin/chmod 0700 /srv/recovery/k3s + task5b_gate0_assert_route marker-mktemp /usr/bin/sudo --non-interactive -- \ + /usr/bin/mktemp --tmpdir=/srv/recovery/k3s .k3slr-marker.tmp.XXXXXXXX + task5b_gate0_assert_route marker-dd /usr/bin/sudo --non-interactive -- \ + /usr/bin/dd of=/srv/recovery/k3s/.k3slr-marker.tmp.FIXTURE1 bs=37 count=1 \ + iflag=fullblock conv=notrunc,fsync status=none + task5b_gate0_assert_route outer-metadata-mktemp /usr/bin/mktemp \ + --tmpdir=/mnt/k3s-recovery-ssd/HyeonworksRecovery \ + .k3slr-volume.tmp.XXXXXXXX + task5b_gate0_assert_route inner-metadata-mktemp \ + /usr/bin/sudo --non-interactive -- /usr/bin/mktemp \ + --tmpdir=/srv/recovery/k3s .k3slr-volume.tmp.XXXXXXXX + task5b_gate0_assert_route outer-metadata-dd /usr/bin/dd \ + of=/mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-volume.tmp.FIXTURE1 \ + bs=4096 count=1 iflag=fullblock conv=notrunc,fsync status=none + task5b_gate0_assert_route inner-metadata-dd \ + /usr/bin/sudo --non-interactive -- /usr/bin/dd \ + if=/mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-volume.tmp.FIXTURE1 \ + of=/srv/recovery/k3s/.k3slr-volume.tmp.FIXTURE1 bs=4096 count=1 \ + iflag=fullblock conv=notrunc,fsync status=none + task5b_gate0_assert_route marker-stage-sync /usr/bin/sudo --non-interactive -- \ + /usr/bin/sync --file-system /srv/recovery/k3s/.k3slr-marker.tmp.FIXTURE1 + task5b_gate0_assert_route marker-publish /usr/bin/sudo --non-interactive -- \ + /usr/bin/ln -- /srv/recovery/k3s/.k3slr-marker.tmp.FIXTURE1 \ + /srv/recovery/k3s/.recovery-marker + task5b_gate0_assert_route outer-stage-sync /usr/bin/sync --file-system \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-volume.tmp.FIXTURE1 + task5b_gate0_assert_route inner-stage-sync /usr/bin/sudo --non-interactive -- \ + /usr/bin/sync --file-system /srv/recovery/k3s/.k3slr-volume.tmp.FIXTURE1 + task5b_gate0_assert_route outer-metadata-publish /usr/bin/ln -- \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-volume.tmp.FIXTURE1 \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/volume.env + task5b_gate0_assert_route inner-metadata-publish \ + /usr/bin/sudo --non-interactive -- /usr/bin/ln -- \ + /srv/recovery/k3s/.k3slr-volume.tmp.FIXTURE1 \ + /srv/recovery/k3s/.recovery-volume.env + task5b_gate0_assert_route marker-stage-unlink \ + /usr/bin/sudo --non-interactive -- /usr/bin/unlink -- \ + /srv/recovery/k3s/.k3slr-marker.tmp.FIXTURE1 + task5b_gate0_assert_route outer-stage-unlink /usr/bin/unlink -- \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/.k3slr-volume.tmp.FIXTURE1 + task5b_gate0_assert_route inner-stage-unlink \ + /usr/bin/sudo --non-interactive -- /usr/bin/unlink -- \ + /srv/recovery/k3s/.k3slr-volume.tmp.FIXTURE1 + task5b_gate0_assert_route inner-parent-sync /usr/bin/sudo --non-interactive -- \ + /usr/bin/sync --file-system /srv/recovery/k3s + task5b_gate0_assert_route inner-umount /usr/bin/sudo --non-interactive -- \ + /usr/bin/umount -- /srv/recovery/k3s + task5b_gate0_assert_route mapping-close /usr/bin/sudo --non-interactive -- \ + /usr/sbin/cryptsetup close k3s-recovery + task5b_gate0_assert_route loop-detach /usr/bin/sudo --non-interactive -- \ + /usr/sbin/losetup --detach /dev/loop37 + + task5b_gate0_assert_rejected 'unknown executable' \ + /fixture/task5b-unknown --execute + } + + # This scan must pass before the first exact hazardous argv is presented to + # the fake. It is independent of every dynamic route assertion below. + task5b_gate0_static_fake_scan || return 1 + task5b_gate0_dynamic_fuse_contract || return 1 + + # The active DEBUG trap exists only inside this foreground nested subshell. + # Returning from the subshell removes it without an allowlisted teardown word. + ( + shopt -s extdebug + set -T + trap 'task5b_gate0_real_exec_fuse' DEBUG + task5b_gate0_active_matrix + ) || matrix_rc=$? + (( matrix_rc == 0 )) || return "$matrix_rc" + + exec 9>&- + /usr/bin/rm -- "$TASK5B_GATE0_STDOUT_PATH" "$TASK5B_GATE0_STDERR_PATH" +) + +# Production lifecycle probes need a second boundary beyond their command +# seam doubles: if production regresses to a direct executable, the DEBUG trap +# must reject that simple command before Bash executes it. Runtime membership +# and the pre-trap static scan share this exact, profile-scoped function set. +readonly -A TASK5B_PRODUCTION_PROBE_ALLOWED_FUNCTIONS=( + [unavailable/_k3slr_lifecycle_action]=1 + [unavailable/_k3slr_command]=1 + [unavailable/task5b_unavailable_probe_payload]=1 + [unavailable/task5b_production_probe_emit_ok]=1 + [legacy_binding/_k3slr_lifecycle_action]=1 + [legacy_binding/_k3slr_command]=1 + [legacy_binding/task5b_legacy_binding_probe_payload]=1 + [legacy_binding/task5b_production_probe_emit_ok]=1 + [api_dispatch/_k3slr_lifecycle_dispatch]=1 + [api_dispatch/_k3slr_lifecycle_step]=1 + [api_dispatch/_k3slr_lifecycle_action]=1 + [api_dispatch/_k3slr_command]=1 + [api_dispatch/task5b_api_dispatch_probe_payload]=1 + [api_dispatch/task5b_production_probe_emit_ok]=1 + [api_direct/_k3slr_lifecycle_action]=1 + [api_direct/_k3slr_command]=1 + [api_direct/task5b_api_direct_probe_payload]=1 + [api_direct/task5b_production_probe_emit_ok]=1 + [api_main_arity/_k3slr_lifecycle_main]=1 + [api_main_arity/task5b_api_main_arity_probe_payload]=1 + [api_main_arity/task5b_production_probe_emit_ok]=1 + [action1_main/_k3slr_lifecycle_main]=1 + [action1_main/_k3slr_load_contract]=1 + [action1_main/_k3slr_parse_contract]=1 + [action1_main/_k3slr_contract_value_is_valid]=1 + [action1_main/_k3slr_uint]=1 + [action1_main/_k3slr_require_execute_tty]=1 + [action1_main/_k3slr_tty_capability]=1 + [action1_main/_k3slr_lifecycle_dispatch]=1 + [action1_main/_k3slr_lifecycle_step]=1 + [action1_main/_k3slr_lifecycle_action]=1 + [action1_main/_k3slr_command]=1 + [action1_main/task5b_gate0_route_equals]=1 + [action1_main/task5b_action1_main_probe_payload]=1 + [action1_main/task5b_production_probe_emit_ok]=1 + [action1_alternate/_k3slr_lifecycle_action]=1 + [action1_alternate/_k3slr_command]=1 + [action1_alternate/task5b_gate0_route_equals]=1 + [action1_alternate/task5b_action1_alternate_probe_payload]=1 + [action1_alternate/task5b_production_probe_emit_ok]=1 + [canary/task5b_production_probe_canary_helper_root]=1 + [canary/task5b_production_probe_canary_helper_leaf]=1 +) +readonly -A TASK5B_PRODUCTION_PROBE_EXPECTED_HASHES=( + [core/task5b_production_probe_profile_is_known]=ed04cce41523a02acbebea479607e696dc5ab7ec1486a4914230b74572cd85af + [core/task5b_production_probe_function_is_allowed]=559b5481974916559ae305975aaab82f88a542dc53547dee4575d599d62903ea + [core/task5b_production_probe_builtin_is_allowed]=58cb03fd47554c9cf9ef4b52372e7125b86c86cd88b23364ff31c41e41e1c84b + [core/task5b_production_probe_debug]=662c89fd5213726938630c16b475d894f5a19ec922a8c09dc7563312c55e38a7 + [core/task5b_production_probe_emit_ok]=278abe2c9fb6b744caa3a5517ebc8b504bee1409df4819e6fb9be1129b22fbd0 + [core/task5b_production_probe_canary_graph_is_pinned]=c3c0724de2a4d7da8d20739d5f66d46dc54bde69c2e791d3171d2e4c72392ac5 + [core/task5b_production_probe_body_is_closed]=bc1d27a01debe57be58252f402ddd958fbd9c4fe73da5a4a2c7cfef7851e067d + [core/task5b_production_probe_arm]=e391120fe268bbfb2b67c8d6cb23591fa9f5635dc1549a9f066c4aba711d5c34 + [core/task5b_gate0_assignment_only_status]=075118e7edcd50ef5722f01874d2b138b4ab6cc483a6c17be517bc9a45df7af8 + [unavailable/_k3slr_command]=8634e9ebe9c34a48eff4381729ce6883b95a69a69c6c7218498b6db8a7806248 + [unavailable/task5b_unavailable_probe_payload]=e83567872c19a73a08baa434eb80e449f493505d6945c4d4f4cae2ffedaa2435 + [unavailable/task5b_production_probe_emit_ok]=278abe2c9fb6b744caa3a5517ebc8b504bee1409df4819e6fb9be1129b22fbd0 + [unavailable/_k3slr_lifecycle_action]=a499b7694c449633f3ce538e8296b1d543d04565c046508587d18b677aa12384 + [legacy_binding/task5b_legacy_binding_probe_payload]=46d73155080f35fdd4c9ec1d531ed50f6ac77447b6389549d934f862e4430316 + [legacy_binding/_k3slr_lifecycle_action]=a499b7694c449633f3ce538e8296b1d543d04565c046508587d18b677aa12384 + [legacy_binding/task5b_production_probe_emit_ok]=278abe2c9fb6b744caa3a5517ebc8b504bee1409df4819e6fb9be1129b22fbd0 + [legacy_binding/_k3slr_command]=8634e9ebe9c34a48eff4381729ce6883b95a69a69c6c7218498b6db8a7806248 + [api_dispatch/_k3slr_command]=8634e9ebe9c34a48eff4381729ce6883b95a69a69c6c7218498b6db8a7806248 + [api_dispatch/_k3slr_lifecycle_step]=1cf91eba1179fbb35f58ca20dbf0e62f2ff3b754cc64f6e4c6547c1a1193b212 + [api_dispatch/task5b_api_dispatch_probe_payload]=341b9a6990134bc9f95fb881e4596a4db94fbfbc4303e8afe61720f1684958e4 + [api_dispatch/_k3slr_lifecycle_action]=a499b7694c449633f3ce538e8296b1d543d04565c046508587d18b677aa12384 + [api_dispatch/_k3slr_lifecycle_dispatch]=e407e698b9a69c47c9969db009211e47614c9753f701fa99f82a6d9e84c82477 + [api_dispatch/task5b_production_probe_emit_ok]=278abe2c9fb6b744caa3a5517ebc8b504bee1409df4819e6fb9be1129b22fbd0 + [api_dispatch/task5b_api_state_debug_observer]=c9bec0cfb5204ee77ac32fab9eb31b00cf033d6fd3ac897a98878991ca13897f + [api_direct/_k3slr_lifecycle_action]=a499b7694c449633f3ce538e8296b1d543d04565c046508587d18b677aa12384 + [api_direct/_k3slr_command]=8634e9ebe9c34a48eff4381729ce6883b95a69a69c6c7218498b6db8a7806248 + [api_direct/task5b_api_direct_probe_payload]=c4723fd1f9a04b2095a1935c21c5cebca01f2e13c75f9ceda59d58ad1a909332 + [api_direct/task5b_production_probe_emit_ok]=278abe2c9fb6b744caa3a5517ebc8b504bee1409df4819e6fb9be1129b22fbd0 + [api_main_arity/task5b_api_main_arity_probe_payload]=5c23d4c2810c534318d210de7c35a6ac7f0fbf52ab88edbd421c30d2c534ff8a + [api_main_arity/_k3slr_lifecycle_main]=677f7f482840af5c5a7dc37c4f7abe6a470a24ed108030cb14fe1f3b1c8ae53f + [api_main_arity/task5b_production_probe_emit_ok]=278abe2c9fb6b744caa3a5517ebc8b504bee1409df4819e6fb9be1129b22fbd0 + [action1_main/task5b_production_probe_emit_ok]=278abe2c9fb6b744caa3a5517ebc8b504bee1409df4819e6fb9be1129b22fbd0 + [action1_main/_k3slr_contract_value_is_valid]=b8eab6a6688c3aac2456bf9a4987eeeefec673a1dccf5e80e785cc9dcbddc7dd + [action1_main/_k3slr_lifecycle_action]=a499b7694c449633f3ce538e8296b1d543d04565c046508587d18b677aa12384 + [action1_main/task5b_action1_main_probe_payload]=ac0431e91d90989ae9eb49dd7ebcecbc12a87f99f50771db0fce62ce9960aa7c + [action1_main/_k3slr_parse_contract]=a27e7c60983ecb4792349c69bc6020ee51dfdd6f0adefbf4359b54b20cf64b0d + [action1_main/_k3slr_uint]=e60445f9cc78399afebda8d2010fbd64ceae23da4cfa15373c60122127df6945 + [action1_main/task5b_gate0_route_equals]=53f6a8cdb6726626a1db2136d7c38e4f2c52c4eaaeb3671506fb504475586a19 + [action1_main/_k3slr_tty_capability]=c59e220dbcc6f3d9af8707cc365c456ee404d20deed8150171a8fb37d87a7497 + [action1_main/_k3slr_load_contract]=71855162c947b01b145e79aa4e30a448c21690efa183c56ccafe97b776912cce + [action1_main/_k3slr_require_execute_tty]=9d194fe3ba36a7bd25aaa5b53eefeafa4d23fe8f307995be7a9039472b42a838 + [action1_main/_k3slr_command]=6e84784f49cc5e8072cdd4c8c0cccbd20ef15e5886228946d1c7c670967a3ec2 + [action1_main/_k3slr_lifecycle_main]=677f7f482840af5c5a7dc37c4f7abe6a470a24ed108030cb14fe1f3b1c8ae53f + [action1_main/_k3slr_lifecycle_dispatch]=e407e698b9a69c47c9969db009211e47614c9753f701fa99f82a6d9e84c82477 + [action1_main/_k3slr_lifecycle_step]=1cf91eba1179fbb35f58ca20dbf0e62f2ff3b754cc64f6e4c6547c1a1193b212 + [action1_main/task5b_action1_debug_observer]=ecaf3a7e1ee93fd5f330301b8fc21c0300834bd5ec376c766af704de83271cee + [action1_alternate/task5b_gate0_route_equals]=53f6a8cdb6726626a1db2136d7c38e4f2c52c4eaaeb3671506fb504475586a19 + [action1_alternate/_k3slr_lifecycle_action]=a499b7694c449633f3ce538e8296b1d543d04565c046508587d18b677aa12384 + [action1_alternate/_k3slr_command]=6e84784f49cc5e8072cdd4c8c0cccbd20ef15e5886228946d1c7c670967a3ec2 + [action1_alternate/task5b_action1_alternate_probe_payload]=aa5693a60539d54eda18fdefef47c7ad20891d94d0478df41a0afa46860565cc + [action1_alternate/task5b_production_probe_emit_ok]=278abe2c9fb6b744caa3a5517ebc8b504bee1409df4819e6fb9be1129b22fbd0 + [canary/task5b_production_probe_canary_helper_root]=d14b146f01b0a6ef5a36be1d613994901683a503661348195b5c2eeca7b9aa9b + [canary/task5b_production_probe_canary_helper_leaf]=a94ebf016f987960b669e5a7c9b776e95b7d305675dc74868b46bc25460810a9 +) + +task5b_production_probe_profile_is_known() { + (( $# == 1 )) || return 1 + case "$1" in + unavailable|legacy_binding|api_dispatch|api_direct|api_main_arity|\ + action1_main|action1_alternate|canary) return 0 ;; + *) return 1 ;; + esac +} + +task5b_production_probe_function_is_allowed() { + local profile="${1-}" function_name="${2-}" + (( $# == 2 )) || return 1 + task5b_production_probe_profile_is_known "$profile" || return 1 + [[ "${TASK5B_PRODUCTION_PROBE_ALLOWED_FUNCTIONS["$profile/$function_name"]+present}" == present ]] +} + +task5b_production_probe_builtin_is_allowed() { + local profile="${1-}" builtin_name="${2-}" + (( $# == 2 )) || return 1 + task5b_production_probe_profile_is_known "$profile" || return 1 + case "$profile" in + canary) + [[ "$builtin_name" == printf ]] + ;; + action1_main) + case "$builtin_name" in + declare|local|printf|return|shift) return 0 ;; + *) return 1 ;; + esac + ;; + *) + case "$builtin_name" in + local|printf|return|shift) return 0 ;; + *) return 1 ;; + esac + ;; + esac +} + +task5b_production_probe_debug() { + local -a saved_bash_rematch=("${BASH_REMATCH[@]}") + local candidate="${1-}" observed_step="${2-}" execution_word='' + local assignment_rc=0 + (( $# == 2 )) || exit 96 + task5b_production_probe_profile_is_known "$task5b_probe_profile" || exit 96 + case "$task5b_probe_profile" in + api_dispatch) + task5b_api_state_debug_observer "$candidate" || { + printf 'TASK5B_PROBE_BLOCK|%s|1\n' "$task5b_probe_profile" >&9 + exit 96 + } + ;; + action1_main) + task5b_action1_debug_observer "$candidate" "$observed_step" || { + printf 'TASK5B_PROBE_BLOCK|%s|1\n' "$task5b_probe_profile" >&9 + exit 96 + } + ;; + esac + candidate="${candidate#"${candidate%%[![:space:]]*}"}" + while [[ "$candidate" =~ ^(if|elif|while|until|then|\!|time)[[:space:]]+(.+)$ ]]; do + candidate="${BASH_REMATCH[2]}" + done + case "$candidate" in + ''|'{'|'}'|'[['*|'(('*|for\ *|case\ *|do|done|esac) + BASH_REMATCH=("${saved_bash_rematch[@]}") + return 0 + ;; + esac + if [[ "$task5b_probe_profile" == action1_main ]]; then + case "$candidate" in + 'IFS= read -r line'|'seen["$key"]=1'|'result["$key"]="$value"') + BASH_REMATCH=("${saved_bash_rematch[@]}") + return 0 + ;; + esac + fi + if [[ "$candidate" =~ ^[a-zA-Z_][a-zA-Z0-9_]*(\[[^]]+\])?\+?= ]]; then + task5b_gate0_assignment_only_status "$candidate" || assignment_rc=$? + if (( assignment_rc == 0 )); then + BASH_REMATCH=("${saved_bash_rematch[@]}") + return 0 + fi + printf 'TASK5B_PROBE_BLOCK|%s|1\n' "$task5b_probe_profile" >&9 + exit 96 + fi + read -r execution_word _ <<<"$candidate" + execution_word="${execution_word%;}" + case "$execution_word" in + \$*|\"\$*|\'\$*|/*|\"/*|\'/*|sudo|command|builtin|exec|eval|source|.|\ + trap|set|shopt|unset|enable|function) + printf 'TASK5B_PROBE_BLOCK|%s|1\n' "$task5b_probe_profile" >&9 + exit 96 + ;; + esac + if task5b_production_probe_builtin_is_allowed "$task5b_probe_profile" \ + "$execution_word" || + task5b_production_probe_function_is_allowed "$task5b_probe_profile" \ + "$execution_word"; then + BASH_REMATCH=("${saved_bash_rematch[@]}") + return 0 + fi + printf 'TASK5B_PROBE_BLOCK|%s|1\n' "$task5b_probe_profile" >&9 + exit 96 +} + +task5b_production_probe_emit_ok() { + (( $# >= 1 )) || return 1 + printf 'TASK5B_PROBE_OK|%s' "$1" >&9 + shift + printf '|%s' "$@" >&9 + printf '\n' >&9 +} + +task5b_production_probe_canary_graph_is_pinned() { + local entry function_name actual_hash='' expected_hash='' function_count=0 + (( $# == 0 )) || return 1 + for entry in "${!TASK5B_PRODUCTION_PROBE_ALLOWED_FUNCTIONS[@]}"; do + [[ "$entry" == canary/* ]] || continue + function_name="${entry#*/}" + (( function_count += 1 )) + expected_hash="${TASK5B_PRODUCTION_PROBE_EXPECTED_HASHES[$entry]-}" + [[ "$expected_hash" =~ ^[0-9a-f]{64}$ ]] || return 1 + task5b_gate0_definition_hash actual_hash "$function_name" || return 1 + [[ "$actual_hash" == "$expected_hash" ]] || return 1 + done + (( function_count == 2 )) +} + +task5b_production_probe_body_is_closed() { + local profile="${1-}" function_name="${2-}" body="${3-}" + local line='' scan_body='' character='' next_character='' quote='' + local mode=normal escaped=0 index=0 length=0 line_number=0 + local allowed_redirection_count=0 expected_redirection_count=0 + (( $# == 3 )) || return 1 + task5b_production_probe_profile_is_known "$profile" || return 1 + [[ "$function_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ && -n "$body" ]] || + return 1 + case "$profile/$function_name" in + */task5b_production_probe_emit_ok) expected_redirection_count=3 ;; + action1_main/_k3slr_parse_contract|*/_k3slr_lifecycle_main) + expected_redirection_count=1 + ;; + esac + while IFS= read -r line; do + if [[ "$function_name" == task5b_production_probe_emit_ok ]]; then + case "$line" in + " printf 'TASK5B_PROBE_OK|%s' \"\$1\" 1>&9;"|\ + " printf '|%s' \"\$@\" 1>&9;"|" printf '\\n' 1>&9") + (( allowed_redirection_count += 1 )) + continue + ;; + esac + elif [[ "$profile/$function_name" == action1_main/_k3slr_parse_contract && + "$line" == " done < \"\$file\";" ]]; then + (( allowed_redirection_count += 1 )) + continue + elif [[ "$function_name" == _k3slr_lifecycle_main && + "$line" == ' declare -F _k3slr_prove_header_restore > /dev/null || return 1;' ]]; then + (( allowed_redirection_count += 1 )) + continue + fi + scan_body+="$line"$'\n' + done <<<"$body" + (( allowed_redirection_count == expected_redirection_count )) || return 1 + [[ "$scan_body" != *'<('* && "$scan_body" != *'>('* && + "$scan_body" != *'trap - DEBUG'* && "$scan_body" != *'set +T'* && + "$scan_body" != *'shopt -u extdebug'* && "$scan_body" != *'unset '* && + "$scan_body" != *'enable '* ]] || return 1 + while IFS= read -r line; do + (( line_number += 1 )) + if (( line_number > 1 )) && + [[ "$line" =~ ^[[:space:]]*(function[[:space:]]+|[a-zA-Z_][a-zA-Z0-9_]*[[:space:]]*\(\)) ]]; then + return 1 + fi + quote='' + mode=normal + escaped=0 + index=0 + length="${#line}" + while (( index < length )); do + character="${line:index:1}" + next_character='' + (( index + 1 >= length )) || next_character="${line:index + 1:1}" + if (( escaped == 1 )); then + escaped=0 + (( index += 1 )) + continue + fi + case "$quote" in + single) + [[ "$character" != "'" ]] || quote='' + (( index += 1 )) + continue + ;; + double) + case "$character" in + '\\') escaped=1 ;; + '"') quote='' ;; + esac + (( index += 1 )) + continue + ;; + esac + case "$mode" in + conditional) + if [[ "$character$next_character" == ']]' ]]; then + mode=normal + (( index += 2 )) + else + (( index += 1 )) + fi + continue + ;; + arithmetic) + if [[ "$character$next_character" == '))' ]]; then + mode=normal + (( index += 2 )) + else + (( index += 1 )) + fi + continue + ;; + esac + case "$character" in + "'") quote=single ;; + '"') quote=double ;; + '\\') escaped=1 ;; + '[') + if [[ "$next_character" == '[' ]]; then + mode=conditional + (( index += 2 )) + continue + fi + ;; + '(') + if [[ "$next_character" == '(' ]]; then + mode=arithmetic + (( index += 2 )) + continue + fi + ;; + '<'|'>') return 1 ;; + esac + (( index += 1 )) + done + done <<<"$scan_body" +} + +task5b_production_probe_core_is_closed() { + local function_name actual_hash='' expected_hash='' + local -a core_functions=( + task5b_production_probe_profile_is_known + task5b_production_probe_function_is_allowed + task5b_production_probe_builtin_is_allowed + task5b_production_probe_debug + task5b_production_probe_emit_ok + task5b_production_probe_canary_graph_is_pinned + task5b_production_probe_body_is_closed + task5b_production_probe_arm + task5b_gate0_assignment_only_status + ) + (( $# == 0 )) || return 1 + for function_name in "${core_functions[@]}"; do + expected_hash="${TASK5B_PRODUCTION_PROBE_EXPECTED_HASHES["core/$function_name"]-}" + [[ "$expected_hash" =~ ^[0-9a-f]{64}$ ]] || return 1 + task5b_gate0_definition_hash actual_hash "$function_name" || return 1 + [[ "$actual_hash" == "$expected_hash" ]] || { + printf 'Task 5B production probe core hash mismatch: %s expected=%s actual=%s\n' \ + "$function_name" "$expected_hash" "$actual_hash" >&2 + return 1 + } + done +} + +task5b_production_probe_static_graph_is_closed() { + local profile="${1-}" entry function_name body='' normalized_body='' + local actual_hash='' expected_hash='' function_count=0 line='' line_number=0 + (( $# == 1 )) || return 1 + task5b_production_probe_profile_is_known "$profile" || return 1 + [[ "$profile" != canary ]] || return 1 + task5b_production_probe_core_is_closed || return 1 + if [[ "$profile" == api_dispatch ]]; then + function_name=task5b_api_state_debug_observer + entry="$profile/$function_name" + expected_hash="${TASK5B_PRODUCTION_PROBE_EXPECTED_HASHES[$entry]-}" + [[ "$expected_hash" =~ ^[0-9a-f]{64}$ ]] || return 1 + task5b_gate0_definition_hash actual_hash "$function_name" || return 1 + [[ "$actual_hash" == "$expected_hash" ]] || { + printf 'Task 5B production probe observer hash mismatch: %s expected=%s actual=%s\n' \ + "$function_name" "$expected_hash" "$actual_hash" >&2 + return 1 + } + elif [[ "$profile" == action1_main ]]; then + function_name=task5b_action1_debug_observer + entry="$profile/$function_name" + expected_hash="${TASK5B_PRODUCTION_PROBE_EXPECTED_HASHES[$entry]-}" + [[ "$expected_hash" =~ ^[0-9a-f]{64}$ ]] || return 1 + task5b_gate0_definition_hash actual_hash "$function_name" || return 1 + [[ "$actual_hash" == "$expected_hash" ]] || { + printf 'Task 5B production probe observer hash mismatch: %s expected=%s actual=%s\n' \ + "$function_name" "$expected_hash" "$actual_hash" >&2 + return 1 + } + fi + for entry in "${!TASK5B_PRODUCTION_PROBE_ALLOWED_FUNCTIONS[@]}"; do + [[ "$entry" == "$profile/"* ]] || continue + function_name="${entry#*/}" + (( function_count += 1 )) + expected_hash="${TASK5B_PRODUCTION_PROBE_EXPECTED_HASHES[$entry]-}" + [[ "$expected_hash" =~ ^[0-9a-f]{64}$ ]] || return 1 + task5b_gate0_definition_hash actual_hash "$function_name" || return 1 + [[ "$actual_hash" == "$expected_hash" ]] || { + printf 'Task 5B production probe profile hash mismatch: %s expected=%s actual=%s\n' \ + "$entry" "$expected_hash" "$actual_hash" >&2 + return 1 + } + body="$(declare -f "$function_name")" || return 1 + task5b_production_probe_body_is_closed "$profile" "$function_name" \ + "$body" || { + printf 'Task 5B production probe body scan failed: %s\n' "$entry" >&2 + return 1 + } + [[ "$body" != *'<('* && "$body" != *'>('* && + "$body" != *'trap - DEBUG'* && "$body" != *'set +T'* && + "$body" != *'shopt -u extdebug'* && "$body" != *'unset '* && + "$body" != *'enable '* ]] || { + printf 'Task 5B production probe forbidden token: %s\n' "$entry" >&2 + return 1 + } + line_number=0 + while IFS= read -r line; do + (( line_number += 1 )) + if (( line_number > 1 )) && + [[ "$line" =~ ^[[:space:]]*(function[[:space:]]+|[a-zA-Z_][a-zA-Z0-9_]*[[:space:]]*\(\)) ]]; then + printf 'Task 5B production probe nested definition: %s\n' "$entry" >&2 + return 1 + fi + done <<<"$body" + task5b_gate0_normalize_static_body_data_lines "$body" normalized_body \ + "$function_name" || { + printf 'Task 5B production probe normalization failed: %s\n' "$entry" >&2 + return 1 + } + if [[ "$function_name" != _k3slr_parse_contract ]]; then + task5b_gate0_static_assignment_prefixes_are_closed "$body" || { + printf 'Task 5B production probe assignment scan failed: %s\n' "$entry" >&2 + return 1 + } + fi + task5b_gate0_static_execution_positions_are_closed "$normalized_body" || + { + printf 'Task 5B production probe execution scan failed: %s\n' "$entry" >&2 + return 1 + } + done + (( function_count > 1 )) || return 1 +} + +task5b_production_probe_arm() { + (( $# == 1 )) || return 1 + task5b_production_probe_profile_is_known "$1" || return 1 + task5b_probe_profile="$1" + shopt -s extdebug + set -T + trap 'task5b_production_probe_debug "${BASH_COMMAND-}" "${step-}"' DEBUG +} + +task5b_production_probe_canary_helper_root() { + task5b_production_probe_canary_helper_leaf +} + +task5b_production_probe_canary_helper_leaf() { + /usr/bin/printf 'TASK5B_ACTION1_HELPER_EXEC_CANARY' +} + +task5b_production_probe_canary_unexpected_helper() { + /usr/bin/printf 'TASK5B_UNEXPECTED_HELPER_EXEC_CANARY' +} + +task5b_production_probe_static_redirection_adversary() { + printf 'TASK5B_REDIRECTION_CANARY' >"$fixture_root/task5b-redirection-canary" +} + +task5b_production_probe_static_process_substitution_adversary() { + : < <(/usr/bin/printf 'TASK5B_PROCESS_SUBSTITUTION_CANARY') +} + +task5b_production_probe_static_redefinition_adversary() { + task5b_nested_redefinition() { :; } +} + +task5b_production_probe_adversary_matrix() ( + local adversary='' record='' probe_rc=0 body='' graph_body='' + local -a adversaries=( + direct helper variable array command-wrapper builtin-wrapper exec-wrapper + eval-wrapper source-wrapper unexpected-helper trap-disable functrace-disable + extdebug-disable unset-wrapper enable-wrapper + ) + task5b_production_probe_core_is_closed || return 1 + task5b_production_probe_canary_graph_is_pinned || return 1 + graph_body="$(declare -f task5b_production_probe_static_graph_is_closed)" || + return 1 + [[ "$graph_body" == *'task5b_production_probe_body_is_closed "$profile" "$function_name" "$body"'* ]] || + fail 'production graph closure uses the same body scanner as static adversaries' + for adversary in "${adversaries[@]}"; do + record='' + probe_rc=0 + record="$( + local runner='' + local -a runners=() + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm canary || return 1 + case "$adversary" in + direct) /usr/bin/printf 'TASK5B_DIRECT_EXEC_CANARY' ;; + helper) task5b_production_probe_canary_helper_root ;; + variable) + runner=/usr/bin/printf + "$runner" 'TASK5B_VARIABLE_EXEC_CANARY' + ;; + array) + runners=(/usr/bin/printf TASK5B_ARRAY_EXEC_CANARY) + "${runners[@]}" + ;; + command-wrapper) command /usr/bin/printf 'TASK5B_COMMAND_EXEC_CANARY' ;; + builtin-wrapper) + builtin command /usr/bin/printf 'TASK5B_BUILTIN_EXEC_CANARY' + ;; + exec-wrapper) exec /usr/bin/printf 'TASK5B_EXEC_CANARY' ;; + eval-wrapper) eval '/usr/bin/printf TASK5B_EVAL_CANARY' ;; + source-wrapper) source /fixture/task5b-source-canary ;; + unexpected-helper) task5b_production_probe_canary_unexpected_helper ;; + trap-disable) trap - DEBUG ;; + functrace-disable) set +T ;; + extdebug-disable) shopt -u extdebug ;; + unset-wrapper) unset task5b_probe_profile ;; + enable-wrapper) enable -n printf ;; + *) return 1 ;; + esac + printf 'TASK5B_CANARY_ESCAPED' + )" || probe_rc=$? + assert_eq 96 "$probe_rc" "production probe fuse blocks adversary: $adversary" + assert_eq 'TASK5B_PROBE_BLOCK|canary|1' "$record" \ + "production probe adversary has one block record and no effect: $adversary" + done + body="$(declare -f task5b_production_probe_static_redirection_adversary)" || + return 1 + assert_fails task5b_production_probe_body_is_closed canary \ + task5b_production_probe_static_redirection_adversary "$body" + body="$(declare -f task5b_production_probe_static_process_substitution_adversary)" || + return 1 + assert_fails task5b_production_probe_body_is_closed canary \ + task5b_production_probe_static_process_substitution_adversary "$body" + body="$(declare -f task5b_production_probe_static_redefinition_adversary)" || + return 1 + assert_fails task5b_production_probe_body_is_closed canary \ + task5b_production_probe_static_redefinition_adversary "$body" +) + +task5b_mountinfo_counts_record() { + local record="${1-}" source_destination="${2-}" outer_destination="${3-}" + local inner_destination="${4-}" name line target source field_index + local source_count=0 outer_count=0 inner_count=0 + local -a fields=() + (( $# == 4 )) || return 1 + for name in "$source_destination" "$outer_destination" "$inner_destination"; do + [[ "$name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1 + done + while IFS= read -r line; do + [[ -n "$line" ]] || continue + read -r -a fields <<<"$line" + (( ${#fields[@]} >= 10 )) || return 1 + target="${fields[4]}" + source='' + for (( field_index=0; field_index<${#fields[@]}; field_index++ )); do + if [[ "${fields[field_index]}" == - ]]; then + (( field_index + 2 < ${#fields[@]} )) || return 1 + source="${fields[field_index + 2]}" + break + fi + done + [[ -n "$source" ]] || return 1 + [[ "$source" != /dev/sda3 ]] || (( source_count += 1 )) + [[ "$target" != /mnt/k3s-recovery-ssd ]] || (( outer_count += 1 )) + [[ "$target" != /srv/recovery/k3s ]] || (( inner_count += 1 )) + done <<<"$record" + printf -v "$source_destination" '%s' "$source_count" + printf -v "$outer_destination" '%s' "$outer_count" + printf -v "$inner_destination" '%s' "$inner_count" +} + +task5b_loop_backing_is_recovery() { + (( $# == 1 )) || return 1 + case "$1" in + /fixture/recovery-partition|\ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/containers/k3s-recovery.luks) + return 0 + ;; + *) return 1 ;; + esac +} + +task5b_read_exact_single_line() { + local destination_name="${1-}" source_path="${2-}" + local -a source_lines=() + (( $# == 2 )) || return 1 + [[ "$destination_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1 + [[ -r "$source_path" ]] || return 1 + mapfile -t source_lines <"$source_path" || return 1 + (( ${#source_lines[@]} == 1 )) || return 1 + [[ -n "${source_lines[0]}" ]] || return 1 + printf -v "$destination_name" '%s' "${source_lines[0]}" +} + +task5b_loop_identity_count_from_record() { + local record="${1-}" recovery_major_minor="${2-}" destination_name="${3-}" + local line loop_name loop_minor back_inode back_major_minor loop_major_minor + local offset sizelimit diskseq_before cached_diskseq diskseq_after + local encoded_filename extra identity_match_count=0 + local -A seen_loop_names=() + (( $# == 3 )) || return 1 + [[ "$destination_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1 + [[ "$recovery_major_minor" =~ ^[0-9]+:[0-9]+$ ]] || return 1 + while IFS= read -r line; do + [[ -n "$line" ]] || continue + IFS='|' read -r loop_name back_inode back_major_minor loop_major_minor \ + offset sizelimit diskseq_before cached_diskseq diskseq_after \ + encoded_filename extra <<<"$line" + [[ -z "${extra:-}" ]] || return 1 + [[ "$loop_name" =~ ^/dev/loop([0-9]+)$ ]] || return 1 + loop_minor="${BASH_REMATCH[1]}" + [[ -z "${seen_loop_names[$loop_name]+present}" ]] || return 1 + seen_loop_names["$loop_name"]=1 + [[ "$back_inode" =~ ^[1-9][0-9]*$ ]] || return 1 + [[ "$back_major_minor" =~ ^[0-9]+:[0-9]+$ ]] || return 1 + [[ "$loop_major_minor" == "7:${loop_minor}" ]] || return 1 + [[ "$offset" =~ ^[0-9]+$ && "$sizelimit" =~ ^[0-9]+$ ]] || return 1 + [[ "$diskseq_before" =~ ^[1-9][0-9]*$ ]] || return 1 + [[ "$cached_diskseq" == "$diskseq_before" ]] || return 1 + [[ "$diskseq_after" == "$diskseq_before" ]] || return 1 + [[ "$encoded_filename" =~ ^([a-zA-Z0-9._/:+-]|\\x[0-9a-fA-F]{2})+$ ]] || return 1 + [[ "$back_major_minor" != "$recovery_major_minor" ]] || + (( identity_match_count += 1 )) + done <<<"$record" + printf -v "$destination_name" '%s' "$identity_match_count" +} + +task5b_loop_live_identity_line() { + local loop_name="${1-}" recovery_major_minor="${2-}" destination_name="${3-}" + local loop_minor + local sysfs_root loop_major_minor_before='' loop_major_minor_after='' + local diskseq_before='' diskseq_after='' backing_before='' backing_after='' + local offset_before='' offset_after='' sizelimit_before='' sizelimit_after='' + local udev_path line value back_major_minor='' back_inode='' + local encoded_filename='' cached_diskseq='' field_name + local back_major_minor_count=0 back_inode_count=0 encoded_filename_count=0 + local cached_diskseq_count=0 + (( $# == 3 )) || return 1 + [[ "$loop_name" =~ ^loop([0-9]+)$ ]] || return 1 + loop_minor="${BASH_REMATCH[1]}" + [[ "$recovery_major_minor" =~ ^[0-9]+:[0-9]+$ ]] || return 1 + [[ "$destination_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1 + sysfs_root="/sys/class/block/${loop_name}" + task5b_read_exact_single_line loop_major_minor_before "${sysfs_root}/dev" || return 1 + task5b_read_exact_single_line diskseq_before "${sysfs_root}/diskseq" || return 1 + task5b_read_exact_single_line backing_before \ + "${sysfs_root}/loop/backing_file" || return 1 + task5b_read_exact_single_line offset_before "${sysfs_root}/loop/offset" || return 1 + task5b_read_exact_single_line sizelimit_before \ + "${sysfs_root}/loop/sizelimit" || return 1 + [[ "$loop_major_minor_before" == "7:${loop_minor}" ]] || return 1 + [[ "$diskseq_before" =~ ^[1-9][0-9]*$ ]] || return 1 + [[ "$offset_before" =~ ^[0-9]+$ && "$sizelimit_before" =~ ^[0-9]+$ ]] || + return 1 + udev_path="/run/udev/data/b${loop_major_minor_before}" + [[ -r "$udev_path" ]] || return 1 + while IFS= read -r line; do + case "$line" in + E:ID_LOOP_BACKING_DEVICE=*) + value="${line#E:ID_LOOP_BACKING_DEVICE=}" + (( back_major_minor_count += 1 )) + back_major_minor="$value" + ;; + E:ID_LOOP_BACKING_INODE=*) + value="${line#E:ID_LOOP_BACKING_INODE=}" + (( back_inode_count += 1 )) + back_inode="$value" + ;; + E:ID_LOOP_BACKING_FILENAME_ENC=*) + value="${line#E:ID_LOOP_BACKING_FILENAME_ENC=}" + (( encoded_filename_count += 1 )) + encoded_filename="$value" + ;; + S:disk/by-diskseq/*) + value="${line#S:disk/by-diskseq/}" + (( cached_diskseq_count += 1 )) + cached_diskseq="$value" + ;; + esac + done <"$udev_path" || return 1 + for field_name in back_major_minor back_inode encoded_filename cached_diskseq; do + [[ -n "${!field_name}" ]] || return 1 + done + (( back_major_minor_count == 1 && back_inode_count == 1 && + encoded_filename_count == 1 && cached_diskseq_count == 1 )) || return 1 + [[ "$back_major_minor" =~ ^[0-9]+:[0-9]+$ ]] || return 1 + [[ "$back_inode" =~ ^[1-9][0-9]*$ ]] || return 1 + [[ "$encoded_filename" =~ ^([a-zA-Z0-9._/:+-]|\\x[0-9a-fA-F]{2})+$ ]] || return 1 + [[ "$cached_diskseq" == "$diskseq_before" ]] || return 1 + task5b_read_exact_single_line loop_major_minor_after "${sysfs_root}/dev" || return 1 + task5b_read_exact_single_line diskseq_after "${sysfs_root}/diskseq" || return 1 + task5b_read_exact_single_line backing_after \ + "${sysfs_root}/loop/backing_file" || return 1 + task5b_read_exact_single_line offset_after "${sysfs_root}/loop/offset" || return 1 + task5b_read_exact_single_line sizelimit_after \ + "${sysfs_root}/loop/sizelimit" || return 1 + [[ "$loop_major_minor_after" == "$loop_major_minor_before" ]] || return 1 + [[ "$diskseq_after" == "$diskseq_before" ]] || return 1 + [[ "$backing_after" == "$backing_before" ]] || return 1 + [[ "$offset_after" == "$offset_before" ]] || return 1 + [[ "$sizelimit_after" == "$sizelimit_before" ]] || return 1 + if task5b_loop_backing_is_recovery "$backing_before"; then + [[ "$back_major_minor" == "$recovery_major_minor" ]] || return 1 + fi + printf -v "$destination_name" '%s' \ + "/dev/${loop_name}|${back_inode}|${back_major_minor}|${loop_major_minor_before}|${offset_before}|${sizelimit_before}|${diskseq_before}|${cached_diskseq}|${diskseq_after}|${encoded_filename}" +} + +task5b_recovery_loop_count() { + local destination_name="${1-}" recovery_major_minor='' + local loop_path loop_name identity_line identity_record='' loop_index + local -a before_paths=() after_paths=() + (( $# == 1 )) || return 1 + [[ "$destination_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1 + task5b_read_exact_single_line recovery_major_minor /sys/class/block/sda3/dev || + return 1 + [[ "$recovery_major_minor" =~ ^[0-9]+:[0-9]+$ ]] || return 1 + shopt -s nullglob + before_paths=(/sys/class/block/loop*/loop/backing_file) + for loop_path in "${before_paths[@]}"; do + loop_name="${loop_path#/sys/class/block/}" + loop_name="${loop_name%%/*}" + task5b_loop_live_identity_line "$loop_name" "$recovery_major_minor" \ + identity_line || return 1 + identity_record+="$identity_line"$'\n' + done + after_paths=(/sys/class/block/loop*/loop/backing_file) + shopt -u nullglob + (( ${#after_paths[@]} == ${#before_paths[@]} )) || return 1 + for (( loop_index=0; loop_index<${#before_paths[@]}; loop_index++ )); do + [[ "${after_paths[loop_index]}" == "${before_paths[loop_index]}" ]] || return 1 + done + task5b_loop_identity_count_from_record "$identity_record" \ + "$recovery_major_minor" "$destination_name" +} + +task5b_path_state_from_observation() { + local destination_name="${1-}" exists="${2-}" symlink="${3-}" state=absent + (( $# == 3 )) || return 1 + [[ "$destination_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1 + [[ "$exists" == 0 || "$exists" == 1 ]] || return 1 + [[ "$symlink" == 0 || "$symlink" == 1 ]] || return 1 + (( exists == 0 && symlink == 0 )) || state=present + printf -v "$destination_name" '%s' "$state" +} + +task5b_path_state() { + local destination_name="${1-}" path="${2-}" exists=0 symlink=0 + (( $# == 2 )) || return 1 + [[ -e "$path" ]] && exists=1 + [[ -L "$path" ]] && symlink=1 + task5b_path_state_from_observation "$destination_name" "$exists" "$symlink" +} + +task5b_fixture_process_argv_matches() { + local current_pid="${1-}" candidate_pid="${2-}" executable='' argument + local test_script_match=0 fixture_marker_match=0 + shift 2 || return 1 + [[ "$current_pid" =~ ^[1-9][0-9]*$ && "$candidate_pid" =~ ^[1-9][0-9]*$ ]] || return 1 + [[ "$candidate_pid" != "$current_pid" ]] || return 1 + (( $# >= 1 )) || return 1 + executable="$1" + [[ "$executable" == bash || "$executable" == /bin/bash || + "$executable" == /usr/bin/bash ]] || return 1 + for argument in "$@"; do + case "$argument" in + "$REPOSITORY_ROOT/scripts/validate/test-k3s-local-recovery.sh"|\ + scripts/validate/test-k3s-local-recovery.sh) + test_script_match=1 + ;; + task5b-fixture|task5b-fixture:*) fixture_marker_match=1 ;; + esac + done + (( test_script_match == 1 || fixture_marker_match == 1 )) +} + +task5b_expected_safe_live_state_record() { + local destination_name="${1-}" + (( $# == 1 )) || return 1 + [[ "$destination_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1 + printf -v "$destination_name" '%s' \ + $'mounts_sda3=0\nmounts_outer=0\nmounts_inner=0\nouter_mount_path=absent\ninner_mount_path=absent\nrecovery_loops=0\nmapper_main=absent\nmapper_proof=absent\nreserved_root=absent\ntask5b_temp_residue=0\ntask5b_fixture_processes=0' +} + +task5b_live_state_record() { + local destination_name="${1-}" line mountinfo_record='' + local mounts_sda3=0 mounts_outer=0 mounts_inner=0 recovery_loops=0 + local outer_mount_path=absent inner_mount_path=absent + local mapper_main=absent mapper_proof=absent reserved_root=absent + local task5b_temp_residue=0 task5b_fixture_processes=0 pid_path candidate_pid + local -a residue_paths=() process_arguments=() + [[ "$destination_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1 + while IFS= read -r line; do + mountinfo_record+="$line"$'\n' + done /dev/null || : + task5b_fixture_process_argv_matches "$BASHPID" "$candidate_pid" \ + "${process_arguments[@]}" || continue + (( task5b_fixture_processes += 1 )) + done + printf -v "$destination_name" \ + 'mounts_sda3=%s\nmounts_outer=%s\nmounts_inner=%s\nouter_mount_path=%s\ninner_mount_path=%s\nrecovery_loops=%s\nmapper_main=%s\nmapper_proof=%s\nreserved_root=%s\ntask5b_temp_residue=%s\ntask5b_fixture_processes=%s' \ + "$mounts_sda3" "$mounts_outer" "$mounts_inner" "$outer_mount_path" \ + "$inner_mount_path" "$recovery_loops" "$mapper_main" "$mapper_proof" \ + "$reserved_root" "$task5b_temp_residue" "$task5b_fixture_processes" +} + +task5b_gate0_review_round1_adversarial_assertions() { + local mount_record mounts_sda3=0 mounts_outer=0 mounts_inner=0 path_state='' + local selector_output='' selector_rc=0 + mount_record=$'36 25 8:3 / /mnt/k3s-recovery-ssd rw - ntfs3 /dev/sda3 rw\n37 25 253:0 / /srv/recovery/k3s rw - ext4 /dev/mapper/k3s-recovery rw\n' + assert_succeeds task5b_mountinfo_counts_record "$mount_record" mounts_sda3 \ + mounts_outer mounts_inner + assert_eq 1 "$mounts_sda3" 'collector parses exact /dev/sda3 source' + assert_eq 1 "$mounts_outer" 'collector parses exact outer mountpoint' + assert_eq 1 "$mounts_inner" 'collector parses exact inner mountpoint' + + assert_succeeds task5b_loop_backing_is_recovery /fixture/recovery-partition + assert_succeeds task5b_loop_backing_is_recovery \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/containers/k3s-recovery.luks + assert_fails task5b_loop_backing_is_recovery /dev/sda3 + assert_fails task5b_loop_backing_is_recovery /other/recovery-partition + assert_fails task5b_loop_backing_is_recovery \ + /mnt/k3s-recovery-ssd/HyeonworksRecovery/containers/k3s-recovery.luks.bak + + assert_succeeds task5b_path_state_from_observation path_state 0 0 + assert_eq absent "$path_state" 'collector treats only nonexistence and nonsymlink as absent' + assert_succeeds task5b_path_state_from_observation path_state 0 1 + assert_eq present "$path_state" 'collector treats a dangling symlink as present' + assert_succeeds task5b_path_state_from_observation path_state 1 0 + assert_eq present "$path_state" 'collector treats an existing path as present' + + assert_fails task5b_fixture_process_argv_matches 100 100 /usr/bin/bash \ + scripts/validate/test-k3s-local-recovery.sh + assert_succeeds task5b_fixture_process_argv_matches 100 101 /usr/bin/bash \ + scripts/validate/test-k3s-local-recovery.sh + assert_succeeds task5b_fixture_process_argv_matches 100 102 /bin/bash -c : \ + task5b-fixture:gate0 + assert_fails task5b_fixture_process_argv_matches 100 103 /usr/bin/bash -c : + assert_fails task5b_fixture_process_argv_matches 100 104 /usr/bin/sleep 10 \ + task5b-fixture:gate0 + + selector_output="$( + TASK5B_FOCUS=not-a-task5b-focus TASK5A_FOCUS=dispatcher \ + /usr/bin/timeout 10s /usr/bin/bash "${BASH_SOURCE[0]}" 2>&1 + )" || selector_rc=$? + assert_eq 2 "$selector_rc" 'unknown Task 5B focus is rejected before legacy selection' + assert_eq 'TEST FAILURE: unsupported TASK5B_FOCUS: not-a-task5b-focus' \ + "$selector_output" 'unknown Task 5B focus emits only the early selector diagnostic' +} + +task5b_gate0_review_round2_adversarial_assertions() { + local record='' match_count=0 mutated_helper_body='' fuse_body='' + + record='/dev/loop37|4242|8:3|7:37|0|0|99|99|99|\x2ffixture\x2frecovery.luks' + assert_succeeds task5b_loop_identity_count_from_record "$record" 8:3 \ + match_count + assert_eq 1 "$match_count" 'loop identity parser counts exact backing MAJ:MIN' + + record='/dev/loop37|4242|8:33|7:37|0|0|99|99|99|\x2froot\x2fsnap' + assert_succeeds task5b_loop_identity_count_from_record "$record" 8:3 \ + match_count + assert_eq 0 "$match_count" 'loop identity parser rejects a nonmatching device' + + record='/dev/loop37|4242|8:3|7:37|0|0|99|99|99|\x2fdeleted\x20backing' + assert_succeeds task5b_loop_identity_count_from_record "$record" 8:3 \ + match_count + assert_eq 1 "$match_count" \ + 'deleted or missing backing pathname cannot hide matching device identity' + + assert_fails task5b_loop_identity_count_from_record \ + '/dev/loop37|4242|8:3|7:37|0|0|99|99' 8:3 match_count + assert_fails task5b_loop_identity_count_from_record \ + '/dev/loop37|4242|8:3|7:37|0|0|99|98|99|\x2fstale' 8:3 match_count + record=$'/dev/loop37|4242|8:3|7:37|0|0|99|99|99|\\x2fone\n/dev/loop37|4243|8:33|7:37|0|0|99|99|99|\\x2ftwo' + assert_fails task5b_loop_identity_count_from_record "$record" 8:3 match_count + + mutated_helper_body="$(declare -f task5b_gate0_assert_rejected)"$'\n'\ +'trap - DEBUG +/usr/bin/printf TASK5B_TRAP_DISABLE_CANARY' || return 1 + assert_fails task5b_gate0_static_body_is_closed "$mutated_helper_body" + fuse_body="$(declare -f task5b_gate0_real_exec_fuse)" || return 1 + [[ "$fuse_body" != *'|trap)'* ]] || + fail 'active fuse must not allowlist DEBUG-trap teardown' +} + +task5b_api_state_trimmed_line_count() { + local body="${1-}" expected_line="${2-}" line count=0 + (( $# == 2 )) || return 1 + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ "$line" != "$expected_line" ]] || (( count += 1 )) + done <<<"$body" + printf '%s\n' "$count" +} + +task5b_api_state_dispatcher_has_obsolete_provider_flow() { + local body="${1-}" line + local provider_token_re='("\$header_provider"|\$header_provider|"\$\{header_provider\}"|\$\{header_provider\})' + local direct_start_re='' control_provider_re='' separator_provider_re='' + (( $# == 1 )) || return 1 + direct_start_re="^[[:space:]]*${provider_token_re}([[:space:];|&]|$)" + control_provider_re="(^|[[:space:]])(if|then|elif|while|until|!)[[:space:]]+${provider_token_re}([[:space:];|&]|$)" + separator_provider_re="(&&|\|\||;)[[:space:]]*${provider_token_re}([[:space:];|&]|$)" + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ "$line" != *'${@:3}'* && "$line" != *provider_arguments* ]] || + return 0 + [[ ! "$line" =~ $direct_start_re && + ! "$line" =~ $control_provider_re && + ! "$line" =~ $separator_provider_re ]] || return 0 + if [[ "$line" == *'$header_provider'* || + "$line" == *'${header_provider}'* ]]; then + case "$line" in + action_arguments=*|action_arguments+=*|_k3slr_lifecycle_action[[:space:]]*) + return 0 + ;; + esac + [[ "$line" =~ ^[[:alnum:]_]*arguments(\+)?= ]] && return 0 + fi + done <<<"$body" + return 1 +} + +# The production breaks this catches are the obsolete caller-provider argv +# forwarding API, a prepare action that still accepts two arguments, or state +# created anywhere except one empty dispatcher-owned associative declaration. +# The production dispatcher/action are observed directly and are never +# replaced by a fixture success stub. +task5b_api_state_contract() ( + local dispatch_body='' action_body='' main_body='' library_source='' + local state_declaration_count=0 state_append_count=0 action_call_count=0 + local main_dispatch_count=0 state_reference_count=0 + local hook_calls=0 action_calls=0 hook_calls_exact=0 action_calls_exact=0 + local command_calls=0 command_calls_exact=0 command_calls_direct=0 + local tty_calls=0 provider_calls=0 action_rc=0 + local dispatch_exact_actual_exec=0 command_direct_actual_exec=0 + local main_arity_actual_exec=0 wrong_action_actual_exec=0 + local dispatch_exact_rc=0 dispatch_invalid_rc=0 dispatch_case='' index=0 + local main_wrong_arity_successes=0 main_contract_sentinel=not-loaded + local probe_rc=0 record='' record_tag='' record_profile='' record_extra='' + local caller_value='' + local -a invalid_dispatch_cases=(zero one prepare-third open-third close-four) + local -a invalid_dispatch_rcs=() invalid_dispatch_hook_calls=() + local -a invalid_dispatch_action_calls=() invalid_dispatch_command_calls=() + local -a invalid_dispatch_actual_exec=() + local -a wrong_action_rcs=() record_fields=() + local -A caller_seed=([caller_provider_argument]=must-not-be-read) + + # shellcheck source=/dev/null + source "$LIBRARY_PATH" + _k3slr_command() { (( command_calls += 1 )); return 97; } + _k3slr_tty_capability() { (( tty_calls += 1 )); return 1; } + _k3slr_read_tty_line() { (( tty_calls += 1 )); return 1; } + task5b_api_state_header_provider() { (( provider_calls += 1 )); return 97; } + task5b_api_state_debug_observer() { + local observed_command="${1-}" + (( $# == 1 )) || return 1 + case "$observed_command" in + _k3slr_lifecycle_step\ *) (( hook_calls += 1 )) ;; + _k3slr_lifecycle_action\ *) (( action_calls += 1 )) ;; + esac + } + task5b_api_dispatch_probe_payload() { + local dispatch_case="${1-}" dispatch_rc=0 hook_calls=0 action_calls=0 + local command_calls=0 provider_calls=0 + local -A caller_seed=([caller_provider_argument]=must-not-be-read) + (( $# == 1 )) || return 1 + case "$dispatch_case" in + exact) + _k3slr_lifecycle_dispatch prepare task5b_api_state_header_provider || + dispatch_rc=$? + ;; + zero) _k3slr_lifecycle_dispatch || dispatch_rc=$? ;; + one) _k3slr_lifecycle_dispatch prepare || dispatch_rc=$? ;; + prepare-third) + _k3slr_lifecycle_dispatch prepare task5b_api_state_header_provider \ + caller_seed || dispatch_rc=$? + ;; + open-third) + _k3slr_lifecycle_dispatch open task5b_api_state_header_provider \ + caller_seed || dispatch_rc=$? + ;; + close-four) + _k3slr_lifecycle_dispatch close task5b_api_state_header_provider \ + caller_seed extra || dispatch_rc=$? + ;; + *) return 1 ;; + esac + task5b_production_probe_emit_ok api_dispatch "$dispatch_rc" "$hook_calls" \ + "$action_calls" "$command_calls" "$provider_calls" \ + "${caller_seed[caller_provider_argument]}" + } + task5b_api_direct_probe_payload() { + local direct_case="${1-}" action_rc=0 command_calls=0 + local -a wrong_action_rcs=() + (( $# == 1 )) || return 1 + case "$direct_case" in + exact) + _k3slr_lifecycle_action prepare sudo _k3slr_prepare_invocation || + action_rc=$? + task5b_production_probe_emit_ok api_direct "$action_rc" "$command_calls" + ;; + wrong-arities) + _k3slr_lifecycle_action || action_rc=$? + wrong_action_rcs+=("$action_rc") + action_rc=0 + _k3slr_lifecycle_action prepare || action_rc=$? + wrong_action_rcs+=("$action_rc") + action_rc=0 + _k3slr_lifecycle_action prepare sudo || action_rc=$? + wrong_action_rcs+=("$action_rc") + action_rc=0 + _k3slr_lifecycle_action prepare sudo _k3slr_prepare_invocation extra || + action_rc=$? + wrong_action_rcs+=("$action_rc") + action_rc=0 + _k3slr_lifecycle_action open || action_rc=$? + wrong_action_rcs+=("$action_rc") + action_rc=0 + _k3slr_lifecycle_action open sudo extra || action_rc=$? + wrong_action_rcs+=("$action_rc") + action_rc=0 + _k3slr_lifecycle_action close || action_rc=$? + wrong_action_rcs+=("$action_rc") + action_rc=0 + _k3slr_lifecycle_action close sudo extra || action_rc=$? + wrong_action_rcs+=("$action_rc") + task5b_production_probe_emit_ok api_direct "$command_calls" \ + "${wrong_action_rcs[@]}" + ;; + *) return 1 ;; + esac + } + task5b_api_main_arity_probe_payload() { + local main_wrong_arity_successes=0 + (( $# == 0 )) || return 1 + _k3slr_lifecycle_main && (( main_wrong_arity_successes += 1 )) + _k3slr_lifecycle_main prepare && (( main_wrong_arity_successes += 1 )) + _k3slr_lifecycle_main prepare execute caller_seed && + (( main_wrong_arity_successes += 1 )) + task5b_production_probe_emit_ok api_main_arity \ + "$main_wrong_arity_successes" "$K3SLR_SCHEMA_VERSION" + } + task5b_production_probe_adversary_matrix || return 1 + task5b_production_probe_static_graph_is_closed api_dispatch || return 1 + task5b_production_probe_static_graph_is_closed api_direct || return 1 + task5b_production_probe_static_graph_is_closed api_main_arity || return 1 + + record="$( + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm api_dispatch || return 1 + task5b_api_dispatch_probe_payload exact + )" || probe_rc=$? + assert_eq 0 "$probe_rc" 'exact dispatcher probe completes without a fuse block' + IFS='|' read -r record_tag record_profile dispatch_exact_rc hook_calls_exact \ + action_calls_exact command_calls_exact provider_calls caller_value \ + record_extra <<<"$record" || return 1 + [[ "$record_tag" == TASK5B_PROBE_OK && "$record_profile" == api_dispatch && + -z "$record_extra" ]] || return 1 + assert_eq 0 "$provider_calls" 'exact dispatcher does not call its provider' + assert_eq must-not-be-read "$caller_value" \ + 'exact dispatcher leaves caller-owned state unchanged' + + for dispatch_case in "${invalid_dispatch_cases[@]}"; do + record='' + probe_rc=0 + record="$( + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm api_dispatch || return 1 + task5b_api_dispatch_probe_payload "$dispatch_case" + )" || probe_rc=$? + assert_eq 0 "$probe_rc" \ + "wrong dispatcher arity probe completes without a fuse block: $dispatch_case" + IFS='|' read -r record_tag record_profile dispatch_invalid_rc hook_calls \ + action_calls command_calls provider_calls caller_value record_extra \ + <<<"$record" || return 1 + [[ "$record_tag" == TASK5B_PROBE_OK && "$record_profile" == api_dispatch && + -z "$record_extra" ]] || return 1 + invalid_dispatch_rcs+=("$dispatch_invalid_rc") + invalid_dispatch_hook_calls+=("$hook_calls") + invalid_dispatch_action_calls+=("$action_calls") + invalid_dispatch_command_calls+=("$command_calls") + invalid_dispatch_actual_exec+=(0) + assert_eq 0 "$provider_calls" \ + "wrong dispatcher arity calls no provider: $dispatch_case" + assert_eq must-not-be-read "$caller_value" \ + "wrong dispatcher arity leaves caller state unchanged: $dispatch_case" + done + + record='' + probe_rc=0 + record="$( + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm api_direct || return 1 + task5b_api_direct_probe_payload exact + )" || probe_rc=$? + assert_eq 0 "$probe_rc" 'exact direct Action 1 probe completes without a fuse block' + IFS='|' read -r record_tag record_profile action_rc command_calls_direct \ + record_extra <<<"$record" || return 1 + [[ "$record_tag" == TASK5B_PROBE_OK && "$record_profile" == api_direct && + -z "$record_extra" ]] || return 1 + + K3SLR_WRAPPER_CONTRACT="$CONTRACT_PATH" + K3SLR_SCHEMA_VERSION="$main_contract_sentinel" + record='' + probe_rc=0 + record="$( + K3SLR_WRAPPER_CONTRACT="$CONTRACT_PATH" + K3SLR_SCHEMA_VERSION="$main_contract_sentinel" + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm api_main_arity || return 1 + task5b_api_main_arity_probe_payload + )" || probe_rc=$? + assert_eq 0 "$probe_rc" 'wrong main arity probe completes without a fuse block' + IFS='|' read -r record_tag record_profile main_wrong_arity_successes \ + K3SLR_SCHEMA_VERSION record_extra <<<"$record" || return 1 + [[ "$record_tag" == TASK5B_PROBE_OK && + "$record_profile" == api_main_arity && -z "$record_extra" ]] || return 1 + + dispatch_body="$(declare -f _k3slr_lifecycle_dispatch)" || return 1 + action_body="$(declare -f _k3slr_lifecycle_action)" || return 1 + main_body="$(declare -f _k3slr_lifecycle_main)" || return 1 + library_source="$(<"$LIBRARY_PATH")" || return 1 + state_declaration_count="$(task5b_api_state_trimmed_line_count "$dispatch_body" \ + 'local -A _k3slr_prepare_invocation=();')" || return 1 + state_append_count="$(task5b_api_state_trimmed_line_count "$dispatch_body" \ + 'action_arguments+=(_k3slr_prepare_invocation);')" || return 1 + action_call_count="$(task5b_api_state_trimmed_line_count "$dispatch_body" \ + '_k3slr_lifecycle_action "${action_arguments[@]}" || return 1;')" || return 1 + main_dispatch_count="$(task5b_api_state_trimmed_line_count "$main_body" \ + '_k3slr_lifecycle_dispatch "$lifecycle" _k3slr_prove_header_restore')" || return 1 + while IFS= read -r line; do + [[ "$line" != *'_k3slr_prepare_invocation'* ]] || + (( state_reference_count += 1 )) + done <<<"$library_source" + + (( hook_calls_exact > 0 )) || + fail 'exact two-argument prepare dispatcher did not reach its production hook' + (( action_calls_exact > 0 )) || + fail 'exact two-argument prepare dispatcher did not reach its production action' + (( dispatch_exact_rc != 0 )) || + fail 'API/state checkpoint production dispatcher is not fail-closed' + for index in "${!invalid_dispatch_cases[@]}"; do + dispatch_case="${invalid_dispatch_cases[$index]}" + (( invalid_dispatch_rcs[index] != 0 )) || + fail "wrong dispatcher arity succeeded: $dispatch_case" + assert_eq 0 "${invalid_dispatch_hook_calls[$index]}" \ + "wrong dispatcher arity reached production hook: $dispatch_case" + assert_eq 0 "${invalid_dispatch_action_calls[$index]}" \ + "wrong dispatcher arity reached production action: $dispatch_case" + assert_eq 0 "${invalid_dispatch_command_calls[$index]}" \ + "wrong dispatcher arity reached the command seam: $dispatch_case" + assert_eq 0 "${invalid_dispatch_actual_exec[$index]}" \ + "wrong dispatcher arity executes no real command: $dispatch_case" + done + assert_eq must-not-be-read "${caller_seed[caller_provider_argument]}" \ + 'rejected caller state/provider argument remains unchanged' + assert_eq 1 "$command_calls_exact" \ + 'exact dispatcher reaches the safe Action 1 command fuse once' + assert_eq 1 "$command_calls_direct" \ + 'exact direct Action 1 reaches the safe command fuse once' + assert_eq 0 "$dispatch_exact_actual_exec" \ + 'exact dispatcher executes no real command' + assert_eq 0 "$command_direct_actual_exec" \ + 'exact direct Action 1 executes no real command' + assert_eq 0 "$main_arity_actual_exec" \ + 'wrong main arity executes no real command' + assert_eq 0 "$tty_calls" 'API/state arity matrix has TTY effect 0' + assert_eq 0 "$provider_calls" 'API/state arity matrix has provider effect 0' + assert_eq 97 "$action_rc" \ + 'exact three-argument Action 1 preserves the safe command-fuse failure' + assert_eq 0 "$main_wrong_arity_successes" \ + 'main rejects zero, one, and three arguments' + assert_eq not-loaded "$K3SLR_SCHEMA_VERSION" \ + 'wrong main arity fails before contract load effects' + + assert_eq 1 "$state_declaration_count" \ + 'dispatcher has one direct empty prepare associative-state declaration' + assert_eq 1 "$state_append_count" \ + 'dispatcher appends the fixed prepare-state name once to action argv' + assert_eq 1 "$action_call_count" \ + 'dispatcher has one production action invocation site' + assert_eq 1 "$main_dispatch_count" \ + 'main has one exact two-argument dispatcher invocation site' + assert_eq 3 "$state_reference_count" \ + 'production has only the prepare-state declaration, fixed action carrier, and Action 1 literal guard' + assert_fails task5b_api_state_dispatcher_has_obsolete_provider_flow \ + "$dispatch_body" + assert_succeeds task5b_api_state_dispatcher_has_obsolete_provider_flow \ + "$dispatch_body"$'\n''${@:3}' + assert_succeeds task5b_api_state_dispatcher_has_obsolete_provider_flow \ + "$dispatch_body"$'\n''provider_arguments=("$header_provider")' + assert_succeeds task5b_api_state_dispatcher_has_obsolete_provider_flow \ + "$dispatch_body"$'\n''"$header_provider" restore' + assert_succeeds task5b_api_state_dispatcher_has_obsolete_provider_flow \ + "$dispatch_body"$'\n''if "$header_provider"; then :; fi' + assert_succeeds task5b_api_state_dispatcher_has_obsolete_provider_flow \ + "$dispatch_body"$'\n'': && "$header_provider" restore' + assert_succeeds task5b_api_state_dispatcher_has_obsolete_provider_flow \ + "$dispatch_body"$'\n''! "$header_provider" restore' + assert_succeeds task5b_api_state_dispatcher_has_obsolete_provider_flow \ + "$dispatch_body"$'\n'': || "$header_provider" restore' + assert_succeeds task5b_api_state_dispatcher_has_obsolete_provider_flow \ + "$dispatch_body"$'\n''action_arguments+=("$header_provider")' + assert_fails task5b_api_state_dispatcher_has_obsolete_provider_flow \ + "$dispatch_body"$'\n''[[ "$header_provider" == _k3slr_prove_header_restore ]] || return 1' + assert_fails task5b_api_state_dispatcher_has_obsolete_provider_flow \ + "$dispatch_body"$'\n''declare -F "$header_provider" >/dev/null || return 1' + [[ "$dispatch_body" == *'header_provider="${2-}"'* ]] || + fail 'dispatcher invocation-local provider identity binding is absent' + [[ "$dispatch_body" != *'${@:3}'* && "$dispatch_body" != *provider_arguments* ]] || + fail 'dispatcher retains obsolete caller provider argument compatibility' + [[ "$dispatch_body" == *'(( $# == 2 )) || return 1'* ]] || + fail 'dispatcher exact two-argument arity guard is absent' + [[ "$action_body" == *'prepare)'*'(( $# == 3 )) || return 1'* && + "$action_body" == *'open | close)'*'(( $# == 2 )) || return 1'* ]] || + fail 'production action handler retains obsolete lifecycle arity' + [[ "$main_body" != *'_k3slr_prepare_invocation'* && + "$main_body" != *'${@:3}'* ]] || + fail 'main creates or forwards prepare invocation state' + [[ "$main_body" == *'(( $# == 2 )) || return 1'* ]] || + fail 'main exact two-argument arity guard is absent' + [[ "${main_body%%_k3slr_load_contract*}" == *'(( $# == 2 )) || return 1'* ]] || + fail 'main arity guard does not precede contract effects' + + record='' + probe_rc=0 + record="$( + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm api_direct || return 1 + task5b_api_direct_probe_payload wrong-arities + )" || probe_rc=$? + assert_eq 0 "$probe_rc" 'wrong action arity probe completes without a fuse block' + IFS='|' read -r -a record_fields <<<"$record" || return 1 + record_tag="${record_fields[0]-}" + record_profile="${record_fields[1]-}" + command_calls="${record_fields[2]-}" + wrong_action_rcs=("${record_fields[@]:3}") + [[ "$record_tag" == TASK5B_PROBE_OK && "$record_profile" == api_direct && + ${#wrong_action_rcs[@]} == 8 ]] || return 1 + for action_rc in "${wrong_action_rcs[@]}"; do + (( action_rc != 0 )) || fail 'wrong action arity unexpectedly succeeded' + done + assert_eq 0 "$command_calls" 'wrong action/dispatcher arity has command effect 0' + assert_eq 0 "$wrong_action_actual_exec" \ + 'wrong action/dispatcher arity has no fuse block' + assert_eq 0 "$tty_calls" 'wrong action/dispatcher arity has TTY effect 0' + assert_eq 0 "$provider_calls" 'wrong action/dispatcher arity has provider effect 0' +) + +# The production breaks this catches are a high-level TTY gate that bypasses +# the low capability seam, fails to preserve its status, or reaches it after a +# wrong-arity call. Only the lowest seam is replaced in this source-loaded +# foreground subshell. +task5b_tty_seam_contract() ( + local tty_capability_calls=0 tty_capability_rc=0 high_rc=0 + + # shellcheck source=/dev/null + source "$LIBRARY_PATH" + _k3slr_tty_capability() { + (( tty_capability_calls += 1 )) + return "$tty_capability_rc" + } + + high_rc=0 + _k3slr_require_execute_tty || high_rc=$? + assert_eq 1 "$tty_capability_calls" \ + 'TTY high helper delegates exactly once on low-capability success' + assert_eq 0 "$high_rc" \ + 'TTY high helper preserves low-capability success' + + tty_capability_calls=0 + tty_capability_rc=73 + high_rc=0 + _k3slr_require_execute_tty || high_rc=$? + assert_eq 1 "$tty_capability_calls" \ + 'TTY high helper delegates exactly once on low-capability failure' + assert_eq 73 "$high_rc" \ + 'TTY high helper preserves low-capability failure' + + tty_capability_calls=0 + tty_capability_rc=0 + high_rc=0 + _k3slr_require_execute_tty unexpected || high_rc=$? + assert_eq 1 "$high_rc" 'TTY high helper rejects wrong arity' + assert_eq 0 "$tty_capability_calls" \ + 'TTY high helper rejects wrong arity before the low seam' +) + +task5b_action1_literal_line_count() { + local body="${1-}" needle="${2-}" destination_name="${3-}" line count=0 + (( $# == 3 )) || return 1 + [[ -n "$needle" && "$destination_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || + return 1 + while IFS= read -r line; do + [[ "$line" != *"$needle"* ]] || (( count += 1 )) + done <<<"$body" + printf -v "$destination_name" '%s' "$count" +} + +# Every literal direct prepare/sudo test call must live in a fixture whose +# command seam fails closed or in the Action 1 exact matcher. The historical +# unavailable-action loop is indirect, so it has its own command fuse and must +# no longer enumerate the now-available prepare/sudo action. +task5b_action1_nonfocused_direct_sudo_paths_are_fused() { + local direct_needle='' test_source='' safe_bodies='' + local dryrun_body='' api_state_body='' action1_body='' unavailable_body='' + local source_count=0 safe_count=0 + direct_needle='_k3slr_lifecycle_action prepare '"sudo" + test_source="$(<"${BASH_SOURCE[0]}")" || return 1 + dryrun_body="$(declare -f task5a_dry_run_and_prepackage_action_binding)" || + return 1 + api_state_body="$(declare -f task5b_api_state_contract)" || return 1 + action1_body="$(declare -f task5b_action1_command_binding_contract)" || + return 1 + unavailable_body="$(declare -f task5a_production_actions_fail_closed)" || + return 1 + safe_bodies="$dryrun_body"$'\n'"$api_state_body"$'\n'"$action1_body" + task5b_action1_literal_line_count "$test_source" "$direct_needle" \ + source_count || return 1 + task5b_action1_literal_line_count "$safe_bodies" "$direct_needle" \ + safe_count || return 1 + (( source_count > 0 && source_count == safe_count )) || return 1 + [[ "$dryrun_body" == *'_k3slr_command ()'* && + "$dryrun_body" == *'return 97'* && + "$dryrun_body" == *'task5b_production_probe_arm legacy_binding'* ]] || + return 1 + [[ "$api_state_body" == *'_k3slr_command ()'* && + "$api_state_body" == *'return 97'* && + "$api_state_body" == *'task5b_production_probe_arm api_direct'* ]] || + return 1 + [[ "$action1_body" == *'task5b_gate0_route_equals sudo-validate "$@"'* && + "$action1_body" == *'return 97'* && + "$action1_body" == *'task5b_production_probe_arm action1_main'* && + "$action1_body" == *'task5b_production_probe_arm action1_alternate'* ]] || + return 1 + [[ "$unavailable_body" == *'_k3slr_command ()'* && + "$unavailable_body" == *'return 97'* && + "$unavailable_body" == *'task5b_production_probe_arm unavailable'* && + "$unavailable_body" != *'prepare_unavailable=(sudo'* ]] || return 1 +} + +# The production breaks this catches are a prepare/sudo handler that omits or +# misframes the exact sudo validation argv, reaches a command after sudo, calls +# the header provider early, or accepts any state name except the dispatcher- +# owned literal. The real main, dispatcher, hook, action, and invocation state +# remain untouched; only the lowest TTY and command seams are replaced. +task5b_action1_command_binding_contract() ( + local tty_calls=0 tty_rc=0 sudo_dispatches=0 sudo_rc=0 + local unexpected_commands=0 actual_exec=0 provider_calls=0 main_rc=0 + local hook_sudo_entries=0 hook_context_entries=0 hook_suffix_entries=0 + local action_sudo_entries=0 action_context_entries=0 action_suffix_entries=0 + local action_rc=0 command_body='' + local probe_record='' record_tag='' record_profile='' record_extra='' + + # shellcheck source=/dev/null + source "$LIBRARY_PATH" + K3SLR_WRAPPER_CONTRACT="$CONTRACT_PATH" + + _k3slr_prove_header_restore() { + (( provider_calls += 1 )) + return 97 + } + _k3slr_tty_capability() { + (( tty_calls += 1 )) + return "$tty_rc" + } + _k3slr_command() { + if task5b_gate0_route_equals sudo-validate "$@"; then + (( sudo_dispatches += 1 )) + return "$sudo_rc" + fi + (( unexpected_commands += 1 )) + return 97 + } + task5b_action1_debug_observer() { + local observed_command="${1-}" observed_step="${2-}" + (( $# == 2 )) || return 1 + case "$observed_command" in + _k3slr_lifecycle_step\ *) + case "$observed_step" in + sudo) (( hook_sudo_entries += 1 )) ;; + context) (( hook_context_entries += 1 )) ;; + *) (( hook_suffix_entries += 1 )) ;; + esac + ;; + _k3slr_lifecycle_action\ *) + case "$observed_step" in + sudo) (( action_sudo_entries += 1 )) ;; + context) (( action_context_entries += 1 )) ;; + *) (( action_suffix_entries += 1 )) ;; + esac + ;; + esac + } + task5b_action1_main_probe_payload() { + local probe_rc=0 tty_calls=0 sudo_dispatches=0 unexpected_commands=0 + local provider_calls=0 hook_sudo_entries=0 hook_context_entries=0 + local hook_suffix_entries=0 action_sudo_entries=0 + local action_context_entries=0 action_suffix_entries=0 + (( $# == 0 )) || return 1 + _k3slr_lifecycle_main prepare execute || probe_rc=$? + task5b_production_probe_emit_ok action1_main "$probe_rc" "$tty_calls" \ + "$sudo_dispatches" "$unexpected_commands" "$provider_calls" \ + "$hook_sudo_entries" "$hook_context_entries" "$hook_suffix_entries" \ + "$action_sudo_entries" "$action_context_entries" \ + "$action_suffix_entries" + } + task5b_action1_alternate_probe_payload() { + local probe_rc=0 tty_calls=0 sudo_dispatches=0 unexpected_commands=0 + local provider_calls=0 hook_sudo_entries=0 hook_context_entries=0 + local hook_suffix_entries=0 action_sudo_entries=0 + local action_context_entries=0 action_suffix_entries=0 + (( $# == 0 )) || return 1 + _k3slr_lifecycle_action prepare sudo task5b_alternate_state || probe_rc=$? + task5b_production_probe_emit_ok action1_alternate "$probe_rc" \ + "$tty_calls" "$sudo_dispatches" "$unexpected_commands" \ + "$provider_calls" "$hook_sudo_entries" "$hook_context_entries" \ + "$hook_suffix_entries" "$action_sudo_entries" \ + "$action_context_entries" "$action_suffix_entries" + } + task5b_action1_run_probe() { + local probe_kind="${1-}" probe_profile='' + local guarded_rc=0 + (( $# == 1 )) || return 1 + case "$probe_kind" in + main) probe_profile=action1_main ;; + alternate) probe_profile=action1_alternate ;; + *) return 1 ;; + esac + case "$probe_kind" in + main) + probe_record="$( + local task5b_probe_profile='' + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm action1_main || return 1 + task5b_action1_main_probe_payload + )" || guarded_rc=$? + ;; + alternate) + probe_record="$( + local task5b_probe_profile='' + exec 9>&1 + exec 2>&1 + task5b_production_probe_arm action1_alternate || return 1 + task5b_action1_alternate_probe_payload + )" || guarded_rc=$? + ;; + esac + if (( guarded_rc != 0 )); then + return "$guarded_rc" + fi + record_tag='' + record_extra='' + IFS='|' read -r record_tag record_profile action_rc tty_calls \ + sudo_dispatches unexpected_commands provider_calls hook_sudo_entries \ + hook_context_entries hook_suffix_entries action_sudo_entries \ + action_context_entries action_suffix_entries record_extra \ + <<<"$probe_record" || return 1 + [[ "$record_tag" == TASK5B_PROBE_OK && + "$record_profile" == "$probe_profile" && -z "$record_extra" ]] || + fail "Action 1 $probe_kind model keeps combined stdout and stderr empty" + actual_exec=0 + } + task5b_action1_run_main() { + task5b_action1_run_probe main || return 1 + main_rc="$action_rc" + } + + task5b_production_probe_adversary_matrix || { + printf 'TEST FAILURE: Action 1 production-probe adversary matrix failed\n' >&2 + return 1 + } + task5b_production_probe_static_graph_is_closed action1_main || { + printf 'TEST FAILURE: Action 1 main exact graph scan failed\n' >&2 + return 1 + } + task5b_production_probe_static_graph_is_closed action1_alternate || { + printf 'TEST FAILURE: Action 1 alternate exact graph scan failed\n' >&2 + return 1 + } + + task5b_action1_run_main || { + printf 'TEST FAILURE: Action 1 main guarded probe did not emit an OK record\n' >&2 + return 1 + } + (( main_rc != 0 )) || fail 'Action 1 success model stops at pending context' + assert_eq 1 "$tty_calls" 'Action 1 full main reaches the low TTY seam once' + (( action_sudo_entries > 0 )) || + fail 'Action 1 full main reaches the production prepare/sudo handler' + assert_eq 1 "$sudo_dispatches" \ + 'Action 1 production handler dispatches exact sudo validation once' + (( hook_sudo_entries > 0 )) || + fail 'Action 1 success model records the sudo lifecycle hook entry' + (( hook_context_entries > 0 )) || + fail 'Action 1 success model reaches the pending context hook entry' + (( action_context_entries > 0 )) || + fail 'Action 1 success model reaches the pending context action entry' + assert_eq 0 "$hook_suffix_entries" \ + 'Action 1 success model reaches no hook after pending context' + assert_eq 0 "$action_suffix_entries" \ + 'Action 1 success model reaches no action after pending context' + assert_eq 0 "$unexpected_commands" \ + 'Action 1 success model dispatches no command after sudo validation' + assert_eq 0 "$actual_exec" 'Action 1 success model executes no real command' + assert_eq 0 "$provider_calls" 'Action 1 success model does not call the provider' + + command_body="$(declare -f _k3slr_command)" || return 1 + assert_succeeds task5b_gate0_static_body_is_closed "$command_body" + assert_succeeds task5b_action1_nonfocused_direct_sudo_paths_are_fused + + tty_calls=0 + tty_rc=71 + sudo_dispatches=0 + sudo_rc=0 + unexpected_commands=0 + provider_calls=0 + hook_sudo_entries=0 + hook_context_entries=0 + hook_suffix_entries=0 + action_sudo_entries=0 + action_context_entries=0 + action_suffix_entries=0 + task5b_action1_run_main + (( main_rc != 0 )) || fail 'Action 1 rejects a failed low TTY capability' + assert_eq 1 "$tty_calls" 'Action 1 TTY failure calls the low seam once' + assert_eq 0 "$hook_sudo_entries" 'Action 1 TTY failure reaches no dispatcher hook' + assert_eq 0 "$action_sudo_entries" 'Action 1 TTY failure reaches no action' + assert_eq 0 "$sudo_dispatches" 'Action 1 TTY failure dispatches no sudo command' + assert_eq 0 "$unexpected_commands" 'Action 1 TTY failure dispatches no command' + assert_eq 0 "$actual_exec" 'Action 1 TTY failure executes no real command' + assert_eq 0 "$provider_calls" 'Action 1 TTY failure calls no provider' + + tty_calls=0 + tty_rc=0 + sudo_dispatches=0 + sudo_rc=97 + unexpected_commands=0 + provider_calls=0 + hook_sudo_entries=0 + hook_context_entries=0 + hook_suffix_entries=0 + action_sudo_entries=0 + action_context_entries=0 + action_suffix_entries=0 + task5b_action1_run_main + (( main_rc != 0 )) || fail 'Action 1 propagates modeled sudo failure' + assert_eq 1 "$tty_calls" 'Action 1 sudo failure calls the low TTY seam once' + (( hook_sudo_entries > 0 )) || + fail 'Action 1 sudo failure reaches the sudo hook entry' + (( action_sudo_entries > 0 )) || + fail 'Action 1 sudo failure reaches the production sudo action entry' + assert_eq 1 "$sudo_dispatches" \ + 'Action 1 sudo failure reaches the exact sudo model once' + assert_eq 0 "$hook_context_entries" \ + 'Action 1 sudo failure reaches no context hook' + assert_eq 0 "$action_context_entries" \ + 'Action 1 sudo failure reaches no context action' + assert_eq 0 "$hook_suffix_entries" 'Action 1 sudo failure reaches no suffix hook' + assert_eq 0 "$action_suffix_entries" \ + 'Action 1 sudo failure reaches no suffix action' + assert_eq 0 "$unexpected_commands" \ + 'Action 1 sudo failure dispatches no unexpected command' + assert_eq 0 "$actual_exec" 'Action 1 sudo failure executes no real command' + assert_eq 0 "$provider_calls" 'Action 1 sudo failure calls no provider' + + task5b_action1_run_probe alternate + assert_eq 1 "$action_rc" 'Action 1 rejects an alternate exact-three state name' + assert_eq 0 "$sudo_dispatches" \ + 'Action 1 alternate state name dispatches no sudo command' + assert_eq 0 "$unexpected_commands" \ + 'Action 1 alternate state name dispatches no unexpected command' + assert_eq 0 "$actual_exec" 'Action 1 alternate state executes no real command' + assert_eq 0 "$provider_calls" 'Action 1 alternate state calls no provider' +) + +task5b_run_focus() { + local focus="${1-}" expected_record before_record after_record focus_rc=0 + (( $# == 1 )) || return 1 + task5b_expected_safe_live_state_record expected_record || return 1 + task5b_live_state_record before_record || return 1 + assert_eq "$expected_record" "$before_record" \ + "Task 5B $focus pre-snapshot is independently all-zero/all-absent" || return 1 + task5b_gate0_contract || return 1 + case "$focus" in + gate0) + task5b_gate0_review_round1_adversarial_assertions || focus_rc=$? + if (( focus_rc == 0 )); then + task5b_gate0_review_round2_adversarial_assertions || focus_rc=$? + fi + ;; + api_state) + task5b_api_state_contract || focus_rc=$? + ;; + tty_seam) + task5b_tty_seam_contract || focus_rc=$? + ;; + action1) + task5b_action1_command_binding_contract || focus_rc=$? + ;; + *) focus_rc=1 ;; + esac + task5b_gate0_contract || return 1 + task5b_live_state_record after_record || return 1 + assert_eq "$expected_record" "$after_record" \ + "Task 5B $focus post-snapshot is independently all-zero/all-absent" || return 1 + assert_eq "$before_record" "$after_record" \ + "Task 5B $focus focus preserves promptless live state/process/temp record" || return 1 + (( focus_rc == 0 )) || return "$focus_rc" +} + +if [[ "${TASK5B_FOCUS:-}" == gate0 || "${TASK5B_FOCUS:-}" == api_state || + "${TASK5B_FOCUS:-}" == tty_seam || "${TASK5B_FOCUS:-}" == action1 ]]; then + assert_succeeds task5b_run_focus "$TASK5B_FOCUS" +elif [[ "${TASK5A_FOCUS:-}" == wrappers ]]; then + assert_succeeds task5a_wrapper_and_guard_contract +elif [[ "${TASK5A_FOCUS:-}" == capabilities ]]; then + assert_succeeds task5a_prepare_provider_capability_gate + assert_succeeds task5a_open_close_capability_gate +elif [[ "${TASK5A_FOCUS:-}" == dispatcher ]]; then + assert_succeeds task5a_lifecycle_dispatch_contract + assert_succeeds task5a_production_actions_fail_closed +elif [[ "${TASK5A_FOCUS:-}" == records ]]; then + assert_succeeds task5a_exact_record_and_pure_parser_contract +elif [[ "${TASK5A_FOCUS:-}" == prepackage ]]; then + assert_succeeds task5a_ntfs_capacity_and_prepackage_seams +elif [[ "${TASK5A_FOCUS:-}" == capture ]]; then + assert_succeeds task5a_exact_process_capture_contract +elif [[ "${TASK5A_FOCUS:-}" == prepackage_system ]]; then + assert_succeeds task5a_prepackage_system_command_matrix + assert_succeeds task5a_ntfsinfo_pipeline_contract +elif [[ "${TASK5A_FOCUS:-}" == dryrun ]]; then + assert_succeeds task5a_dry_run_and_prepackage_action_binding +elif [[ "${TASK4_REVIEW5_FOCUS:-}" == priority ]]; then + assert_succeeds task4_review5_pending_priority +elif [[ "${TASK4_REVIEW5_FOCUS:-}" == feas_cleanup ]]; then + assert_succeeds task4_review5_feasibility_cleanup_failure_is_not_signal_success +elif [[ "${TASK4_REVIEW5_FOCUS:-}" == unpinned ]]; then + assert_succeeds task4_review5_published_unpinned_cleanup +elif [[ "${TASK4_REVIEW5_FOCUS:-}" == parser ]]; then + assert_succeeds task4_review5_exact_proc_record_contract +elif [[ "${TASK4_REVIEW5_FOCUS:-}" == anchor ]]; then + assert_succeeds task4_review5_default_supervisor_anchor + assert_succeeds task4_review5_feasibility_default_supervisor_anchor +elif [[ "${TASK4_REVIEW5_FOCUS:-}" == phase_fixture_cleanup ]]; then + assert_succeeds task4_review5_phase_fixture_failure_cleanup_contract +elif [[ "${TASK4_REVIEW4_FOCUS:-}" == phases ]]; then + assert_succeeds task4_review4_signal_phase_matrix +elif [[ "${TASK4_REVIEW4_FOCUS:-}" == boundaries ]]; then + assert_succeeds task4_review4_stale_cached_group_boundary + assert_succeeds task4_review4_direct_fallback_boundaries + assert_succeeds task4_review4_term_ignoring_group_escalates + assert_succeeds task4_review4_wait_clear_is_atomic +elif [[ "${TASK4_REVIEW4_FOCUS:-}" == stale_group ]]; then + assert_succeeds task4_review4_stale_cached_group_boundary +elif [[ "${TASK4_REVIEW3_FOCUS:-}" == signal ]]; then + assert_succeeds task4_review3_nested_pipeline_signal_cleanup +elif [[ "${TASK4_REVIEW3_FOCUS:-}" == early_signal ]]; then + assert_succeeds task4_review3_launch_window_signal_cleanup +elif [[ "${TASK4_REVIEW2_FOCUS:-}" == critical ]]; then + assert_succeeds task4_review2_tty_and_bounded_validated_pipe +elif [[ "${TASK4_REVIEW2_FOCUS:-}" == show_protected ]]; then + assert_succeeds task4_review2_show_protected_and_readonly +elif [[ "${TASK4_REVIEW2_FOCUS:-}" == feas_show_protected ]]; then + assert_succeeds task4_review2_feasibility_show_protected +elif [[ "${TASK4_REVIEW2_FOCUS:-}" == feas_attachment_listing ]]; then + assert_succeeds task4_review2_feasibility_attachment_listing + assert_succeeds task4_review2_attachment_parser_contract +elif [[ "${TASK4_REVIEW2_FOCUS:-}" == capture ]]; then + assert_succeeds task4_review2_capture_one_line_exact_record +elif [[ "${TASK4_REVIEW2_FOCUS:-}" == atomic ]]; then + assert_succeeds task4_review2_atomic_save_transition +elif [[ "${TASK4_REVIEW1_FOCUS:-}" == critical ]]; then + assert_succeeds task4_review1_producer_must_finish_before_consumer +elif [[ "${TASK4_REVIEW1_FOCUS:-}" == lineage ]]; then + assert_succeeds task4_review1_preflight_and_lineage_gate +elif [[ "${TASK4_REVIEW1_FOCUS:-}" == constants_lf ]]; then + assert_succeeds task4_review1_exact_constants_become_readonly + assert_succeeds task4_review1_generated_password_exact_lf +elif [[ "${TASK4_REVIEW1_FOCUS:-}" == signal ]]; then + assert_succeeds task4_review1_blocking_child_is_reaped +elif [[ "${TASK4_REVIEW1_FOCUS:-}" == noclobber ]]; then + assert_succeeds task4_review1_noclobber_lifecycle +elif [[ "${TASK4_REVIEW1_FOCUS:-}" == parser ]]; then + assert_succeeds task4_review1_raw_package_record +elif [[ "${TASK4_REVIEW1_FOCUS:-}" == leak ]]; then + assert_succeeds task4_review1_two_sentinel_leak_contract +else + assert_succeeds task5a_wrapper_and_guard_contract + assert_succeeds task5a_prepare_provider_capability_gate + assert_succeeds task5a_open_close_capability_gate + assert_succeeds task5a_lifecycle_dispatch_contract + assert_succeeds task5a_production_actions_fail_closed + assert_succeeds task5a_exact_record_and_pure_parser_contract + assert_succeeds task5a_ntfs_capacity_and_prepackage_seams + assert_succeeds task5a_exact_process_capture_contract + assert_succeeds task5a_prepackage_system_command_matrix + assert_succeeds task5a_ntfsinfo_pipeline_contract + assert_succeeds task5a_dry_run_and_prepackage_action_binding + assert_succeeds task4_pipeline_contract + assert_succeeds task4_tty_and_secret_api_contract + assert_succeeds task4_feasibility_cleanup_and_leak_contract + assert_succeeds task4_review1_producer_must_finish_before_consumer + assert_succeeds task4_review1_preflight_and_lineage_gate + assert_succeeds task4_review1_exact_constants_become_readonly + assert_succeeds task4_review1_generated_password_exact_lf + assert_succeeds task4_review1_blocking_child_is_reaped + assert_succeeds task4_review1_noclobber_lifecycle + assert_succeeds task4_review1_raw_package_record + assert_succeeds task4_review1_two_sentinel_leak_contract + assert_succeeds task4_review2_tty_and_bounded_validated_pipe + assert_succeeds task4_review2_show_protected_and_readonly + assert_succeeds task4_review2_feasibility_show_protected + assert_succeeds task4_review2_feasibility_attachment_listing + assert_succeeds task4_review2_attachment_parser_contract + assert_succeeds task4_review2_capture_one_line_exact_record + assert_succeeds task4_review2_atomic_save_transition + assert_succeeds task4_review3_nested_pipeline_signal_cleanup + assert_succeeds task4_review3_launch_window_signal_cleanup + assert_succeeds task4_review4_stale_cached_group_boundary + assert_succeeds task4_review4_direct_fallback_boundaries + assert_succeeds task4_review4_term_ignoring_group_escalates + assert_succeeds task4_review4_wait_clear_is_atomic + assert_succeeds task4_review4_signal_phase_matrix + assert_succeeds task4_review5_exact_proc_record_contract + assert_succeeds task4_review5_default_supervisor_anchor + assert_succeeds task4_review5_feasibility_default_supervisor_anchor + assert_succeeds task4_review5_phase_fixture_failure_cleanup_contract + assert_succeeds task4_review5_published_unpinned_cleanup + assert_succeeds task4_review5_pending_priority + assert_succeeds task4_review5_feasibility_cleanup_failure_is_not_signal_success +fi + +printf 'PASS: local recovery contract, capacity, and validator fixtures\n' diff --git a/scripts/validate/test-k3s-secret-encryption-restore-evidence.sh b/scripts/validate/test-k3s-secret-encryption-restore-evidence.sh new file mode 100755 index 0000000..782ab85 --- /dev/null +++ b/scripts/validate/test-k3s-secret-encryption-restore-evidence.sh @@ -0,0 +1,705 @@ +#!/usr/bin/env bash + +# k3s Secret 복구 증거의 parser, 세 mode와 권한 경계를 검증한다. +# 이 파일은 source-safe 함수만 호출하며 live host나 Kubernetes API를 건드리지 않는다. +set -Eeuo pipefail + +readonly TEST_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly BOOTSTRAP_PATH="${TEST_ROOT}/scripts/bootstrap/apply-k3s-secret-encryption.sh" +readonly RESTORE_VALIDATOR_PATH="${TEST_ROOT}/scripts/validate/k3s-secret-encryption-restore-evidence.sh" + +fail() { + printf 'RESTORE EVIDENCE TEST FAILURE: %s\n' "$*" >&2 + exit 1 +} + +assert_eq() { + local expected="$1" actual="$2" description="$3" + [[ "$actual" == "$expected" ]] || fail "$description" +} + +# 잘못된 legacy field를 쓰는 producer 변경을 잡는다. 기대값은 Task 5 계약에서 +# 손으로 적었으며 production parser나 formatter로 만들지 않는다. +probe_canonical_bundle_metadata_producer() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + declare -F _kseb_render_bundle_metadata >/dev/null || return 1 + local actual + actual="$(_kseb_render_bundle_metadata \ + '01234567-89ab-4cde-8fab-0123456789ab' post embedded-etcd \ + '2026-08-01T00:00:00Z' 7 'v1.36.2+k3s1')" || return 1 + assert_eq $'schema=platform-k3s-bundle-v1\nbundle_id=01234567-89ab-4cde-8fab-0123456789ab\nphase=post\nk3s_version=v1.36.2+k3s1\ndatastore=embedded-etcd\ncreated_at_utc=2026-08-01T00:00:00Z\nsecret_count=7' \ + "$actual" 'canonical bundle metadata fields/order' +) + +probe_canonical_bundle_metadata_producer || fail 'canonical bundle metadata producer' + +# 아래 production 변경이 깨질 때 잡는 결함은 allowlist parser 우회, mode 간 +# 검증 누락, sudo 전 입력 검증 누락과 기존 evidence overwrite다. +[[ -f "$RESTORE_VALIDATOR_PATH" ]] || fail 'restore evidence validator is absent' +# shellcheck source=/dev/null +source "$RESTORE_VALIDATOR_PATH" + +fixture_root="$(mktemp -d "${TMPDIR:-/tmp}/k3s-restore-evidence-test.XXXXXX")" +cleanup() { + case "$fixture_root" in + /tmp/k3s-restore-evidence-test.*|"${TMPDIR:-/tmp}"/k3s-restore-evidence-test.*) + rm -rf -- "$fixture_root" + ;; + *) fail 'unexpected fixture root' ;; + esac +} +trap cleanup EXIT + +readonly FIXTURE_BUNDLE_ID='01234567-89ab-4cde-8fab-0123456789ab' +readonly FIXTURE_OTHER_ID='11234567-89ab-4cde-8fab-0123456789ab' +readonly FIXTURE_LOCAL_NODE_UID='aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee' +readonly FIXTURE_OTHER_NODE_UID='11111111-2222-4333-8444-555555555555' +readonly FIXTURE_TESTED_AT='2026-08-01T00:00:00Z' +readonly FIXTURE_RECORDED_AT='2026-08-01T01:00:00Z' + +# Fix round 1 / finding 4: invalid compact timestamp는 production transaction의 +# directory/backup/marker mutation 전에 거부해야 한다. +probe_invalid_timestamp_precedes_transaction_mutation() ( + local parent="${fixture_root}/invalid-stamp" root data fake_k3s command_log rc=0 + parent="$(mktemp -d "${fixture_root}/invalid-stamp.XXXXXX")" + root="$parent/recovery"; data="$parent/data"; fake_k3s="$parent/k3s" + command_log="$parent/commands.log" + mkdir -p "$root" "$data/server/cred" + chmod 0700 "$root" + printf 'fixture\n' >"$data/server/token" + printf '{}\n' >"$data/server/cred/encryption-config.json" + : >"$command_log" + printf '%s\n' '#!/usr/bin/env bash' \ + 'printf "called\n" >>"${KSEB_INVALID_STAMP_LOG:?}"' \ + 'exit 1' >"$fake_k3s" + chmod 0700 "$fake_k3s" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" + _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } + _kseb_transaction_data_dir() { printf '%s\n' "$data"; } + _kseb_transaction_config_dir() { printf '%s\n' "$parent/no-config"; } + _kseb_transaction_systemd_paths() { :; } + _kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; } + _kseb_transaction_post_marker() { printf '%s\n' "$parent/post-marker"; } + _kseb_transaction_restore_evidence() { printf '%s\n' "$parent/restore-evidence"; } + export KSEB_INVALID_STAMP_LOG="$command_log" + _kseb_pinned_phase_transaction "$root" bundle/pre embedded-etcd \ + "$FIXTURE_BUNDLE_ID" pre 20261301T000000Z 7 v1.36.2+k3s1 \ + >/dev/null 2>&1 || rc=$? + [[ "$rc" -ne 0 && ! -e "$root/bundle" && ! -s "$command_log" ]] +) + +write_metadata() { + local path="$1" phase="${2:-post}" bundle_id="${3:-$FIXTURE_BUNDLE_ID}" + local version="${4:-v1.36.2+k3s1}" datastore="${5:-embedded-etcd}" + local created="${6:-2026-07-31T23:00:00Z}" count="${7:-7}" + printf 'schema=platform-k3s-bundle-v1\nbundle_id=%s\nphase=%s\nk3s_version=%s\ndatastore=%s\ncreated_at_utc=%s\nsecret_count=%s\n' \ + "$bundle_id" "$phase" "$version" "$datastore" "$created" "$count" >"$path" + chmod 0600 "$path" +} + +write_result() { + local path="$1" bundle_id="${2:-$FIXTURE_BUNDLE_ID}" version="${3:-v1.36.2+k3s1}" + local datastore="${4:-embedded-etcd}" node="${5:-pass}" isolation="${6:-pass}" + local integrity="${7:-match}" count="${8:-pass}" tested="${9:-$FIXTURE_TESTED_AT}" + printf 'schema=platform-k3s-restore-result-v1\nbundle_id=%s\nk3s_version=%s\ndatastore=%s\nstatus_class=enabled_stable\nrotation_stage=reencrypt_finished\nhashes=match\nlocal_integrity=%s\napi_ready=pass\nnode_ready=%s\nisolation=%s\nsecret_count_match=%s\ntested_at_utc=%s\n' \ + "$bundle_id" "$version" "$datastore" "$integrity" "$node" "$isolation" "$count" "$tested" >"$path" + chmod 0600 "$path" +} + +write_evidence() { + local path="$1" bundle_id="${2:-$FIXTURE_BUNDLE_ID}" version="${3:-v1.36.2+k3s1}" + local datastore="${4:-embedded-etcd}" integrity="${5:-match}" destroyed="${6:-confirmed}" + local tested="${7:-$FIXTURE_TESTED_AT}" recorded="${8:-$FIXTURE_RECORDED_AT}" + printf 'schema=platform-k3s-restore-evidence-v1\nbundle_id=%s\nk3s_version=%s\ndatastore=%s\nrotation_stage=reencrypt_finished\nlocal_integrity=%s\nnode_ready=pass\nisolation=pass\ntested_at_utc=%s\nrecorded_at_utc=%s\ndestroyed=%s\n' \ + "$bundle_id" "$version" "$datastore" "$integrity" "$tested" "$recorded" "$destroyed" >"$path" + chmod 0600 "$path" +} + +assert_fails() { + if "$@" >/dev/null 2>&1; then fail "expected failure: $*"; fi +} + +assert_succeeds() { + "$@" || fail "expected success: $*" +} + +assert_succeeds probe_invalid_timestamp_precedes_transaction_mutation + +metadata="${fixture_root}/bundle.env" +result="${fixture_root}/result.env" +evidence_fixture="${fixture_root}/evidence.env" +write_metadata "$metadata" +write_result "$result" +write_evidence "$evidence_fixture" + +probe_parse() ( + local kind="$1" path="$2" + declare -A parsed=() + _ksre_parse_external_file "$kind" "$path" parsed +) + +assert_succeeds probe_parse bundle "$metadata" +assert_succeeds probe_parse result "$result" +assert_succeeds probe_parse evidence "$evidence_fixture" + +# Fix round 1 / finding 1: local restore validator와 current-user API가 +# 서로 다른 cluster를 보면 production identity gate가 실패해야 한다. +matching_nodes='{"items":[{"metadata":{"name":"isolated-ready","uid":"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}' +mismatched_nodes='{"items":[{"metadata":{"name":"other-ready","uid":"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}' +multiple_ready_nodes='{"items":[{"metadata":{"name":"isolated-ready","uid":"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"other-ready","uid":"11111111-2222-4333-8444-555555555555"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}' +assert_succeeds _ksre_require_same_ready_node isolated-ready "$FIXTURE_LOCAL_NODE_UID" "$matching_nodes" +assert_fails _ksre_require_same_ready_node isolated-ready "$FIXTURE_LOCAL_NODE_UID" "$mismatched_nodes" +assert_fails _ksre_require_same_ready_node isolated-ready "$FIXTURE_LOCAL_NODE_UID" "$multiple_ready_nodes" + +# Fix round 2 / finding 1: 같은 UID가 복제된 다른 API도 허용하지 않도록 +# current-user context의 effective endpoint는 exact loopback이며 proxy가 없어야 한다. +assert_succeeds _ksre_validate_current_connection 'https://127.0.0.1:6443|' +assert_fails _ksre_validate_current_connection 'https://192.0.2.10:6443|' +assert_fails _ksre_validate_current_connection 'https://localhost:6443|' +assert_fails _ksre_validate_current_connection 'https://127.0.0.1:6443/|' +assert_fails _ksre_validate_current_connection 'https://127.0.0.1:6443|http://127.0.0.1:8080' + +probe_live_inventory_cluster_binding() ( + local api_node="$1" api_node_uid="$2" output + declare -F _ksre_current_connection >/dev/null || return 1 + declare -F _ksre_require_same_connection >/dev/null || return 1 + declare -F _ksre_run_restore_status_validator >/dev/null || return 1 + declare -F _ksre_current_user_api_ready >/dev/null || return 1 + declare -F _ksre_current_user_nodes_json >/dev/null || return 1 + declare -F _ksre_current_user_secret_count >/dev/null || return 1 + _ksre_run_restore_status_validator() { + local handoff="$1" + printf 'version=v1.36.2+k3s1\nserver_count=1\nnode_name=isolated-ready\nnode_uid=%s\nnode_ready=Ready\ndatastore=embedded-etcd\nencryption=Enabled\nrotation=reencrypt_finished\nserver_hashes=match\nintegrity=match\napi_readyz=pass\n' \ + "$FIXTURE_LOCAL_NODE_UID" \ + >"$handoff/inventory.env" + chmod 0600 "$handoff/inventory.env" + } + _ksre_current_connection() { + [[ "${1:-}" == fixture-context ]] || return 1 + printf 'https://127.0.0.1:6443|\n' + } + _ksre_current_user_api_ready() { + [[ "${1:-}" == fixture-context && "${2:-}" == https://127.0.0.1:6443 ]] + } + _ksre_current_user_nodes_json() { + [[ "${1:-}" == fixture-context && "${2:-}" == https://127.0.0.1:6443 ]] || return 1 + printf '{"items":[{"metadata":{"name":"%s","uid":"%s"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}\n' \ + "$api_node" "$api_node_uid" + } + _ksre_current_user_secret_count() { + [[ "${1:-}" == fixture-context && "${2:-}" == https://127.0.0.1:6443 ]] || return 1 + printf '7\n' + } + output="$(_ksre_live_restore_inventory fixture-context 'https://127.0.0.1:6443|')" || return 1 + [[ "$output" == 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|match|pass|pass|7' ]] +) + +assert_succeeds probe_live_inventory_cluster_binding isolated-ready "$FIXTURE_LOCAL_NODE_UID" +assert_fails probe_live_inventory_cluster_binding isolated-ready "$FIXTURE_OTHER_NODE_UID" +assert_fails probe_live_inventory_cluster_binding other-ready "$FIXTURE_LOCAL_NODE_UID" + +# Fix round 1 / finding 2: Bash read가 버리는 NUL도 raw byte 단계에서 거부한다. +nul_metadata="${fixture_root}/metadata-nul.env" +{ + printf 'schema=platform-k3s-bundle-v1\n' + printf 'bundle_id=%s\n' "$FIXTURE_BUNDLE_ID" + printf 'phase=po\0st\n' + printf 'k3s_version=v1.36.2+k3s1\n' + printf 'datastore=embedded-etcd\n' + printf 'created_at_utc=2026-07-31T23:00:00Z\n' + printf 'secret_count=7\n' +} >"$nul_metadata" +chmod 0600 "$nul_metadata" +assert_fails probe_parse bundle "$nul_metadata" + +# Fix round 1 / finding 3: pathname이 아니라 실제로 열린 FD identity를 검사한다. +probe_pinned_external_fd_survives_path_swap() ( + local input="${fixture_root}/toctou-valid.env" moved="${fixture_root}/toctou-valid-opened.env" + local input_fd + write_metadata "$input" + exec {input_fd}<"$input" + /bin/mv -- "$input" "$moved" + printf 'unsafe replacement\n' >"$input" + chmod 0644 "$input" + declare -A parsed=() + _ksre_parse_external_fd bundle "$input_fd" parsed || return 1 + exec {input_fd}<&- + [[ "${parsed[bundle_id]}" == "$FIXTURE_BUNDLE_ID" ]] +) + +probe_unsafe_opened_fd_cannot_borrow_safe_path() ( + local input="${fixture_root}/toctou-unsafe.env" moved="${fixture_root}/toctou-unsafe-opened.env" + local input_fd + write_metadata "$input" + chmod 0644 "$input" + exec {input_fd}<"$input" + /bin/mv -- "$input" "$moved" + write_metadata "$input" + declare -A parsed=() + if _ksre_parse_external_fd bundle "$input_fd" parsed; then return 1; fi + exec {input_fd}<&- +) + +assert_succeeds probe_pinned_external_fd_survives_path_swap +assert_succeeds probe_unsafe_opened_fd_cannot_borrow_safe_path + +# Fix round 2 / finding 2: production final-component open의 정확한 순간을 +# test-only interposer로 멈춘다. precheck 뒤 symlink로 바꾸고 open 뒤 같은 +# target inode의 hard link로 되돌려도 parser가 반드시 실패해야 한다. +race_interposer_source="${fixture_root}/nofollow-race.c" +race_interposer="${fixture_root}/nofollow-race.so" +cat >"$race_interposer_source" <<'RACE_INTERPOSER' +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +static int triggered; +static int (*real_open_fn)(const char *, int, ...); +static int (*real_open64_fn)(const char *, int, ...); +static int (*real_openat_fn)(int, const char *, int, ...); +static int (*real_openat64_fn)(int, const char *, int, ...); + +static void resolve_symbols(void) { + if (!real_open_fn) real_open_fn = dlsym(RTLD_NEXT, "open"); + if (!real_open64_fn) real_open64_fn = dlsym(RTLD_NEXT, "open64"); + if (!real_openat_fn) real_openat_fn = dlsym(RTLD_NEXT, "openat"); + if (!real_openat64_fn) real_openat64_fn = dlsym(RTLD_NEXT, "openat64"); +} + +static void marker_path(char *buffer, size_t size, const char *name) { + const char *directory = getenv("KSRE_RACE_SYNC"); + if (!directory || snprintf(buffer, size, "%s/%s", directory, name) >= (int)size) _exit(125); +} + +static void touch_marker(const char *name) { + char path[PATH_MAX]; + int fd; + marker_path(path, sizeof(path), name); + resolve_symbols(); + fd = real_open_fn(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd < 0) _exit(125); + dprintf(fd, "%ld\n", (long)getpid()); + close(fd); +} + +static void wait_for_marker(const char *name) { + char path[PATH_MAX]; + marker_path(path, sizeof(path), name); + while (access(path, F_OK) != 0) usleep(1000); +} + +static int should_race(const char *path) { + const char *target = getenv("KSRE_RACE_PATH"); + if (!triggered && target && path && strcmp(path, target) == 0) { + triggered = 1; + touch_marker("entered-open"); + wait_for_marker("continue-open"); + return 1; + } + return 0; +} + +static void after_open(void) { + touch_marker("opened"); + wait_for_marker("continue-postcheck"); +} + +int open(const char *path, int flags, ...) { + mode_t mode = 0; + int raced, result; + va_list arguments; + resolve_symbols(); + if ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) { + va_start(arguments, flags); mode = va_arg(arguments, mode_t); va_end(arguments); + } + raced = should_race(path); + result = ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) + ? real_open_fn(path, flags, mode) : real_open_fn(path, flags); + if (raced) after_open(); + return result; +} + +int open64(const char *path, int flags, ...) { + mode_t mode = 0; + int raced, result; + va_list arguments; + resolve_symbols(); + if ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) { + va_start(arguments, flags); mode = va_arg(arguments, mode_t); va_end(arguments); + } + raced = should_race(path); + if (!real_open64_fn) real_open64_fn = real_open_fn; + result = ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) + ? real_open64_fn(path, flags, mode) : real_open64_fn(path, flags); + if (raced) after_open(); + return result; +} + +int openat(int directory_fd, const char *path, int flags, ...) { + mode_t mode = 0; + int raced, result; + va_list arguments; + resolve_symbols(); + if ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) { + va_start(arguments, flags); mode = va_arg(arguments, mode_t); va_end(arguments); + } + raced = should_race(path); + result = ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) + ? real_openat_fn(directory_fd, path, flags, mode) : real_openat_fn(directory_fd, path, flags); + if (raced) after_open(); + return result; +} + +int openat64(int directory_fd, const char *path, int flags, ...) { + mode_t mode = 0; + int raced, result; + va_list arguments; + resolve_symbols(); + if ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) { + va_start(arguments, flags); mode = va_arg(arguments, mode_t); va_end(arguments); + } + raced = should_race(path); + if (!real_openat64_fn) real_openat64_fn = real_openat_fn; + result = ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) + ? real_openat64_fn(directory_fd, path, flags, mode) : real_openat64_fn(directory_fd, path, flags); + if (raced) after_open(); + return result; +} +RACE_INTERPOSER +/usr/bin/cc -shared -fPIC -O2 -o "$race_interposer" "$race_interposer_source" -ldl || + fail 'compile no-follow race interposer' + +wait_for_race_marker() { + local marker="$1" process_id="$2" attempt + for ((attempt = 0; attempt < 5000; attempt++)); do + [[ -e "$marker" ]] && return 0 + /bin/kill -0 "$process_id" 2>/dev/null || return 1 + /usr/bin/sleep 0.001 + done + return 1 +} + +probe_open_instant_symlink_replacement_rejected() ( + local target="${fixture_root}/race-target.env" input="${fixture_root}/race-input.env" + local original="${fixture_root}/race-input-original.env" sync="${fixture_root}/race-sync" + local process_id parse_rc=0 + write_metadata "$target" + write_metadata "$input" + mkdir "$sync" + /usr/bin/env KSRE_RACE_PATH="$input" KSRE_RACE_SYNC="$sync" LD_PRELOAD="$race_interposer" \ + /usr/bin/timeout --signal=TERM --kill-after=1s 10s \ + /usr/bin/bash -c ' + set -Eeuo pipefail + source "$1" + declare -A parsed=() + _ksre_parse_external_file bundle "$2" parsed + ' bash "$RESTORE_VALIDATOR_PATH" "$input" & + process_id=$! + if ! wait_for_race_marker "$sync/entered-open" "$process_id"; then + : >"$sync/continue-open"; : >"$sync/continue-postcheck" + wait "$process_id" 2>/dev/null || : + return 1 + fi + /bin/mv -- "$input" "$original" + /bin/ln -s -- "$target" "$input" + : >"$sync/continue-open" + if ! wait_for_race_marker "$sync/opened" "$process_id"; then + : >"$sync/continue-postcheck" + wait "$process_id" 2>/dev/null || : + return 1 + fi + /bin/rm -- "$input" + /bin/ln -- "$target" "$input" + : >"$sync/continue-postcheck" + if wait "$process_id"; then parse_rc=0; else parse_rc=$?; fi + [[ "$parse_rc" -ne 0 ]] +) + +assert_succeeds probe_open_instant_symlink_replacement_rejected + +# Fix round 3: production broker가 readiness 전에 멈춰도 parent 자체 상한 안에 +# nonzero로 끝나고 partial success output과 살아 있는 broker를 남기면 안 된다. +probe_broker_readiness_delay_is_bounded() ( + local input="${fixture_root}/broker-delay.env" sync="${fixture_root}/broker-delay-sync" + local partial_output="${fixture_root}/broker-delay.out" broker_process='' parser_rc=0 attempt + write_metadata "$input" + mkdir "$sync" + /usr/bin/env KSRE_RACE_PATH="$input" KSRE_RACE_SYNC="$sync" LD_PRELOAD="$race_interposer" \ + /usr/bin/timeout --signal=TERM --kill-after=0.5s 4s \ + /usr/bin/bash -c ' + set -Eeuo pipefail + source "$1" + declare -A parsed=() + if _ksre_parse_external_file bundle "$2" parsed; then + printf "unexpected-success\n" + exit 0 + fi + exit 23 + ' bash "$RESTORE_VALIDATOR_PATH" "$input" >"$partial_output" 2>/dev/null || parser_rc=$? + [[ -s "$sync/entered-open" ]] || return 1 + broker_process="$(<"$sync/entered-open")" + [[ "$broker_process" =~ ^[0-9]+$ ]] || return 1 + : >"$sync/continue-open" + : >"$sync/continue-postcheck" + for ((attempt = 0; attempt < 1000; attempt++)); do + /bin/kill -0 "$broker_process" 2>/dev/null || break + /usr/bin/sleep 0.001 + done + if [[ "$parser_rc" == 124 || "$parser_rc" == 137 ]]; then + printf 'broker readiness required the outer watchdog\n' >&2 + return 1 + fi + [[ "$parser_rc" -ne 0 && ! -s "$partial_output" ]] + ! /bin/kill -0 "$broker_process" 2>/dev/null +) + +assert_succeeds probe_broker_readiness_delay_is_bounded + +# Fix round 2 pre-review concern: writer가 없는 FIFO는 regular-file 검사 전에 +# block하지 않고 즉시 거부되어야 한다. timeout은 실패가 아니라 RED 증거다. +probe_fifo_rejected_without_blocking() ( + local fifo="${fixture_root}/external-input.fifo" parser_rc=0 + /usr/bin/mkfifo -- "$fifo" + /usr/bin/chmod 0600 "$fifo" + /usr/bin/timeout --signal=TERM --kill-after=0.25s 0.25s \ + /usr/bin/bash -c ' + set -Eeuo pipefail + source "$1" + declare -A parsed=() + _ksre_parse_external_file bundle "$2" parsed + ' bash "$RESTORE_VALIDATOR_PATH" "$fifo" >/dev/null 2>&1 || parser_rc=$? + if [[ "$parser_rc" == 124 || "$parser_rc" == 137 ]]; then + /usr/bin/timeout --signal=TERM --kill-after=0.25s 1s \ + /usr/bin/bash -c 'printf x >"$1"' bash "$fifo" >/dev/null 2>&1 || : + printf 'FIFO parser blocked before regular-file rejection\n' >&2 + return 1 + fi + [[ "$parser_rc" -ne 0 ]] +) + +assert_succeeds probe_fifo_rejected_without_blocking + +secret_stream="${fixture_root}/secret-stream.json" +secret_trace="${fixture_root}/secret-stream.trace" +printf '{"items":[{"data":{"value":"fixture-sensitive-payload"}}]}\n' >"$secret_stream" +probe_stream_secret_count() ( + exec 9>"$secret_trace" + BASH_XTRACEFD=9 + set -x + local count + count="$(_ksre_secret_count_from_stream <"$secret_stream")" + set +x + [[ "$count" == 1 ]] +) +assert_succeeds probe_stream_secret_count +[[ "$(<"$secret_trace")" != *fixture-sensitive-payload* ]] || fail 'Secret payload reached xtrace' + +# Parser/security matrix. 각 fixture는 parser의 실제 branch를 지나며 privileged +# helper가 한 번이라도 호출되면 별도 marker를 남기도록 mode fixture에서 확인한다. +for attack in command_substitution backtick duplicate unknown empty control malformed trailing; do + candidate="${fixture_root}/metadata-${attack}.env" + write_metadata "$candidate" + case "$attack" in + command_substitution) /usr/bin/sed -i '$c\secret_count=$(id)' "$candidate" ;; + backtick) /usr/bin/sed -i '$c\secret_count=`id`' "$candidate" ;; + duplicate) printf 'phase=post\n' >>"$candidate" ;; + unknown) printf 'credential=forbidden\n' >>"$candidate" ;; + empty) /usr/bin/sed -i 's/^phase=.*/phase=/' "$candidate" ;; + control) /usr/bin/sed -i 's/^phase=.*/phase=post\r/' "$candidate" ;; + malformed) printf 'not-an-assignment\n' >>"$candidate" ;; + trailing) printf 'trailing-data\n' >>"$candidate" ;; + esac + assert_fails probe_parse bundle "$candidate" +done + +empty_file="${fixture_root}/empty.env"; : >"$empty_file"; chmod 0600 "$empty_file" +assert_fails probe_parse bundle "$empty_file" +mode_file="${fixture_root}/mode.env"; write_metadata "$mode_file"; chmod 0644 "$mode_file" +assert_fails probe_parse bundle "$mode_file" +symlink_file="${fixture_root}/link.env"; ln -s -- "$metadata" "$symlink_file" +assert_fails probe_parse bundle "$symlink_file" + +probe_mode_rejects_before_privilege() ( + local path="$1" marker="${fixture_root}/unexpected-privilege" + _ksre_current_context() { printf 'default\n'; } + _ksre_read_authoritative_bundle() { : >"$marker"; return 1; } + _ksre_install_evidence() { : >"$marker"; return 1; } + _ksre_record "$path" "$result" <<<"DESTROYED default" >/dev/null 2>&1 && return 1 + [[ ! -e "$marker" ]] +) +for attack in command_substitution backtick duplicate unknown control malformed trailing; do + assert_succeeds probe_mode_rejects_before_privilege "${fixture_root}/metadata-${attack}.env" +done +assert_succeeds probe_mode_rejects_before_privilege "$mode_file" +assert_succeeds probe_mode_rejects_before_privilege "$symlink_file" + +mode_events="${fixture_root}/mode-events.log" +installed_evidence="${fixture_root}/installed-evidence.env" + +install_mode_fakes() { + : >"$mode_events" + rm -f -- "$installed_evidence" + active_evidence_file="$evidence_fixture" + _ksre_current_context() { printf 'context\n' >>"$mode_events"; printf 'default\n'; } + _ksre_current_connection() { printf 'connection\n' >>"$mode_events"; printf 'https://127.0.0.1:6443|\n'; } + _ksre_require_same_context() { printf 'context-recheck\n' >>"$mode_events"; } + _ksre_now_rfc3339() { printf '%s\n' "$FIXTURE_RECORDED_AT"; } + _ksre_now_epoch() { printf '1785546000\n'; } + _ksre_live_restore_inventory() { + printf 'live\n' >>"$mode_events" + printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|match|pass|pass|7\n' + } + _ksre_isolation_attested() { printf 'isolation\n' >>"$mode_events"; } + _ksre_read_authoritative_bundle() { + local destination="$1" + printf 'authority\n' >>"$mode_events" + _ksre_parse_trusted_content bundle "$(<"$metadata")" "$destination" + } + _ksre_read_installed_evidence() { + local destination="$1" + printf 'evidence\n' >>"$mode_events" + _ksre_parse_trusted_content evidence "$(<"$active_evidence_file")" "$destination" + } + _ksre_evidence_target_absent() { printf 'target-check\n' >>"$mode_events"; } + _ksre_prompt_destroyed() { printf 'destroyed\n' >>"$mode_events"; } + _ksre_install_evidence() { + local content="$1" + printf 'install\n' >>"$mode_events" + printf '%s\n' "$content" >"$installed_evidence" + chmod 0600 "$installed_evidence" + } +} + +probe_emit_success() ( + install_mode_fakes + local output="${fixture_root}/emitted.env" + rm -f -- "$output" + _ksre_emit_result "$metadata" "$output" >/dev/null + declare -A parsed=() + _ksre_parse_external_file result "$output" parsed + [[ "${parsed[bundle_id]}" == "$FIXTURE_BUNDLE_ID" && + "${parsed[secret_count_match]}" == pass && -z "${parsed[destroyed]+present}" && + "$(<"$mode_events")" == $'context\nconnection\nlive\nisolation\nconnection\ncontext-recheck' ]] +) +assert_succeeds probe_emit_success + +probe_emit_relative_output() ( + install_mode_fakes + cd -- "$fixture_root" + rm -f -- relative-result.env + _ksre_emit_result "$metadata" relative-result.env >/dev/null + [[ -f relative-result.env ]] +) +assert_succeeds probe_emit_relative_output + +probe_emit_existing_output() ( + install_mode_fakes + local output="${fixture_root}/existing-output.env" + printf 'keep\n' >"$output"; chmod 0600 "$output" + _ksre_emit_result "$metadata" "$output" >/dev/null 2>&1 && return 1 + [[ "$(<"$output")" == keep ]] +) +assert_succeeds probe_emit_existing_output + +probe_emit_live_failure() ( + local field="$1" + install_mode_fakes + local output="${fixture_root}/emit-failure-${field}.env" + _ksre_live_restore_inventory() { + case "$field" in + count) printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|match|pass|pass|6\n' ;; + node) printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|match|pass|fail|7\n' ;; + integrity) printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|mismatch|pass|pass|7\n' ;; + esac + } + if [[ "$field" == isolation ]]; then _ksre_isolation_attested() { return 1; }; fi + _ksre_emit_result "$metadata" "$output" >/dev/null 2>&1 && return 1 + [[ ! -e "$output" ]] +) +for field in count node isolation integrity; do assert_succeeds probe_emit_live_failure "$field"; done + +probe_record_success() ( + install_mode_fakes + _ksre_record "$metadata" "$result" <<<"DESTROYED default" >/dev/null + declare -A parsed=() + _ksre_parse_external_file evidence "$installed_evidence" parsed + [[ "${parsed[destroyed]}" == confirmed && "$(head -n 1 "$mode_events")" == context ]] +) +assert_succeeds probe_record_success + +probe_record_failure() ( + local scenario="$1" local_metadata="${fixture_root}/record-${scenario}-metadata.env" + local local_result="${fixture_root}/record-${scenario}-result.env" + install_mode_fakes + write_metadata "$local_metadata"; write_result "$local_result" + case "$scenario" in + metadata_id) write_metadata "$local_metadata" post "$FIXTURE_OTHER_ID" ;; + result_id) write_result "$local_result" "$FIXTURE_OTHER_ID" ;; + result_version) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.1+k3s1 ;; + result_backend) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 sqlite ;; + pre) write_metadata "$local_metadata" pre ;; + stale) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd pass pass match pass 2026-07-30T00:00:00Z ;; + secret_count) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd pass pass match fail ;; + node) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd fail ;; + isolation) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd pass fail ;; + integrity) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd pass pass mismatch ;; + authority_id) + _ksre_read_authoritative_bundle() { + local destination="$1" alternate="${fixture_root}/alternate-authority.env" + write_metadata "$alternate" post "$FIXTURE_OTHER_ID" + _ksre_parse_trusted_content bundle "$(<"$alternate")" "$destination" + } + ;; + destroyed) _ksre_prompt_destroyed() { return 1; } ;; + target_exists) _ksre_evidence_target_absent() { return 1; } ;; + esac + _ksre_record "$local_metadata" "$local_result" <<<"DESTROYED default" >/dev/null 2>&1 && return 1 + [[ ! -e "$installed_evidence" ]] +) +for scenario in metadata_id result_id result_version result_backend pre stale secret_count node isolation integrity authority_id destroyed target_exists; do + assert_succeeds probe_record_failure "$scenario" +done + +probe_check_success() ( + install_mode_fakes + _ksre_check >/dev/null + [[ "$(head -n 1 "$mode_events")" == context ]] +) +assert_succeeds probe_check_success + +probe_check_failure() ( + local scenario="$1" altered="${fixture_root}/check-${scenario}.env" + install_mode_fakes + case "$scenario" in + authority_id) + _ksre_read_authoritative_bundle() { + local destination="$1" alternate="${fixture_root}/check-authority.env" + write_metadata "$alternate" post "$FIXTURE_OTHER_ID" + _ksre_parse_trusted_content bundle "$(<"$alternate")" "$destination" + } + ;; + version) write_evidence "$altered" "$FIXTURE_BUNDLE_ID" v1.36.1+k3s1; active_evidence_file="$altered" ;; + backend) write_evidence "$altered" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 sqlite; active_evidence_file="$altered" ;; + integrity) write_evidence "$altered" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd mismatch; active_evidence_file="$altered" ;; + destroyed) write_evidence "$altered" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd match pending; active_evidence_file="$altered" ;; + stale) write_evidence "$altered" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd match confirmed 2026-06-01T00:00:00Z 2026-06-01T01:00:00Z; active_evidence_file="$altered" ;; + live_stage) _ksre_live_restore_inventory() { printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|start|match|match|pass|pass|7\n'; } ;; + live_integrity) _ksre_live_restore_inventory() { printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|mismatch|pass|pass|7\n'; } ;; + esac + _ksre_check >/dev/null 2>&1 && return 1 + return 0 +) +for scenario in authority_id version backend integrity destroyed stale live_stage live_integrity; do + assert_succeeds probe_check_failure "$scenario" +done + +printf 'K3S SECRET ENCRYPTION RESTORE EVIDENCE TEST PASS\n' diff --git a/scripts/validate/test-k3s-secret-encryption-status.sh b/scripts/validate/test-k3s-secret-encryption-status.sh new file mode 100644 index 0000000..40db1d7 --- /dev/null +++ b/scripts/validate/test-k3s-secret-encryption-status.sh @@ -0,0 +1,2360 @@ +#!/usr/bin/env bash + +# Regression coverage for parser branches that must fail closed. No command in +# this file contacts a k3s service, systemd, or the Kubernetes API. +set -Eeuo pipefail + +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly LIBRARY_PATH="${REPOSITORY_ROOT}/scripts/lib/k3s-secret-encryption.sh" +readonly VALIDATOR_PATH="${REPOSITORY_ROOT}/scripts/validate/k3s-secret-encryption.sh" +readonly ORIGINAL_PATH="$PATH" + +fail() { + printf 'TEST FAILURE: %s\n' "$*" >&2 + exit 1 +} + +assert_eq() { + local expected="$1" + local actual="$2" + local description="$3" + [[ "$actual" == "$expected" ]] || fail "$description" +} + +assert_succeeds() { + "$@" || fail "expected success: $*" +} + +assert_fails() { + if "$@"; then + fail "expected failure: $*" + fi +} + +assert_exit() { + local expected="$1" + shift + local actual=0 + "$@" || actual=$? + [[ "$actual" == "$expected" ]] || fail "expected exit ${expected}, got ${actual}: $*" +} + +fixture_root="$(mktemp -d "${TMPDIR:-/tmp}/k3s-secret-encryption-test.XXXXXX")" +verified_output_dir='' +rejected_output_dir='' +race_output_dir='' +finished_output_dir='' +transition_output_dir='' +enabled_output_dir='' +symlink_output_parent='' +cleanup() { + if [[ -n "$verified_output_dir" ]]; then + rm -rf -- "$verified_output_dir" + fi + if [[ -n "$rejected_output_dir" ]]; then + rm -rf -- "$rejected_output_dir" + fi + if [[ -n "$race_output_dir" ]]; then + rm -rf -- "$race_output_dir" + fi + if [[ -n "$finished_output_dir" ]]; then + rm -rf -- "$finished_output_dir" + fi + if [[ -n "$transition_output_dir" ]]; then + rm -rf -- "$transition_output_dir" + fi + if [[ -n "$enabled_output_dir" ]]; then + rm -rf -- "$enabled_output_dir" + fi + if [[ -n "$symlink_output_parent" && -L "$symlink_output_parent" ]]; then + unlink -- "$symlink_output_parent" + fi + case "$fixture_root" in + /tmp/k3s-secret-encryption-test.*|"${TMPDIR:-/tmp}"/k3s-secret-encryption-test.*) + rm -rf -- "$fixture_root" + ;; + *) + fail 'refusing to remove an unexpected fixture directory' + ;; + esac +} +trap cleanup EXIT + +# The production change that this test catches is a missing status parser. +# This source must fail RED until the source-only library is created. +# shellcheck source=/dev/null +source "$LIBRARY_PATH" + +# Regression: GNU stat describes a zero-byte regular file as "regular empty +# file". The root text reader must accept the file by inode kind, ownership, +# and mode instead of depending on that presentation string. +assert_succeeds /usr/bin/bash -c ' + set -Eeuo pipefail + source "$1" + _kse_privileged_capture() { + local binary="$1" + shift + if [[ "$binary" == "$KSE_STAT" ]]; then + case "${1:-}" in + --format=%u:%a:%F) printf "0:600:regular empty file\n" ;; + --format=%u:%a) printf "0:600\n" ;; + *) return 1 ;; + esac + return 0 + fi + if [[ "$binary" == "$KSE_CAT" && "${*: -1}" == /etc/systemd/system/k3s.service.env ]]; then + return 0 + fi + return 1 + } + _kse_privileged_quiet() { + [[ "$1" == "$KSE_TEST" ]] || return 1 + shift + case "$*" in + "-f /etc/systemd/system/k3s.service.env"|"! -L /etc/systemd/system/k3s.service.env") return 0 ;; + *) return 1 ;; + esac + } + value="$(_kse_read_root_text /etc/systemd/system/k3s.service.env)" + [[ -z "$value" ]] +' bash "$VALIDATOR_PATH" + +disabled_json='{"stage":"","activekey":""}' +start_json='{"stage":"start","activekey":"","enable":false,"hashmatch":true}' +enabled_start_json='{"stage":"start","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}' +enabled_finished_json='{"stage":"reencrypt_finished","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}' +active_json='{"stage":"reencrypt_active","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}' +mismatch_json='{"stage":"reencrypt_finished","activekey":"AES-CBC fixture-key","enable":true,"hasherror":"hash mismatch"}' +secretbox_json='{"stage":"start","activekey":"XSalsa20-POLY1305 fixture-key","enable":true,"hashmatch":true}' + +# Wrong branches here would weaken the state-machine fail-stop gate. +assert_eq disabled_no_config "$(classify_encryption_status "$disabled_json")" 'disabled fixture classification' +assert_eq transition_start "$(classify_encryption_status "$start_json")" 'late-enable transition classification' +assert_eq enabled_stable "$(classify_encryption_status "$enabled_start_json")" 'enabled start classification' +assert_eq enabled_stable "$(classify_encryption_status "$enabled_finished_json")" 'enabled finished classification' +assert_eq unsafe_transition "$(classify_encryption_status "$active_json")" 'active re-encryption classification' +assert_eq hash_mismatch "$(classify_encryption_status "$mismatch_json")" 'hash mismatch classification' +assert_eq invalid "$(classify_encryption_status '{"stage":"start","activekey":"","enable":false,"hashmatch":true,"inactivekeys":["unexpected"]}')" 'non-empty transition inactive keys rejection' +assert_eq invalid "$(classify_encryption_status '{"stage":"","activekey":"","hasherror":"unexpected"}')" 'disabled status hash error rejection' +assert_eq invalid "$(classify_encryption_status "${disabled_json}"$'\n'"${disabled_json}")" 'concatenated JSON documents rejection' +assert_eq invalid "$(classify_encryption_status "${disabled_json} trailing")" 'JSON trailing garbage rejection' +assert_eq aescbc "$(classify_encryption_provider "$enabled_start_json")" 'AES-CBC provider classification' +assert_eq secretbox "$(classify_encryption_provider "$secretbox_json")" 'secretbox provider classification' +assert_eq invalid "$(classify_encryption_provider '{"stage":"start","activekey":"AES-CBC","enable":true,"hashmatch":true}')" 'provider exact-prefix rejection' + +assert_succeeds version_supports_late_enable v1.33.10+k3s1 +assert_succeeds version_supports_late_enable v1.34.6+k3s1 +assert_succeeds version_supports_late_enable v1.35.3+k3s1 +assert_succeeds version_supports_late_enable v1.36.2+k3s1 +assert_fails version_supports_late_enable v1.33.9+k3s1 +assert_fails version_supports_late_enable v1.34.5+k3s1 +assert_fails version_supports_late_enable v1.35.2+k3s1 + +assert_succeeds require_exact_encryption_state enabled_stable "$enabled_finished_json" +assert_fails require_exact_encryption_state enabled_stable "$active_json" + +fixture_config="${fixture_root}/encryption-config.json" +fixture_state="${fixture_root}/encryption-state.json" +fixture_stale_state="${fixture_root}/encryption-state-stale.json" +printf '{"fixture":"config"}\n' >"$fixture_config" +fixture_hash="$(sha256sum -- "$fixture_config" | awk '{print $1}')" +fixture_annotation="reencrypt_finished-${fixture_hash}" +printf '%s' "$fixture_annotation" >"$fixture_state" +printf '%s' "start-${fixture_hash}" >"$fixture_stale_state" +chmod 0600 "$fixture_config" "$fixture_state" "$fixture_stale_state" + +assert_eq match "$(verify_local_encryption_config_integrity_evidence \ + '0:600:regular file' '0:600:regular file' "$fixture_hash" \ + "$fixture_annotation" "$fixture_annotation" reencrypt_finished)" \ + 'privileged evidence integrity match' + +# The shim changes only the metadata an unprivileged fixture cannot create; +# content, hashes, symlink checks, and all parser behavior stay real. +mkdir -p "${fixture_root}/stat-shim" +printf '%s\n' '#!/usr/bin/env bash' \ + 'if [[ "$1" == "--format=%u:%a:%F" ]]; then printf "0:600:regular file\\n"; exit 0; fi' \ + 'exec /usr/bin/stat "$@"' >"${fixture_root}/stat-shim/stat" +chmod 0700 "${fixture_root}/stat-shim/stat" +PATH="${fixture_root}/stat-shim:${ORIGINAL_PATH}" +assert_eq match "$(verify_local_encryption_config_integrity "$fixture_config" "$fixture_state" "$fixture_annotation" reencrypt_finished)" 'matching root metadata fixture' +assert_eq mismatch "$(verify_local_encryption_config_integrity "$fixture_config" "$fixture_stale_state" "$fixture_annotation" reencrypt_finished || true)" 'stale state rejection' +PATH="$ORIGINAL_PATH" +assert_eq mismatch "$(verify_local_encryption_config_integrity "$fixture_config" "$fixture_state" "$fixture_annotation" reencrypt_finished || true)" 'user-owned config rejection' +ln -s -- "$fixture_config" "${fixture_root}/config-link" +assert_eq mismatch "$(verify_local_encryption_config_integrity "${fixture_root}/config-link" "$fixture_state" "$fixture_annotation" reencrypt_finished || true)" 'symlink config rejection' + +fake_bin="${fixture_root}/bin" +mkdir -p "$fake_bin" +printf '%s\n' '#!/usr/bin/env bash' \ + 'printf "%s\n" "${SYSTEMCTL_SHOW_FIXTURE:-}"; [[ "${SYSTEMCTL_SHOW_FIXTURE:-}" == *$'"'"'\nEnvironment='"'"'* ]] || printf "Environment=\n"; [[ "${SYSTEMCTL_SHOW_FIXTURE:-}" == *$'"'"'\nEnvironmentFiles='"'"'* ]] || printf "EnvironmentFiles=\n"' >"${fake_bin}/systemctl" +printf '%s\n' '#!/usr/bin/env bash' \ + 'if [[ "${K3S_HANG:-0}" == 1 ]]; then while :; do /usr/bin/sleep 1; done; fi; if [[ "${1:-}" == --version ]]; then printf "k3s version v1.36.2+k3s1 (fixture)\n"; exit 0; fi; if [[ "${1:-}" == kubectl ]]; then case "${2:-}" in get) if [[ "${3:-}" == nodes ]]; then printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"fixture-server\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"},\"annotations\":{\"k3s.io/encryption-config-hash\":\"token-password-encryption-config\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}"; exit 0; fi; if [[ "${3:-}" == --raw=/readyz ]]; then printf "token-password-encryption-config\n"; [[ "${K3S_API_RC:-0}" == 0 ]] && exit 0 || exit 1; fi ;; esac; exit 1; fi; case "${K3S_STATUS_FIXTURE:-}" in disabled) printf "%s\n" "{\"stage\":\"\",\"activekey\":\"\"}"; exit 0 ;; finished) stage=reencrypt_finished ;; start) stage=start ;; unsupported) stage=unexpected_stage ;; mismatch) printf "%s\n" "{\"stage\":\"reencrypt_finished\",\"activekey\":\"AES-CBC fixture-key\",\"enable\":true,\"hasherror\":\"mismatch\"}"; exit 0 ;; malformed) printf "%s\n" "{"; exit 0 ;; active_then_finished) if [[ -e "${K3S_STATUS_COUNTER:?}" ]]; then stage=reencrypt_finished; else : >"$K3S_STATUS_COUNTER"; stage=reencrypt_active; fi ;; *) exit 1 ;; esac; printf "%s\n" "{\"stage\":\"${stage}\",\"activekey\":\"AES-CBC fixture-key\",\"enable\":true,\"hashmatch\":true}"' >"${fake_bin}/k3s" +printf '%s\n' '#!/usr/bin/env bash' \ + '{ printf "sudo"; printf " <%s>" "$@"; printf "\n"; } >>"${FAKE_ARG_LOG:?}"' \ + 'if [[ "${SUDO_HANG:-0}" == 1 ]]; then while :; do /usr/bin/sleep 1; done; fi' \ + '[[ "${1:-}" == --non-interactive ]] && shift' \ + 'if [[ "${1:-}" == /usr/bin/env && " $* " == *" --privileged-probe "* && -n "${K3S_VALIDATOR_FAKE_PATH:-}" ]]; then shift; [[ "${1:-}" == -i ]] && shift; [[ "${1:-}" == PATH=* ]] && shift; [[ "${1:-}" == LC_ALL=* ]] && shift; exec /usr/bin/env PATH="$K3S_VALIDATOR_FAKE_PATH" "$@"; fi' \ + 'exec "$@"' >"${fake_bin}/sudo" +printf '%s\n' '#!/usr/bin/env bash' \ + '{ printf "timeout"; printf " <%s>" "$@"; printf "\n"; } >>"${FAKE_ARG_LOG:?}"' \ + 'shift 3' \ + 'if [[ "${SUDO_HANG:-0}" == 1 ]]; then exec /usr/bin/timeout --signal=TERM --kill-after=0.1s 0.05s "$@"; fi' \ + 'if [[ "${K3S_HANG:-0}" == 1 ]]; then if [[ -e "${K3S_TIMEOUT_COUNTER:?}" ]]; then exit 124; fi; : >"$K3S_TIMEOUT_COUNTER"; exec /usr/bin/timeout --signal=TERM --kill-after=0.1s 0.05s "$@"; fi' \ + 'exec "$@"' >"${fake_bin}/timeout" +printf '%s\n' '#!/usr/bin/env bash' 'exit 0' >"${fake_bin}/sleep" +printf '%s\n' '#!/usr/bin/env bash' 'exit 1' >"${fake_bin}/kubectl" +printf '%s\n' '#!/usr/bin/env bash' \ + 'if [[ "${HANDOFF_RACE:-0}" == 1 && " $* " == *" inventory.env status.sha256 "* ]]; then : >unexpected-race; fi' \ + 'exec /usr/bin/stat "$@"' >"${fake_bin}/stat" +chmod 0700 "${fake_bin}/systemctl" "${fake_bin}/k3s" "${fake_bin}/sudo" "${fake_bin}/timeout" "${fake_bin}/sleep" "${fake_bin}/kubectl" "${fake_bin}/stat" +PATH="${fake_bin}:${ORIGINAL_PATH}" +FAKE_ARG_LOG="${fixture_root}/fake-argv.log" +export SYSTEMCTL_SHOW_FIXTURE K3S_STATUS_FIXTURE FAKE_ARG_LOG K3S_API_RC + +config_root="${fixture_root}/config-root" +mkdir -p "${config_root}/config.yaml.d" +printf 'secrets-encryption: false\n' >"${config_root}/config.yaml" +printf 'secrets-encryption: true\n' >"${config_root}/config.yaml.d/40-encryption.yaml" +export K3S_CONFIG_DIR="$config_root" +SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server' +assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'default YAML drop-in merge and implicit provider' + +cli_config="${fixture_root}/cli.yaml" +env_config="${fixture_root}/env.yaml" +printf 'secrets-encryption: true\nsecrets-encryption-provider: aescbc\n' >"$cli_config" +printf 'secrets-encryption: false\n' >"$env_config" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${cli_config} +Environment=K3S_CONFIG_FILE=${env_config}" +assert_eq "aescbc/${cli_config}" "$(detect_effective_encryption_config_owner)" 'CLI config precedence' +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +Environment=K3S_CONFIG_FILE=${cli_config}" +assert_eq "aescbc/${cli_config}" "$(detect_effective_encryption_config_owner)" 'K3S_CONFIG_FILE config selection' +printf 'secrets-encryption: true\n' >"$env_config" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${env_config} +Environment=K3S_SECRETS_ENCRYPTION_PROVIDER=secretbox" +assert_eq secretbox/environment "$(detect_effective_encryption_config_owner)" 'environment provider overrides YAML provider absence' +printf 'secrets-encryption-provider: secretbox\n' >"$env_config" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${env_config} +Environment=K3S_SECRETS_ENCRYPTION=true" +assert_eq "secretbox/${env_config}" "$(detect_effective_encryption_config_owner)" 'YAML provider remains effective when environment only enables' +SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --secrets-encryption=true --secrets-encryption-provider=aescbc' +assert_eq aescbc/command-line "$(detect_effective_encryption_config_owner)" 'ExecStart encryption key precedence' +SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --secrets-encryption=true ; ignore_errors=no ; }' +assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'systemctl show ExecStart serialization' + +# These fixtures catch accepting anything except one systemd ExecStart record +# whose known metadata appears once, in the production order, with valid types. +SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +Environment=K3S_SECRETS_ENCRYPTION=true' +assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'production systemctl show ExecStart serialization' +SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } { path=/usr/bin/false ; argv[]=/usr/bin/false ; ignore_errors=no ; } +Environment=K3S_SECRETS_ENCRYPTION=true' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart extra record rejection' +SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; argv[]=/usr/local/bin/k3s server ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +Environment=K3S_SECRETS_ENCRYPTION=true' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart duplicate argv rejection' +SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; unknown=0 ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +Environment=K3S_SECRETS_ENCRYPTION=true' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart unknown metadata rejection' +SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; pid=0 ; code=(null) ; status=0/0 } +Environment=K3S_SECRETS_ENCRYPTION=true' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart duplicate metadata rejection' +SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; stop_time=[n/a] ; start_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +Environment=K3S_SECRETS_ENCRYPTION=true' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart metadata order rejection' +SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=not-a-number ; code=(null) ; status=0/0 } +Environment=K3S_SECRETS_ENCRYPTION=true' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart malformed metadata rejection' +SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 +Environment=K3S_SECRETS_ENCRYPTION=true' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart malformed brace rejection' + +env_file="${fixture_root}/k3s.env" +printf 'K3S_SECRETS_ENCRYPTION=true\n' >"$env_file" +K3S_CONFIG_DIR="${fixture_root}/empty-config-root" +mkdir -p "$K3S_CONFIG_DIR" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${env_file}" +assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'EnvironmentFile enable source' +printf '# comment\nK3S_SECRETS_ENCRYPTION=true\n' >"${fixture_root}/comment.env" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${fixture_root}/comment.env" +assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'EnvironmentFile comments ignored' +printf 'K3S_SECRETS_ENCRYPTION="true"\n' >"${fixture_root}/quoted.env" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${fixture_root}/quoted.env" +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'quoted EnvironmentFile assignment rejection' +printf 'K3S_SECRETS_ENCRYPTION\n' >"${fixture_root}/reset.env" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${fixture_root}/reset.env" +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'EnvironmentFile reset rejection' +printf "OTHER='multiline\nK3S_SECRETS_ENCRYPTION=true\nclosed'\n" >"${fixture_root}/multiline-other.env" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${fixture_root}/multiline-other.env (ignore_errors=no)" +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'non-target multiline record does not create assignment' +printf '%s\n' 'OTHER="first \"' 'K3S_SECRETS_ENCRYPTION=true' 'continued"' >"${fixture_root}/escaped-quote-other.env" +assert_eq '' "$(_k3s_envfile_values "${fixture_root}/escaped-quote-other.env" K3S_SECRETS_ENCRYPTION || true)" 'escaped quote keeps target-looking line inside non-target logical record' +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${fixture_root}/escaped-quote-other.env (ignore_errors=no)" +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'escaped quoted EnvironmentFile record fails closed without target assignment' +printf '%s\n' 'OTHER=prefix\' 'K3S_SECRETS_ENCRYPTION=true' >"${fixture_root}/continued-other.env" +assert_eq '' "$(_k3s_envfile_values "${fixture_root}/continued-other.env" K3S_SECRETS_ENCRYPTION || true)" 'backslash continuation keeps target-looking line inside non-target logical record' +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${fixture_root}/continued-other.env (ignore_errors=no)" +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'continued EnvironmentFile record fails closed without target assignment' +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${env_file} (ignore_errors=no) ${fixture_root}/comment.env (ignore_errors=no)" +assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'later EnvironmentFile assignment wins' +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +Environment=K3S_SECRETS_ENCRYPTION=false +EnvironmentFiles=${env_file} (ignore_errors=no)" +assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'EnvironmentFile overrides Environment assignment' +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${fixture_root}/absent.env (ignore_errors=yes) ${env_file} (ignore_errors=no)" +assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'optional missing EnvironmentFile skipped' +provider_env_file="${fixture_root}/k3s-provider.env" +printf 'K3S_SECRETS_ENCRYPTION=true\nK3S_SECRETS_ENCRYPTION_PROVIDER=secretbox\n' >"$provider_env_file" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${provider_env_file}" +assert_eq secretbox/environment "$(detect_effective_encryption_config_owner)" 'EnvironmentFile provider source' +SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server +Environment=K3S_SECRETS_ENCRYPTION=true' +assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'direct environment enable source' +dropin_dir="${fixture_root}/k3s.service.d" +mkdir -p "$dropin_dir" +printf '[Service]\nEnvironment=K3S_SECRETS_ENCRYPTION=true\n' >"${dropin_dir}/40-encryption.conf" +K3S_SYSTEMD_DROPIN_DIR="$dropin_dir" +SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server' +assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'systemd drop-in environment source' +K3S_SYSTEMD_DROPIN_DIR="${fixture_root}/no-dropins" +SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server +Environment=K3S_SECRETS_ENCRYPTION=true K3S_SECRETS_ENCRYPTION=false' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'same precedence environment conflict' +SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server +Environment=K3S_SECRETS_ENCRYPTION=true K3S_SECRETS_ENCRYPTION_PROVIDER=aescbc K3S_SECRETS_ENCRYPTION_PROVIDER=secretbox' +assert_eq secretbox/environment "$(detect_effective_encryption_config_owner)" 'later Environment provider assignment wins' +SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --config=$CONFIG +Environment=K3S_SECRETS_ENCRYPTION=true' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'dynamic config expansion rejection' +SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --config=' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'empty CLI config rejection' +SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --secrets-encryption-provider=' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'empty CLI provider rejection' +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${fixture_root}/missing.env" +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'unreadable EnvironmentFile rejection' +SYSTEMCTL_SHOW_FIXTURE='ExecStart=' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'blank ExecStart rejection' +SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/bin/false' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'unrelated ExecStart rejection' +SYSTEMCTL_SHOW_FIXTURE=$'ExecStart=/usr/local/bin/k3s server\nExecStart=/usr/local/bin/k3s server' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'multiple ExecStart rejection' +SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --secrets-encryption=true ; ignore_errors=no ; } { path=/usr/bin/false ; argv[]=/usr/bin/false ; ignore_errors=no ; } +Environment= +EnvironmentFiles=' +assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'multiple serialized ExecStart command records rejection' + +data_dir="${fixture_root}/data" +mkdir -p "${data_dir}/server/db" +touch "${data_dir}/server/db/state.db" +( + _k3s_datastore_local_evidence() { printf 'sqlite\n'; } + SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server' + assert_eq sqlite "$(detect_k3s_datastore)" 'default data-dir sqlite evidence' +) +( + _k3s_datastore_local_evidence() { printf 'embedded-etcd\n'; } + SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server' + assert_eq embedded-etcd "$(detect_k3s_datastore)" 'default data-dir embedded etcd evidence' +) +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir}" +assert_eq ambiguous "$(detect_k3s_datastore)" 'custom CLI data-dir rejection' +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir} --datastore-endpoint=" +assert_eq ambiguous "$(detect_k3s_datastore)" 'empty CLI datastore endpoint rejection' +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir} --datastore-endpoint=fixture" +assert_eq ambiguous "$(detect_k3s_datastore)" 'ExecStart external endpoint conflict rejection' +alt_data_dir="${fixture_root}/alt-data" +mkdir -p "${alt_data_dir}/server/db" +touch "${alt_data_dir}/server/db/state.db" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +Environment=K3S_DATA_DIR=${alt_data_dir}" +assert_eq ambiguous "$(detect_k3s_datastore)" 'custom environment data-dir rejection' +printf 'data-dir: %s\ndatastore-endpoint: fixture\n' "$alt_data_dir" >"$env_config" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${env_config}" +assert_eq ambiguous "$(detect_k3s_datastore)" 'merged YAML data-dir and endpoint conflict rejection' +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server +EnvironmentFiles=${fixture_root}/config-path.env" +printf 'K3S_CONFIG_FILE=%s\n' "$env_config" >"${fixture_root}/config-path.env" +assert_eq ambiguous "$(detect_k3s_datastore)" 'EnvironmentFile config selection' +datastore_env_file="${fixture_root}/datastore.env" +printf 'K3S_DATASTORE_ENDPOINT=fixture\n' >"$datastore_env_file" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir} +EnvironmentFiles=${datastore_env_file}" +assert_eq ambiguous "$(detect_k3s_datastore)" 'EnvironmentFile external datastore conflict rejection' +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir}" +rm -f -- "${data_dir}/server/db/state.db" +mkdir -p "${data_dir}/server/db/etcd" +assert_eq ambiguous "$(detect_k3s_datastore)" 'custom embedded-etcd path rejection' +rm -rf -- "${data_dir}/server/db/etcd" +SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir} +Environment=K3S_DATASTORE_ENDPOINT=fixture" +assert_eq ambiguous "$(detect_k3s_datastore)" 'custom path takes precedence over external classification' +touch "${data_dir}/server/db/state.db" +mkdir -p "${data_dir}/server/db/etcd" +assert_eq ambiguous "$(detect_k3s_datastore)" 'conflicting datastore evidence rejection' + +printf 'secrets-encryption: true\nsecrets-encryption-provider: aescbc\n' >"${config_root}/config.yaml" +printf 'secrets-encryption-provider: secretbox\n' >"${config_root}/config.yaml.d/40-provider.yaml" +K3S_CONFIG_DIR="$config_root" +SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server' +assert_eq "secretbox/${config_root}/config.yaml.d/40-provider.yaml" "$(detect_effective_encryption_config_owner)" 'provider fragment owner' + +virtual_config="${fixture_root}/not-user-listable/config.yaml" +( + _k3s_list_yaml_files() { printf '%s\n' "${config_root}/config.yaml.d/40-provider.yaml"; } + assert_eq "${config_root}/config.yaml.d/40-provider.yaml|secretbox" \ + "$(_k3s_yaml_value "$virtual_config" secrets-encryption-provider)" \ + 'privileged YAML listing callback' +) + +# A correct helper returns non-zero before this test-only outer guard fires. +# Exit 124 means the helper did not bound its own privileged command. +assert_bounded_hang_failure() { + local helper="$1" argument="$2" counter rc=0 + counter="${fixture_root}/${helper}-timeout-counter" + rm -f -- "$counter" + /usr/bin/timeout --signal=TERM --kill-after=0.1s 1s env \ + PATH="$PATH" K3S_HANG=1 K3S_TIMEOUT_COUNTER="$counter" \ + FAKE_ARG_LOG="$FAKE_ARG_LOG" \ + bash -c 'set -Eeuo pipefail; source "$1"; "$2" "$3"' \ + bash "$LIBRARY_PATH" "$helper" "$argument" || rc=$? + [[ "$rc" -ne 0 && "$rc" -ne 124 ]] || fail "$helper did not bound a hanging privileged command" +} + +assert_bounded_hang_failure wait_for_reencrypt_finished 0 +assert_bounded_hang_failure wait_for_k3s_api ignored + +# A 599-second first call leaves too little budget for another bounded call. +# The fake clock makes the ten-minute deadline boundary deterministic and fast. +assert_deadline_stops_second_call() { + local helper="$1" argument="$2" + local clock_file="${fixture_root}/${helper}-clock" calls_file="${fixture_root}/${helper}-deadline-calls" call_count + printf '0\n' >"$clock_file" + : >"$calls_file" + ( + _k3s_now_seconds() { printf '%s\n' "$(<"$clock_file")"; } + timeout() { + local now + printf 'call\n' >>"$calls_file" + now="$(<"$clock_file")" + printf '%s\n' "$((now + 599))" >"$clock_file" + if [[ " $* " == *' secrets-encrypt '* ]]; then + printf '%s\n' '{"stage":"reencrypt_active","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}' + return 0 + fi + return 1 + } + sleep() { + local now + now="$(<"$clock_file")" + printf '%s\n' "$((now + $1))" >"$clock_file" + } + assert_fails "$helper" "$argument" + ) + call_count="$(wc -l <"$calls_file" | tr -d '[:space:]')" + assert_eq 1 "$call_count" "$helper respects the overall ten-minute deadline (calls=$call_count)" +} + +assert_deadline_stops_second_call wait_for_reencrypt_finished 0 +assert_deadline_stops_second_call wait_for_k3s_api ignored + +K3S_STATUS_FIXTURE=finished +: >"$FAKE_ARG_LOG" +assert_succeeds wait_for_reencrypt_finished 0 +assert_eq $'timeout <--signal=TERM> <--kill-after=1s> <9s> <--non-interactive> <--output> \nsudo <--non-interactive> <--output> ' "$(<"$FAKE_ARG_LOG")" 'bounded authoritative status reader argv' +K3S_STATUS_FIXTURE=active_then_finished +K3S_STATUS_COUNTER="${fixture_root}/status-counter" +export K3S_STATUS_COUNTER +assert_succeeds wait_for_reencrypt_finished 1 +K3S_STATUS_FIXTURE=start +assert_fails wait_for_reencrypt_finished 0 +K3S_STATUS_FIXTURE=mismatch +assert_fails wait_for_reencrypt_finished 0 +K3S_STATUS_FIXTURE=malformed +assert_fails wait_for_reencrypt_finished 0 +K3S_STATUS_FIXTURE=unsupported +assert_fails wait_for_reencrypt_finished 0 +K3S_API_RC=0 +: >"$FAKE_ARG_LOG" +assert_succeeds wait_for_k3s_api +assert_eq $'timeout <--signal=TERM> <--kill-after=1s> <9s> <--non-interactive> <--raw=/readyz>\nsudo <--non-interactive> <--raw=/readyz>' "$(<"$FAKE_ARG_LOG")" 'bounded credentialed readyz argv' +K3S_API_RC=1 +assert_fails wait_for_k3s_api + +# Validator entrypoint fixtures source the production script in a fresh shell, +# override only its privileged command boundary, and call the real main. There +# is no production environment switch that enables this harness. +run_validator_fixture() { + /usr/bin/env \ + KSE_FIX_STATUS="${KSE_FIX_STATUS:-disabled}" \ + KSE_FIX_INTEGRITY="${KSE_FIX_INTEGRITY:-match}" \ + KSE_FIX_VERSION="${KSE_FIX_VERSION:-v1.36.2+k3s1}" \ + KSE_FIX_NODES="${KSE_FIX_NODES:-valid}" \ + KSE_FIX_API_RC="${KSE_FIX_API_RC:-0}" \ + KSE_FIX_SUDO_RC="${KSE_FIX_SUDO_RC:-0}" \ + KSE_FIX_ARG_LOG="$FAKE_ARG_LOG" \ + /usr/bin/bash -c ' + set -Eeuo pipefail + source "$1" + _kse_validate_runtime_dependencies() { return 0; } + _kse_validate_sudo_credentials() { + printf " <--signal=TERM> <--kill-after=1s> <9s> <--non-interactive> <--validate>\n" >>"$KSE_FIX_ARG_LOG" + [[ "$KSE_FIX_SUDO_RC" == 0 ]] + } + _kse_detect_datastore() { printf "sqlite\n"; } + _kse_privileged_capture() { + local binary="$1" fixture_hash stage annotation + shift + { printf "<%s>" "$binary"; printf " <%s>" "$@"; printf "\n"; } >>"$KSE_FIX_ARG_LOG" + fixture_hash=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + if [[ "$binary" == "$KSE_K3S" && "${1:-}" == --version ]]; then + printf "k3s version %s (fixture)\n" "$KSE_FIX_VERSION"; return 0 + fi + if [[ "$binary" == "$KSE_K3S" && " ${*} " == *" secrets-encrypt status --output json "* ]]; then + case "$KSE_FIX_STATUS" in + disabled) printf "%s\n" "{\"stage\":\"\",\"activekey\":\"\"}" ;; + transition) printf "%s\n" "{\"stage\":\"start\",\"activekey\":\"\",\"enable\":false,\"hashmatch\":true}" ;; + enabled_start) printf "%s\n" "{\"stage\":\"start\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hashmatch\":true}" ;; + finished) printf "%s\n" "{\"stage\":\"reencrypt_finished\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hashmatch\":true}" ;; + active) printf "%s\n" "{\"stage\":\"reencrypt_active\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hashmatch\":true}" ;; + mismatch) printf "%s\n" "{\"stage\":\"reencrypt_finished\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hasherror\":\"password-token\"}" ;; + malformed) printf "{\n" ;; + *) return 1 ;; + esac + return 0 + fi + if [[ "$binary" == "$KSE_K3S" && " ${*} " == *" kubectl get nodes -o json "* ]]; then + case "$KSE_FIX_STATUS" in disabled) annotation="" ;; transition|enabled_start) annotation="start-${fixture_hash}" ;; *) annotation="reencrypt_finished-${fixture_hash}" ;; esac + case "$KSE_FIX_NODES" in + valid) printf "{\"items\":[{\"metadata\":{\"name\":\"donghyeon-system-product-name\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"},\"annotations\":{%s}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}\n" "${annotation:+\"k3s.io/encryption-config-hash\":\"${annotation}\"}" ;; + different_ready) printf "{\"items\":[{\"metadata\":{\"name\":\"isolated-restore\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"},\"annotations\":{%s}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}\n" "${annotation:+\"k3s.io/encryption-config-hash\":\"${annotation}\"}" ;; + wrong_node) printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"wrong-node\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}" ;; + not_ready) printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"donghyeon-system-product-name\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"False\"}]}}]}" ;; + two_servers) printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"donghyeon-system-product-name\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}},{\"metadata\":{\"name\":\"second\",\"uid\":\"11111111-2222-4333-8444-555555555555\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}" ;; + esac + return 0 + fi + if [[ "$binary" == "$KSE_SYSTEMCTL" ]]; then + printf "%s\n" "ExecStart=/usr/local/bin/k3s server" "Environment=K3S_TOKEN=token-password-encryption-config" "EnvironmentFiles="; return 0 + fi + if [[ "$binary" == "$KSE_STAT" ]]; then printf "0:600:regular file\n"; return 0; fi + if [[ "$binary" == "$KSE_SHA256SUM" ]]; then printf "%s %s\n" "$fixture_hash" "$KSE_CONFIG_PATH"; return 0; fi + if [[ "$binary" == "$KSE_CAT" && " ${*} " == *" $KSE_STATE_PATH "* ]]; then + case "$KSE_FIX_STATUS" in transition|enabled_start) stage=start ;; *) stage=reencrypt_finished ;; esac + [[ "$KSE_FIX_INTEGRITY" == match ]] || fixture_hash=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + printf "%s-%s" "$stage" "$fixture_hash"; return 0 + fi + return 1 + } + _kse_privileged_quiet() { + if [[ "$1" == "$KSE_K3S" && " ${*:2} " == *" kubectl get --raw=/readyz "* ]]; then + { printf "<%s>" "$1"; shift; printf " <%s>" "$@"; printf "\n"; } >>"$KSE_FIX_ARG_LOG" + [[ "$KSE_FIX_API_RC" == 0 ]]; return + fi + _kse_privileged_capture "$@" >/dev/null + } + k3s_secret_encryption_main "${@:2}" + ' bash "$VALIDATOR_PATH" "$@" +} + +# A user-owned executable can never satisfy the production root-owned binary +# gate, even when it is absolute and executable. +assert_fails /usr/bin/bash -c 'source "$1"; _kse_trusted_executable "$2"' \ + bash "$VALIDATOR_PATH" "${fake_bin}/k3s" + +assert_validator_fixture_fails() { + if run_validator_fixture "$@" >"${fixture_root}/validator-failure.out" 2>"${fixture_root}/validator-failure.err"; then + fail "validator fixture unexpectedly succeeded: $*" + fi +} + +assert_sensitive_free() { + local text="$1" + [[ "$text" != *token* && "$text" != *password* && + "$text" != *encryption-config* && "$text" != *'AES-CBC '* ]] +} + +# Mutation probe: the sensitive-output assertion must reject this literal. +assert_fails assert_sensitive_free 'token-password-encryption-config' 'sensitive assertion mutation probe' + +# A denied noninteractive sudo credential gate must be actionable and must stop +# before the first privileged evidence command. +: >"$FAKE_ARG_LOG" +sudo_denial_rc=0 +KSE_FIX_SUDO_RC=1 run_validator_fixture --expect-disabled \ + >"${fixture_root}/sudo-denial.out" 2>"${fixture_root}/sudo-denial.err" || sudo_denial_rc=$? +[[ "$sudo_denial_rc" -ne 0 ]] || fail 'sudo denial fixture unexpectedly succeeded' +assert_eq 'ERROR: noninteractive sudo authorization is unavailable; run sudo -v in the same terminal, then rerun the validator' \ + "$(<"${fixture_root}/sudo-denial.err")" 'sudo denial actionable error' +assert_eq ' <--signal=TERM> <--kill-after=1s> <9s> <--non-interactive> <--validate>' \ + "$(<"$FAKE_ARG_LOG")" 'sudo denial stops before privileged evidence' + +assert_fixture_stdout_safe() { + local status="$1" output expected + shift + output="$(KSE_FIX_STATUS="$status" run_validator_fixture "$@" 2>"${fixture_root}/validator-safe.err")" || fail "$status safe stdout fixture" + assert_sensitive_free "$output" || fail "$status stdout leaks sensitive fixture data" + case "$status" in + disabled) expected=$'Encryption status: Disabled\nRotation stage: none' ;; + transition) expected=$'Encryption status: Disabled\nRotation stage: start' ;; + enabled_start) expected=$'Encryption status: Enabled\nRotation stage: start' ;; + finished) expected=$'Encryption status: Enabled\nRotation stage: reencrypt_finished' ;; + esac + [[ "$output" == *"$expected"* ]] || fail "$status stdout inventory class" +} + +assert_failed_fixture_stdout_safe() { + local status="$1" output rc=0 expected + shift + output="$(KSE_FIX_STATUS="$status" run_validator_fixture "$@" 2>"${fixture_root}/validator-safe-failure.err")" || rc=$? + [[ "$rc" -ne 0 ]] || fail "$status fixture unexpectedly succeeds" + assert_sensitive_free "$output" || fail "$status failing stdout leaks sensitive fixture data" + case "$status" in + active|mismatch) + expected=$'Encryption status: Enabled\nRotation stage: unsafe' + [[ "$output" == *"$expected"* ]] || fail "$status failing stdout inventory class" + ;; + malformed) [[ -z "$output" ]] || fail 'malformed status writes inventory stdout' ;; + esac +} + +validator_stdout='' +: >"$FAKE_ARG_LOG" +KSE_FIX_STATUS=disabled validator_stdout="$(run_validator_fixture --expect-disabled)" || fail 'disabled fixture accepts --expect-disabled' +[[ "$validator_stdout" != *token* && "$validator_stdout" != *password* && "$validator_stdout" != *encryption-config* ]] || fail 'validator stdout leaks fixture secret material' +[[ "$(<"$FAKE_ARG_LOG")" != *"${VALIDATOR_PATH}"* ]] || fail 'validator sudo-executes workspace code' +[[ "$(<"$FAKE_ARG_LOG")" == *' <--version>'* ]] || fail 'validator does not use the absolute k3s binary' +[[ "$(<"$FAKE_ARG_LOG")" == *' <-o> '* ]] || fail 'validator node inventory argv' +[[ "$(<"$FAKE_ARG_LOG")" == *' <--raw=/readyz>'* ]] || fail 'validator readyz argv' + +KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-enabled +KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-transition-start +KSE_FIX_STATUS=transition run_validator_fixture --expect-transition-start >/dev/null || fail 'transition fixture expectation' +KSE_FIX_STATUS=transition assert_validator_fixture_fails --expect-disabled +KSE_FIX_STATUS=enabled_start run_validator_fixture --expect-enabled >/dev/null || fail 'enabled/start fixture expectation' +KSE_FIX_STATUS=enabled_start assert_validator_fixture_fails --expect-reencrypted +KSE_FIX_STATUS=finished run_validator_fixture --expect-enabled >/dev/null || fail 'enabled/finished fixture expectation' +KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match run_validator_fixture --expect-reencrypted >/dev/null || fail 'reencrypted integrity match fixture' +KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match KSE_FIX_NODES=different_ready \ + run_validator_fixture --expect-reencrypted-restore >/dev/null || fail 'isolated restore accepts a different ready server hostname' +KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match KSE_FIX_NODES=not_ready \ + assert_validator_fixture_fails --expect-reencrypted-restore +KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=mismatch assert_validator_fixture_fails --expect-reencrypted +KSE_FIX_STATUS=active assert_validator_fixture_fails +KSE_FIX_STATUS=mismatch assert_validator_fixture_fails +KSE_FIX_STATUS=malformed assert_validator_fixture_fails +KSE_FIX_STATUS=disabled KSE_FIX_VERSION=v1.36.1+k3s1 assert_validator_fixture_fails --expect-disabled +KSE_FIX_STATUS=disabled KSE_FIX_VERSION=v1.36.2+k3s1suffix assert_validator_fixture_fails --expect-disabled +KSE_FIX_STATUS=disabled KSE_FIX_NODES=two_servers assert_validator_fixture_fails --expect-disabled +KSE_FIX_STATUS=disabled KSE_FIX_NODES=wrong_node assert_validator_fixture_fails --expect-disabled +KSE_FIX_STATUS=disabled KSE_FIX_NODES=not_ready assert_validator_fixture_fails --expect-disabled +KSE_FIX_STATUS=disabled KSE_FIX_API_RC=1 assert_validator_fixture_fails --expect-disabled +KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir '' + +# Exercise the sensitive-output contract for every validator inventory class, +# including states that correctly return non-zero. +assert_fixture_stdout_safe disabled --expect-disabled +assert_fixture_stdout_safe transition --expect-transition-start +assert_fixture_stdout_safe enabled_start --expect-enabled +assert_fixture_stdout_safe finished --expect-enabled +assert_failed_fixture_stdout_safe active +assert_failed_fixture_stdout_safe mismatch +assert_failed_fixture_stdout_safe malformed + +# Failed expectations must leave a valid empty handoff untouched. +rejected_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" +chmod 0700 "$rejected_output_dir" +KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-enabled --verified-output-dir "$rejected_output_dir" +[[ -z "$(find "$rejected_output_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'failed expectation leaves verified output' + +# Only a physical direct child of /tmp is accepted. +mkdir -p "${rejected_output_dir}/nested" +KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "$rejected_output_dir" +KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "${rejected_output_dir}/nested" +symlink_output_parent="/tmp/platform-k3s-encryption.link${RANDOM}${RANDOM}" +ln -s -- "$fixture_root" "$symlink_output_parent" +KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "${symlink_output_parent}/nested" +race_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" +chmod 0750 "$race_output_dir" +KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "$race_output_dir" + +verified_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" +chmod 0700 "$verified_output_dir" +KSE_FIX_STATUS=disabled validator_stdout="$(run_validator_fixture --expect-disabled --verified-output-dir "$verified_output_dir")" || fail 'verified output fixture accepts safe directory' +assert_eq 600 "$(stat --format='%a' "$verified_output_dir/inventory.env")" 'inventory handoff mode' +assert_eq 600 "$(stat --format='%a' "$verified_output_dir/status.sha256")" 'status SHA handoff mode' +[[ "$(<"$verified_output_dir/inventory.env")" == *'node_uid=aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'* ]] || fail 'verified inventory omits immutable node UID' +assert_eq 4786ad5101dd5c0de42b13d276e2cb0968448fd2b5c16abb6164ffeb69fafd2b "$(<"$verified_output_dir/status.sha256")" 'canonical disabled status SHA' +[[ "$(<"$verified_output_dir/inventory.env")" != *token* && "$(<"$verified_output_dir/inventory.env")" != *password* && "$(<"$verified_output_dir/inventory.env")" != *encryption-config* ]] || fail 'verified inventory leaks fixture secret material' +assert_sensitive_free "$(<"$verified_output_dir/inventory.env")$'\n'$(<"$verified_output_dir/status.sha256")" || fail 'disabled handoff leaks sensitive fixture data' + +transition_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" +chmod 0700 "$transition_output_dir" +KSE_FIX_STATUS=transition run_validator_fixture --expect-transition-start --verified-output-dir "$transition_output_dir" >/dev/null || fail 'transition verified output fixture' +assert_sensitive_free "$(<"$transition_output_dir/inventory.env")$'\n'$(<"$transition_output_dir/status.sha256")" || fail 'transition handoff leaks sensitive fixture data' + +enabled_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" +chmod 0700 "$enabled_output_dir" +KSE_FIX_STATUS=enabled_start run_validator_fixture --expect-enabled --verified-output-dir "$enabled_output_dir" >/dev/null || fail 'enabled/start verified output fixture' +assert_sensitive_free "$(<"$enabled_output_dir/inventory.env")$'\n'$(<"$enabled_output_dir/status.sha256")" || fail 'enabled/start handoff leaks sensitive fixture data' + +finished_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" +chmod 0700 "$finished_output_dir" +KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match run_validator_fixture --expect-reencrypted --verified-output-dir "$finished_output_dir" >/dev/null || fail 'reencrypted verified output fixture' +assert_eq 9f963cbcdaa79fd76d8c6df50d1108cee96be0e9e947308cb18a033c8eb24595 "$(<"$finished_output_dir/status.sha256")" 'canonical reencrypted status SHA' +[[ "$(<"$finished_output_dir/inventory.env")" != *token* && "$(<"$finished_output_dir/inventory.env")" != *password* && "$(<"$finished_output_dir/inventory.env")" != *encryption-config* ]] || fail 'reencrypted handoff leaks fixture secret material' +assert_sensitive_free "$(<"$finished_output_dir/inventory.env")$'\n'$(<"$finished_output_dir/status.sha256")" || fail 'reencrypted handoff leaks sensitive fixture data' + +# Caller PATH commands must never participate in validation or handoff. +malicious_bin="${fixture_root}/malicious-bin" +malicious_marker="${fixture_root}/malicious-path-used" +mkdir -p "$malicious_bin" +for command_name in dirname stat find id timeout sudo; do + printf '%s\n' '#!/usr/bin/env bash' ": >\"${malicious_marker}\"" 'exit 99' >"${malicious_bin}/${command_name}" + chmod 0700 "${malicious_bin}/${command_name}" +done +malicious_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" +chmod 0700 "$malicious_output_dir" +PATH="${malicious_bin}:${ORIGINAL_PATH}" KSE_FIX_STATUS=disabled run_validator_fixture --expect-disabled --verified-output-dir "$malicious_output_dir" >/dev/null || fail 'validator ignores caller PATH commands' +[[ ! -e "$malicious_marker" ]] || fail 'validator trusted a caller PATH command' +rm -rf -- "$malicious_output_dir" + +# The host drop-in must contain exactly the two encryption settings and no +# Kubernetes resource metadata or unrelated configuration. +awk ' + /^[[:space:]]*($|#)/ { next } + $0 == "secrets-encryption: true" { enabled++; next } + $0 == "secrets-encryption-provider: aescbc" { provider++; next } + { unexpected++ } + END { exit !(enabled == 1 && provider == 1 && unexpected == 0) } +' infrastructure/security/k3s/40-secrets-encryption.yaml + +# Bootstrap state-machine regression coverage. The production script is +# sourced and its host boundaries are replaced in this subshell only; there is +# deliberately no environment-controlled fake path in production. +readonly BOOTSTRAP_PATH="${REPOSITORY_ROOT}/scripts/bootstrap/apply-k3s-secret-encryption.sh" +bootstrap_fixture_root="${fixture_root}/bootstrap" +bootstrap_backup_root="${bootstrap_fixture_root}/recovery" +mkdir -p "$bootstrap_backup_root" + +run_bootstrap_fixture() ( + local scenario="$1" + shift + local log="${bootstrap_fixture_root}/${scenario}.log" + : >"$log" + + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + + _kseb_load_inventory() { + printf 'inventory\n' >>"$log" + case "$scenario" in + disabled_sqlite|local_success|local_plain_ext4|local_wrong_outer_disk|pre_capacity_failure|post_capacity_failure|backup_failure|service_recovery_failure|post_backup_failure|marker_failure|final_validation_failure|timeout_finished|timeout_start|root_failure|enable_failure|dropin_failure|initial_restart_failure|final_restart_failure|estimate|estimate_invalid) + printf 'disabled_no_config|none|sqlite|none|none|mismatch|mismatch\n' + ;; + disabled_etcd) + printf 'disabled_no_config|none|embedded-etcd|none|none|mismatch|mismatch\n' + ;; + disabled_backend_drift_external|disabled_backend_drift_ambiguous) + if [[ -e "${bootstrap_fixture_root}/${scenario}.counter" ]]; then + if [[ "$scenario" == disabled_backend_drift_external ]]; then + printf 'disabled_no_config|none|external|none|none|mismatch|mismatch\n' + else + printf 'disabled_no_config|none|ambiguous|none|none|mismatch|mismatch\n' + fi + else + : >"${bootstrap_fixture_root}/${scenario}.counter" + printf 'disabled_no_config|none|sqlite|none|none|mismatch|mismatch\n' + fi + ;; + mismatch) + printf 'hash_mismatch|unsafe|sqlite|invalid|ambiguous|mismatch|mismatch\n' + ;; + external) + printf 'disabled_no_config|none|external|none|none|mismatch|mismatch\n' + ;; + ambiguous_backend) + printf 'disabled_no_config|none|ambiguous|none|none|mismatch|mismatch\n' + ;; + enabled_finished) + printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|aescbc/config-owner|match|match\n' + ;; + enabled_finished_drift_provider|enabled_finished_drift_owner) + if [[ -e "${bootstrap_fixture_root}/${scenario}.counter" ]]; then + if [[ "$scenario" == enabled_finished_drift_provider ]]; then + printf 'enabled_stable|reencrypt_finished|sqlite|secretbox|secretbox/config-owner|match|match\n' + else + printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|ambiguous|match|match\n' + fi + else + : >"${bootstrap_fixture_root}/${scenario}.counter" + printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|aescbc/config-owner|match|match\n' + fi + ;; + enabled_wrong_provider) + printf 'enabled_stable|reencrypt_finished|sqlite|secretbox|secretbox/config-owner|match|match\n' + ;; + enabled_wrong_owner) + printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|ambiguous|match|match\n' + ;; + enabled_bad_integrity) + printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|aescbc/config-owner|mismatch|match\n' + ;; + enabled_start|enabled_start_rotate) + printf 'enabled_stable|start|sqlite|aescbc|aescbc/config-owner|match|match\n' + ;; + enabled_start_drift) + if [[ -e "${bootstrap_fixture_root}/enabled-start-drift.counter" ]]; then + printf 'enabled_stable|start|sqlite|aescbc|aescbc/config-owner|mismatch|mismatch\n' + else + : >"${bootstrap_fixture_root}/enabled-start-drift.counter" + printf 'enabled_stable|start|sqlite|aescbc|aescbc/config-owner|match|match\n' + fi + ;; + *) return 97 ;; + esac + } + # Keep the legacy override during the RED stage so the old implementation + # cannot reach a real terminal. The new implementation must use the split + # confirmation boundaries below instead. + _kseb_confirm_execute() { printf 'legacy-confirm\n' >>"$log"; } + _kseb_prepare_execute_context() { printf 'default\n'; } + _kseb_confirm_apply() { printf 'APPLY\n' >>"$log"; } + _kseb_validate_backup_root() { + printf 'generic-root\n' >>"$log" + [[ "$scenario" != root_failure ]] + } + _kseb_validate_recovery_policy() { + local policy="$1" + printf 'policy-root:%s\n' "$policy" >>"$log" + case "$scenario" in + local_plain_ext4|local_wrong_outer_disk) return 1 ;; + *) return 0 ;; + esac + } + _kseb_confirm_recovery() { + local context="$1" policy="$2" + [[ "$context" == default ]] + printf 'RECOVERY\nENCRYPTED\n' >>"$log" + if [[ "$policy" == local-separate-disk-luks ]]; then + printf 'LOCAL_RISK_ACCEPTED\n' >>"$log" + fi + } + _kseb_backup_phase() { + local phase="$1" backend="$2" + if [[ "$scenario" == pre_capacity_failure && "$phase" == pre ]]; then + printf 'capacity:pre\n' >>"$log" + return 1 + fi + if [[ "$scenario" == post_capacity_failure && "$phase" == post ]]; then + printf 'capacity:post\n' >>"$log" + return 1 + fi + printf 'backup:%s:%s\n' "$phase" "$backend" >>"$log" + [[ "$scenario" != backup_failure || "$phase" != pre ]] && + [[ "$scenario" != post_backup_failure || "$phase" != post ]] || return $? + if [[ "$scenario" == service_recovery_failure && "$phase" == pre ]]; then return 75; fi + if [[ "$scenario" == marker_failure && "$phase" == post ]]; then return 76; fi + } + _kseb_enable() { printf 'enable\n' >>"$log"; [[ "$scenario" != enable_failure ]]; } + _kseb_install_dropin() { printf 'install-dropin\n' >>"$log"; [[ "$scenario" != dropin_failure ]]; } + _kseb_restart() { + local counter="${bootstrap_fixture_root}/${scenario}.restart-counter" + printf 'restart\n' >>"$log" + if [[ "$scenario" == initial_restart_failure ]]; then return 1; fi + if [[ "$scenario" == final_restart_failure ]]; then + if [[ -e "$counter" ]]; then return 1; fi + : >"$counter" + fi + } + _kseb_rotate_keys() { + printf 'rotate\n' >>"$log" + case "$scenario" in timeout_finished|timeout_start) return 124 ;; *) return 0 ;; esac + } + _kseb_read_status_json() { + local stage counter="${bootstrap_fixture_root}/${scenario}.status-counter" + case "$scenario" in + timeout_finished) + if [[ -e "$counter" ]]; then stage=reencrypt_finished; else stage=reencrypt_active; : >"$counter"; fi + ;; + timeout_start) stage=start ;; + *) stage=reencrypt_finished ;; + esac + printf 'poll:%s\n' "$stage" >>"$log" + printf '{"stage":"%s","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}\n' "$stage" + } + sleep() { :; } + _kseb_validate_expectation() { + printf 'validate:%s\n' "$1" >>"$log" + [[ "$scenario" != final_validation_failure || "$1" != --expect-reencrypted ]] + } + _kseb_dry_run() { printf 'dry-run\n' >>"$log"; } + _kseb_estimate_phase_bytes() { + printf 'estimate\n' >>"$log" + if [[ "$scenario" == estimate_invalid ]]; then + printf '9223372036854775808\n' + else + printf '1048576\n' + fi + } + + apply_k3s_secret_encryption_main "$@" \ + >"${bootstrap_fixture_root}/${scenario}.stdout" \ + 2>"${bootstrap_fixture_root}/${scenario}.stderr" +) + +assert_bootstrap_order() { + local scenario="$1" expected="$2" + assert_eq "$expected" "$(<"${bootstrap_fixture_root}/${scenario}.log")" "$scenario bootstrap order" +} + +# These tests protect the required, closed recovery-policy parser and split +# confirmation dispatch. Missing or malformed policy input must stop before +# inventory collection, confirmation, or mutation dispatch. +assert_exit 2 run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" +assert_bootstrap_order disabled_sqlite '' +assert_exit 2 run_bootstrap_fixture disabled_sqlite --recovery-policy encrypted-off-host +assert_bootstrap_order disabled_sqlite '' +assert_exit 2 run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \ + --recovery-policy unknown +assert_bootstrap_order disabled_sqlite '' +assert_exit 2 run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \ + --recovery-policy encrypted-off-host --recovery-policy encrypted-off-host +assert_bootstrap_order disabled_sqlite '' + +assert_succeeds run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \ + --recovery-policy encrypted-off-host +assert_bootstrap_order disabled_sqlite $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite' + +assert_succeeds run_bootstrap_fixture local_success --execute --backup-root "$bootstrap_backup_root" \ + --recovery-policy local-separate-disk-luks +assert_bootstrap_order local_success $'inventory\nAPPLY\ngeneric-root\npolicy-root:local-separate-disk-luks\nRECOVERY\nENCRYPTED\nLOCAL_RISK_ACCEPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite' + +assert_fails run_bootstrap_fixture local_plain_ext4 --execute --backup-root "$bootstrap_backup_root" \ + --recovery-policy local-separate-disk-luks +assert_bootstrap_order local_plain_ext4 $'inventory\nAPPLY\ngeneric-root\npolicy-root:local-separate-disk-luks' +assert_fails run_bootstrap_fixture local_wrong_outer_disk --execute --backup-root "$bootstrap_backup_root" \ + --recovery-policy local-separate-disk-luks +assert_bootstrap_order local_wrong_outer_disk $'inventory\nAPPLY\ngeneric-root\npolicy-root:local-separate-disk-luks' + +# The production change that makes these probes pass is the real local policy +# boundary: exact inner root, one validator call, then pinned-root revalidation. +probe_recovery_policy_dispatch() ( + local policy="$1" root="$2" scenario="$3" rc=0 + local log="${bootstrap_fixture_root}/policy-dispatch-${scenario}.log" + : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_local_recovery_root() { printf '/fixture/local-recovery\n'; } + _kseb_run_local_recovery_validator() { + printf 'local-validator\n' >>"$log" + [[ "$scenario" != validator_failure ]] + } + _kseb_revalidate_backup_root() { + printf 'root-revalidated\n' >>"$log" + [[ "$scenario" != identity_drift ]] + } + _kseb_validate_recovery_policy "$policy" "$root" || rc=$? + printf 'rc=%s\n' "$rc" >>"$log" + command cat -- "$log" +) +assert_eq $'local-validator\nroot-revalidated\nrc=0' \ + "$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery success)" \ + 'local recovery policy validates exact root and revalidates identity' +assert_eq 'rc=1' \ + "$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery/child child_root)" \ + 'local recovery policy rejects a child root before validation' +assert_eq $'local-validator\nrc=1' \ + "$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery validator_failure)" \ + 'local recovery policy rejects validator failure before root revalidation' +assert_eq $'local-validator\nroot-revalidated\nrc=1' \ + "$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery identity_drift)" \ + 'local recovery policy rejects root identity drift' +assert_eq 'rc=0' \ + "$(probe_recovery_policy_dispatch encrypted-off-host /fixture/off-host off_host)" \ + 'off-host policy never dispatches the local validator' + +local_contract_fixture="${bootstrap_fixture_root}/local-recovery-contract.env" +awk -F= ' + $1 == "K3SLR_INNER_MOUNT" { print "K3SLR_INNER_MOUNT=/fixture/contract-root"; next } + { print } +' infrastructure/security/k3s/local-recovery.env >"$local_contract_fixture" +probe_local_recovery_contract_binding() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_local_recovery_contract() { printf '%s\n' "$local_contract_fixture"; } + _kseb_local_recovery_root +) +assert_eq '/fixture/contract-root' "$(probe_local_recovery_contract_binding)" \ + 'local policy root is read from the authoritative recovery contract' + +probe_recovery_policy_contract_drift() ( + local marker="${bootstrap_fixture_root}/contract-drift.marker" rc=0 + local log="${bootstrap_fixture_root}/contract-drift.log" + rm -f -- "$marker"; : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_local_recovery_root() { + if [[ -e "$marker" ]]; then + printf '/fixture/root-after\n' + else + : >"$marker" + printf '/fixture/root-before\n' + fi + } + _kseb_run_local_recovery_validator() { printf 'local-validator\n' >>"$log"; } + _kseb_revalidate_backup_root() { printf 'unexpected-root-revalidation\n' >>"$log"; } + _kseb_validate_recovery_policy local-separate-disk-luks /fixture/root-before || rc=$? + printf 'rc=%s\n' "$rc" >>"$log" + command cat -- "$log" +) +assert_eq $'local-validator\nrc=1' "$(probe_recovery_policy_contract_drift)" \ + 'local policy rejects contract root drift after validator success' + +duplicate_contract_fixture="${bootstrap_fixture_root}/local-recovery-contract-duplicate.env" +awk ' + { print } + END { print "K3SLR_INNER_MOUNT=/fixture/duplicate-root" } +' infrastructure/security/k3s/local-recovery.env >"$duplicate_contract_fixture" +probe_duplicate_local_recovery_contract() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_local_recovery_contract() { printf '%s\n' "$duplicate_contract_fixture"; } + _kseb_local_recovery_root +) +assert_fails probe_duplicate_local_recovery_contract + +probe_local_validator_boundary() ( + local scenario="$1" rc=0 + local command_log="${bootstrap_fixture_root}/local-validator-${scenario}.argv" + local public_out="${bootstrap_fixture_root}/local-validator-${scenario}.stdout" + local public_err="${bootstrap_fixture_root}/local-validator-${scenario}.stderr" + : >"$command_log"; : >"$public_out"; : >"$public_err" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + if [[ "$scenario" == cleanup_failure ]]; then + _kseb_safe_remove_local_validator_handoff() { + /usr/bin/rm -- "$1/stdout" "$1/stderr" >/dev/null 2>&1 || true + /usr/bin/rmdir -- "$1" >/dev/null 2>&1 || true + return 1 + } + fi + _kseb_local_validator_command() { + printf '%s\n' "$@" >"$command_log" + case "$scenario" in + success|cleanup_failure) + printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' + ;; + extra_stdout) + printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\nUNEXPECTED\n' + ;; + stderr_output) + printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' + printf 'RAW-IDENTIFIER-SENTINEL\n' >&2 + ;; + command_failure) return 1 ;; + oversize) /usr/bin/yes X | /usr/bin/head --bytes=16384 ;; + *) return 97 ;; + esac + } + _kseb_run_local_recovery_validator >"$public_out" 2>"$public_err" || rc=$? + [[ ! -s "$public_out" && ! -s "$public_err" ]] || return 1 + printf 'rc=%s\n' "$rc" +) +assert_eq 'rc=0' "$(probe_local_validator_boundary success)" \ + 'local validator accepts only its exact safe success record' +assert_eq $'/usr/bin/env\n-i\nPATH=/usr/sbin:/usr/bin:/sbin:/bin\nLC_ALL=C\n/usr/bin/timeout\n--signal=TERM\n--kill-after=1s\n60s\n/usr/bin/bash\n--noprofile\n--norc\n--\n'"${REPOSITORY_ROOT}"$'/scripts/validate/k3s-local-recovery.sh\n--expect-open' \ + "$(<"${bootstrap_fixture_root}/local-validator-success.argv")" \ + 'local validator command uses fixed binaries, clean environment, and expect-open' +assert_eq 'rc=1' "$(probe_local_validator_boundary extra_stdout)" \ + 'local validator rejects extra stdout without exposing it' +assert_eq 'rc=1' "$(probe_local_validator_boundary stderr_output)" \ + 'local validator rejects stderr without exposing it' +assert_eq 'rc=1' "$(probe_local_validator_boundary command_failure)" \ + 'local validator rejects a nonzero command result' +assert_eq 'rc=1' "$(probe_local_validator_boundary cleanup_failure)" \ + 'local validator treats capture cleanup failure as policy failure' +local_validator_tmp_count() { + /usr/bin/find /tmp -maxdepth 1 -type d -name 'platform-k3s-local-validator.*' -printf . | /usr/bin/wc -c +} +local_validator_tmp_before="$(local_validator_tmp_count)" +assert_eq 'rc=1' "$(probe_local_validator_boundary oversize)" \ + 'local validator bounds oversized output and returns failure' +assert_eq "$local_validator_tmp_before" "$(local_validator_tmp_count)" \ + 'oversized local validator output leaves no temporary directory' + +probe_local_validator_result_matrix() ( + local scenario="$1" directory="${bootstrap_fixture_root}/validator-result-${1}" + local stdout_file="$directory/stdout" stderr_file="$directory/stderr" command_rc=0 + rm -rf -- "$directory"; mkdir -m 0700 -- "$directory" + : >"$stderr_file" + case "$scenario" in + exact|timeout|signal|stderr) + printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file" + ;; + embedded_nul) + printf 'Recovery device: matc\0\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file" + ;; + carriage_return) + printf 'Recovery device: matc\r\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file" + ;; + short) + printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file" + /usr/bin/truncate --size=85 -- "$stdout_file" + ;; + long) + printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\nX' >"$stdout_file" + ;; + *) return 97 ;; + esac + case "$scenario" in + timeout) command_rc=124 ;; + signal) command_rc=137 ;; + stderr) printf 'unexpected\n' >"$stderr_file" ;; + esac + chmod 0600 -- "$stdout_file" "$stderr_file" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_local_validator_result_is_exact "$command_rc" "$stdout_file" "$stderr_file" +) +assert_succeeds probe_local_validator_result_matrix exact +assert_fails probe_local_validator_result_matrix embedded_nul +assert_fails probe_local_validator_result_matrix carriage_return +assert_fails probe_local_validator_result_matrix short +assert_fails probe_local_validator_result_matrix long +assert_fails probe_local_validator_result_matrix timeout +assert_fails probe_local_validator_result_matrix signal +assert_fails probe_local_validator_result_matrix stderr + +probe_local_validator_exact_cleanup() ( + local scenario="$1" directory + directory="$(/usr/bin/mktemp -d /tmp/platform-k3s-local-validator.XXXXXX)" || return 1 + case "$directory" in /tmp/platform-k3s-local-validator.[A-Za-z0-9]*) ;; *) return 1 ;; esac + trap '/usr/bin/rm -f -- "$directory/stdout" "$directory/stderr" "$directory/extra"; /usr/bin/rmdir -- "$directory" 2>/dev/null || true' EXIT + : >"$directory/stdout"; : >"$directory/stderr" + if [[ "$scenario" == extra ]]; then : >"$directory/extra"; fi + chmod 0600 -- "$directory"/* + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + if [[ "$scenario" == success ]]; then + _kseb_safe_remove_local_validator_handoff "$directory" || return 1 + [[ ! -e "$directory" ]] + else + if _kseb_safe_remove_local_validator_handoff "$directory"; then return 1; fi + [[ -d "$directory" && -f "$directory/extra" && ! -e "$directory/stdout" && ! -e "$directory/stderr" ]] + fi +) +assert_succeeds probe_local_validator_exact_cleanup success +assert_succeeds probe_local_validator_exact_cleanup extra + +probe_recovery_confirmation() ( + local policy="$1" answers="$2" context_after="${3:-default}" rc=0 output + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_current_context() { printf '%s\n' "$context_after"; } + output="$(_kseb_confirm_recovery default "$policy" <<<"$answers" 2>&1)" || rc=$? + printf 'rc=%s|offhost=%s|localrisk=%s\n' "$rc" \ + "$([[ "$output" == *off-host* ]] && printf 1 || printf 0)" \ + "$([[ "$output" == *LOCAL_RISK_ACCEPTED* ]] && printf 1 || printf 0)" +) +assert_eq 'rc=0|offhost=1|localrisk=0' \ + "$(probe_recovery_confirmation encrypted-off-host $'RECOVERY default\nENCRYPTED default')" \ + 'off-host policy keeps its two explicit confirmations' +assert_eq 'rc=0|offhost=0|localrisk=1' \ + "$(probe_recovery_confirmation local-separate-disk-luks $'RECOVERY default\nENCRYPTED default\nLOCAL_RISK_ACCEPTED default')" \ + 'local policy uses truthful encryption and local-risk confirmations' +assert_eq 'rc=1|offhost=0|localrisk=1' \ + "$(probe_recovery_confirmation local-separate-disk-luks $'RECOVERY default\nENCRYPTED default')" \ + 'local policy refuses a missing local-risk confirmation' +assert_eq 'rc=1|offhost=0|localrisk=0' \ + "$(probe_recovery_confirmation encrypted-off-host $'WRONG default\nENCRYPTED default')" \ + 'recovery confirmation rejects the wrong RECOVERY record' +assert_eq 'rc=1|offhost=1|localrisk=0' \ + "$(probe_recovery_confirmation encrypted-off-host $'RECOVERY default\nWRONG default')" \ + 'recovery confirmation rejects the wrong ENCRYPTED record' +assert_eq 'rc=1|offhost=1|localrisk=0' \ + "$(probe_recovery_confirmation encrypted-off-host $'RECOVERY default\nENCRYPTED default' changed)" \ + 'recovery confirmation rejects context drift' + +probe_apply_confirmation() ( + local answer="$1" rc=0 + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_confirm_apply default <<<"$answer" >/dev/null 2>&1 || rc=$? + printf 'rc=%s\n' "$rc" +) +assert_eq 'rc=0' "$(probe_apply_confirmation 'APPLY default')" 'APPLY confirmation exact success' +assert_eq 'rc=1' "$(probe_apply_confirmation 'APPLY other')" 'APPLY confirmation rejects another context' +assert_eq 'rc=1' "$(probe_apply_confirmation 'apply default')" 'APPLY confirmation is case-sensitive' + +# Existing bootstrap state-machine regressions continue below. Once the +# implementation exists, these literals protect the entire mutation order. +assert_succeeds run_bootstrap_fixture disabled_etcd --execute --backup-root "$bootstrap_backup_root" \ + --recovery-policy encrypted-off-host +[[ "$(<"${bootstrap_fixture_root}/disabled_etcd.log")" == *$'backup:pre:embedded-etcd'*$'backup:post:embedded-etcd'* ]] || fail 'embedded-etcd backend routing' + +assert_succeeds run_bootstrap_fixture timeout_finished --execute --backup-root "$bootstrap_backup_root" \ + --recovery-policy encrypted-off-host +assert_bootstrap_order timeout_finished $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_active\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite' +assert_eq 1 "$(awk '$0 == "rotate" { count++ } END { print count + 0 }' "${bootstrap_fixture_root}/timeout_finished.log")" 'timeout never retries rotate-keys' + +assert_fails run_bootstrap_fixture timeout_start --execute --backup-root "$bootstrap_backup_root" \ + --recovery-policy encrypted-off-host +assert_bootstrap_order timeout_start $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:start' + +assert_fails run_bootstrap_fixture mismatch --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order mismatch 'inventory' +assert_fails run_bootstrap_fixture external --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order external 'inventory' +assert_fails run_bootstrap_fixture ambiguous_backend --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order ambiguous_backend 'inventory' +assert_fails run_bootstrap_fixture backup_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order backup_failure $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite' +assert_fails run_bootstrap_fixture root_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order root_failure $'inventory\nAPPLY\ngeneric-root' +assert_fails run_bootstrap_fixture final_validation_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +[[ "$(<"${bootstrap_fixture_root}/final_validation_failure.log")" != *'backup:post:'* ]] || fail 'post backup before final verification' + +assert_succeeds run_bootstrap_fixture enabled_finished --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order enabled_finished $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-reencrypted\ninventory\nbackup:post:sqlite' +assert_fails run_bootstrap_fixture enabled_wrong_provider --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order enabled_wrong_provider 'inventory' +assert_fails run_bootstrap_fixture enabled_wrong_owner --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order enabled_wrong_owner 'inventory' +assert_fails run_bootstrap_fixture enabled_bad_integrity --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order enabled_bad_integrity 'inventory' + +assert_fails run_bootstrap_fixture enabled_start --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order enabled_start 'inventory' +assert_succeeds run_bootstrap_fixture enabled_start_rotate --execute --rotate-existing --backup-root "$bootstrap_backup_root" \ + --recovery-policy encrypted-off-host +assert_bootstrap_order enabled_start_rotate $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-enabled\nbackup:pre:sqlite\ninventory\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite' + +assert_succeeds run_bootstrap_fixture disabled_sqlite +assert_bootstrap_order disabled_sqlite $'inventory\ndry-run' +assert_fails run_bootstrap_fixture disabled_sqlite --rotate-existing +assert_fails run_bootstrap_fixture disabled_sqlite --execute --backup-root relative/path --recovery-policy encrypted-off-host +assert_fails run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \ + --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host + +# These CLI regressions catch estimator dispatch through any mutating branch, +# accepting an invalid estimate, or emitting anything except the public line. +assert_succeeds run_bootstrap_fixture estimate --estimate-recovery-bytes +assert_eq 'phase_bytes=1048576' "$(<"${bootstrap_fixture_root}/estimate.stdout")" \ + 'read-only phase estimator output' +assert_bootstrap_order estimate $'inventory\nestimate' +assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --execute +assert_exit 2 run_bootstrap_fixture estimate --execute --estimate-recovery-bytes +assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --rotate-existing +assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --backup-root "$bootstrap_backup_root" +assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --estimate-recovery-bytes +assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --help +assert_exit 2 run_bootstrap_fixture estimate --help --estimate-recovery-bytes +assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --recovery-policy auto +assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --recovery-policy encrypted-off-host +assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --recovery-policy local-separate-disk-luks +assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --unknown +assert_fails run_bootstrap_fixture estimate_invalid --estimate-recovery-bytes +assert_eq '' "$(<"${bootstrap_fixture_root}/estimate_invalid.stdout")" \ + 'invalid estimate has no public output' + +probe_estimator_fixed_sources() ( + local backend="$1" log="${bootstrap_fixture_root}/estimate-${1}-sources.log" + : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_privileged_estimator_source_kind() { printf 'present\n'; } + _kseb_estimator_source_identity() { + case "$1" in + /usr/lib/systemd/system/k3s.service|/lib/systemd/system/k3s.service) printf '41:8\n' ;; + *) printf '41:%s\n' "$(printf '%s' "$1" | /usr/bin/cksum | awk '{print $1}')" ;; + esac + } + _kseb_privileged_estimator_du() { + printf '%s\n' "$1" >>"$log" + printf '1\t%s\n' "$1" + } + _kseb_estimate_phase_bytes "$backend" +) +assert_eq 16777224 "$(probe_estimator_fixed_sources sqlite)" \ + 'SQLite estimator sums fixed backup inputs once' +assert_eq $'/var/lib/rancher/k3s/server/db\n/var/lib/rancher/k3s/server/token\n/etc/rancher/k3s\n/etc/systemd/system/k3s.service\n/etc/systemd/system/k3s.service.env\n/etc/systemd/system/k3s.service.d\n/usr/lib/systemd/system/k3s.service\n/var/lib/rancher/k3s/server/cred/encryption-config.json' \ + "$(<"${bootstrap_fixture_root}/estimate-sqlite-sources.log")" \ + 'SQLite estimator reads each effective fixed source once' +assert_eq 16777224 "$(probe_estimator_fixed_sources embedded-etcd)" \ + 'embedded-etcd estimator sums fixed backup inputs once' +assert_eq '/var/lib/rancher/k3s/server/db/etcd' \ + "$(sed -n '1p' "${bootstrap_fixture_root}/estimate-embedded-etcd-sources.log")" \ + 'embedded-etcd estimator uses current local data directory' + +probe_estimator_du_contract() ( + local scenario="$1" marker="${bootstrap_fixture_root}/estimate-identity-${1}.counter" + rm -f -- "$marker" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_privileged_estimator_source_kind() { printf 'present\n'; } + _kseb_estimator_source_identity() { + if [[ "$scenario" == identity_change && -e "$marker" ]]; then printf '41:2\n' + else printf '41:1\n'; : >"$marker"; fi + } + _kseb_privileged_estimator_du() { + case "$scenario" in + malformed) printf 'not-a-number\t%s\n' "$1" ;; + multiline) printf '1\t%s\n2\t%s\n' "$1" "$1" ;; + trailing_blank) printf '1\t%s\n\n' "$1" ;; + wrong_path) printf '1\t/etc/shadow\n' ;; + *) printf '1\t%s\n' "$1" ;; + esac + } + _kseb_estimate_source_bytes sqlite /var/lib/rancher/k3s/server/token required +) +assert_fails probe_estimator_du_contract malformed +assert_fails probe_estimator_du_contract multiline +assert_fails probe_estimator_du_contract trailing_blank +assert_fails probe_estimator_du_contract wrong_path +assert_fails probe_estimator_du_contract identity_change + +probe_estimator_symlink_parent() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_privileged_estimator_source_kind() { printf 'present\n'; } + _kseb_privileged_estimator_lstat() { + case "$1" in + /|/var) printf '41:1|41ed\n' ;; + /var/lib) printf '41:2|a1ff\n' ;; + *) printf '41:3|41ed\n' ;; + esac + } + _kseb_privileged_estimator_du() { printf '1\t%s\n' "$1"; } + _kseb_estimate_source_bytes sqlite /var/lib/rancher/k3s/server/db required +) +assert_fails probe_estimator_symlink_parent + +probe_estimator_numeric_mode_metadata() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_privileged_estimator_source_kind() { printf 'present\n'; } + _kseb_privileged_estimator_lstat() { + if [[ "$1" == /var/lib/rancher/k3s/server/token ]]; then + printf '41:9|81a4\n' + else + printf '41:8|41ed\n' + fi + } + _kseb_privileged_estimator_du() { printf '1\t%s\n' "$1"; } + _kseb_estimate_source_bytes sqlite /var/lib/rancher/k3s/server/token required +) +assert_eq 1 "$(probe_estimator_numeric_mode_metadata)" \ + 'numeric stat mode accepts directories and a regular-file source' + +probe_estimator_numeric_stat_argv() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_privileged_estimator_capture() { + printf '<%s>' "$@" + printf '\n' + } + _kseb_privileged_estimator_lstat /var/lib/rancher/k3s/server/token +) +assert_eq '<--format=%d:%i|%f><-->' \ + "$(probe_estimator_numeric_stat_argv 2>/dev/null || true)" \ + 'privileged stat uses locale-independent numeric mode data' + +probe_estimator_out_of_bound() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_privileged_estimator_source_kind() { printf 'present\n'; } + _kseb_estimator_source_identity() { printf '41:1\n'; } + _kseb_privileged_estimator_du() { printf '1\t%s\n' "$1"; } + _kseb_estimate_source_bytes sqlite /tmp/not-a-k3s-source required +) +assert_fails probe_estimator_out_of_bound + +probe_estimator_overflow() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_estimator_source_records() { + printf 'required|/var/lib/rancher/k3s/server/db\n' + printf 'required|/var/lib/rancher/k3s/server/token\n' + } + _kseb_estimator_source_probe() { + [[ "$1" == */server/db ]] && printf '41:1\n' || printf '41:2\n' + } + _kseb_estimate_source_bytes() { + [[ "$2" == */server/db ]] && printf '9223372036854775800\n' || printf '8\n' + } + _kseb_estimate_phase_bytes sqlite +) +assert_fails probe_estimator_overflow +assert_fails probe_estimator_fixed_sources external + +# The production change that makes these arithmetic cases pass is an +# overflow-safe P + ceil(P / 4) reserve calculation. Expected values are +# hand-derived literals rather than values computed with production helpers. +probe_required_recovery_bytes() ( + local phase="$1" phase_bytes="$2" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_required_recovery_bytes "$phase" "$phase_bytes" +) +assert_eq 10737418240 "$(probe_required_recovery_bytes post 0)" 'zero-byte post reserve' +assert_eq 10737418240 "$(probe_required_recovery_bytes pre 0)" 'zero-byte pre reserve' +assert_eq 10737418242 "$(probe_required_recovery_bytes post 1)" 'one-byte post reserve rounds upward' +assert_eq 10737418244 "$(probe_required_recovery_bytes pre 1)" 'one-byte pre reserve rounds upward twice' +assert_eq 10737418245 "$(probe_required_recovery_bytes post 0004)" 'normalized four-byte post reserve' +assert_eq 10737418250 "$(probe_required_recovery_bytes pre 4)" 'four-byte pre reserve' +assert_eq 11811160065 "$(probe_required_recovery_bytes post 858993460)" 'post capacity literal' +assert_eq 12884901890 "$(probe_required_recovery_bytes pre 858993460)" 'pre capacity literal' +assert_eq 9223372036854775807 "$(probe_required_recovery_bytes post 7378697620893886053)" \ + 'maximum signed post reserve' +assert_eq 9223372036854775806 "$(probe_required_recovery_bytes pre 3689348810446943026)" \ + 'maximum signed pre reserve' +assert_fails probe_required_recovery_bytes post 7378697620893886054 +assert_fails probe_required_recovery_bytes pre 3689348810446943027 +assert_fails probe_required_recovery_bytes post 7378697629483820646 +assert_fails probe_required_recovery_bytes post 9223372036854775807 +assert_fails probe_required_recovery_bytes invalid 1 +assert_fails probe_required_recovery_bytes pre -1 +assert_fails probe_required_recovery_bytes post 1.25 +assert_fails probe_required_recovery_bytes post 9223372036854775808 + +probe_phase_capacity_gate() ( + local phase="$1" fixture_phase_bytes="$2" fixture_free_bytes="$3" rc=0 + local log="${bootstrap_fixture_root}/capacity-${phase}-${fixture_free_bytes}.log" + : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_estimate_phase_bytes() { printf 'estimate:%s\n' "$1" >>"$log"; printf '%s\n' "$fixture_phase_bytes"; } + _kseb_privileged_free_bytes() { printf 'free:%s\n' "$1" >>"$log"; printf '%s\n' "$fixture_free_bytes"; } + _kseb_require_phase_capacity "$phase" sqlite /fixture-root || rc=$? + printf 'rc=%s\n' "$rc" >>"$log" + command cat -- "$log" +) +assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=0' \ + "$(probe_phase_capacity_gate pre 858993460 12884901890)" 'pre capacity exact threshold' +assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=1' \ + "$(probe_phase_capacity_gate pre 858993460 12884901889)" 'pre capacity one byte short' +assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=0' \ + "$(probe_phase_capacity_gate post 858993460 11811160065)" 'post capacity exact threshold' +assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=1' \ + "$(probe_phase_capacity_gate post 858993460 11811160064)" 'post capacity one byte short' +assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=0' \ + "$(probe_phase_capacity_gate post 858993460 00011811160065)" 'free-byte normalization' +assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=1' \ + "$(probe_phase_capacity_gate post 858993460 9223372036854775808)" 'free-byte signed overflow rejection' + +# Focused regression: API-derived metadata must be captured before entering the +# backend routine that stops SQLite. The backend deliberately returns failure +# before any filesystem command, so this exercises production orchestration +# without a backup write or host call. +probe_backup_metadata_order() ( + local log="${bootstrap_fixture_root}/metadata-order.log" + : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_require_phase_capacity() { + printf 'capacity:%s:%s:%s\n' "$1" "$2" "$3" >>"$log" + } + _kseb_secret_count() { printf 'secret-count\n' >>"$log"; printf '7\n'; } + _kseb_version() { printf 'version\n' >>"$log"; printf 'v1.36.2+k3s1\n'; } + _kseb_revalidate_backup_root() { :; } + _kseb_pinned_phase_transaction() { + printf 'transaction:%s:count=%s:version=%s\n' "${5:-missing}" "${7:-missing}" "${8:-missing}" >>"$log" + return 1 + } + _kseb_backup_phase pre sqlite /fixture-root /fixture-bundle \ + 00000000-0000-0000-0000-000000000000 20260801T000000Z >/dev/null 2>&1 || true + _kseb_backup_phase post sqlite /fixture-root /fixture-bundle \ + 00000000-0000-0000-0000-000000000000 20260801T000000Z >/dev/null 2>&1 || true + command cat -- "$log" +) +assert_eq $'capacity:pre:sqlite:/fixture-root\nsecret-count\nversion\ntransaction:pre:count=7:version=v1.36.2+k3s1\ncapacity:post:sqlite:/fixture-root\nsecret-count\nversion\ntransaction:post:count=7:version=v1.36.2+k3s1' \ + "$(probe_backup_metadata_order)" 'pre and post each recalculate capacity before stopped-service backend' + +probe_backup_capacity_failure() ( + local phase="$1" log="${bootstrap_fixture_root}/capacity-failure-${1}.log" rc=0 + : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_revalidate_backup_root() { :; } + _kseb_require_phase_capacity() { printf 'capacity:%s\n' "$1" >>"$log"; return 1; } + _kseb_secret_count() { printf 'unexpected-secret-count\n' >>"$log"; } + _kseb_version() { printf 'unexpected-version\n' >>"$log"; } + _kseb_pinned_phase_transaction() { printf 'unexpected-transaction\n' >>"$log"; } + _kseb_backup_phase "$phase" sqlite /fixture-root /fixture-bundle \ + 00000000-0000-0000-0000-000000000000 20260801T000000Z >/dev/null 2>&1 || rc=$? + printf 'rc=%s\n' "$rc" >>"$log" + command cat -- "$log" +) +assert_eq $'capacity:pre\nrc=1' "$(probe_backup_capacity_failure pre)" \ + 'pre capacity failure stops before API metadata and backup transaction' +assert_eq $'capacity:post\nrc=1' "$(probe_backup_capacity_failure post)" \ + 'post capacity failure stops before API metadata and backup transaction' + +# Focused regression: the bootstrap owns a bounded API waiter whose command +# boundary can use the absolute k3s binary. The fake readiness probe is the +# only injected dependency and succeeds on its second call. +probe_bootstrap_api_waiter() ( + local log="${bootstrap_fixture_root}/api-wait.log" counter="${bootstrap_fixture_root}/api-wait.counter" + : >"$log" + rm -f -- "$counter" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_api_ready_once() { + printf 'probe\n' >>"$log" + if [[ -e "$counter" ]]; then return 0; fi + : >"$counter" + return 1 + } + sleep() { printf 'sleep:%s\n' "$1" >>"$log"; } + _kseb_wait_for_api + command cat -- "$log" +) +assert_eq $'probe\nsleep:10\nprobe' "$(probe_bootstrap_api_waiter 2>/dev/null || true)" \ + 'bootstrap bounded absolute-command API waiter' + +# A root-only recovery mount is not traversable by the invoking user, so phase +# collision decisions must be made from privileged evidence rather than shell +# -e checks. This pure production helper makes those decisions testable. +probe_phase_layout_policy() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_phase_layout_allowed pre absent absent absent || return 1 + if _kseb_phase_layout_allowed pre directory '0:0:700:directory' absent; then return 1; fi + _kseb_phase_layout_allowed post absent absent absent || return 1 + _kseb_phase_layout_allowed post directory '0:0:700:directory' directory || return 1 + if _kseb_phase_layout_allowed post directory '0:0:755:directory' directory; then return 1; fi + if _kseb_phase_layout_allowed post directory '0:0:700:directory' absent; then return 1; fi +) +assert_succeeds probe_phase_layout_policy + +# Enabled/start can drift during confirmation or the pre-backup. The second +# inventory is the immediate pre-rotation gate and must stop all later mutation. +assert_fails run_bootstrap_fixture enabled_start_drift --execute --rotate-existing --backup-root "$bootstrap_backup_root" \ + --recovery-policy encrypted-off-host +assert_bootstrap_order enabled_start_drift $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-enabled\nbackup:pre:sqlite\ninventory' + +# The guarded SQLite lifecycle must attempt start after every stop attempt, +# including a stop error or copy failure, and only run recovery after a +# confirmed start. These dependencies are local functions in the probe. +probe_sqlite_guard() ( + local scenario="$1" log="${bootstrap_fixture_root}/sqlite-guard-${1}.log" rc=0 + : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_stop_k3s() { printf 'stop\n' >>"$log"; [[ "$scenario" != stop_failure ]]; } + _kseb_start_k3s() { printf 'start\n' >>"$log"; [[ "$scenario" != start_failure ]]; } + _kseb_sqlite_copy_and_verify() { printf 'copy\n' >>"$log"; [[ "$scenario" != copy_failure ]]; } + _kseb_before_start_dispatch() { + if [[ "$scenario" == pre_start_term ]]; then kill -s TERM "$BASHPID"; fi + } + _kseb_recovery_check() { printf 'recovery\n' >>"$log"; } + _kseb_backup_sqlite_guarded /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$? + printf 'rc=%s\n' "$rc" >>"$log" + command cat -- "$log" +) +assert_eq $'stop\ncopy\nstart\nrecovery\nrc=0' "$(probe_sqlite_guard success 2>/dev/null || true)" \ + 'SQLite guarded success lifecycle' +assert_eq $'stop\ncopy\nstart\nrc=1' "$(probe_sqlite_guard copy_failure 2>/dev/null || true)" \ + 'SQLite copy failure restarts service' +assert_eq $'stop\nstart\nrc=1' "$(probe_sqlite_guard stop_failure 2>/dev/null || true)" \ + 'SQLite stop error still attempts service start' +assert_eq $'stop\ncopy\nstart\nrc=75' "$(probe_sqlite_guard start_failure 2>/dev/null || true)" \ + 'SQLite start failure is not reported as recovery success' +assert_eq $'stop\ncopy\nstart\nrc=143' "$(probe_sqlite_guard pre_start_term 2>/dev/null || true)" \ + 'SQLite pre-start TERM dispatches start once and stops flow' + +probe_quiet_command_boundary() ( + local output rc=0 + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + noisy_backup_failure() { + printf 'sentinel-recovery-absolute-path\n' + printf 'sentinel-server-token-and-encryption-config\n' >&2 + return 42 + } + output="$(_kseb_run_quiet noisy_backup_failure 2>&1)" || rc=$? + [[ "$rc" == 42 && -z "$output" ]] +) +assert_succeeds probe_quiet_command_boundary + +probe_relative_manifest_builder() ( + local fixture="${bootstrap_fixture_root}/manifest-fixture" output="${bootstrap_fixture_root}/manifest-output" + mkdir -p "$fixture/sub" + : >"$fixture/alpha"; : >"$fixture/sub/beta" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_manifest_list_files() { printf '%s\0%s\0' "$fixture/alpha" "$fixture/sub/beta"; } + _kseb_manifest_hash_file() { + case "$1" in */alpha) printf '%064d\n' 1 ;; */beta) printf '%064d\n' 2 ;; *) return 1 ;; esac + } + _kseb_build_relative_manifest "$fixture" "$output" + assert_eq $'0000000000000000000000000000000000000000000000000000000000000001 ./alpha\n0000000000000000000000000000000000000000000000000000000000000002 ./sub/beta' \ + "$(<"$output")" 'relative manifest entries' +) +assert_succeeds probe_relative_manifest_builder + +probe_manifest_hash_failure() ( + local fixture="${bootstrap_fixture_root}/manifest-failure" output="${bootstrap_fixture_root}/manifest-failure-output" + mkdir -p "$fixture" + : >"$fixture/alpha" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_manifest_list_files() { printf '%s\0' "$fixture/alpha"; } + _kseb_manifest_hash_file() { return 1; } + ! _kseb_build_relative_manifest "$fixture" "$output" +) +assert_succeeds probe_manifest_hash_failure + +probe_manifest_list_failure() ( + local fixture="${bootstrap_fixture_root}/manifest-list-failure" output="${bootstrap_fixture_root}/manifest-list-failure-output" + mkdir -p "$fixture" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_manifest_list_files() { return 1; } + if _kseb_build_relative_manifest "$fixture" "$output"; then return 1; fi +) +assert_succeeds probe_manifest_list_failure + +probe_manifest_install_check() ( + local scenario="$1" log="${bootstrap_fixture_root}/manifest-check-${1}.log" + : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_privileged_manifest_lifecycle() { + printf 'build\ninstall\ncheck\n' >>"$log" + [[ "$scenario" != check_failure ]] + } + if [[ "$scenario" == check_failure ]]; then + ! _kseb_write_verification_manifest /phase + else + _kseb_write_verification_manifest /phase + fi + command cat -- "$log" +) +assert_eq $'build\ninstall\ncheck' "$(probe_manifest_install_check success 2>/dev/null || true)" \ + 'manifest installed then checked' +assert_eq $'build\ninstall\ncheck' "$(probe_manifest_install_check check_failure 2>/dev/null || true)" \ + 'manifest check failure propagates' + +probe_backup_root_chain_policy() ( + local scenario="$1" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_path_lstat() { + case "$scenario/$1" in + writable//safe) printf '0:0:777:directory\n' ;; + symlink//safe) printf '0:0:777:symbolic link\n' ;; + *) printf '0:0:755:directory\n' ;; + esac + } + _kseb_backup_root_chain_safe /safe/recovery +) +assert_succeeds probe_backup_root_chain_policy safe +assert_fails probe_backup_root_chain_policy writable +assert_fails probe_backup_root_chain_policy symlink + +probe_backup_root_identity_policy() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_backup_root_identity_matches '41:99' '41:99' || return 1 + if _kseb_backup_root_identity_matches '41:99' '41:100'; then return 1; fi +) +assert_succeeds probe_backup_root_identity_policy + +probe_backend_command_contract() ( + local log="${bootstrap_fixture_root}/backend-command-contract.log" + : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_privileged_exec() { printf '<%s>\n' "$*" >>"$log"; } + _kseb_etcd_snapshot_save pre 20260801T000000Z /phase-pre + _kseb_etcd_snapshot_list /phase-pre + _kseb_etcd_snapshot_save post 20260801T000000Z /phase-post + _kseb_etcd_snapshot_list /phase-post + _kseb_archive_host_config /phase-pre + _kseb_copy_sqlite_datastore /phase-pre + _kseb_compare_sqlite_datastore /phase-pre + command cat -- "$log" +) +assert_eq $'\n\n\n\n\n\n' \ + "$(probe_backend_command_contract 2>/dev/null || true)" 'production backend exact argv contract' + +probe_production_backend_order() ( + local backend="$1" scenario="$2" log="${bootstrap_fixture_root}/production-${1}-${2}.log" rc=0 + : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_copy_sqlite_datastore() { printf 'sqlite-copy\n' >>"$log"; } + _kseb_compare_sqlite_datastore() { printf 'sqlite-compare\n' >>"$log"; } + _kseb_etcd_snapshot_save() { printf 'etcd-save:%s\n' "$1" >>"$log"; } + _kseb_etcd_snapshot_list() { printf 'etcd-list\n' >>"$log"; } + _kseb_copy_common_recovery() { printf 'common-copy\n' >>"$log"; } + _kseb_write_bundle_metadata() { printf 'metadata\n' >>"$log"; } + _kseb_write_verification_manifest() { + printf 'manifest\n' >>"$log" + [[ "$scenario" != manifest_failure ]] + } + _kseb_recovery_check() { printf 'recovery\n' >>"$log"; } + if [[ "$backend" == sqlite ]]; then + _kseb_sqlite_copy_and_verify /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$? + else + _kseb_backup_etcd /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$? + fi + printf 'rc=%s\n' "$rc" >>"$log" + command cat -- "$log" +) +assert_eq $'sqlite-copy\ncommon-copy\nsqlite-compare\nmetadata\nmanifest\nrc=0' \ + "$(probe_production_backend_order sqlite success)" 'production SQLite helper order' +assert_eq $'sqlite-copy\ncommon-copy\nsqlite-compare\nmetadata\nmanifest\nrc=1' \ + "$(probe_production_backend_order sqlite manifest_failure)" 'SQLite manifest failure propagation' +assert_eq $'etcd-save:pre\netcd-list\ncommon-copy\nmetadata\nmanifest\nrecovery\nrc=0' \ + "$(probe_production_backend_order etcd success)" 'production embedded-etcd helper order' +assert_eq $'etcd-save:pre\netcd-list\ncommon-copy\nmetadata\nmanifest\nrc=1' \ + "$(probe_production_backend_order etcd manifest_failure)" 'etcd manifest failure blocks recovery' + +probe_terminal_guidance_contract() ( + local output rc=0 injected + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + output="$(_kseb_terminal_failure reencrypt_stage_start 2>&1)" || rc=$? + [[ "$rc" == 1 && "$output" == 'ERROR: current-state=reencrypt_stage_start recovery=bootstrap/manual/k3s-secret-encryption.md' ]] || return 1 + injected="$(_kseb_terminal_failure 'unsafe-/sentinel-recovery-path-server-token' 2>&1 || true)" + [[ "$injected" == 'ERROR: current-state=unknown recovery=bootstrap/manual/k3s-secret-encryption.md' ]] +) +assert_succeeds probe_terminal_guidance_contract + +assert_public_failure_guidance() { + local scenario="$1" expected_state="$2" output + output="$(<"${bootstrap_fixture_root}/${scenario}.stderr")" + [[ "$output" == *"ERROR: current-state=${expected_state} recovery=bootstrap/manual/k3s-secret-encryption.md"* ]] || + fail "$scenario terminal guidance" + [[ "$output" != *'sentinel-recovery-absolute-path'* && "$output" != *'server-token'* && + "$output" != *'encryption-config'* ]] || fail "$scenario terminal guidance leak" +} +assert_public_failure_guidance timeout_start reencrypt_stage_start +assert_public_failure_guidance mismatch hash_mismatch +assert_public_failure_guidance final_validation_failure final_validation_failed +assert_public_failure_guidance backup_failure pre_backup_failed +assert_fails run_bootstrap_fixture pre_capacity_failure --execute --backup-root "$bootstrap_backup_root" \ + --recovery-policy encrypted-off-host +assert_bootstrap_order pre_capacity_failure $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\ncapacity:pre' +assert_public_failure_guidance pre_capacity_failure pre_backup_failed +assert_fails run_bootstrap_fixture post_capacity_failure --execute --backup-root "$bootstrap_backup_root" \ + --recovery-policy encrypted-off-host +assert_bootstrap_order post_capacity_failure $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\ncapacity:post' +assert_public_failure_guidance post_capacity_failure post_backup_failed +assert_fails run_bootstrap_fixture service_recovery_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_public_failure_guidance service_recovery_failure service_recovery_failed +assert_fails run_bootstrap_fixture post_backup_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_public_failure_guidance post_backup_failure post_backup_failed +assert_fails run_bootstrap_fixture marker_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_public_failure_guidance marker_failure marker_install_failed + +assert_fails run_bootstrap_fixture enable_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_public_failure_guidance enable_failure partial_enable_failed +assert_fails run_bootstrap_fixture dropin_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_public_failure_guidance dropin_failure partial_dropin_failed +assert_fails run_bootstrap_fixture initial_restart_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_public_failure_guidance initial_restart_failure partial_initial_restart_failed +assert_fails run_bootstrap_fixture final_restart_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_public_failure_guidance final_restart_failure partial_final_restart_failed + +probe_recovery_budget_policy() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_recovery_budget_valid 430 130 30 || return 1 + if _kseb_recovery_budget_valid 441 130 30; then return 1; fi +) +assert_succeeds probe_recovery_budget_policy + +probe_workload_ready_policy() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + local empty_status='{"items":[{"status":{"phase":"Running","containerStatuses":[]},"spec":{"containers":[{"name":"app"}],"initContainers":[]}}]}' + local ready='{"items":[{"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":true}],"initContainerStatuses":[]},"spec":{"containers":[{"name":"app"}],"initContainers":[]}}]}' + if _kseb_workloads_ready_json "$empty_status"; then return 1; fi + _kseb_workloads_ready_json "$ready" +) +assert_succeeds probe_workload_ready_policy + +# Regression: after a K3s restart, a transient non-ready Pod must be retried +# inside the existing 30-second workload budget instead of failing the backup +# on the first observation. Clock, sleep, and API reads are file-backed +# because command substitutions execute the read seam in a child shell. +probe_workload_wait_transient() ( + local state_dir="${bootstrap_fixture_root}/workload-wait-transient" + local clock_file="${state_dir}/clock" counter_file="${state_dir}/counter" + local log="${state_dir}/log" + local nonready ready + mkdir -p "$state_dir" + printf '0\n' >"$clock_file" + printf '0\n' >"$counter_file" + : >"$log" + + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + nonready='{"items":[{"metadata":{"name":"fixture","namespace":"default"},"spec":{"containers":[{"name":"app"}],"initContainers":[]},"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":false}],"initContainerStatuses":[]}}]}' + ready='{"items":[{"metadata":{"name":"fixture","namespace":"default"},"spec":{"containers":[{"name":"app"}],"initContainers":[]},"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":true}],"initContainerStatuses":[]}}]}' + + _kseb_now_seconds() { command cat -- "$clock_file"; } + _kseb_read_workloads_json() { + local command_timeout="$1" count + printf 'read:%s\n' "$command_timeout" >>"$log" + count="$(command cat -- "$counter_file")" + count=$((count + 1)) + printf '%s\n' "$count" >"$counter_file" + if (( count == 1 )); then printf '%s\n' "$nonready"; else printf '%s\n' "$ready"; fi + } + sleep() { + local delay="$1" now + printf 'sleep:%s\n' "$delay" >>"$log" + now="$(command cat -- "$clock_file")" + printf '%s\n' "$((now + delay))" >"$clock_file" + } + + _kseb_wait_for_workloads || return 1 + [[ "$(command cat -- "$log")" == $'read:9\nsleep:5\nread:9' ]] +) +assert_succeeds probe_workload_wait_transient + +probe_workload_wait_deadline() ( + local state_dir="${bootstrap_fixture_root}/workload-wait-deadline" + local clock_file="${state_dir}/clock" log="${state_dir}/log" + local nonready + mkdir -p "$state_dir" + printf '0\n' >"$clock_file" + : >"$log" + + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + nonready='{"items":[{"metadata":{"name":"fixture","namespace":"default"},"spec":{"containers":[{"name":"app"}],"initContainers":[]},"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":false}],"initContainerStatuses":[]}}]}' + + _kseb_now_seconds() { command cat -- "$clock_file"; } + _kseb_read_workloads_json() { + printf 'read:%s\n' "$1" >>"$log" + printf '%s\n' "$nonready" + } + sleep() { + local delay="$1" now + printf 'sleep:%s\n' "$delay" >>"$log" + now="$(command cat -- "$clock_file")" + printf '%s\n' "$((now + delay))" >"$clock_file" + } + + if _kseb_wait_for_workloads; then return 1; fi + [[ "$(command cat -- "$log")" == $'read:9\nsleep:5\nread:9\nsleep:5\nread:9\nsleep:5\nread:9\nsleep:5\nread:9\nsleep:5\nread:4\nsleep:5' ]] +) +assert_succeeds probe_workload_wait_deadline + +if [[ "${KSEB_WORKLOAD_WAIT_FOCUS:-}" == source ]]; then + printf 'K3S WORKLOAD WAIT SOURCE TEST PASS\n' + exit 0 +fi + +probe_sqlite_signal_guard() ( + local signal="$1" log="${bootstrap_fixture_root}/sqlite-signal-${1}.log" rc=0 + : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_stop_k3s() { printf 'stop\n' >>"$log"; } + _kseb_start_k3s() { printf 'start\n' >>"$log"; } + _kseb_sqlite_copy_and_verify() { + printf 'copy\n' >>"$log" + kill -s "$signal" "$BASHPID" + printf 'continued-after-signal\n' >>"$log" + } + _kseb_recovery_check() { printf 'recovery\n' >>"$log"; } + _kseb_backup_sqlite_guarded /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$? + printf 'rc=%s\n' "$rc" >>"$log" + command cat -- "$log" +) +assert_eq $'stop\ncopy\nstart\nrc=143' "$(probe_sqlite_signal_guard TERM 2>/dev/null || true)" \ + 'SQLite TERM guard starts once and stops flow' +assert_eq $'stop\ncopy\nstart\nrc=130' "$(probe_sqlite_signal_guard INT 2>/dev/null || true)" \ + 'SQLite INT guard starts once and stops flow' + +# Round 1: a documented root:root 0700 mount cannot be traversed by the +# invoking user. Exercise the production decision with only privileged +# evidence readers replaced. +probe_privileged_root0700_validation() ( + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_privileged_root_kind() { printf 'directory\n'; } + _kseb_privileged_realpath() { printf '%s\n' "$1"; } + _kseb_privileged_root_metadata() { printf '0:0:700:directory\n'; } + _kseb_privileged_backup_root_chain_safe() { return 0; } + _kseb_privileged_mount_source() { [[ "$1" == "$KSEB_DATA_DIR" ]] && printf 'data-source\n' || printf 'backup-source\n'; } + _kseb_privileged_free_bytes() { printf '10737418240\n'; } + _kseb_privileged_root_identity() { printf '41:99\n'; } + _kseb_validate_backup_root /secure/recovery + [[ "$KSEB_BACKUP_ROOT_IDENTITY" == '41:99' ]] +) +assert_succeeds probe_privileged_root0700_validation + +probe_pinned_root_swap_boundary() ( + local scenario="$1" dispatched=false + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + KSEB_BACKUP_ROOT_IDENTITY='41:99' + _kseb_privileged_backup_root_chain_safe() { return 0; } + _kseb_privileged_root_identity() { + [[ "$scenario" == swapped ]] && printf '41:100\n' || printf '41:99\n' + } + _kseb_privileged_relative_components_safe() { return 0; } + _kseb_privileged_pinned_dispatch() { dispatched=true; } + if [[ "$scenario" == swapped ]]; then + if _kseb_pinned_root_operation /secure/recovery bundle/pre verify; then return 1; fi + [[ "$dispatched" == false ]] + else + _kseb_pinned_root_operation /secure/recovery bundle/pre verify + [[ "$dispatched" == true ]] + fi +) +assert_succeeds probe_pinned_root_swap_boundary stable +assert_succeeds probe_pinned_root_swap_boundary swapped + +probe_actual_pinned_root_swap() ( + local parent="${bootstrap_fixture_root}/actual-swap" root dispatched=false + mkdir -p "$parent/recovery/bundle/pre" "$parent/replacement/bundle/pre" + root="$parent/recovery" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" + _kseb_privileged_backup_root_chain_safe() { + mv -- "$root" "$parent/original" + mv -- "$parent/replacement" "$root" + } + _kseb_privileged_root_identity() { stat --format='%d:%i' -- "$1"; } + _kseb_privileged_relative_components_safe() { return 0; } + _kseb_privileged_pinned_dispatch() { dispatched=true; } + if _kseb_pinned_root_operation "$root" bundle/pre verify; then return 1; fi + [[ "$dispatched" == false ]] +) +assert_succeeds probe_actual_pinned_root_swap + +probe_root_only_manifest_entrypoint() ( + local log="${bootstrap_fixture_root}/root-only-manifest.log" + : >"$log" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + mktemp() { printf 'user-mktemp\n' >>"$log"; return 1; } + _kseb_privileged_manifest_lifecycle() { printf 'root-lifecycle\n' >>"$log"; } + _kseb_write_verification_manifest /secure/recovery/bundle/pre + assert_eq 'root-lifecycle' "$(<"$log")" 'manifest lifecycle remains root-only' +) +assert_succeeds probe_root_only_manifest_entrypoint + +probe_production_mismatch_load_boundary() ( + local inventory + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + bash() { return 1; } + _kseb_read_status_json() { + printf '%s\n' '{"stage":"reencrypt_finished","activekey":"AES-CBC fixture-key","enable":true,"hasherror":"mismatch"}' + } + inventory="$(_kseb_load_inventory)" || return 1 + [[ "$inventory" == hash_mismatch\|unsafe\|ambiguous\|invalid\|ambiguous\|mismatch\|mismatch ]] +) +assert_succeeds probe_production_mismatch_load_boundary + +probe_unexpected_stage_output_safety() ( + local output rc=0 output_file="${bootstrap_fixture_root}/unexpected-stage.out" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + _kseb_read_status_json() { + printf '%s\n' '{"stage":"unexpected\nSENTINEL-STAGE","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}' + } + : >"$output_file" + _kseb_wait_for_reencrypt 124 >"$output_file" 2>&1 || rc=$? + output="$(<"$output_file")" + [[ "$rc" != 0 && -z "$output" && "$KSEB_LAST_SAFE_STATE" == reencrypt_stage_unexpected ]] +) +assert_succeeds probe_unexpected_stage_output_safety + +assert_fails run_bootstrap_fixture enabled_finished_drift_provider --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order enabled_finished_drift_provider $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-reencrypted\ninventory' +assert_fails run_bootstrap_fixture enabled_finished_drift_owner --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order enabled_finished_drift_owner $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-reencrypted\ninventory' + +assert_fails run_bootstrap_fixture disabled_backend_drift_external --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order disabled_backend_drift_external $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory' +assert_fails run_bootstrap_fixture disabled_backend_drift_ambiguous --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host +assert_bootstrap_order disabled_backend_drift_ambiguous $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory' + +# Round 2: the production privileged dispatcher must dereference both the root +# and phase directory FDs. Only privilege elevation is replaced; the fixed +# dispatcher and its stat decisions remain production code. +probe_production_fd_dereference() ( + local root="${bootstrap_fixture_root}/fd-dereference" identity metadata + mkdir -p "$root/bundle/pre" + chmod 0700 "$root" "$root/bundle" "$root/bundle/pre" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + identity="$(stat --format='%d:%i' -- "$root")" + declare -F _kseb_run_privileged_shell >/dev/null || return 1 + _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } + metadata="$(_kseb_privileged_pinned_dispatch "$root" "$identity" bundle/pre inspect)" + [[ "$metadata" == "$identity|$(id -u):$(id -g):700:directory" ]] +) +if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == fd ]]; then + assert_succeeds probe_production_fd_dereference +fi + +write_round2_fake_k3s() { + local path="$1" + printf '%s\n' '#!/usr/bin/env bash' \ + 'set -Eeuo pipefail' \ + 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == save ]]; then' \ + ' for argument in "$@"; do case "$argument" in --dir=*) destination=${argument#--dir=} ;; esac; done' \ + ' if [[ -n "${KSEB_SWAP_ROOT:-}" ]]; then mv -- "$KSEB_SWAP_ROOT" "$KSEB_SWAP_ORIGINAL"; mv -- "$KSEB_SWAP_REPLACEMENT" "$KSEB_SWAP_ROOT"; fi' \ + ' if [[ -n "${KSEB_SWAP_PHASE:-}" ]]; then mv -- "$KSEB_SWAP_PHASE" "$KSEB_SWAP_PHASE_ORIGINAL"; mv -- "$KSEB_SWAP_PHASE_REPLACEMENT" "$KSEB_SWAP_PHASE"; fi' \ + ' : >"${destination:?}/fixture-snapshot"; exit 0' \ + 'fi' \ + 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == list ]]; then exit 0; fi' \ + 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == --raw=/readyz ]]; then exit 0; fi' \ + 'if [[ "${1:-}" == kubectl && "${2:-}" == wait ]]; then exit 0; fi' \ + 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == pods ]]; then printf "%s\\n" "{\"items\":[]}"; exit 0; fi' \ + 'exit 1' >"$path" + chmod 0700 "$path" +} + +# This fixture renames and replaces the public root after the production +# transaction has opened it. Snapshot, common copy, manifest, sizing and the +# post marker must all stay on the original FD-derived tree. +probe_complete_transaction_actual_swap() ( + local parent="${bootstrap_fixture_root}/round2-swap" root original replacement + local data config fake_k3s marker evidence output expected_metadata + parent="$(mktemp -d "${bootstrap_fixture_root}/round2-swap.XXXXXX")" + root="$parent/recovery"; original="$parent/original"; replacement="$parent/replacement" + data="$parent/data"; config="$parent/config"; fake_k3s="$parent/k3s" + marker="$parent/platform-post-bundle.env"; evidence="$parent/platform-restore-evidence.env" + mkdir -p "$root" "$replacement" "$data/server/cred" "$config" + chmod 0700 "$root" "$replacement" + printf 'token\n' >"$data/server/token" + printf '{}\n' >"$data/server/cred/encryption-config.json" + printf 'config\n' >"$config/config.yaml" + : >"$evidence" + write_round2_fake_k3s "$fake_k3s" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" + declare -F _kseb_pinned_phase_transaction >/dev/null || return 1 + _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } + _kseb_transaction_data_dir() { printf '%s\n' "$data"; } + _kseb_transaction_config_dir() { printf '%s\n' "$config"; } + _kseb_transaction_systemd_paths() { :; } + _kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; } + _kseb_transaction_post_marker() { printf '%s\n' "$marker"; } + _kseb_transaction_restore_evidence() { printf '%s\n' "$evidence"; } + export KSEB_SWAP_ROOT="$root" KSEB_SWAP_ORIGINAL="$original" KSEB_SWAP_REPLACEMENT="$replacement" + output="$(_kseb_pinned_phase_transaction "$root" bundle/post embedded-etcd \ + 01234567-89ab-4cde-8fab-0123456789ab post 20260801T000000Z 7 v1.36.2+k3s1)" + expected_metadata=$'schema=platform-k3s-bundle-v1\nbundle_id=01234567-89ab-4cde-8fab-0123456789ab\nphase=post\nk3s_version=v1.36.2+k3s1\ndatastore=embedded-etcd\ncreated_at_utc=2026-08-01T00:00:00Z\nsecret_count=7' + [[ "$output" =~ ^[0-9]+$ && -f "$original/bundle/post/fixture-snapshot" && + -f "$original/bundle/post/server-token" && -f "$original/bundle/post/verification.manifest" && + -f "$marker" && "$(<"$marker")" == "$expected_metadata" && + "$(<"$original/bundle/post/bundle.env")" == "$expected_metadata" && + ! -e "$root/bundle" ]] +) +if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == transaction ]]; then + assert_succeeds probe_complete_transaction_actual_swap +fi + +# The manifest lifecycle must open its phase directory relative to the pinned +# root FD and validate the dereferenced directory target before hashing. +probe_manifest_from_pinned_handle() ( + local parent root data config fake_k3s phase original_phase replacement_phase output + local rc=0 + parent="$(mktemp -d "${bootstrap_fixture_root}/round2-manifest.XXXXXX")" + root="$parent/recovery"; data="$parent/data"; config="$parent/config"; fake_k3s="$parent/k3s" + phase="$root/bundle/pre"; original_phase="$root/bundle/original-pre" + replacement_phase="$parent/replacement-pre" + mkdir -p "$root" "$data/server/cred" "$config" "$replacement_phase" + chmod 0700 "$root" + printf 'token\n' >"$data/server/token" + printf '{}\n' >"$data/server/cred/encryption-config.json" + printf 'config\n' >"$config/config.yaml" + write_round2_fake_k3s "$fake_k3s" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" + declare -F _kseb_pinned_phase_transaction >/dev/null || return 1 + _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } + _kseb_transaction_data_dir() { printf '%s\n' "$data"; } + _kseb_transaction_config_dir() { printf '%s\n' "$config"; } + _kseb_transaction_systemd_paths() { :; } + _kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; } + export KSEB_SWAP_ROOT='' KSEB_SWAP_PHASE="$phase" KSEB_SWAP_PHASE_ORIGINAL="$original_phase" + export KSEB_SWAP_PHASE_REPLACEMENT="$replacement_phase" + output="$(_kseb_pinned_phase_transaction "$root" bundle/pre embedded-etcd bundle pre 20260801T000000Z 7 v1.36.2+k3s1 2>&1)" || rc=$? + [[ "$rc" != 0 && "$output" != *"$parent"* && -d "$original_phase" && + -z "$(find "$original_phase" -mindepth 1 -print -quit)" && -d "$phase" && + ! -e "$phase/verification.manifest" ]] +) +if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == manifest ]]; then + assert_succeeds probe_manifest_from_pinned_handle +fi + +write_round2_fake_systemctl() { + local path="$1" + printf '%s\n' '#!/usr/bin/env bash' \ + 'set -Eeuo pipefail' \ + 'printf "%s\\n" "$1" >>"${KSEB_SYSTEMCTL_LOG:?}"' \ + 'if [[ "$1" == start && "${KSEB_SIGNAL_DURING_START:-0}" == 1 ]]; then kill -s TERM "$PPID"; fi' \ + 'exit 0' >"$path" + chmod 0700 "$path" +} + +write_workload_wait_fake_k3s() { + local path="$1" + printf '%s\n' '#!/usr/bin/env bash' \ + 'set -Eeuo pipefail' \ + 'printf "%s|%s %s %s\n" "${KSEB_WORKLOAD_SCENARIO:?}" "${1:-}" "${2:-}" "${3:-}" >>"${KSEB_K3S_CALL_LOG:?}"' \ + 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == save ]]; then' \ + ' for argument in "$@"; do case "$argument" in --dir=*) destination=${argument#--dir=} ;; esac; done' \ + ' : >"${destination:?}/fixture-snapshot"' \ + ' exit 0' \ + 'fi' \ + 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == list ]]; then exit 0; fi' \ + 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == --raw=/readyz ]]; then' \ + ' if [[ "${KSEB_WORKLOAD_SCENARIO:?}" == api_failure ]]; then exit 9; fi' \ + ' exit 0' \ + 'fi' \ + 'if [[ "${1:-}" == kubectl && "${2:-}" == wait ]]; then exit 0; fi' \ + 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == pods ]]; then' \ + ' count=$(<"${KSEB_WORKLOAD_COUNTER:?}")' \ + ' count=$((count + 1))' \ + ' printf "%s\n" "$count" >"$KSEB_WORKLOAD_COUNTER"' \ + ' printf "read\n" >>"${KSEB_WORKLOAD_LOG:?}"' \ + ' if [[ "$KSEB_WORKLOAD_SCENARIO" == transient && "$count" == 1 ]]; then' \ + ' ready=false' \ + ' elif [[ "$KSEB_WORKLOAD_SCENARIO" == persistent ]]; then' \ + ' ready=false' \ + ' else' \ + ' ready=true' \ + ' fi' \ + ' printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"fixture\",\"namespace\":\"default\"},\"spec\":{\"containers\":[{\"name\":\"app\"}],\"initContainers\":[]},\"status\":{\"phase\":\"Running\",\"containerStatuses\":[{\"name\":\"app\",\"ready\":${ready}}],\"initContainerStatuses\":[]}}]}"' \ + ' exit 0' \ + 'fi' \ + 'exit 1' >"$path" + chmod 0700 "$path" +} + +probe_pinned_workload_recovery() ( + local scenario="$1" parent root data config fake_k3s fake_systemctl + local systemctl_log workload_log workload_counter k3s_call_log marker evidence stderr_log output='' rc=0 + parent="$(mktemp -d "${bootstrap_fixture_root}/pinned-workload-${scenario}.XXXXXX")" + root="$parent/recovery"; data="$parent/data"; config="$parent/config" + fake_k3s="$parent/k3s"; fake_systemctl="$parent/systemctl" + systemctl_log="$parent/systemctl.log"; workload_log="$parent/workload.log" + k3s_call_log="$parent/k3s-call.log" + workload_counter="$parent/workload.counter" + marker="$parent/platform-post-bundle.env"; evidence="$parent/platform-restore-evidence.env" + stderr_log="$parent/transaction.stderr" + mkdir -p "$root" "$data/server/db" "$config" + chmod 0700 "$root" + printf 'state\n' >"$data/server/db/state.db" + printf 'token\n' >"$data/server/token" + printf 'config\n' >"$config/config.yaml" + : >"$systemctl_log"; : >"$workload_log"; : >"$k3s_call_log"; printf '0\n' >"$workload_counter" + write_workload_wait_fake_k3s "$fake_k3s" + write_round2_fake_systemctl "$fake_systemctl" + + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" + _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } + _kseb_transaction_data_dir() { printf '%s\n' "$data"; } + _kseb_transaction_config_dir() { printf '%s\n' "$config"; } + _kseb_transaction_systemd_paths() { :; } + _kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; } + _kseb_transaction_systemctl_binary() { printf '%s\n' "$fake_systemctl"; } + _kseb_transaction_post_marker() { printf '%s\n' "$marker"; } + _kseb_transaction_restore_evidence() { printf '%s\n' "$evidence"; } + export KSEB_SYSTEMCTL_LOG="$systemctl_log" KSEB_SIGNAL_DURING_START=0 + export KSEB_WORKLOAD_SCENARIO="$scenario" KSEB_WORKLOAD_COUNTER="$workload_counter" + export KSEB_WORKLOAD_LOG="$workload_log" KSEB_K3S_CALL_LOG="$k3s_call_log" + + output="$(_kseb_privileged_pinned_dispatch "$root" "$KSEB_BACKUP_ROOT_IDENTITY" \ + bundle/pre transaction "$data" "$config" "$fake_k3s" "$fake_systemctl" \ + "$marker" "$evidence" '' sqlite 01234567-89ab-4cde-8fab-0123456789ab \ + pre 20260801T000000Z 7 v1.36.2+k3s1 pre 2>"$stderr_log")" || rc=$? + + case "$scenario" in + transient) + if [[ "$rc" != 0 || ! "$output" =~ ^[0-9]+$ || + "$(command cat -- "$systemctl_log")" != $'stop\nstart' || + "$(command cat -- "$workload_log")" != $'read\nread' || + ! -f "$root/bundle/pre/datastore/state.db" || + ! -f "$root/bundle/pre/verification.manifest" ]]; then + printf 'PINNED_TRANSIENT_DIAG rc=%s systemctl_lines=%s workload_lines=%s k3s_calls=%s manifest=%s stderr_lines=%s\n' \ + "$rc" "$(wc -l <"$systemctl_log" | tr -d '[:space:]')" \ + "$(wc -l <"$workload_log" | tr -d '[:space:]')" \ + "$(wc -l <"$k3s_call_log" | tr -d '[:space:]')" \ + "$([[ -f "$root/bundle/pre/verification.manifest" ]] && printf yes || printf no)" \ + "$(wc -l <"$stderr_log" | tr -d '[:space:]')" >&2 + command cat -- "$k3s_call_log" >&2 + return 1 + fi + ;; + persistent) + if [[ "$rc" != 75 || -n "$output" || + "$(command cat -- "$systemctl_log")" != $'stop\nstart' || + "$(wc -l <"$workload_log" | tr -d '[:space:]')" -lt 2 || + ! -d "$root/bundle/pre" || + -n "$(find "$root/bundle/pre" -mindepth 1 -print -quit)" ]]; then + printf 'PINNED_PERSISTENT_DIAG rc=%s output_bytes=%s systemctl_lines=%s workload_lines=%s k3s_calls=%s phase_exists=%s phase_entries=%s\n' \ + "$rc" "${#output}" "$(wc -l <"$systemctl_log" | tr -d '[:space:]')" \ + "$(wc -l <"$workload_log" | tr -d '[:space:]')" \ + "$(wc -l <"$k3s_call_log" | tr -d '[:space:]')" \ + "$([[ -d "$root/bundle/pre" ]] && printf yes || printf no)" \ + "$(find "$root/bundle/pre" -mindepth 1 -printf . 2>/dev/null | wc -c | tr -d '[:space:]')" >&2 + command cat -- "$k3s_call_log" >&2 + return 1 + fi + ;; + api_failure) + if [[ "$rc" != 75 || -n "$output" || + "$(command cat -- "$systemctl_log")" != $'stop\nstart' || + -s "$workload_log" || ! -d "$root/bundle/pre" || + -n "$(find "$root/bundle/pre" -mindepth 1 -print -quit)" ]]; then + printf 'PINNED_API_FAILURE_DIAG rc=%s output_bytes=%s systemctl_lines=%s workload_lines=%s k3s_calls=%s phase_exists=%s phase_entries=%s\n' \ + "$rc" "${#output}" "$(wc -l <"$systemctl_log" | tr -d '[:space:]')" \ + "$(wc -l <"$workload_log" | tr -d '[:space:]')" \ + "$(wc -l <"$k3s_call_log" | tr -d '[:space:]')" \ + "$([[ -d "$root/bundle/pre" ]] && printf yes || printf no)" \ + "$(find "$root/bundle/pre" -mindepth 1 -printf . 2>/dev/null | wc -c | tr -d '[:space:]')" >&2 + command cat -- "$k3s_call_log" >&2 + return 1 + fi + ;; + *) return 1 ;; + esac +) +assert_succeeds probe_pinned_workload_recovery transient +assert_succeeds probe_pinned_workload_recovery persistent +assert_succeeds probe_pinned_workload_recovery api_failure + +if [[ "${KSEB_WORKLOAD_WAIT_FOCUS:-}" == pinned ]]; then + printf 'K3S WORKLOAD WAIT PINNED TEST PASS\n' + exit 0 +fi + +# Trigger failure after stop so EXIT cleanup performs the real production start +# dispatch. TERM arrives from inside that command; cleanup must not be cut off +# and start must be attempted exactly once. +probe_cleanup_window_signal() ( + local parent data root systemctl_log fake_systemctl rc=0 + parent="$(mktemp -d "${bootstrap_fixture_root}/round2-cleanup.XXXXXX")" + data="$parent/data"; root="$parent/recovery"; systemctl_log="$parent/systemctl.log" + fake_systemctl="$parent/systemctl" + mkdir -p "$data/server/db" "$root" + chmod 0700 "$root" + : >"$systemctl_log" + write_round2_fake_systemctl "$fake_systemctl" + # Deliberately omit server/token so the pinned transaction enters cleanup. + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" + _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } + _kseb_transaction_data_dir() { printf '%s\n' "$data"; } + _kseb_transaction_config_dir() { printf '%s\n' "$parent/no-config"; } + _kseb_transaction_systemd_paths() { :; } + _kseb_transaction_systemctl_binary() { printf '%s\n' "$fake_systemctl"; } + export KSEB_SYSTEMCTL_LOG="$systemctl_log" KSEB_SIGNAL_DURING_START=1 + _kseb_pinned_phase_transaction "$root" bundle/pre sqlite bundle pre 20260801T000000Z 7 v1.36.2+k3s1 >/dev/null 2>&1 || rc=$? + [[ "$rc" != 0 && "$(<"$systemctl_log")" == $'stop\nstart' && + -d "$root/bundle/pre" && -z "$(find "$root/bundle/pre" -mindepth 1 -print -quit)" ]] +) +if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == cleanup ]]; then + assert_succeeds probe_cleanup_window_signal +fi + +write_round3_fake_k3s() { + local path="$1" + printf '%s\n' '#!/usr/bin/env bash' \ + 'set -Eeuo pipefail' \ + 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == save ]]; then' \ + ' for argument in "$@"; do case "$argument" in --dir=*) destination=${argument#--dir=} ;; esac; done' \ + ' : >"${destination:?}/partial-before-swap"' \ + ' mv -- "${KSEB_R3_PHASE:?}" "${KSEB_R3_ORIGINAL:?}"' \ + ' mv -- "${KSEB_R3_REPLACEMENT:?}" "${KSEB_R3_PHASE:?}"' \ + ' case "${KSEB_R3_SCENARIO:?}" in' \ + ' snapshot_failure) exit 9 ;;' \ + ' manifest_failure) : >"${destination}/invalid manifest name" ;;' \ + ' commit_swap) : >"${destination}/valid-after-swap" ;;' \ + ' *) exit 91 ;;' \ + ' esac' \ + ' exit 0' \ + 'fi' \ + 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == list ]]; then exit 0; fi' \ + 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == --raw=/readyz ]]; then exit 0; fi' \ + 'if [[ "${1:-}" == kubectl && "${2:-}" == wait ]]; then exit 0; fi' \ + 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == pods ]]; then printf "%s\\n" "{\"items\":[]}"; exit 0; fi' \ + 'exit 1' >"$path" + chmod 0700 "$path" +} + +# A production transaction must never clean or commit through the old phase +# pathname after that pathname has been replaced. Only privilege elevation +# and the k3s external effect are replaced; FD creation, manifest, cleanup, +# recovery, marker and commit decisions are the real fixed dispatcher. +probe_round3_phase_swap_identity() ( + local scenario="$1" parent root data config fake_k3s phase original_phase replacement_phase + local marker evidence output rc=0 + parent="$(mktemp -d "${bootstrap_fixture_root}/round3-${scenario}.XXXXXX")" + root="$parent/recovery"; data="$parent/data"; config="$parent/config"; fake_k3s="$parent/k3s" + phase="$root/bundle/post"; original_phase="$root/bundle/original-post" + replacement_phase="$parent/replacement-post" + marker="$parent/platform-post-bundle.env"; evidence="$parent/platform-restore-evidence.env" + mkdir -p "$root" "$data/server/cred" "$config" "$replacement_phase" + chmod 0700 "$root" "$replacement_phase" + printf 'token\n' >"$data/server/token" + printf '{}\n' >"$data/server/cred/encryption-config.json" + printf 'config\n' >"$config/config.yaml" + printf 'replacement-untouched\n' >"$replacement_phase/replacement-sentinel" + printf 'restore-untouched\n' >"$evidence" + write_round3_fake_k3s "$fake_k3s" + # shellcheck source=/dev/null + source "$BOOTSTRAP_PATH" + KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" + _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } + _kseb_transaction_data_dir() { printf '%s\n' "$data"; } + _kseb_transaction_config_dir() { printf '%s\n' "$config"; } + _kseb_transaction_systemd_paths() { :; } + _kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; } + _kseb_transaction_post_marker() { printf '%s\n' "$marker"; } + _kseb_transaction_restore_evidence() { printf '%s\n' "$evidence"; } + export KSEB_R3_SCENARIO="$scenario" KSEB_R3_PHASE="$phase" KSEB_R3_ORIGINAL="$original_phase" + export KSEB_R3_REPLACEMENT="$replacement_phase" + output="$(_kseb_pinned_phase_transaction "$root" bundle/post embedded-etcd bundle post 20260801T000000Z 7 v1.36.2+k3s1 2>&1)" || rc=$? + [[ "$rc" != 0 && "$output" != *"$parent"* && -d "$phase" && + "$(<"$phase/replacement-sentinel")" == replacement-untouched && + -d "$original_phase" && -z "$(find "$original_phase" -mindepth 1 -print -quit)" && + ! -e "$marker" && "$(<"$evidence")" == restore-untouched ]] +) + +if [[ -z "${KSEB_ROUND3_FOCUS:-}" || "${KSEB_ROUND3_FOCUS:-}" == failure ]]; then + assert_succeeds probe_round3_phase_swap_identity snapshot_failure +fi +if [[ -z "${KSEB_ROUND3_FOCUS:-}" || "${KSEB_ROUND3_FOCUS:-}" == manifest ]]; then + assert_succeeds probe_round3_phase_swap_identity manifest_failure +fi +if [[ -z "${KSEB_ROUND3_FOCUS:-}" || "${KSEB_ROUND3_FOCUS:-}" == commit ]]; then + assert_succeeds probe_round3_phase_swap_identity commit_swap +fi + +bash "${REPOSITORY_ROOT}/scripts/validate/test-k3s-secret-encryption-restore-evidence.sh" + +printf 'K3S SECRET ENCRYPTION STATUS TEST PASS\n' diff --git a/scripts/validate/test-observability-alerting.sh b/scripts/validate/test-observability-alerting.sh new file mode 100755 index 0000000..11a858a --- /dev/null +++ b/scripts/validate/test-observability-alerting.sh @@ -0,0 +1,273 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly ALERTING_ROOT="$ROOT/services/observability/alerting" +readonly ALERTMANAGER_CONFIG="$ALERTING_ROOT/platform-alertmanager.yaml" +readonly PUBLIC_HTTPS_POLICY="$ALERTING_ROOT/alertmanager-public-https-egress.yaml" +readonly ALERTING_KUSTOMIZATION="$ALERTING_ROOT/kustomization.yaml" +readonly OBSERVABILITY_KUSTOMIZATION="$ROOT/services/observability/kustomization.yaml" +readonly KPS_ROOT="$ROOT/services/observability/kube-prometheus-stack" +readonly KPS_VALUES="$ROOT/services/observability/kube-prometheus-stack/values/home.yaml" + +usage() { + printf 'usage: %s [--server-dry-run]\n' "${0##*/}" >&2 + exit 2 +} + +server_dry_run=false +case $# in + 0) ;; + 1) + [[ "$1" == --server-dry-run ]] || usage + server_dry_run=true + ;; + *) usage ;; +esac + +python3 - \ + "$ALERTMANAGER_CONFIG" \ + "$PUBLIC_HTTPS_POLICY" \ + "$ALERTING_KUSTOMIZATION" \ + "$OBSERVABILITY_KUSTOMIZATION" \ + "$KPS_VALUES" <<'PY' +import pathlib +import sys + +import yaml + + +def fail(reason): + print(f"FAIL: observability alerting contract: {reason}", file=sys.stderr) + raise SystemExit(1) + + +def load_one(path): + source = pathlib.Path(path) + try: + with source.open(encoding="utf-8") as stream: + documents = [item for item in yaml.safe_load_all(stream) if item is not None] + except (OSError, yaml.YAMLError): + fail(f"{source.name} is missing or invalid YAML") + if len(documents) != 1 or not isinstance(documents[0], dict): + fail(f"{source.name} must contain exactly one object") + return documents[0] + + +alert_path, policy_path, child_kustomization_path, root_kustomization_path, values_path = sys.argv[1:] +alert = load_one(alert_path) +policy = load_one(policy_path) +child = load_one(child_kustomization_path) +root = load_one(root_kustomization_path) +values = load_one(values_path) + +if alert != { + "apiVersion": "monitoring.coreos.com/v1alpha1", + "kind": "AlertmanagerConfig", + "metadata": { + "name": "platform-alertmanager", + "namespace": "observability", + "labels": {"observability.hyeonworks.com/instance": "home"}, + }, + "spec": { + "route": { + "receiver": "platform-slack", + "groupBy": ["cluster", "namespace", "alertname", "severity"], + "groupWait": "30s", + "groupInterval": "5m", + "repeatInterval": "4h", + "routes": [ + { + "receiver": "platform-null", + "matchers": [ + { + "name": "alertname", + "matchType": "=", + "value": "InfoInhibitor", + } + ], + } + ], + }, + "inhibitRules": [ + { + "sourceMatch": [ + { + "name": "alertname", + "matchType": "=", + "value": "InfoInhibitor", + } + ], + "targetMatch": [ + { + "name": "severity", + "matchType": "=", + "value": "info", + } + ], + "equal": ["namespace"], + } + ], + "receivers": [ + {"name": "platform-null"}, + { + "name": "platform-slack", + "slackConfigs": [ + { + "apiURL": { + "name": "alertmanager-slack-webhook", + "key": "url", + }, + "sendResolved": True, + "linkNames": False, + "mrkdwnIn": ["text", "fields"], + "fallback": '{{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}', + "title": '{{ if eq .Status "firing" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}', + "titleLink": "https://grafana.learn.hyeonworks.com/", + "color": '{{ if eq .Status "resolved" }}good{{ else if or (eq .CommonLabels.severity "emergency") (eq .CommonLabels.severity "critical") }}danger{{ else if eq .CommonLabels.severity "warning" }}warning{{ else }}#439FE0{{ end }}', + "fields": [ + { + "title": "Status", + "value": "{{ .Status | toUpper }}", + "short": True, + }, + { + "title": "Severity", + "value": "{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}", + "short": True, + }, + { + "title": "Location", + "value": "{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}", + "short": True, + }, + { + "title": "Alert count", + "value": "{{ len .Alerts }}", + "short": True, + }, + ], + "text": "{{ range .Alerts }}\n*Alert status:* {{ .Status | toUpper }}\n*Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}\n*Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}\n*Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}\n*Started:* {{ .StartsAt.Format \"2006-01-02T15:04:05Z07:00\" }}\n{{ if eq .Status \"resolved\" }}*Ended:* {{ .EndsAt.Format \"2006-01-02T15:04:05Z07:00\" }}{{ end }}\n{{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}\n\n{{ end }}\n · ", + "footer": "hyeonworks observability · Alertmanager", + } + ], + } + ], + }, +}: + fail("platform-alertmanager must match the exact global Slack contract") + +if policy != { + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": "observability-allow-alertmanager-public-https", + "namespace": "observability", + }, + "spec": { + "podSelector": { + "matchLabels": { + "app.kubernetes.io/name": "alertmanager", + "app.kubernetes.io/instance": "observability-core-kube-pr-alertmanager", + } + }, + "policyTypes": ["Egress"], + "egress": [ + { + "to": [ + { + "ipBlock": { + "cidr": "0.0.0.0/0", + "except": [ + "10.0.0.0/8", + "100.64.0.0/10", + "172.16.0.0/12", + "192.168.0.0/16", + ], + } + } + ], + "ports": [{"protocol": "TCP", "port": 443}], + } + ], + }, +}: + fail("Alertmanager egress must allow only public IPv4 TCP/443") + +if child.get("apiVersion") != "kustomize.config.k8s.io/v1beta1" or child.get("kind") != "Kustomization": + fail("alerting kustomization identity is invalid") +if child.get("namespace") != "observability": + fail("alerting kustomization namespace must be observability") +if child.get("resources") != [ + "platform-alertmanager.yaml", + "alertmanager-public-https-egress.yaml", +]: + fail("alerting kustomization resources are not exact") +if set(child) != {"apiVersion", "kind", "namespace", "resources"}: + fail("alerting kustomization contains unapproved behavior") + +root_resources = root.get("resources") +if not isinstance(root_resources, list) or len(root_resources) != len(set(root_resources)): + fail("observability root resources must be a unique list") +for required in ("targets", "rules/platform", "alerting", "dashboards/platform"): + if root_resources.count(required) != 1: + fail(f"observability root must include {required} exactly once") + +try: + alertmanager_spec = values["alertmanager"]["alertmanagerSpec"] +except (KeyError, TypeError): + fail("kube-prometheus-stack alertmanagerSpec is missing") +if alertmanager_spec.get("alertmanagerConfiguration") != {"name": "platform-alertmanager"}: + fail("global alertmanagerConfiguration reference is not exact") +for forbidden in ("alertmanagerConfigSelector", "alertmanagerConfigNamespaceSelector"): + if forbidden in alertmanager_spec: + fail(f"{forbidden} must remain absent when using the global reference") + +for document, label in ((alert, "AlertmanagerConfig"), (policy, "NetworkPolicy")): + if document.get("kind") == "Secret": + fail(f"{label} source must not contain a Secret") + +print("OBSERVABILITY ALERTING STATIC CONTRACT PASS") +PY + +render_dir="$(mktemp -d /tmp/platform-observability-alerting-test.XXXXXX)" +cleanup() { + case "$render_dir" in + /tmp/platform-observability-alerting-test.*) rm -rf -- "$render_dir" ;; + esac +} +trap cleanup EXIT +chmod 0700 "$render_dir" +kubectl kustomize --enable-helm "$KPS_ROOT" >"$render_dir/kps.yaml" +chmod 0600 "$render_dir/kps.yaml" +python3 - "$render_dir/kps.yaml" <<'PY' +import sys + +import yaml + + +with open(sys.argv[1], encoding="utf-8") as stream: + items = [item for item in yaml.safe_load_all(stream) if item is not None] +alertmanagers = [item for item in items if item.get("kind") == "Alertmanager"] +if len(alertmanagers) != 1: + raise SystemExit("FAIL: rendered KPS must contain exactly one Alertmanager") +spec = alertmanagers[0].get("spec") or {} +if spec.get("alertmanagerConfiguration") != {"name": "platform-alertmanager"}: + raise SystemExit("FAIL: rendered Alertmanager global reference is not exact") +for forbidden in ("alertmanagerConfigSelector", "alertmanagerConfigNamespaceSelector"): + if forbidden in spec: + raise SystemExit(f"FAIL: rendered Alertmanager retains forbidden {forbidden}") +print("OBSERVABILITY ALERTMANAGER GLOBAL RENDER CONTRACT PASS") +PY + +if [[ "$server_dry_run" == true ]]; then + readonly KUBECTL_BIN="${PLATFORM_KUBECTL_BIN:-/usr/local/bin/kubectl}" + [[ "$KUBECTL_BIN" == /* && -f "$KUBECTL_BIN" && -x "$KUBECTL_BIN" && ! -L "$KUBECTL_BIN" ]] || { + printf 'FAIL: PLATFORM_KUBECTL_BIN must be an executable absolute regular file\n' >&2 + exit 1 + } + "$KUBECTL_BIN" --request-timeout=10s apply --dry-run=server -f "$ALERTMANAGER_CONFIG" >/dev/null + "$KUBECTL_BIN" --request-timeout=10s apply --dry-run=server -f "$PUBLIC_HTTPS_POLICY" >/dev/null + printf 'OBSERVABILITY ALERTING SERVER DRY RUN PASS\n' +fi diff --git a/scripts/validate/test-observability-dashboards.sh b/scripts/validate/test-observability-dashboards.sh new file mode 100755 index 0000000..2101e0f --- /dev/null +++ b/scripts/validate/test-observability-dashboards.sh @@ -0,0 +1,612 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P) +DASHBOARD_ROOT="$ROOT/services/observability/dashboards/platform" +METRIC_ROOT=${PLATFORM_OBSERVABILITY_METRIC_ROOT:-/tmp/platform-observability-metrics.VUpsZn} +TARGET_INVENTORY="$METRIC_ROOT/target-initial/inventory.json" +POST_INVENTORY="$METRIC_ROOT/post-substrate/inventory.json" +PROMETHEUS_IMAGE='quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b' +WORK_DIR='' +PROMETHEUS_BASE_URL='' +PORT_FORWARD_PID='' + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +pass() { + printf 'PASS: %s\n' "$*" +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "required command is unavailable: $1" +} + +usage() { + printf 'Usage: %s [--prometheus-base-url http://127.0.0.1:PORT]\n' "${0##*/}" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --prometheus-base-url) + [[ $# -ge 2 ]] || fail '--prometheus-base-url requires a value' + [[ -z "$PROMETHEUS_BASE_URL" ]] || fail '--prometheus-base-url may be specified only once' + PROMETHEUS_BASE_URL=$2 + shift 2 + ;; + --help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac + done + + if [[ -n "$PROMETHEUS_BASE_URL" ]]; then + [[ "$PROMETHEUS_BASE_URL" =~ ^http://(127\.0\.0\.1|localhost):([1-9][0-9]{0,4})$ ]] \ + || fail '--prometheus-base-url must be an exact loopback HTTP origin without a path' + (( BASH_REMATCH[2] <= 65535 )) \ + || fail '--prometheus-base-url port exceeds 65535' + fi +} + +cleanup() { + local rc=$? + trap - EXIT + if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" 2>/dev/null; then + kill "$PORT_FORWARD_PID" 2>/dev/null || true + wait "$PORT_FORWARD_PID" 2>/dev/null || true + fi + if [[ -n "$WORK_DIR" && -d "$WORK_DIR" ]]; then + rm -rf -- "$WORK_DIR" + fi + exit "$rc" +} + +verify_inventory() { + local phase=$1 + local inventory="$METRIC_ROOT/$phase/inventory.json" + local checksum="$METRIC_ROOT/$phase/inventory.sha256" + local expected actual + + [[ -f "$inventory" ]] || fail "$phase inventory is absent" + [[ -f "$checksum" ]] || fail "$phase inventory checksum is absent" + jq -e --arg phase "$phase" ' + .schema == "platform-observability-metric-inventory/v1" + and .phase == $phase + and (.targets | type == "array" and length > 0) + and all(.targets[]; .health == "up" and (.metrics | type == "array" and length > 0)) + ' "$inventory" >/dev/null || fail "$phase inventory contract is invalid" + + read -r expected checksum_name < "$checksum" + [[ "$checksum_name" == 'inventory.json' ]] || fail "$phase checksum names an unexpected file" + actual=$(sha256sum "$inventory" | awk '{print $1}') + [[ "$actual" == "$expected" ]] || fail "$phase inventory checksum mismatch" + pass "$phase inventory contract and checksum" +} + +inventory_has_metric() { + local inventory=$1 + local metric=$2 + jq -e --arg metric "$metric" 'any(.targets[].metrics[]; .name == $metric)' "$inventory" >/dev/null +} + +inventory_metric_has_label() { + local inventory=$1 + local metric=$2 + local label=$3 + jq -e --arg metric "$metric" --arg label "$label" ' + any(.targets[].metrics[]; .name == $metric and (.label_names | index($label) != null)) + ' "$inventory" >/dev/null +} + +validate_inventory_references() { + local dashboard=$1 + local inventory=$2 + local expression selector metric matchers matcher label token + local inventory_metrics referenced_metrics found + + inventory_metrics=$(jq -r '[.targets[].metrics[].name] | unique[]' "$inventory") + + while IFS= read -r expression; do + [[ -n "$expression" ]] || fail "empty PromQL expression in $dashboard" + referenced_metrics='' + + while IFS= read -r selector; do + [[ -n "$selector" ]] || continue + metric=${selector%%\{*} + grep -Fxq "$metric" <<<"$inventory_metrics" \ + || fail "$dashboard references absent metric $metric" + if ! grep -Fxq "$metric" <<<"$referenced_metrics"; then + referenced_metrics+="${metric}"$'\n' + fi + matchers=${selector#*\{} + matchers=${matchers%\}} + while IFS= read -r matcher; do + matcher=${matcher#"${matcher%%[![:space:]]*}"} + [[ -n "$matcher" ]] || continue + label=$(sed -E 's/^([A-Za-z_][A-Za-z0-9_]*).*/\1/' <<<"$matcher") + [[ "$label" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] \ + || fail "$dashboard has an unauditable label matcher in $expression" + inventory_metric_has_label "$inventory" "$metric" "$label" \ + || fail "$dashboard references absent label $metric.$label" + done < <(tr ',' '\n' <<<"$matchers") + done < <(grep -oE '[A-Za-z_:][A-Za-z0-9_:]*\{[^}]*\}' <<<"$expression" || true) + + [[ -n "$referenced_metrics" ]] || fail "$dashboard expression has no inventory metric: $expression" + + while IFS= read -r token; do + [[ -n "$token" ]] || continue + if grep -Fxq "$token" <<<"$inventory_metrics"; then + grep -Fxq "$token" <<<"$referenced_metrics" \ + || fail "$dashboard uses inventory metric $token without an auditable selector" + fi + done < <(grep -oE '[A-Za-z_:][A-Za-z0-9_:]*' <<<"$expression" | sort -u) + + while IFS= read -r label; do + [[ -n "$label" ]] || continue + found=0 + while IFS= read -r metric; do + [[ -n "$metric" ]] || continue + if inventory_metric_has_label "$inventory" "$metric" "$label"; then + found=1 + break + fi + done <<<"$referenced_metrics" + [[ "$found" -eq 1 ]] || fail "$dashboard groups by absent inventory label $label" + done < <( + grep -oE '(by|without)[[:space:]]*\([^)]*\)' <<<"$expression" \ + | sed -E 's/^[^(]*\(([^)]*)\)$/\1/' \ + | tr ',' '\n' \ + | sed -E 's/^[[:space:]]+|[[:space:]]+$//g' \ + || true + ) + done < <(jq -r '.panels[].targets[].expr' "$dashboard") +} + +validate_dashboard() { + local file=$1 + local expected_uid=$2 + local inventory=$3 + local path="$DASHBOARD_ROOT/$file" + + [[ -f "$path" ]] || fail "dashboard source is absent: $file" + jq -e --arg uid "$expected_uid" ' + type == "object" + and .uid == $uid + and (.title | type == "string" and length > 0) + and .schemaVersion == 42 + and .editable == false + and .refresh == "30s" + and (.templating.list == []) + and (.panels | type == "array" and length > 0) + and ([.panels[].id] | length == (unique | length)) + and all(.panels[]; + .type != "row" + and (.title | type == "string" and length > 0) + and (.gridPos.w > 0 and .gridPos.h > 0) + and .datasource.type == "prometheus" + and .datasource.uid == "prometheus" + and (.targets | type == "array" and length > 0) + and all(.targets[]; + .datasource.type == "prometheus" + and .datasource.uid == "prometheus" + and (.expr | type == "string" and length > 0) + and (.refId | type == "string" and length > 0) + ) + ) + ' "$path" >/dev/null || fail "dashboard structure is invalid: $file" + + if jq -r '.title, .tags[], .panels[].title' "$path" \ + | grep -Eiq '(^|[^[:alnum:]_])(spring|jvm|kafka|batch|backup)([^[:alnum:]_]|$)'; then + fail "dashboard contains a forbidden product area: $file" + fi + if jq -r '.panels[].targets[].expr' "$path" \ + | grep -Eiq '(^|[^[:alnum:]_])(spring_|jvm_|kafka_|batch_|backup_)'; then + fail "dashboard contains a forbidden product query: $file" + fi + if jq -r '.. | strings' "$path" \ + | grep -Eiq '(request[_ -]?id|trace[_ -]?id|username|raw[_ -]?url)'; then + fail "dashboard contains forbidden or high-cardinality content: $file" + fi + + validate_inventory_references "$path" "$inventory" + pass "$file structure and inventory references" +} + +validate_rendered_configmaps() { + local rendered=$1 + local file name block + local -a files=( + kubernetes-node + workload-health + platform-services + observability-backends + https-endpoints + ) + + kubectl kustomize "$DASHBOARD_ROOT" > "$rendered" + [[ $(grep -c '^kind: ConfigMap$' "$rendered") -eq 5 ]] \ + || fail 'kustomization must render exactly five ConfigMaps' + + for file in "${files[@]}"; do + name="grafana-dashboard-$file" + block=$(awk -v name="$name" ' + BEGIN { RS="---" } + $0 ~ "name: " name "([[:space:]]|$)" { print } + ' "$rendered") + [[ -n "$block" ]] || fail "rendered ConfigMap is absent: $name" + grep -Eq 'grafana_dashboard: ("1"|1)$' <<<"$block" \ + || fail "$name lacks grafana_dashboard=1" + grep -q 'observability.hyeonworks.com/instance: home' <<<"$block" \ + || fail "$name lacks the observability instance label" + grep -q 'observability.hyeonworks.com/owner: platform-observability' <<<"$block" \ + || fail "$name lacks the observability owner label" + grep -q " $file.json:" <<<"$block" \ + || fail "$name does not embed $file.json" + done + pass 'kustomization renders five stable labeled dashboard ConfigMaps' +} + +validate_inventory_driven_omissions() { + local workload="$DASHBOARD_ROOT/workload-health.json" + local services="$DASHBOARD_ROOT/platform-services.json" + local endpoints="$DASHBOARD_ROOT/https-endpoints.json" + + if inventory_has_metric "$TARGET_INVENTORY" kube_pod_container_status_last_terminated_reason; then + fail 'the OOM omission contract no longer matches target-initial inventory' + fi + if jq -r '.panels[].targets[].expr' "$workload" \ + | grep -Fq 'kube_pod_container_status_last_terminated_reason'; then + fail 'workload dashboard guessed an OOM metric absent from target-initial inventory' + fi + pass 'OOM panel omitted: kube_pod_container_status_last_terminated_reason is absent from target-initial inventory' + + jq -e ' + ([.panels[].targets[].expr] | index("sum(kube_persistentvolumeclaim_status_phase{phase=\"Pending\"})") != null) + and + ([.panels[].targets[].expr] | index("sum(kube_persistentvolumeclaim_status_phase{phase=\"Lost\"})") != null) + ' "$workload" >/dev/null \ + || fail 'PVC health panel must retain zero-valued Pending and Lost series instead of filtering healthy state away' + pass 'PVC health panel keeps visible zero-valued Pending and Lost aggregates' + + if jq -e ' + any(.targets[].metrics[]; + (.name | startswith("traefik_")) + and (.name | endswith("_bucket")) + and (.name | test("duration|request")) + ) + ' "$TARGET_INVENTORY" >/dev/null; then + fail 'the Traefik latency omission contract no longer matches target-initial inventory' + fi + if jq -r '.panels[].targets[].expr' "$services" \ + | grep -Eq 'histogram_quantile|traefik_.*_bucket'; then + fail 'service dashboard guessed a Traefik latency histogram absent from target-initial inventory' + fi + pass 'Traefik p95/p99 omitted: no request duration histogram bucket exists in target-initial inventory' + + inventory_has_metric "$POST_INVENTORY" probe_success \ + || fail 'post-substrate inventory lacks probe_success for the platform boundary summary' + jq -e ' + any(.panels[].targets[]; + .expr == "min by (job) (probe_success{})" + ) + ' "$services" >/dev/null \ + || fail 'platform service dashboard lacks the inventory-backed boundary health summary' + pass 'platform service boundary summary uses post-substrate probe_success' + + jq -e ' + any(.panels[].targets[]; + .expr == "max by (instance) (probe_http_status_code{job=\"blackbox-private-edge\"})" + ) + ' "$endpoints" >/dev/null \ + || fail 'HTTPS dashboard does not isolate the observed private-edge 403 boundary' + pass 'HTTPS dashboard isolates the observed blackbox-private-edge status boundary' +} + +validate_promql_syntax() { + local rules=$1 + local file expression index=0 + local -a files=( + kubernetes-node.json + workload-health.json + platform-services.json + observability-backends.json + https-endpoints.json + ) + + { + printf 'groups:\n' + printf ' - name: dashboard.promql.syntax\n' + printf ' rules:\n' + for file in "${files[@]}"; do + while IFS= read -r expression; do + index=$((index + 1)) + printf ' - record: dashboard_syntax_%d\n' "$index" + printf ' expr: %s\n' "$(jq -Rn --arg expr "$expression" '$expr')" + done < <(jq -r '.panels[].targets[].expr' "$DASHBOARD_ROOT/$file") + done + } > "$rules" + + if [[ -n ${PROMTOOL_BIN:-} ]]; then + "$PROMTOOL_BIN" check rules "$rules" >/dev/null + else + require_command docker + docker run --rm \ + --entrypoint=/bin/promtool \ + -v "$rules:/tmp/dashboard-rules.yaml:ro" \ + "$PROMETHEUS_IMAGE" check rules /tmp/dashboard-rules.yaml >/dev/null + fi + pass "promtool parsed $index dashboard expressions" +} + +validate_traefik_low_traffic_ratio() { + local rules="$WORK_DIR/traefik-ratio-rules.yaml" + local tests="$WORK_DIR/traefik-ratio-tests.yaml" + local expression + + expression=$(jq -er ' + .panels[] + | select(.title == "Traefik 5xx Ratio") + | .targets[] + | select(.refId == "A") + | .expr + ' "$DASHBOARD_ROOT/platform-services.json") \ + || fail 'Traefik 5xx ratio expression is absent' + + { + printf 'groups:\n' + printf ' - name: dashboard.traefik.ratio\n' + printf ' interval: 1m\n' + printf ' rules:\n' + printf ' - record: dashboard_traefik_5xx_ratio_percent\n' + printf ' expr: %s\n' "$(jq -Rn --arg expr "$expression" '$expr')" + } > "$rules" + + { + printf 'rule_files:\n' + printf ' - traefik-ratio-rules.yaml\n' + printf 'evaluation_interval: 1m\n' + printf 'tests:\n' + printf ' - interval: 1m\n' + printf ' input_series:\n' + printf ' - series: '\''traefik_entrypoint_requests_total{code="200",entrypoint="websecure"}'\''\n' + printf ' values: '\''0+1x10'\''\n' + printf ' - series: '\''traefik_entrypoint_requests_total{code="500",entrypoint="websecure"}'\''\n' + printf ' values: '\''0+1x10'\''\n' + printf ' promql_expr_test:\n' + printf ' - expr: dashboard_traefik_5xx_ratio_percent\n' + printf ' eval_time: 10m\n' + printf ' exp_samples:\n' + printf ' - labels: '\''dashboard_traefik_5xx_ratio_percent{}'\''\n' + printf ' value: 50\n' + } > "$tests" + + if [[ -n ${PROMTOOL_BIN:-} ]]; then + (cd "$WORK_DIR" && "$PROMTOOL_BIN" test rules traefik-ratio-tests.yaml) >/dev/null + else + require_command docker + docker run --rm \ + --entrypoint=/bin/promtool \ + -v "$WORK_DIR:/tmp/dashboard-validation:ro" \ + -w /tmp/dashboard-validation \ + "$PROMETHEUS_IMAGE" test rules traefik-ratio-tests.yaml >/dev/null + fi + pass 'Traefik 5xx ratio preserves 50 percent at low request rates' +} + +start_prometheus_port_forward() { + local log="$WORK_DIR/prometheus-port-forward.log" + local port='' + local attempt + + kubectl -n observability port-forward \ + --address=127.0.0.1 \ + service/observability-core-kube-pr-prometheus \ + :9090 > "$log" 2>&1 & + PORT_FORWARD_PID=$! + + for ((attempt = 1; attempt <= 100; attempt++)); do + if grep -Eq '^Forwarding from 127\.0\.0\.1:[0-9]+ -> 9090$' "$log"; then + port=$(sed -nE 's/^Forwarding from 127\.0\.0\.1:([0-9]+) -> 9090$/\1/p' "$log" | head -n 1) + break + fi + if ! kill -0 "$PORT_FORWARD_PID" 2>/dev/null; then + fail "Prometheus port-forward exited before readiness: $(tr '\n' ' ' < "$log")" + fi + sleep 0.1 + done + + [[ "$port" =~ ^[1-9][0-9]*$ ]] \ + || fail 'Prometheus port-forward did not bind a loopback port within 10 seconds' + PROMETHEUS_BASE_URL="http://127.0.0.1:$port" +} + +prometheus_get() { + local endpoint=$1 + shift + curl \ + --fail \ + --silent \ + --show-error \ + --connect-timeout 2 \ + --max-time 10 \ + --get \ + "$PROMETHEUS_BASE_URL$endpoint" \ + "$@" +} + +wait_for_prometheus() { + local attempt + for ((attempt = 1; attempt <= 50; attempt++)); do + if prometheus_get '/-/ready' >/dev/null 2>&1; then + return 0 + fi + sleep 0.1 + done + fail 'Prometheus did not become ready within 5 seconds' +} + +live_query_nonempty() { + local expression=$1 + local context=$2 + local expected_scalar=${3:-} + local quiet=${4:-0} + local response="$WORK_DIR/live-query-response.json" + + if ! prometheus_get '/api/v1/query' \ + --data-urlencode "query=$expression" > "$response"; then + fail "live Prometheus query request failed: $context" + fi + jq -e ' + .status == "success" + and (.data.resultType == "vector" or .data.resultType == "scalar") + and (.data.result | type == "array" and length > 0) + ' "$response" >/dev/null \ + || fail "live Prometheus query returned no vector/scalar result: $context" + + if [[ -n "$expected_scalar" ]] && ! jq -e --arg expected "$expected_scalar" ' + .data.resultType == "scalar" + and .data.result[1] == $expected + ' "$response" >/dev/null; then + if [[ "$quiet" -eq 1 ]]; then + return 1 + fi + fail "live Prometheus scalar result was not exact $expected_scalar: $context" + fi +} + +validate_scalar_one_gate_regression() { + if (live_query_nonempty 'scalar(vector(0))' 'scalar-zero regression fixture' 1 1); then + fail 'scalar-zero comparison fixture passed the success gate' + fi + live_query_nonempty 'scalar(vector(1))' 'scalar-one regression fixture' 1 + pass 'success comparison gate rejects scalar 0 and accepts exact scalar 1' +} + +validate_live_dashboard_queries() { + local dashboard uid panel ref expression + local count=0 + + for dashboard in "$DASHBOARD_ROOT"/*.json; do + while IFS=$'\t' read -r uid panel ref expression; do + live_query_nonempty "$expression" "$uid / $panel / $ref" + count=$((count + 1)) + done < <( + jq -r ' + .uid as $uid + | .panels[] + | .title as $panel + | .targets[] + | [$uid, $panel, .refId, .expr] + | @tsv + ' "$dashboard" + ) + done + + [[ "$count" -eq 37 ]] || fail "expected 37 live dashboard queries, got $count" + pass 'live Prometheus accepted 37 dashboard queries with nonempty vector/scalar results' +} + +validate_live_matcher_selectors() { + local dashboard expression selector + local selectors="$WORK_DIR/dashboard-matchers.txt" + local count=0 + + : > "$selectors" + for dashboard in "$DASHBOARD_ROOT"/*.json; do + while IFS= read -r expression; do + grep -oE '[A-Za-z_:][A-Za-z0-9_:]*\{[^}]+\}' <<<"$expression" >> "$selectors" || true + done < <(jq -r '.panels[].targets[].expr' "$dashboard") + done + sort -u -o "$selectors" "$selectors" + + while IFS= read -r selector; do + [[ -n "$selector" ]] || continue + live_query_nonempty "$selector" "live matcher selector $selector" + count=$((count + 1)) + done < "$selectors" + + [[ "$count" -gt 0 ]] || fail 'no dashboard matcher selectors were discovered' + pass "live Prometheus found series for $count dashboard matcher selectors" +} + +validate_live_matcher_values() { + local -a presence_contracts=( + 'node_cpu_seconds_total{mode="idle"}' + 'node_filesystem_avail_bytes{mountpoint="/"}' + 'node_filesystem_size_bytes{mountpoint="/"}' + 'node_filesystem_files_free{mountpoint="/"}' + 'node_filesystem_files{mountpoint="/"}' + 'kube_persistentvolumeclaim_status_phase{phase="Pending"}' + 'kube_persistentvolumeclaim_status_phase{phase="Lost"}' + ) + local -a success_contracts=( + 'scalar(count(count by (bucket) (minio_cluster_usage_buckets_total_bytes{bucket=~"loki|tempo"}))) == bool 2' + 'scalar(count(count by (bucket) (minio_cluster_usage_buckets_quota_total_bytes{bucket=~"loki|tempo"}))) == bool 2' + 'scalar(min(probe_success{job="blackbox-private-edge"})) == bool 1' + 'scalar(count(probe_http_status_code{job="blackbox-private-edge"})) == bool scalar(count(probe_http_status_code{job="blackbox-private-edge"} == 403))' + ) + local contract + + for contract in "${presence_contracts[@]}"; do + live_query_nonempty "$contract" "live matcher-value contract $contract" + done + for contract in "${success_contracts[@]}"; do + live_query_nonempty "$contract" "live matcher-value contract $contract" 1 + done + pass 'live matcher values cover idle CPU, root filesystems, PVC phases, both buckets, and private-edge 403' +} + +validate_live_prometheus() { + require_command curl + if [[ -z "$PROMETHEUS_BASE_URL" ]]; then + start_prometheus_port_forward + fi + wait_for_prometheus + validate_scalar_one_gate_regression + validate_live_dashboard_queries + validate_live_matcher_selectors + validate_live_matcher_values +} + +main() { + local work rendered rules + parse_args "$@" + require_command jq + require_command sha256sum + require_command kubectl + + verify_inventory target-initial + verify_inventory post-substrate + [[ -d "$DASHBOARD_ROOT" ]] || fail 'dashboard source directory is absent' + + validate_dashboard kubernetes-node.json platform-kubernetes-node "$TARGET_INVENTORY" + validate_dashboard workload-health.json platform-workload-health "$TARGET_INVENTORY" + validate_dashboard platform-services.json platform-services "$POST_INVENTORY" + validate_dashboard observability-backends.json platform-observability-backends "$TARGET_INVENTORY" + validate_dashboard https-endpoints.json platform-https-endpoints "$POST_INVENTORY" + validate_inventory_driven_omissions + + work=$(mktemp -d) + WORK_DIR=$work + chmod 0755 "$WORK_DIR" + trap cleanup EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + rendered="$work/rendered.yaml" + rules="$work/dashboard-rules.yaml" + validate_rendered_configmaps "$rendered" + validate_promql_syntax "$rules" + validate_traefik_low_traffic_ratio + validate_live_prometheus + + pass 'observability dashboard contract' +} + +main "$@" diff --git a/scripts/validate/test-observability-slack-risk-policy.sh b/scripts/validate/test-observability-slack-risk-policy.sh new file mode 100644 index 0000000..2c67e87 --- /dev/null +++ b/scripts/validate/test-observability-slack-risk-policy.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This closed policy check intentionally reports only labels and token counts. +readonly ROOT='/home/donghyeon/workspace/platform' +readonly CENTRAL_DOCS='/home/donghyeon/workspace/docs/platform' +readonly MASTER="$CENTRAL_DOCS/plans/2026-07-31-observability-phase4.md" +readonly ACCESS="$CENTRAL_DOCS/plans/2026-07-31-observability-access-alerting.md" +readonly MANUAL="$ROOT/bootstrap/manual/phase4-observability-access.md" +readonly WORKLOG="$CENTRAL_DOCS/runbooks/2026-08-13-observability-phase4-resume-worklog.md" +readonly HISTORICAL_PLAN="$CENTRAL_DOCS/plans/2026-08-11-gitea-metrics-checkpoint.md" +readonly HISTORICAL_DESIGN="$CENTRAL_DOCS/plans/2026-08-11-gitea-metrics-checkpoint-design.md" + +failures=0 + +fail() { + printf 'FAIL: %s\n' "$1" >&2 + failures=$((failures + 1)) +} + +require_file() { + [[ -f "$1" && ! -L "$1" ]] || fail "$2 is not a regular file" +} + +count_token() { + local file=$1 token=$2 + grep -Foc -- "$token" "$file" || true +} + +require_count() { + local file=$1 label=$2 token=$3 expected=$4 actual + actual="$(count_token "$file" "$token")" + [[ "$actual" == "$expected" ]] || fail "$label token_count=$actual expected=$expected" +} + +require_hash() { + local file=$1 label=$2 expected=$3 actual + actual="$(sha256sum -- "$file" | awk '{print $1}')" + [[ "$actual" == "$expected" ]] || fail "$label sha256=$actual expected=$expected" +} + +require_execution_relationships() { + python3 - "$ACCESS" "$MANUAL" <<'PY' +import pathlib +import sys + +access = pathlib.Path(sys.argv[1]).read_text(encoding='utf-8') +manual = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8') + +def commands(text): + result = [] + lines = text.splitlines() + index = 0 + while index < len(lines): + line = lines[index] + if line.lstrip().startswith('bash scripts/bootstrap/create-observability-secrets.sh'): + start = index + parts = [line.strip()] + while parts[-1].rstrip().endswith('\\') and index + 1 < len(lines): + index += 1 + parts.append(lines[index].strip()) + result.append((start, ' '.join(parts))) + index += 1 + return result + +def slack_execute(items): + return [item for item in items if '--execute' in item[1] and '--slack-webhook' in item[1]] + +def checker_after_count(text, command_index): + return sum( + index > command_index and '--check-slack-deployment-evidence' in line + for index, line in enumerate(text.splitlines()) + ) + +failures = 0 +def require(label, actual, expected): + global failures + if actual != expected: + print(f'FAIL: {label} count={actual} expected={expected}') + failures += 1 + +hostile = '''padding padding padding padding padding padding padding padding padding padding padding padding +--check-slack-deployment-evidence +bash scripts/bootstrap/create-observability-secrets.sh --execute --slack-webhook --accept-no-off-host-slack-recovery +''' +hostile_slack = slack_execute(commands(hostile)) +require('hostile checker-before-Slack ordering', checker_after_count(hostile, hostile_slack[0][0]), 0) + +step = '- [ ] **Step 3: 실행 순서를 manual 문서에 고정한다**' +start = access.find(step) +if start < 0: + require('authoritative access cutover block', 0, 1) + block = '' +else: + fence_start = access.find(' ```bash', start) + fence_end = access.find(' ```', fence_start + len(' ```bash')) + block = access[fence_start:fence_end] if fence_start >= 0 and fence_end >= 0 else '' + require('authoritative access cutover block', int(bool(block)), 1) + +access_slack = slack_execute(commands(block)) +require('authoritative access Slack execute command', len(access_slack), 1) +if len(access_slack) == 1: + require('authoritative access Slack risk flag', int('--accept-no-off-host-slack-recovery' in access_slack[0][1]), 1) + +manual_slack = slack_execute(commands(manual)) +require('manual Slack execute command', len(manual_slack), 1) +if len(manual_slack) == 1: + command_index, command = manual_slack[0] + require('manual Slack risk flag', int('--accept-no-off-host-slack-recovery' in command), 1) + require('manual deployment checker after Slack bootstrap', checker_after_count(manual, command_index), 1) + +sys.exit(1 if failures else 0) +PY +} + +require_file "$MASTER" master +require_file "$ACCESS" access +require_file "$MANUAL" manual +require_file "$WORKLOG" worklog +require_file "$HISTORICAL_PLAN" historical_plan +require_file "$HISTORICAL_DESIGN" historical_design + +if (( failures == 0 )); then + require_count "$MASTER" 'master deployment gate' '--check-slack-deployment-evidence' 1 + require_count "$ACCESS" 'access deployment gate' '--check-slack-deployment-evidence' 2 + require_count "$ACCESS" 'access strict recovery gate' '--check-slack-recovery-evidence' 1 + require_count "$MANUAL" 'manual deployment gate' '--check-slack-deployment-evidence' 1 + require_count "$MANUAL" 'manual strict recovery gate' '--check-slack-recovery-evidence' 1 + require_count "$MASTER" 'master strict DR-only wording' 'off-host Slack DR-complete 판정에만 사용' 1 + require_count "$ACCESS" 'access strict DR-only wording' 'operational apply를 허용하지 않는다' 1 + require_count "$MANUAL" 'manual risk-path DR-deferred wording' '이 경로는 DR-complete을 주장하지 않습니다' 1 + require_count "$MANUAL" 'manual risk flag' '--accept-no-off-host-slack-recovery' 1 + require_count "$MANUAL" 'manual exact risk confirmation' 'ACCEPT NO OFF-HOST SLACK RECOVERY default' 1 + require_count "$MANUAL" 'manual kubectl prohibition' 'kubectl로 Secret을 수동 생성하지 않습니다' 1 + require_count "$MANUAL" 'manual false recovery prohibition' '거짓 `RECOVERY SLACK default` 확인을 입력하지 않습니다' 1 + require_count "$MANUAL" 'manual two deployment-gate calls' '두 번 호출' 1 + require_count "$MANUAL" 'manual v2 acceptance schema' 'platform-observability-rules-alerts-v2' 1 + require_count "$MANUAL" 'manual bare risk ledger value' '`RISK_ACCEPTED`' 1 + require_count "$MANUAL" 'manual bare recovery ledger value' '`RECOVERY`' 1 + require_count "$ACCESS" 'access risk flag order' '--accept-no-off-host-slack-recovery' 3 + require_count "$MASTER" 'master operational acceptance status' 'Phase 4 operational acceptance: complete with active Slack DR exception' 1 + require_count "$MASTER" 'master deferred DR status' 'Slack off-host disaster recovery: deferred / not complete' 1 + require_count "$ACCESS" 'access operational acceptance status' 'Phase 4 operational acceptance: complete with active Slack DR exception' 1 + require_count "$ACCESS" 'access deferred DR status' 'Slack off-host disaster recovery: deferred / not complete' 1 + require_count "$WORKLOG" 'worklog user decision' 'Slack risk acceptance decision' 1 + require_count "$WORKLOG" 'worklog KeePass deferred' 'KeePass work remains deferred' 1 + require_count "$WORKLOG" 'worklog absent Slack Secret' 'alertmanager-slack-webhook Secret remains absent' 1 + require_hash "$HISTORICAL_PLAN" 'historical plan byte preservation' '10ccc38835928304bd631a60680ecc8e6c1d565d51cbe7047169eb609aa38ce5' + require_hash "$HISTORICAL_DESIGN" 'historical design byte preservation' 'a9fd69e0241e5b56c5328076648a8ecf4a6f71d416aae7d53453ce7ff969249d' + if ! require_execution_relationships; then + failures=$((failures + 1)) + fi +fi + +if [[ -d "$ROOT/docs" ]]; then + fail 'platform docs boundary token_count=1 expected=0' +fi + +if (( failures != 0 )); then + exit 1 +fi + +printf 'PASS: observability Slack risk policy token_counts=exact\n' diff --git a/scripts/validate/test-observability-smoke.sh b/scripts/validate/test-observability-smoke.sh new file mode 100644 index 0000000..2925c23 --- /dev/null +++ b/scripts/validate/test-observability-smoke.sh @@ -0,0 +1,1111 @@ +#!/usr/bin/env bash + +# Focused tests for the observability acceptance state machine. The production +# artifact is executed for CLI behavior and sourced for pure validators and +# lifecycle orchestration; no live command, sudo, Secret, or mutation is used. +set -Eeuo pipefail +set +x +umask 077 + +readonly TEST_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly SMOKE="$TEST_ROOT/scripts/validate/observability-smoke.sh" +TEST_WORK='' +RAW_DELETE_WORK='' +ASSERTIONS=0 + +fail() { printf 'OBSERVABILITY SMOKE TEST FAILURE: %s\n' "$*" >&2; exit 1; } +pass() { ASSERTIONS=$((ASSERTIONS + 1)); printf 'PASS: %s\n' "$1"; } +assert_eq() { [[ "$1" == "$2" ]] || fail "$3: expected=[$1] actual=[$2]"; } +assert_contains() { [[ "$1" == *"$2"* ]] || fail "$3"; } +assert_not_contains() { [[ "$1" != *"$2"* ]] || fail "$3"; } +expect_failure() { local rc=0; "$@" >/dev/null 2>&1 || rc=$?; (( rc != 0 )) || fail "expected failure: $*"; } + +cleanup() { + trap - EXIT HUP INT TERM + case "$RAW_DELETE_WORK" in /tmp/platform-observability-smoke.????????) rm -rf -- "$RAW_DELETE_WORK" ;; esac + case "$TEST_WORK" in /tmp/platform-observability-smoke-test.??????) rm -rf -- "$TEST_WORK" ;; esac +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +[[ -f "$SMOKE" && ! -L "$SMOKE" ]] || + fail 'production smoke is missing (RED: create observability-smoke.sh)' + +TEST_WORK="$(mktemp -d /tmp/platform-observability-smoke-test.XXXXXX)" +chmod 0700 "$TEST_WORK" + +readonly EXPECTED_DRY_RUN=$'OBSERVABILITY_SMOKE_DRY_RUN=PASS\nCHECK_01=Grafana public/private DNS exactness\nCHECK_02=Grafana certificate chain,hostname,one-SAN,expiry\nCHECK_03=LAN,Tailscale,denied-source,public-edge HTTPS status\nCHECK_04=Gitea,Grafana,Keycloak public metrics denial\nCHECK_05=unknown-SNI and Traefik NodePort boundary\nCHECK_06=exact Prometheus target allowlist and health\nCHECK_07=Gitea,Keycloak,CNPG,AIStor,Traefik metric contracts\nCHECK_08=untrusted-Pod AIStor and blackbox denial\nCHECK_09=public,private,internal,TLS blackbox series\nCHECK_10=Grafana datasource health and Loki-to-Tempo trace link\nCHECK_11=AIStor quota and recording-rule series\nCHECK_12=Grafana,Alertmanager restart persistence\nCHECK_13=Certbot dry-run deploy-hook and Nginx health\nHUMAN_OIDC=admin,viewer,denied,break-glass,session-revoke,relogin-denied\nHUMAN_SLACK=firing,resolved\nHUMAN_EXTERNAL_CLIENT=required\nTEMPORARY_OBJECTS=NetworkPolicy,Pod,AlertmanagerSilence,PrometheusRule\nPRESERVE=Secret,PVC,preexisting-Keycloak-membership\nMUTATION=NOT_REQUESTED' + +# Mutation caught: moving a command check or external read before the no-arg +# branch makes the command trace non-empty and breaks the exact output. +mkdir -m 0700 "$TEST_WORK/bin" +for command_name in dirname kubectl curl dig openssl nc sudo; do + printf '#!/usr/bin/env bash\nprintf %s\\n "%s" >>"%s"\nexit 97\n' \ + "'%s'" "$command_name" "$TEST_WORK/external.log" >"$TEST_WORK/bin/$command_name" + chmod 0755 "$TEST_WORK/bin/$command_name" +done +: >"$TEST_WORK/external.log" +dry_output="$(PATH="$TEST_WORK/bin:/usr/bin:/bin" bash "$SMOKE")" +assert_eq "$EXPECTED_DRY_RUN" "$dry_output" 'default exact plan' +[[ ! -s "$TEST_WORK/external.log" ]] || fail 'default invocation touched an external boundary' +pass 'default invocation is exact and no-contact' + +# Mutation caught: accepting an extra/duplicated flag could accidentally enter +# an execute path the operator did not request. +for argument in --wrong --execute=1 --context default; do + rc=0; PATH="$TEST_WORK/bin:/usr/bin:/bin" bash "$SMOKE" $argument >/dev/null 2>&1 || rc=$? + assert_eq 1 "$rc" "unsupported argument status: $argument" +done +[[ ! -s "$TEST_WORK/external.log" ]] || fail 'bad CLI touched an external boundary' +pass 'CLI rejects every non-contract argument before contact' + +override_rc=0 +override_output="$(PLATFORM_OBSERVABILITY_SMOKE_KUBECTL=/tmp/not-kubectl bash "$SMOKE" 2>&1)" || override_rc=$? +assert_eq 1 "$override_rc" 'production override status' +assert_contains "$override_output" 'production boundary override is forbidden' 'production override message' +source_only_rc=0 +source_only_output="$(PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 bash "$SMOKE" 2>&1)" || source_only_rc=$? +assert_eq 1 "$source_only_rc" 'source-only environment bypass status' +assert_contains "$source_only_output" 'production boundary override is forbidden' 'source-only environment bypass message' +pass 'production invocation rejects test seams' + +# Source-only mode exposes pure validators without running main. +PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$SMOKE" + +# Production command constants must themselves satisfy the non-symlink +# executable boundary used by execute preflight on this host. +declare -F validate_command_boundaries >/dev/null || fail 'command-boundary validator is missing' +validate_command_boundaries +pass 'production command boundaries resolve to fixed regular executables' + +mkdir -m 0700 "$TEST_WORK/hostile-bin" +cat >"$TEST_WORK/hostile-bin/stat" <<'SH' +#!/usr/bin/env bash +printf 'hostile-stat\n' >>"${OBS_SMOKE_HOSTILE_PATH_LOG:?}" +exec /usr/bin/stat "$@" +SH +chmod 0755 "$TEST_WORK/hostile-bin/stat" +: >"$TEST_WORK/hostile-path.log" +RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)" +chmod 0700 "$RAW_DELETE_WORK" +: >"$RAW_DELETE_WORK/file"; chmod 0600 "$RAW_DELETE_WORK/file" +( + WORK="$RAW_DELETE_WORK" + PATH="$TEST_WORK/hostile-bin:/usr/bin:/bin" OBS_SMOKE_HOSTILE_PATH_LOG="$TEST_WORK/hostile-path.log" \ + safe_work_file "$WORK/file" +) || fail 'fixed utility boundary setup' +[[ ! -s "$TEST_WORK/hostile-path.log" ]] || fail 'security utility resolved through hostile PATH' +rm -rf -- "$RAW_DELETE_WORK"; RAW_DELETE_WORK='' +pass 'security utility boundaries ignore hostile PATH' + +# A signal after the long-lived API proxy PID is assigned but before its +# process-start identity is captured must be deferred. Otherwise EXIT cleanup +# cannot prove ownership and the proxy is leaked. +cat >"$TEST_WORK/bin/kubectl-proxy-signal" <<'SH' +#!/usr/bin/env bash +trap 'exit 0' TERM INT +while :; do /usr/bin/sleep 1; done +SH +chmod 0755 "$TEST_WORK/bin/kubectl-proxy-signal" +RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)" +chmod 0700 "$RAW_DELETE_WORK" +rm -f -- "$TEST_WORK/proxy-signal.marker" "$TEST_WORK/proxy-signal.pid" +proxy_signal_rc=0 +( + WORK="$RAW_DELETE_WORK"; KUBECTL_BIN="$TEST_WORK/bin/kubectl-proxy-signal" + API_PROXY_PID=''; API_PROXY_START=''; API_PROXY_SOCKET='' + MUTATION_CRITICAL_DEPTH=0; DEFERRED_SIGNAL_RC=0 + OBS_SMOKE_PROXY_SIGNAL_TARGET=$BASHPID + OBS_SMOKE_PROXY_SIGNAL_MARKER="$TEST_WORK/proxy-signal.marker" + export OBS_SMOKE_PROXY_SIGNAL_TARGET OBS_SMOKE_PROXY_SIGNAL_MARKER + process_start_time() { + if [[ ! -e "$OBS_SMOKE_PROXY_SIGNAL_MARKER" ]]; then + : >"$OBS_SMOKE_PROXY_SIGNAL_MARKER" + /bin/kill -INT "$OBS_SMOKE_PROXY_SIGNAL_TARGET" + fi + printf '12345' + } + trap on_exit EXIT + trap 'handle_termination_signal 130' INT + start_api_proxy +) || proxy_signal_rc=$? +assert_eq 130 "$proxy_signal_rc" 'API proxy deferred signal status' +[[ -f "$TEST_WORK/proxy-signal.marker" ]] || fail 'API proxy signal fixture did not reach identity capture' +proxy_pid="$(pgrep -f "^bash $TEST_WORK/bin/kubectl-proxy-signal" | head -n1 || true)" +if [[ -n "$proxy_pid" ]] && /bin/kill -0 "$proxy_pid" 2>/dev/null; then + /bin/kill -KILL "$proxy_pid" 2>/dev/null || true + fail 'API proxy process leaked across the signal ownership window' +fi +RAW_DELETE_WORK='' +pass 'API proxy signal window closes after PID start-time ownership capture' + +declare -F validate_commit_environment >/dev/null || fail 'full repeat commit environment gate is missing' +( + EXPECTED_NODE_UID='' + NODE_FIXTURE_UID='node-uid-one' + validate_command_boundaries() { printf 'commands\n' >>"$TEST_WORK/commit-gate.log"; } + kubectl_bounded() { + printf '%q ' "$@" >>"$TEST_WORK/commit-gate.log"; printf '\n' >>"$TEST_WORK/commit-gate.log" + case "$1 $2" in + 'config current-context') printf 'default' ;; + 'config view') printf 'https://127.0.0.1:6443' ;; + 'get --raw=/readyz') : ;; + 'get node') jq -n --arg name "$EXPECTED_NODE" --arg uid "$NODE_FIXTURE_UID" '{metadata:{name:$name,uid:$uid},status:{conditions:[{type:"Ready",status:"True"}]}}' ;; + 'auth can-i') printf 'yes' ;; + *) return 91 ;; + esac + } + : >"$TEST_WORK/commit-gate.log" + validate_commit_environment + validate_commit_environment + NODE_FIXTURE_UID='node-uid-replacement' + expect_failure validate_commit_environment +) || fail 'full repeat commit environment gate flow' +assert_eq 3 "$(grep -c '^commands$' "$TEST_WORK/commit-gate.log")" 'command boundary repeat count' +[[ "$(grep -c '^auth can-i ' "$TEST_WORK/commit-gate.log")" -gt 20 ]] || fail 'authority set was not repeated' +pass 'commit gate repeats commands, context, node identity, API and authority' + +declare -F kubectl_wait_90_bounded >/dev/null || fail '90-second Kubernetes convergence wrapper is missing' +declare -F kubectl_wait_180_bounded >/dev/null || fail '180-second Kubernetes convergence wrapper is missing' +cat >"$TEST_WORK/bin/kubectl-convergence" <<'SH' +#!/usr/bin/env bash +set -Eeuo pipefail +/usr/bin/tr '\0' ' ' <"/proc/$PPID/cmdline" >>"${OBS_SMOKE_CONVERGENCE_LOG:?}" +printf '\n' >>"$OBS_SMOKE_CONVERGENCE_LOG" +printf '%q ' "$@" >>"$OBS_SMOKE_CONVERGENCE_LOG"; printf '\n' >>"$OBS_SMOKE_CONVERGENCE_LOG" +SH +chmod 0755 "$TEST_WORK/bin/kubectl-convergence" +: >"$TEST_WORK/convergence.log" +KUBECTL_BIN="$TEST_WORK/bin/kubectl-convergence" OBS_SMOKE_CONVERGENCE_LOG="$TEST_WORK/convergence.log" \ + kubectl_wait_90_bounded -n observability wait --timeout=90s pod/example +KUBECTL_BIN="$TEST_WORK/bin/kubectl-convergence" OBS_SMOKE_CONVERGENCE_LOG="$TEST_WORK/convergence.log" \ + kubectl_wait_180_bounded -n observability rollout status deployment/grafana --timeout=180s +convergence_log="$(<"$TEST_WORK/convergence.log")" +assert_contains "$convergence_log" '--kill-after=2s 100s' '90-second outer convergence deadline' +assert_contains "$convergence_log" '--request-timeout=95s -n observability wait --timeout=90s pod/example' '90-second request deadline' +assert_contains "$convergence_log" '--kill-after=2s 190s' '180-second outer convergence deadline' +assert_contains "$convergence_log" '--request-timeout=185s -n observability rollout status deployment/grafana --timeout=180s' '180-second request deadline' +KUBECTL_BIN="$KUBECTL_DEFAULT" +pass 'Kubernetes convergence honors declared waits within fixed outer deadlines' + +# A second-resolution timestamp cannot identify ownership under concurrent +# runs. Production must add a cryptographically random, non-secret nonce. +declare -F new_smoke_run_id >/dev/null || fail 'unique smoke run-id generator is missing' +run_id_one="$(new_smoke_run_id network)" +run_id_two="$(new_smoke_run_id network)" +[[ "$run_id_one" =~ ^network-[0-9]{8}t[0-9]{6}z-[0-9a-f]{32}$ ]] || fail 'first smoke run ID is not exact' +[[ "$run_id_two" =~ ^network-[0-9]{8}t[0-9]{6}z-[0-9a-f]{32}$ ]] || fail 'second smoke run ID is not exact' +[[ "$run_id_one" != "$run_id_two" ]] || fail 'concurrent smoke run IDs collided' +pass 'temporary object identity uses a unique non-secret nonce' + +# The full run ID is safe as a label value but, when prefixed again, exceeds +# Kubernetes' 63-character DNS label limit. Resource names must retain the +# full 128-bit nonce while remaining valid exact DNS labels. +declare -F kubernetes_smoke_name >/dev/null || fail 'bounded Kubernetes smoke-name helper is missing' +kubernetes_name="$(kubernetes_smoke_name "$run_id_one")" +[[ "$kubernetes_name" =~ ^observability-smoke-[0-9a-f]{32}$ ]] || fail 'Kubernetes smoke name is not nonce-bound' +(( ${#kubernetes_name} <= 63 )) || fail 'Kubernetes smoke name exceeds DNS label length' +expect_failure kubernetes_smoke_name 'network-20260812t091011z-not-a-nonce' +pass 'temporary Kubernetes names retain ownership entropy within DNS limits' + +# Mutation caught: accepting public/private DNS drift or more than one SAN. +validate_dns_contract '' '' 192.168.0.107 '' 100.92.240.34 '' 192.168.0.107 '' +expect_failure validate_dns_contract 203.0.113.10 '' 192.168.0.107 '' 100.92.240.34 '' 192.168.0.107 '' +expect_failure validate_dns_contract '' '' 192.168.0.108 '' 100.92.240.34 '' 192.168.0.107 '' +validate_certificate_contract $'DNS:grafana.learn.hyeonworks.com' 0 0 +expect_failure validate_certificate_contract $'DNS:grafana.learn.hyeonworks.com\nDNS:extra.example' 0 0 +expect_failure validate_certificate_contract 'DNS:grafana.learn.hyeonworks.com' 1 0 +expect_failure validate_certificate_contract 'DNS:grafana.learn.hyeonworks.com' 0 1 +declare -F validate_certificate_extension_contract >/dev/null || + fail 'full certificate GeneralNames validator is missing' +validate_certificate_extension_contract \ + $'X509v3 Subject Alternative Name:\n DNS:grafana.learn.hyeonworks.com' 0 0 +for extra_name in \ + 'IP Address:192.168.0.107' \ + 'email:admin@learn.hyeonworks.com' \ + 'URI:https://grafana.learn.hyeonworks.com/' \ + 'othername: UPN::grafana'; do + expect_failure validate_certificate_extension_contract \ + "X509v3 Subject Alternative Name: + DNS:grafana.learn.hyeonworks.com, $extra_name" 0 0 +done +pass 'DNS and certificate validators reject drift' + +# Production machine leaf: exercise its real bounded dig argv construction, +# normalization, and response-to-contract flow at the command boundary. +cat >"$TEST_WORK/bin/dig-leaf" <<'SH' +#!/usr/bin/env bash +set -Eeuo pipefail +printf '%q ' "$@" >>"${OBS_SMOKE_LEAF_LOG:?}" +printf '\n' >>"${OBS_SMOKE_LEAF_LOG:?}" +[[ "$1" == +time=2 && "$2" == +tries=1 && "$3" == +short && "$6" == grafana.learn.hyeonworks.com ]] +resolver=${4#@}; record=$5 +if [[ "${OBS_SMOKE_LEAF_DRIFT:-0}" == 1 && "$resolver" == 192.168.0.107 && "$record" == A ]]; then + printf '192.168.0.108\n' +elif [[ "$resolver" == 192.168.0.107 && "$record" == A ]]; then + printf '\n192.168.0.107\n192.168.0.107\n' +elif [[ "$resolver" == 100.92.240.34 && "$record" == A ]]; then + printf '100.92.240.34\n' +fi +SH +chmod 0755 "$TEST_WORK/bin/dig-leaf" +: >"$TEST_WORK/dig-leaf.log" +DIG_BIN="$TEST_WORK/bin/dig-leaf" +OBS_SMOKE_LEAF_LOG="$TEST_WORK/dig-leaf.log" check_dns_host >/dev/null +assert_eq 6 "$(wc -l <"$TEST_WORK/dig-leaf.log" | tr -d '[:space:]')" 'DNS leaf command count' +assert_contains "$(<"$TEST_WORK/dig-leaf.log")" '+time=2 +tries=1 +short @1.1.1.1 A grafana.learn.hyeonworks.com' 'public A argv contract' +assert_contains "$(<"$TEST_WORK/dig-leaf.log")" '+time=2 +tries=1 +short @192.168.0.107 AAAA grafana.learn.hyeonworks.com' 'LAN AAAA argv contract' +assert_contains "$(<"$TEST_WORK/dig-leaf.log")" '+time=2 +tries=1 +short @100.92.240.34 A grafana.learn.hyeonworks.com' 'Tailscale A argv contract' +expect_failure env OBS_SMOKE_LEAF_LOG="$TEST_WORK/dig-leaf.log" OBS_SMOKE_LEAF_DRIFT=1 \ + bash -c 'PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$1"; DIG_BIN="$2"; check_dns_host' \ + bash "$SMOKE" "$TEST_WORK/bin/dig-leaf" +DIG_BIN="$DIG_DEFAULT" +pass 'production DNS leaf uses bounded exact argv and rejects parsed drift' + +cat >"$TEST_WORK/traefik-service.json" <<'JSON' +{"apiVersion":"v1","kind":"Service","metadata":{"name":"traefik","namespace":"kube-system","uid":"traefik-service-uid"},"spec":{"type":"NodePort","externalTrafficPolicy":"Cluster","selector":{"app.kubernetes.io/name":"traefik","app.kubernetes.io/instance":"traefik-kube-system"},"ports":[{"name":"web","port":80,"protocol":"TCP","targetPort":"web","nodePort":30080},{"name":"websecure","port":443,"protocol":"TCP","targetPort":"websecure","nodePort":30443}]}} +JSON +chmod 0600 "$TEST_WORK/traefik-service.json" +declare -F check_nodeport_identity >/dev/null || fail 'Traefik NodePort identity leaf is missing' +( + WORK="$TEST_WORK" + kubectl_bounded() { cat -- "$TEST_WORK/traefik-service.json"; } + http_status() { + printf '%q ' "$@" >"$TEST_WORK/nodeport-http.argv" + printf '404' + } + check_nodeport_identity >/dev/null +) || fail 'exact Traefik NodePort identity flow' +assert_contains "$(<"$TEST_WORK/nodeport-http.argv")" '--header Host:\ observability-smoke.invalid http://127.0.0.1:30080/' 'NodePort exact HTTP argv' +expect_failure bash -c ' + PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$1" + WORK="$(dirname -- "$2")" + SERVICE_FIXTURE=$2 + kubectl_bounded() { cat -- "$SERVICE_FIXTURE"; } + http_status() { printf 200; } + check_nodeport_identity +' bash "$SMOKE" "$TEST_WORK/traefik-service.json" +jq '.spec.selector["app.kubernetes.io/name"]="foreign"' "$TEST_WORK/traefik-service.json" >"$TEST_WORK/traefik-service-foreign.json" +expect_failure bash -c ' + PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$1" + WORK="$(dirname -- "$2")" + SERVICE_FIXTURE=$2 + kubectl_bounded() { cat -- "$SERVICE_FIXTURE"; } + http_status() { printf 404; } + check_nodeport_identity +' bash "$SMOKE" "$TEST_WORK/traefik-service-foreign.json" +pass 'loopback NodePort proves exact Traefik Service and HTTP response' + +# A hostile ambient curlrc must not influence either public HTTPS calls or the +# private Unix-socket API client. `--disable` has to be curl's first argument. +mkdir -m 0700 "$TEST_WORK/curl-home" +cat >"$TEST_WORK/curl-home/.curlrc" </dev/null +[[ ! -e "$TEST_WORK/hostile-curl.trace" ]] || fail 'http_status loaded hostile curlrc' +RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)" +chmod 0700 "$RAW_DELETE_WORK" +proxy_rc=0 +( + WORK="$RAW_DELETE_WORK" + API_PROXY_SOCKET="$WORK/nonexistent.sock" + HOME="$TEST_WORK/curl-home" CURL_HOME="$TEST_WORK/curl-home" \ + proxy_request GET '/api/v1/namespaces/observability/pods' "$WORK/output.json" +) >/dev/null 2>&1 || proxy_rc=$? +assert_eq 1 "$proxy_rc" 'failed-socket proxy status' +[[ ! -e "$TEST_WORK/hostile-curl.trace" ]] || fail 'proxy_request loaded hostile curlrc' +rm -rf -- "$RAW_DELETE_WORK" +RAW_DELETE_WORK='' +pass 'curl boundaries ignore hostile ambient curlrc' + +cat >"$TEST_WORK/targets.json" <<'JSON' +{"status":"success","data":{"activeTargets":[ +{"scrapePool":"podMonitor/platform-data/platform-postgres/0","health":"up"}, +{"scrapePool":"probe/observability/platform-private-edge","health":"up"}, +{"scrapePool":"probe/observability/platform-private-edge","health":"up"}, +{"scrapePool":"probe/observability/platform-private-edge","health":"up"}, +{"scrapePool":"probe/observability/platform-private-internal","health":"up"}, +{"scrapePool":"probe/observability/platform-private-internal","health":"up"}, +{"scrapePool":"probe/observability/platform-private-internal","health":"up"}, +{"scrapePool":"probe/observability/platform-public-edge","health":"up"}, +{"scrapePool":"probe/observability/platform-public-edge","health":"up"}, +{"scrapePool":"serviceMonitor/gitea/gitea/0","health":"up"}, +{"scrapePool":"serviceMonitor/keycloak/keycloak/0","health":"up"}, +{"scrapePool":"serviceMonitor/kube-system/traefik/0","health":"up"}, +{"scrapePool":"serviceMonitor/object-storage/aistor-bucket-usage/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability-agent/alloy/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability-agent/node-exporter/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability/blackbox-exporter/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability/grafana/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability/loki/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-alertmanager/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-alertmanager/1","health":"up"}, +{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-apiserver/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-coredns/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-kubelet/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-kubelet/1","health":"up"}, +{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-kubelet/2","health":"up"}, +{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-operator/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-prometheus/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-prometheus/1","health":"up"}, +{"scrapePool":"serviceMonitor/observability/observability-core-kube-state-metrics/0","health":"up"}, +{"scrapePool":"serviceMonitor/observability/tempo/0","health":"up"} +]}} +JSON +python3 -I -S - "$TEST_WORK/targets.json" <<'PY' +import json,pathlib,sys +path=pathlib.Path(sys.argv[1]); item=json.loads(path.read_text()) +urls={ + 'podMonitor/platform-data/platform-postgres/0':'http://10.42.0.10:9187/metrics', + 'serviceMonitor/gitea/gitea/0':'http://10.42.0.11:3000/metrics', + 'serviceMonitor/keycloak/keycloak/0':'http://10.42.0.12:9000/metrics', + 'serviceMonitor/kube-system/traefik/0':'http://10.42.0.13:9100/metrics', + 'serviceMonitor/object-storage/aistor-bucket-usage/0':'http://10.42.0.14:9000/minio/metrics/v3/cluster/usage/buckets', + 'serviceMonitor/observability-agent/alloy/0':'http://10.42.0.21:12345/metrics', + 'serviceMonitor/observability-agent/node-exporter/0':'http://192.168.0.107:9100/metrics', + 'serviceMonitor/observability/blackbox-exporter/0':'http://10.42.0.22:9115/metrics', + 'serviceMonitor/observability/grafana/0':'http://10.42.0.23:3000/metrics', + 'serviceMonitor/observability/loki/0':'http://10.42.0.24:3100/metrics', + 'serviceMonitor/observability/observability-core-kube-pr-alertmanager/0':'http://10.42.0.25:9093/metrics', + 'serviceMonitor/observability/observability-core-kube-pr-alertmanager/1':'http://10.42.0.25:8080/metrics', + 'serviceMonitor/observability/observability-core-kube-pr-apiserver/0':'https://192.168.0.107:6443/metrics', + 'serviceMonitor/observability/observability-core-kube-pr-coredns/0':'http://10.42.0.26:9153/metrics', + 'serviceMonitor/observability/observability-core-kube-pr-kubelet/0':'https://192.168.0.107:10250/metrics', + 'serviceMonitor/observability/observability-core-kube-pr-kubelet/1':'https://192.168.0.107:10250/metrics/cadvisor', + 'serviceMonitor/observability/observability-core-kube-pr-kubelet/2':'https://192.168.0.107:10250/metrics/probes', + 'serviceMonitor/observability/observability-core-kube-pr-operator/0':'http://10.42.0.27:8080/metrics', + 'serviceMonitor/observability/observability-core-kube-pr-prometheus/0':'http://10.42.0.28:9090/metrics', + 'serviceMonitor/observability/observability-core-kube-pr-prometheus/1':'http://10.42.0.28:8080/metrics', + 'serviceMonitor/observability/observability-core-kube-state-metrics/0':'http://10.42.0.29:8080/metrics', + 'serviceMonitor/observability/tempo/0':'http://10.42.0.30:3200/metrics', +} +jobs={ + 'podMonitor/platform-data/platform-postgres/0':'platform-postgres', + 'serviceMonitor/gitea/gitea/0':'gitea', + 'serviceMonitor/keycloak/keycloak/0':'keycloak-service', + 'serviceMonitor/kube-system/traefik/0':'traefik', + 'serviceMonitor/object-storage/aistor-bucket-usage/0':'minio-aistor', + 'serviceMonitor/observability-agent/alloy/0':'alloy', + 'serviceMonitor/observability-agent/node-exporter/0':'prometheus-node-exporter', + 'serviceMonitor/observability/blackbox-exporter/0':'blackbox-exporter', + 'serviceMonitor/observability/grafana/0':'grafana', + 'serviceMonitor/observability/loki/0':'observability/loki', + 'serviceMonitor/observability/observability-core-kube-pr-alertmanager/0':'observability-core-kube-pr-alertmanager', + 'serviceMonitor/observability/observability-core-kube-pr-alertmanager/1':'observability-core-kube-pr-alertmanager', + 'serviceMonitor/observability/observability-core-kube-pr-apiserver/0':'apiserver', + 'serviceMonitor/observability/observability-core-kube-pr-coredns/0':'coredns', + 'serviceMonitor/observability/observability-core-kube-pr-kubelet/0':'kubelet', + 'serviceMonitor/observability/observability-core-kube-pr-kubelet/1':'kubelet', + 'serviceMonitor/observability/observability-core-kube-pr-kubelet/2':'kubelet', + 'serviceMonitor/observability/observability-core-kube-pr-operator/0':'observability-core-kube-pr-operator', + 'serviceMonitor/observability/observability-core-kube-pr-prometheus/0':'observability-core-kube-pr-prometheus', + 'serviceMonitor/observability/observability-core-kube-pr-prometheus/1':'observability-core-kube-pr-prometheus', + 'serviceMonitor/observability/observability-core-kube-state-metrics/0':'kube-state-metrics', + 'serviceMonitor/observability/tempo/0':'tempo', +} +namespaces={pool:pool.split('/')[1] for pool in urls} +namespaces.update({ + 'serviceMonitor/observability/observability-core-kube-pr-apiserver/0':'default', + 'serviceMonitor/observability/observability-core-kube-pr-coredns/0':'kube-system', + 'serviceMonitor/observability/observability-core-kube-pr-kubelet/0':'kube-system', + 'serviceMonitor/observability/observability-core-kube-pr-kubelet/1':'kube-system', + 'serviceMonitor/observability/observability-core-kube-pr-kubelet/2':'kube-system', +}) +probe_targets={ + 'probe/observability/platform-private-edge':iter([ + ('https://db-admin.learn.hyeonworks.com/','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fdb-admin.learn.hyeonworks.com%2F','blackbox-private-edge','private-edge'), + ('https://grafana.learn.hyeonworks.com/','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fgrafana.learn.hyeonworks.com%2F','blackbox-private-edge','private-edge'), + ('https://storage-admin.learn.hyeonworks.com/','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fstorage-admin.learn.hyeonworks.com%2F','blackbox-private-edge','private-edge')]), + 'probe/observability/platform-private-internal':iter([ + ('http://pgadmin.platform-admin.svc.cluster.local/misc/ping','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=db-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fpgadmin.platform-admin.svc.cluster.local%2Fmisc%2Fping','blackbox-private-internal','private-internal'), + ('http://grafana.observability.svc.cluster.local/api/health','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=grafana.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fgrafana.observability.svc.cluster.local%2Fapi%2Fhealth','blackbox-private-internal','private-internal'), + ('http://minio-aistor-console.object-storage.svc.cluster.local:9090/','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=storage-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fminio-aistor-console.object-storage.svc.cluster.local%3A9090%2F','blackbox-private-internal','private-internal')]), + 'probe/observability/platform-public-edge':iter([ + ('https://git.learn.hyeonworks.com/api/healthz','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fgit.learn.hyeonworks.com%2Fapi%2Fhealthz','blackbox-public-edge','public-edge'), + ('https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fid.learn.hyeonworks.com%2Frealms%2Fhyeonworks%2F.well-known%2Fopenid-configuration','blackbox-public-edge','public-edge')]), +} +for target in item['data']['activeTargets']: + pool=target['scrapePool'] + if pool.startswith('probe/'): + instance,url,job,group=next(probe_targets[pool]) + target['scrapeUrl']=url + target['labels']={'instance':instance,'job':job,'namespace':'observability','observability.hyeonworks.com/probe-group':group} + else: + from urllib.parse import urlsplit + target['scrapeUrl']=target['globalUrl']=urls[pool] + parsed=urlsplit(target['scrapeUrl']) + target['discoveredLabels']={'__address__':parsed.netloc,'__scheme__':parsed.scheme,'__metrics_path__':parsed.path} + target['labels']={'instance':parsed.netloc,'job':jobs[pool],'namespace':namespaces[pool]} +path.write_text(json.dumps(item,separators=(',',':'))+'\n') +PY +chmod 0600 "$TEST_WORK/targets.json" +validate_target_contract "$TEST_WORK/targets.json" +jq '(.data.activeTargets[0].health)="down"' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-down.json" +expect_failure validate_target_contract "$TEST_WORK/targets-down.json" +jq '.data.activeTargets += [{"scrapePool":"serviceMonitor/kube-system/kube-scheduler/0","health":"up"}]' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-extra.json" +expect_failure validate_target_contract "$TEST_WORK/targets-extra.json" +jq '(.data.activeTargets[]|select(.scrapePool=="serviceMonitor/keycloak/keycloak/0").scrapeUrl)="http://10.42.0.12:8080/metrics"' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-wrong-port.json" +expect_failure validate_target_contract "$TEST_WORK/targets-wrong-port.json" +jq '(.data.activeTargets[]|select(.scrapePool=="serviceMonitor/observability/grafana/0").scrapeUrl)="http://10.42.0.99:9998/metrics"' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-core-wrong-port.json" +expect_failure validate_target_contract "$TEST_WORK/targets-core-wrong-port.json" +jq '(.data.activeTargets[]|select(.scrapePool=="serviceMonitor/observability/grafana/0")) |= (.scrapeUrl="http://10.42.0.99:3000/metrics" | .globalUrl=.scrapeUrl | .discoveredLabels.__address__="10.42.0.99:3000")' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-same-port-substitution.json" +expect_failure validate_target_contract "$TEST_WORK/targets-same-port-substitution.json" +jq '(.data.activeTargets[]|select(.scrapePool=="probe/observability/platform-public-edge").labels.instance)="https://git.learn.hyeonworks.com/api/healthz"' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-substituted-instance.json" +expect_failure validate_target_contract "$TEST_WORK/targets-substituted-instance.json" +pass 'target validator enforces exact 30-target/25-pool allowlist' + +cat >"$TEST_WORK/series.json" <<'JSON' +{"status":"success","data":{"resultType":"vector","result":[ +{"metric":{"bucket":"loki","job":"minio-aistor","namespace":"object-storage","instance":"10.42.0.4:9000"},"value":[1,"1"]}, +{"metric":{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage","instance":"10.42.0.4:9000"},"value":[1,"1"]} +]}} +JSON +chmod 0600 "$TEST_WORK/series.json" +validate_series_contract "$TEST_WORK/series.json" 2 'bucket,instance,job,namespace' 'loki,tempo' bucket +expect_failure validate_series_contract "$TEST_WORK/series.json" 1 'bucket,instance,job,namespace' 'loki,tempo' bucket +jq 'del(.data.result[0].metric.instance)' "$TEST_WORK/series.json" >"$TEST_WORK/series-bad.json" +expect_failure validate_series_contract "$TEST_WORK/series-bad.json" 2 'bucket,instance,job,namespace' 'loki,tempo' bucket +declare -F validate_series_identity_contract >/dev/null || fail 'exact series identity validator is missing' +validate_series_identity_contract "$TEST_WORK/series.json" \ + '[{"bucket":"loki","job":"minio-aistor","namespace":"object-storage"},{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage"}]' \ + '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$' +jq '(.data.result[0].metric.job)="wrong"' "$TEST_WORK/series.json" >"$TEST_WORK/series-wrong-job.json" +expect_failure validate_series_identity_contract "$TEST_WORK/series-wrong-job.json" \ + '[{"bucket":"loki","job":"minio-aistor","namespace":"object-storage"},{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage"}]' \ + '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$' +pass 'metric-series validator enforces count, labels, and exact values' + +: >"$TEST_WORK/metric-query-contract.log" +( + check_one_query() { printf '%s\037' "$@" >>"$TEST_WORK/metric-query-contract.log"; printf '\n' >>"$TEST_WORK/metric-query-contract.log"; } + check_metric_contracts >/dev/null +) || fail 'metric query contract renderer' +cnpg_contract="$(grep '^cnpg' "$TEST_WORK/metric-query-contract.log")" +keycloak_contract="$(grep '^keycloak' "$TEST_WORK/metric-query-contract.log")" +assert_contains "$keycloak_contract" 'process_uptime_seconds{job="keycloak-service",namespace="keycloak"}' 'Keycloak query live job identity' +assert_contains "$keycloak_contract" '[{"job":"keycloak-service","namespace":"keycloak"}]' 'Keycloak result live job identity' +assert_contains "$cnpg_contract" 'cnpg_collector_up{job="platform-postgres",namespace="platform-data",cluster="platform-postgres"}' 'CNPG query cluster identity' +assert_contains "$cnpg_contract" '[{"cluster":"platform-postgres","job":"platform-postgres","namespace":"platform-data"}]' 'CNPG result cluster identity' +pass 'CNPG metric query binds the exact cluster identity' + +cat >"$TEST_WORK/datasources.json" <<'JSON' +{"prometheus":{"status":"OK"},"loki":{"status":"OK"},"tempo":{"status":"OK"}, + "loki_config":{"uid":"loki","jsonData":{"derivedFields":[{"name":"trace_id","datasourceUid":"tempo","url":"${__value.raw}"}]}}, + "tempo_config":{"uid":"tempo","jsonData":{"tracesToLogsV2":{"datasourceUid":"loki","filterByTraceID":true}}}, + "selected_trace":"0123456789abcdef0123456789abcdef", + "tempo_search":{"traces":[{"traceID":"0123456789abcdef0123456789abcdef","rootServiceName":"platform-smoke"}]}, + "loki_query":{"status":"success","data":{"result":[{"values":[["1","{\"trace_id\":\"0123456789abcdef0123456789abcdef\"}"]]}]}}, + "tempo_query":{"batches":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"platform-smoke"}}]},"scopeSpans":[{"spans":[{"traceId":"ASNFZ4mrze8BI0VniavN7w=="}]}]}]}} +JSON +chmod 0600 "$TEST_WORK/datasources.json" +validate_datasource_contract "$TEST_WORK/datasources.json" +jq '.tempo.status="ERROR"' "$TEST_WORK/datasources.json" >"$TEST_WORK/datasources-bad.json" +expect_failure validate_datasource_contract "$TEST_WORK/datasources-bad.json" +jq '.loki_query.data.result=[]' "$TEST_WORK/datasources.json" >"$TEST_WORK/datasources-no-link.json" +expect_failure validate_datasource_contract "$TEST_WORK/datasources-no-link.json" +declare -F synthetic_loki_query >/dev/null || fail 'core synthetic Loki selector builder is missing' +assert_eq '{cluster="home",namespace="observability"} | json | trace_id="0123456789abcdef0123456789abcdef"' \ + "$(synthetic_loki_query 0123456789abcdef0123456789abcdef)" 'core synthetic Loki selector' +expect_failure synthetic_loki_query 'bad"trace' +jq '(.tempo_query.batches[0].resource.attributes[0].value.stringValue)="wrong-service"' \ + "$TEST_WORK/datasources.json" >"$TEST_WORK/datasources-wrong-synthetic.json" +expect_failure validate_datasource_contract "$TEST_WORK/datasources-wrong-synthetic.json" +pass 'datasource validator requires health and a live Loki-to-Tempo link' + +# Mutation caught: incomplete human evidence must never produce full PASS. +complete_tokens=$'OIDC_ADMIN_CONFIRMED\nOIDC_VIEWER_CONFIRMED\nOIDC_DENIED_CONFIRMED\nBREAK_GLASS_CONFIRMED\nREMOVAL_VIEWER_CONFIRMED\nSESSION_REVOKED\nRELOGIN_DENIED\nFIRING\nRESOLVED\nEXTERNAL_CLIENT_VERIFIED' +validate_human_tokens <<<"$complete_tokens" +expect_failure bash -c 'PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$1"; validate_human_tokens' bash "$SMOKE" <<<"${complete_tokens%$'\nEXTERNAL_CLIENT_VERIFIED'}" +pass 'human acceptance requires every exact token' + +declare -F external_attestation_token >/dev/null || fail 'external-client bound attestation token is missing' +assert_eq 'EXTERNAL_CLIENT_VERIFIED public_ip=203.0.113.10 status=403 source=outside-lan-tailscale proxy=disabled' \ + "$(external_attestation_token 203.0.113.10)" 'external attestation fields' +expect_failure external_attestation_token '203.0.113.10;touch /tmp/no' +external_instructions="$(render_external_attestation 203.0.113.10)" +assert_contains "$external_instructions" "--resolve $GRAFANA_HOST:443:203.0.113.10" 'external exact resolve command' +assert_contains "$external_instructions" 'Expected exact result: HTTP 403' 'external exact expected result' +assert_contains "$external_instructions" 'outside both LAN and Tailscale' 'external source requirement' +pass 'external-client evidence binds exact command, source and result fields' + +declare -F confirmation_remaining_seconds >/dev/null || fail 'shared confirmation deadline calculator is missing' +assert_eq 300 "$(confirmation_remaining_seconds 1300 1000)" 'shared deadline initial window' +assert_eq 1 "$(confirmation_remaining_seconds 1300 1299)" 'shared deadline last second' +expect_failure confirmation_remaining_seconds 1300 1300 +expect_failure confirmation_remaining_seconds 1300 1301 +pass 'session revoke and relogin confirmations share one five-minute deadline' + +declare -F capture_keycloak_absent_baseline >/dev/null || fail 'stable full Keycloak membership baseline helper is missing' +declare -F add_keycloak_viewer_membership >/dev/null || fail 'response-safe Keycloak membership add helper is missing' +( + WORK="$TEST_WORK"; KEYCLOAK_USER_ID='user-owned-id'; KEYCLOAK_VIEWER_GROUP_ID='viewer-owned-id' + KEYCLOAK_MEMBERSHIP_BEFORE="$TEST_WORK/keycloak-full-baseline.json" + : >"$TEST_WORK/keycloak-pagination.log" + keycloak_request() { + local output=$3 first + printf '%s\n' "$2" >>"$TEST_WORK/keycloak-pagination.log" + first="$(sed -n 's/.*[?&]first=\([0-9]*\).*/\1/p' <<<"$2")" + if [[ "$first" == 0 ]]; then + jq -n '[range(0;100)|{id:("group-"+tostring),name:("group-"+tostring),path:("/group-"+tostring)}]' >"$output" + else + jq -n '[{id:"viewer-owned-id",name:"platform-observability-viewers",path:"/platform-observability-viewers"}]' >"$output" + fi + chmod 0600 -- "$output"; printf 200 + } + expect_failure capture_keycloak_absent_baseline admin-owned-id + [[ "$(wc -l <"$TEST_WORK/keycloak-pagination.log" | tr -d '[:space:]')" -ge 2 ]] +) || fail 'full Keycloak pagination detects a viewer membership after page one' + +keycloak_signal_rc=0 +( + WORK="$TEST_WORK"; KEYCLOAK_USER_ID='user-owned-id'; KEYCLOAK_VIEWER_GROUP_ID='viewer-owned-id' + KEYCLOAK_MEMBERSHIP_BEFORE="$TEST_WORK/keycloak-signal-baseline.json" + printf '[]\n' >"$KEYCLOAK_MEMBERSHIP_BEFORE"; chmod 0600 -- "$KEYCLOAK_MEMBERSHIP_BEFORE" + KEYCLOAK_TEST_ADDED=false; KEYCLOAK_MEMBERSHIP_PENDING=false; KEYCLOAK_MEMBERSHIP_REMOVED=false + MUTATION_CRITICAL_DEPTH=0; DEFERRED_SIGNAL_RC=0; SIGNAL_TARGET=$BASHPID + trap 'handle_termination_signal 130' INT + trap 'printf "%s|%s\n" "$KEYCLOAK_TEST_ADDED" "$KEYCLOAK_MEMBERSHIP_PENDING" >"$TEST_WORK/keycloak-signal.capture"' EXIT + keycloak_request() { + local method=$1 output=$3 + if [[ "$method" == PUT ]]; then + : >"$output"; chmod 0600 -- "$output"; /bin/kill -INT "$SIGNAL_TARGET"; return 28 + fi + return 91 + } + keycloak_memberships() { + jq -n '[{id:"viewer-owned-id",name:"platform-observability-viewers",path:"/platform-observability-viewers"}]' >"$1" + chmod 0600 -- "$1" + } + add_keycloak_viewer_membership +) || keycloak_signal_rc=$? +assert_eq 130 "$keycloak_signal_rc" 'deferred Keycloak membership signal status' +assert_eq 'true|false' "$(<"$TEST_WORK/keycloak-signal.capture")" 'signal-safe Keycloak membership ownership capture' +pass 'Keycloak membership add uses full baseline and response-safe ownership capture' + +# Production Keycloak cleanup leaf: a preexisting membership is never touched; +# a test-added membership is removed by exact user/group IDs and verified +# absent even when the DELETE response was lost. +( + WORK="$TEST_WORK" + KEYCLOAK_USER_ID='user-owned-id'; KEYCLOAK_VIEWER_GROUP_ID='viewer-owned-id' + KEYCLOAK_MEMBERSHIP_BEFORE="$TEST_WORK/keycloak-cleanup-baseline.json" + printf '[]\n' >"$KEYCLOAK_MEMBERSHIP_BEFORE"; chmod 0600 -- "$KEYCLOAK_MEMBERSHIP_BEFORE" + KEYCLOAK_TEST_ADDED=false; KEYCLOAK_MEMBERSHIP_PENDING=false; KEYCLOAK_MEMBERSHIP_REMOVED=false + keycloak_request() { printf 'unexpected\n' >>"$TEST_WORK/keycloak-cleanup.log"; return 1; } + : >"$TEST_WORK/keycloak-cleanup.log" + cleanup_keycloak_membership + [[ ! -s "$TEST_WORK/keycloak-cleanup.log" ]] + + KEYCLOAK_TEST_ADDED=true + rm -f -- "$TEST_WORK/keycloak-deleted.marker" + keycloak_request() { + printf '%s|%s\n' "$1" "$2" >>"$TEST_WORK/keycloak-cleanup.log" + : >"$TEST_WORK/keycloak-deleted.marker" + return 28 + } + keycloak_memberships() { + if [[ ! -e "$TEST_WORK/keycloak-deleted.marker" ]]; then + jq -n '[{id:"viewer-owned-id",name:"platform-observability-viewers",path:"/platform-observability-viewers"}]' >"$1" + else + printf '[]\n' >"$1" + fi + chmod 0600 -- "$1" + } + cleanup_keycloak_membership + [[ "$KEYCLOAK_MEMBERSHIP_REMOVED" == true ]] +) || fail 'test-added Keycloak membership cleanup flow' +assert_eq 'DELETE|/admin/realms/hyeonworks/users/user-owned-id/groups/viewer-owned-id' \ + "$(<"$TEST_WORK/keycloak-cleanup.log")" 'exact Keycloak membership cleanup path' +pass 'Keycloak cleanup touches only test-added exact membership' + +# Production Slack lifecycle/render leaf: firing proof precedes confirmation, +# exact owned rule cleanup precedes resolved confirmation, and the alert carries +# every grouping/runbook label without a webhook payload. +cat >"$TEST_WORK/bin/date-slack" <<'SH' +#!/usr/bin/env bash +set -Eeuo pipefail +[[ " $* " == *' +%Y%m%dT%H%M%SZ '* ]] +printf '20260812T091011Z\n' +SH +chmod 0755 "$TEST_WORK/bin/date-slack" +( + WORK="$TEST_WORK"; DATE_BIN="$TEST_WORK/bin/date-slack" + new_smoke_run_id() { printf 'alert-20260812t091011z-00000000000000000000000000000001'; } + CONFIRMATION_QUEUE=(FIRING RESOLVED); CONFIRMATION_INDEX=0; TEST_MODE=true + create_owned_from_file() { cp -- "$1" "$TEST_WORK/slack-rule.capture"; printf 'create\n' >>"$TEST_WORK/slack-order.log"; } + wait_for_smoke_alert() { printf 'wait-%s\n' "$2" >>"$TEST_WORK/slack-order.log"; } + require_confirmation() { printf 'confirm-%s\n' "$1" >>"$TEST_WORK/slack-order.log"; [[ "${CONFIRMATION_QUEUE[$CONFIRMATION_INDEX]}" == "$1" ]] && CONFIRMATION_INDEX=$((CONFIRMATION_INDEX + 1)); } + cleanup_owned_objects() { printf 'cleanup\n' >>"$TEST_WORK/slack-order.log"; } + : >"$TEST_WORK/slack-order.log" + run_slack_acceptance >"$TEST_WORK/slack-output.capture" +) || fail 'Slack lifecycle render flow' +assert_eq $'create\nwait-present\nconfirm-FIRING\ncleanup\nconfirm-RESOLVED\nwait-absent' \ + "$(<"$TEST_WORK/slack-order.log")" 'Slack lifecycle order' +jq -e --arg runbook "$RUNBOOK_URL" ' + .kind=="PrometheusRule" and .metadata.name=="observability-smoke-00000000000000000000000000000001" and + .metadata.namespace=="observability" and + .metadata.labels["platform.hyeonworks.com/smoke-run"]=="alert-20260812t091011z-00000000000000000000000000000001" and + (.spec.groups|length)==1 and (.spec.groups[0].rules|length)==1 and + .spec.groups[0].rules[0].alert=="PlatformObservabilitySmoke" and + .spec.groups[0].rules[0].expr=="vector(1)" and + .spec.groups[0].rules[0].labels=={"cluster":"home","namespace":"observability","instance":"alert-20260812t091011z-00000000000000000000000000000001","severity":"info"} and + .spec.groups[0].rules[0].annotations.runbook_url==$runbook +' "$TEST_WORK/slack-rule.capture" >/dev/null || fail 'Slack rule exact payload contract' +pass 'Slack lifecycle renders and cleans only one exact temporary rule' + +# Production network transaction leaf: first prove the exact deployed ingress +# policies, then explicitly allow the test Pod's egress to both targets. A +# failed connect can no longer be manufactured by a test-owned egress deny. +cat >"$TEST_WORK/fixture-aistor-ingress-policy.json" <<'JSON' +{"apiVersion":"networking.k8s.io/v1","kind":"NetworkPolicy","metadata":{"name":"object-storage-allow-prometheus-metrics","namespace":"object-storage","uid":"aistor-policy-uid","resourceVersion":"11"},"spec":{"podSelector":{"matchLabels":{"aistor.min.io/objectStore":"minio-aistor"}},"policyTypes":["Ingress"],"ingress":[{"from":[{"namespaceSelector":{"matchLabels":{"kubernetes.io/metadata.name":"observability"}},"podSelector":{"matchLabels":{"app.kubernetes.io/name":"prometheus","app.kubernetes.io/instance":"observability-core-kube-pr-prometheus"}}}],"ports":[{"protocol":"TCP","port":9000}]}]}} +JSON +cat >"$TEST_WORK/fixture-blackbox-ingress-policy.json" <<'JSON' +{"apiVersion":"networking.k8s.io/v1","kind":"NetworkPolicy","metadata":{"name":"observability-allow-prometheus-to-blackbox","namespace":"observability","uid":"blackbox-policy-uid","resourceVersion":"12"},"spec":{"podSelector":{"matchLabels":{"app.kubernetes.io/name":"prometheus-blackbox-exporter","app.kubernetes.io/instance":"blackbox-exporter"}},"policyTypes":["Ingress"],"ingress":[{"from":[{"podSelector":{"matchLabels":{"app.kubernetes.io/name":"prometheus","app.kubernetes.io/instance":"observability-core-kube-pr-prometheus"}}}],"ports":[{"protocol":"TCP","port":9115}]}]}} +JSON +chmod 0600 "$TEST_WORK/fixture-aistor-ingress-policy.json" "$TEST_WORK/fixture-blackbox-ingress-policy.json" +declare -F validate_deployed_ingress_policy_contracts >/dev/null || + fail 'deployed ingress policy exact validator is missing' +validate_deployed_ingress_policy_contracts "$TEST_WORK/fixture-aistor-ingress-policy.json" "$TEST_WORK/fixture-blackbox-ingress-policy.json" +jq '.spec.ingress[0].from[0].podSelector.matchLabels["app.kubernetes.io/name"]="foreign"' \ + "$TEST_WORK/fixture-aistor-ingress-policy.json" >"$TEST_WORK/aistor-ingress-policy-drift.json" +expect_failure validate_deployed_ingress_policy_contracts \ + "$TEST_WORK/aistor-ingress-policy-drift.json" "$TEST_WORK/fixture-blackbox-ingress-policy.json" +( + WORK="$TEST_WORK"; DATE_BIN="$TEST_WORK/bin/date-slack" + new_smoke_run_id() { printf 'network-20260812t091011z-00000000000000000000000000000001'; } + kubectl_bounded() { + if [[ "$1 $2 $3 $4 $5" == '-n object-storage get networkpolicy object-storage-allow-prometheus-metrics' ]]; then + printf 'precondition-aistor\n' >>"$TEST_WORK/network-order.log"; cat -- "$TEST_WORK/fixture-aistor-ingress-policy.json"; return 0 + fi + if [[ "$1 $2 $3 $4 $5" == '-n observability get networkpolicy observability-allow-prometheus-to-blackbox' ]]; then + printf 'precondition-blackbox\n' >>"$TEST_WORK/network-order.log"; cat -- "$TEST_WORK/fixture-blackbox-ingress-policy.json"; return 0 + fi + if [[ "$1 $2 $3 $4" == '-n object-storage get service' && "$5" == minio ]]; then printf '10.43.0.41'; return 0; fi + if [[ "$1 $2 $3 $4" == '-n observability get service' && "$5" == blackbox-exporter ]]; then printf '10.43.0.42'; return 0; fi + if [[ "$1 $2" == '-n observability' && "$3" == wait ]]; then printf 'wait\n' >>"$TEST_WORK/network-order.log"; return 0; fi + if [[ "$1 $2" == '-n observability' && "$3" == logs ]]; then printf 'OBSERVABILITY_NETWORK_BOUNDARY_PASS\n'; return 0; fi + return 91 + } + kubectl_wait_90_bounded() { kubectl_bounded "$@"; } + create_owned_from_file() { + case "$2" in + NetworkPolicy) cp -- "$1" "$TEST_WORK/network-policy.capture" ;; + Pod) cp -- "$1" "$TEST_WORK/network-pod.capture" ;; + esac + printf 'create-%s\n' "$2" >>"$TEST_WORK/network-order.log" + } + cleanup_owned_objects() { printf 'cleanup\n' >>"$TEST_WORK/network-order.log"; } + : >"$TEST_WORK/network-order.log" + run_network_boundary_transaction >"$TEST_WORK/network-output.capture" +) || fail "network transaction render flow: $(<"$TEST_WORK/network-order.log")" +assert_eq $'precondition-aistor\nprecondition-blackbox\ncreate-NetworkPolicy\ncreate-Pod\nwait\ncleanup' "$(<"$TEST_WORK/network-order.log")" 'network transaction order' +jq -e ' + .kind=="NetworkPolicy" and .metadata.namespace=="observability" and + .spec.policyTypes==["Egress"] and (.spec.egress|length)==3 and + .spec.egress[0].ports==[{"protocol":"UDP","port":53},{"protocol":"TCP","port":53}] and + (.spec.egress[0].to|length)==1 and + .spec.egress[0].to[0].namespaceSelector.matchLabels["kubernetes.io/metadata.name"]=="kube-system" and + .spec.egress[0].to[0].podSelector.matchLabels["k8s-app"]=="kube-dns" and + .spec.egress[1]=={"to":[{"namespaceSelector":{"matchLabels":{"kubernetes.io/metadata.name":"object-storage"}},"podSelector":{"matchLabels":{"aistor.min.io/objectStore":"minio-aistor"}}}],"ports":[{"protocol":"TCP","port":9000}]} and + .spec.egress[2]=={"to":[{"namespaceSelector":{"matchLabels":{"kubernetes.io/metadata.name":"observability"}},"podSelector":{"matchLabels":{"app.kubernetes.io/name":"prometheus-blackbox-exporter","app.kubernetes.io/instance":"blackbox-exporter"}}}],"ports":[{"protocol":"TCP","port":9115}]} +' "$TEST_WORK/network-policy.capture" >/dev/null || fail 'network-policy explicit target egress contract' +jq -e --arg image "$BUSYBOX_IMAGE" ' + .kind=="Pod" and .spec.automountServiceAccountToken==false and .spec.restartPolicy=="Never" and + (.spec.containers|length)==1 and .spec.containers[0].image==$image and + .spec.containers[0].securityContext.allowPrivilegeEscalation==false and + .spec.containers[0].securityContext.readOnlyRootFilesystem==true and + .spec.containers[0].securityContext.capabilities.drop==["ALL"] and + (.spec.containers[0].command[2]|contains("nc -z -w 4 10.43.0.41 9000")) and + (.spec.containers[0].command[2]|contains("nc -z -w 4 10.43.0.42 9115")) +' "$TEST_WORK/network-pod.capture" >/dev/null || fail 'network Pod exact restriction contract' +pass 'network transaction proves deployed ingress denial with explicit test egress' + +unbounded_timeout_lines="$(rg -n '\"\$TIMEOUT_BIN\"(?!.*--kill-after)' "$SMOKE" --pcre2 | grep -v '\"\$SUDO_BIN\".*\"\$DATE_BIN\"' || true)" +[[ -z "$unbounded_timeout_lines" ]] || fail "timeout without kill-after remains: $unbounded_timeout_lines" +raw_silence_date_lines="$(rg -n 'start=\"\$\(\"\$DATE_BIN\"|end=\"\$\(\"\$DATE_BIN\"' "$SMOKE" || true)" +[[ -z "$raw_silence_date_lines" ]] || fail "raw silence date subprocess remains: $raw_silence_date_lines" +pass 'all fixed subprocess boundaries include terminal kill deadlines' + +# Production Certbot leaf: exact bounded dry-run includes deploy hooks and only +# succeeds when Nginx remains active. +cat >"$TEST_WORK/bin/sudo-certbot" <<'SH' +#!/usr/bin/env bash +set -Eeuo pipefail +printf '%q ' "$@" >>"${OBS_SMOKE_CERTBOT_LOG:?}"; printf '\n' >>"${OBS_SMOKE_CERTBOT_LOG:?}" +[[ "${1:-}" == -v ]] && exit 0 +if [[ "${1:-}" == -n && "${2:-}" == /usr/bin/true ]]; then exit 0; fi +if [[ "${1:-}" == -n && "${2:-}" == /usr/bin/systemctl && "${3:-}" == is-active && "${4:-}" == nginx ]]; then printf 'active\n'; exit 0; fi +[[ " $* " == *' /snap/bin/certbot renew --dry-run --run-deploy-hooks '* ]] +SH +chmod 0755 "$TEST_WORK/bin/sudo-certbot" +: >"$TEST_WORK/certbot.log" +SUDO_BIN="$TEST_WORK/bin/sudo-certbot" OBS_SMOKE_CERTBOT_LOG="$TEST_WORK/certbot.log" run_certbot_acceptance >/dev/null +assert_contains "$(<"$TEST_WORK/certbot.log")" '-n /usr/bin/timeout --signal=TERM --kill-after=5s 600s /snap/bin/certbot renew --dry-run --run-deploy-hooks' 'Certbot bounded deploy-hook argv' +SUDO_BIN="$SUDO_DEFAULT" +pass 'Certbot leaf runs exact dry-run deploy-hook and Nginx health gate' + +cat >"$TEST_WORK/bin/sudo-stall-unless-bounded" <<'SH' +#!/usr/bin/env bash +set -Eeuo pipefail +parent="$(/usr/bin/tr '\0' ' ' <"/proc/$PPID/cmdline")" +if [[ "$parent" == *'/usr/bin/timeout'* ]]; then exit 17; fi +/usr/bin/sleep 5 +SH +chmod 0755 "$TEST_WORK/bin/sudo-stall-unless-bounded" +stall_rc=0 +/usr/bin/timeout --signal=TERM --kill-after=1s 1s bash -c ' + PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$1" + SUDO_BIN="$2" + run_certbot_acceptance +' bash "$SMOKE" "$TEST_WORK/bin/sudo-stall-unless-bounded" >/dev/null 2>&1 || stall_rc=$? +[[ "$stall_rc" != 124 ]] || fail 'privileged subprocess escaped its internal timeout' +pass 'privileged Certbot and systemd subprocesses are internally bounded' + +# Regression RED: silence create/delete is response-outcome safe. A lost create +# response is recovered only from one exact unique-run silence; a false 204 +# cannot clear ownership while that exact silence remains. +cat >"$TEST_WORK/bin/date-silence" <<'SH' +#!/usr/bin/env bash +set -Eeuo pipefail +case " $* " in + *' +%Y%m%dT%H%M%SZ '*) printf '20260812T091011Z\n' ;; + *" -d +20 minutes +%Y-%m-%dT%H:%M:%SZ "*) printf '2026-08-12T09:30:11Z\n' ;; + *' +%Y-%m-%dT%H:%M:%SZ '*) printf '2026-08-12T09:10:11Z\n' ;; + *) exit 91 ;; +esac +SH +chmod 0755 "$TEST_WORK/bin/date-silence" +( + WORK="$TEST_WORK"; DATE_BIN="$TEST_WORK/bin/date-silence" + new_smoke_run_id() { printf 'persistence-20260812t091011z-00000000000000000000000000000001'; } + SILENCE_ID=''; SILENCE_RUN=''; SILENCE_POLL_ATTEMPTS=2; SILENCE_POLL_DELAY=0 + MANUAL_RECOVERY=false + rm -f -- "$TEST_WORK/silence-post-attempted" + proxy_request() { + local method=$1 output=$3 + case "$method" in + POST) + cp -- "$4" "$TEST_WORK/silence-create-body.capture" + : >"$output"; chmod 0600 -- "$output" + : >"$TEST_WORK/silence-post-attempted" + return 28 + ;; + GET) + if [[ ! -f "$TEST_WORK/silence-post-attempted" ]]; then printf '[]\n' >"$output"; else jq -n --arg run 'persistence-20260812t091011z-00000000000000000000000000000001' '[{ + id:"silence-owned-id",status:{state:"active"},createdBy:"platform-observability-smoke", + comment:("Temporary restart-persistence acceptance "+$run), + matchers:[ + {name:"alertname",value:"PlatformObservabilityPersistenceSmoke",isRegex:false,isEqual:true}, + {name:"instance",value:$run,isRegex:false,isEqual:true} + ] + }]' >"$output"; fi + chmod 0600 -- "$output"; printf '200' + ;; + esac + } + create_persistence_silence + printf '%s|%s\n' "$SILENCE_ID" "$SILENCE_RUN" >"$TEST_WORK/silence-create-recovered.capture" +) || fail 'lost-response silence create was not reconciled' +assert_eq 'silence-owned-id|persistence-20260812t091011z-00000000000000000000000000000001' "$(<"$TEST_WORK/silence-create-recovered.capture")" 'recovered silence identity' +jq -e ' + .createdBy=="platform-observability-smoke" and + .comment=="Temporary restart-persistence acceptance persistence-20260812t091011z-00000000000000000000000000000001" and + ([.matchers[]|select(.name=="instance" and .value=="persistence-20260812t091011z-00000000000000000000000000000001" and .isRegex==false and .isEqual==true)]|length)==1 +' "$TEST_WORK/silence-create-body.capture" >/dev/null || fail 'unique silence create payload' + +false_delete_rc=0 +( + WORK="$TEST_WORK"; SILENCE_ID='silence-owned-id'; SILENCE_RUN='persistence-20260812t091011z-00000000000000000000000000000001' + SILENCE_POLL_ATTEMPTS=2; SILENCE_POLL_DELAY=0; MANUAL_RECOVERY=false + emit_manual_recovery() { printf 'manual\n' >"$TEST_WORK/silence-delete-manual.capture"; MANUAL_RECOVERY=true; } + proxy_request() { + local method=$1 output=$3 + if [[ "$method" == DELETE ]]; then : >"$output"; chmod 0600 -- "$output"; printf '204'; return 0; fi + jq -n --arg run "$SILENCE_RUN" '[{ + id:"silence-owned-id",status:{state:"active"},createdBy:"platform-observability-smoke", + comment:("Temporary restart-persistence acceptance "+$run), + matchers:[{name:"alertname",value:"PlatformObservabilityPersistenceSmoke",isRegex:false,isEqual:true},{name:"instance",value:$run,isRegex:false,isEqual:true}] + }]' >"$output"; chmod 0600 -- "$output"; printf '200' + } + delete_persistence_silence +) || false_delete_rc=$? +assert_eq 1 "$false_delete_rc" 'false-204 silence delete status' +[[ -f "$TEST_WORK/silence-delete-manual.capture" ]] || fail 'false-204 silence delete omitted manual recovery' + +expired_delete_rc=0 +( + WORK="$TEST_WORK"; SILENCE_ID='silence-owned-id'; SILENCE_RUN='persistence-20260812t091011z-00000000000000000000000000000001' + SILENCE_POLL_ATTEMPTS=2; SILENCE_POLL_DELAY=0; MANUAL_RECOVERY=false + proxy_request() { + local method=$1 output=$3 + if [[ "$method" == DELETE ]]; then : >"$output"; chmod 0600 -- "$output"; printf '204'; return 0; fi + jq -n --arg run "$SILENCE_RUN" '[{ + id:"silence-owned-id",status:{state:"expired"},createdBy:"platform-observability-smoke", + comment:("Temporary restart-persistence acceptance "+$run), + matchers:[{name:"alertname",value:"PlatformObservabilityPersistenceSmoke",isRegex:false,isEqual:true},{name:"instance",value:$run,isRegex:false,isEqual:true}] + }]' >"$output"; chmod 0600 -- "$output"; printf '200' + } + delete_persistence_silence +) || expired_delete_rc=$? +assert_eq 0 "$expired_delete_rc" 'stable expired silence delete outcome' +pass 'silence lifecycle reconciles response loss and rejects false delete success' + +signal_silence_rc=0 +( + WORK="$TEST_WORK"; DATE_BIN="$TEST_WORK/bin/date-silence" + new_smoke_run_id() { printf 'persistence-20260812t091011z-00000000000000000000000000000002'; } + SILENCE_ID=''; SILENCE_RUN=''; SILENCE_POLL_ATTEMPTS=2; SILENCE_POLL_DELAY=0 + MUTATION_CRITICAL_DEPTH=0; DEFERRED_SIGNAL_RC=0 + SIGNAL_TARGET=$BASHPID + trap 'handle_termination_signal 130' INT + trap 'printf "%s|%s\n" "$SILENCE_ID" "$SILENCE_RUN" >"$TEST_WORK/signal-silence.capture"' EXIT + rm -f -- "$TEST_WORK/signal-silence-posted" + proxy_request() { + local method=$1 output=$3 + case "$method" in + POST) + : >"$output"; chmod 0600 -- "$output" + : >"$TEST_WORK/signal-silence-posted" + /bin/kill -INT "$SIGNAL_TARGET" + return 28 + ;; + GET) + if [[ ! -f "$TEST_WORK/signal-silence-posted" ]]; then + printf '[]\n' >"$output" + else + jq -n --arg run "$SILENCE_RUN" '[{id:"silence-signal-id",status:{state:"active"},createdBy:"platform-observability-smoke",comment:("Temporary restart-persistence acceptance "+$run),matchers:[{name:"alertname",value:"PlatformObservabilityPersistenceSmoke",isRegex:false,isEqual:true},{name:"instance",value:$run,isRegex:false,isEqual:true}]}]' >"$output" + fi + chmod 0600 -- "$output"; printf '200' + ;; + esac + } + create_persistence_silence +) || signal_silence_rc=$? +assert_eq 130 "$signal_silence_rc" 'deferred silence signal status' +assert_eq 'silence-signal-id|persistence-20260812t091011z-00000000000000000000000000000002' \ + "$(<"$TEST_WORK/signal-silence.capture")" 'signal-safe silence ownership capture' +pass 'silence signal window closes only after exact ownership capture' + +# Production raw delete leaf: keep its real API-path and UID DeleteOptions +# builder while replacing only transport and post-delete reads. Transport loss +# is success only after stable absence; a foreign replacement fails closed. +RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)" +chmod 0700 "$RAW_DELETE_WORK" +( + WORK="$RAW_DELETE_WORK" + proxy_request() { + printf '%s|%s\n' "$1" "$2" >"$WORK/raw-delete-call" + cp -- "$4" "$WORK/raw-delete-body" + : >"$3" + printf '200' + } + owned_object_state() { return 1; } + delete_owned_uid Pod observability smoke-pod uid-pod +) +assert_eq 'DELETE|/api/v1/namespaces/observability/pods/smoke-pod' "$(<"$RAW_DELETE_WORK/raw-delete-call")" 'raw delete API path' +jq -e '.apiVersion=="meta.k8s.io/v1" and .kind=="DeleteOptions" and .propagationPolicy=="Background" and .preconditions=={"uid":"uid-pod"}' "$RAW_DELETE_WORK/raw-delete-body" >/dev/null || fail 'raw UID DeleteOptions contract' +ambiguity_rc=0 +( + WORK="$RAW_DELETE_WORK" + proxy_request() { return 28; } + owned_object_state() { return 1; } + delete_owned_uid PrometheusRule observability smoke-rule uid-rule +) || ambiguity_rc=$? +assert_eq 0 "$ambiguity_rc" 'transport ambiguity followed by absence' +foreign_rc=0 +( + WORK="$RAW_DELETE_WORK" + proxy_request() { return 28; } + owned_object_state() { printf '%s\n' 'foreign-uid|foreign-run'; return 0; } + delete_owned_uid PrometheusRule observability smoke-rule uid-rule +) || foreign_rc=$? +assert_eq 1 "$foreign_rc" 'foreign replacement raw delete status' +rm -rf -- "$RAW_DELETE_WORK" +RAW_DELETE_WORK='' +pass 'production raw UID delete reconciles ambiguity and preserves replacement' + +# Regression RED: an ambiguous create may be followed by an unrelated object +# with copied name/labels. Server defaults are allowed, but every desired field +# and the exact standalone-container boundary must match before ownership. +cat >"$TEST_WORK/owned-desired.json" <<'JSON' +{"apiVersion":"v1","kind":"Pod","metadata":{"name":"smoke-pod","namespace":"observability","labels":{"app.kubernetes.io/managed-by":"platform-observability-smoke","platform.hyeonworks.com/smoke-run":"run-1"}},"spec":{"automountServiceAccountToken":false,"restartPolicy":"Never","containers":[{"name":"network-smoke","image":"busybox@sha256:fixture","command":["sh","-ec","true"],"securityContext":{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"capabilities":{"drop":["ALL"]}}}]}} +JSON +jq '.metadata.uid="uid-pod" | .metadata.resourceVersion="7" | .spec.dnsPolicy="ClusterFirst" | .spec.containers[0].terminationMessagePolicy="File" | .status={"phase":"Pending"}' \ + "$TEST_WORK/owned-desired.json" >"$TEST_WORK/owned-live-defaulted.json" +chmod 0600 "$TEST_WORK/owned-desired.json" "$TEST_WORK/owned-live-defaulted.json" +declare -F validate_live_matches_desired >/dev/null || + fail 'ambiguous-create desired/live ownership validator is missing' +validate_live_matches_desired "$TEST_WORK/owned-desired.json" "$TEST_WORK/owned-live-defaulted.json" +jq '.spec.containers += [{"name":"foreign","image":"busybox:latest"}]' \ + "$TEST_WORK/owned-live-defaulted.json" >"$TEST_WORK/owned-live-sidecar.json" +expect_failure validate_live_matches_desired "$TEST_WORK/owned-desired.json" "$TEST_WORK/owned-live-sidecar.json" +jq '.spec.containers[0].command=["sh","-ec","exfiltrate"]' \ + "$TEST_WORK/owned-live-defaulted.json" >"$TEST_WORK/owned-live-drift.json" +expect_failure validate_live_matches_desired "$TEST_WORK/owned-desired.json" "$TEST_WORK/owned-live-drift.json" +pass 'ambiguous create ownership requires desired spec, not copied labels' + +# Production create leaf: exercise server dry-run, successful identity capture, +# transport-ambiguous re-read, and a copied-label foreign-spec rejection. +RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)" +chmod 0700 "$RAW_DELETE_WORK" +( + WORK="$RAW_DELETE_WORK" + scenario=success + owned_object_state() { return 1; } + kubectl_bounded() { + if [[ "$1" == create && "$2" == --dry-run=server ]]; then return 0; fi + if [[ "$1" == create ]]; then + cat -- "$TEST_WORK/owned-live-defaulted.json" + [[ "$scenario" == success ]] && return 0 + return 28 + fi + if [[ "$1" == -n && "$3" == get ]]; then + if [[ "$scenario" == foreign ]]; then cat -- "$TEST_WORK/owned-live-sidecar.json"; else cat -- "$TEST_WORK/owned-live-defaulted.json"; fi + return 0 + fi + return 91 + } + OWNED_KIND=(); OWNED_NAMESPACE=(); OWNED_NAME=(); OWNED_UID=(); OWNED_RUN=() + create_owned_from_file "$TEST_WORK/owned-desired.json" Pod observability smoke-pod run-1 + printf '%s|%s|%s|%s|%s\n' "${OWNED_KIND[0]}" "${OWNED_NAMESPACE[0]}" "${OWNED_NAME[0]}" "${OWNED_UID[0]}" "${OWNED_RUN[0]}" >"$TEST_WORK/create-success.capture" + + OWNED_KIND=(); OWNED_NAMESPACE=(); OWNED_NAME=(); OWNED_UID=(); OWNED_RUN=(); scenario=ambiguous + ambiguous_create_rc=0 + create_owned_from_file "$TEST_WORK/owned-desired.json" Pod observability smoke-pod run-1 || ambiguous_create_rc=$? + printf '%s|%s|%s\n' "$ambiguous_create_rc" "${#OWNED_UID[@]}" "${OWNED_UID[0]:-}" >"$TEST_WORK/create-ambiguous.capture" + + OWNED_KIND=(); OWNED_NAMESPACE=(); OWNED_NAME=(); OWNED_UID=(); OWNED_RUN=(); scenario=foreign + foreign_create_rc=0 + emit_manual_recovery() { printf 'manual\n' >"$TEST_WORK/create-foreign-manual.capture"; } + create_owned_from_file "$TEST_WORK/owned-desired.json" Pod observability smoke-pod run-1 || foreign_create_rc=$? + printf '%s|%s\n' "$foreign_create_rc" "${#OWNED_UID[@]}" >"$TEST_WORK/create-foreign.capture" +) +assert_eq 'Pod|observability|smoke-pod|uid-pod|run-1' "$(<"$TEST_WORK/create-success.capture")" 'successful create ownership capture' +assert_eq '1|1|uid-pod' "$(<"$TEST_WORK/create-ambiguous.capture")" 'ambiguous create owned-state capture' +assert_eq '1|0' "$(<"$TEST_WORK/create-foreign.capture")" 'foreign create ownership rejection' +[[ -f "$TEST_WORK/create-foreign-manual.capture" ]] || fail 'foreign ambiguous create omitted manual-recovery evidence' +rm -rf -- "$RAW_DELETE_WORK" +RAW_DELETE_WORK='' +pass 'production create leaf fails closed and records only exact owned state' + +# A signal delivered after the API commits but before the response is recorded +# must be deferred until the exact created UID is in the cleanup ledger. +declare -F handle_termination_signal >/dev/null || fail 'mutation signal deferral is missing' +RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)" +chmod 0700 "$RAW_DELETE_WORK" +signal_create_rc=0 +( + WORK="$RAW_DELETE_WORK" + trap 'handle_termination_signal 130' INT + trap 'printf "%s|%s\n" "${#OWNED_UID[@]}" "${OWNED_UID[0]:-}" >"$TEST_WORK/signal-create.capture"' EXIT + owned_object_state() { return 1; } + kubectl_bounded() { + if [[ "$1" == create && "$2" == --dry-run=server ]]; then return 0; fi + if [[ "$1" == create ]]; then + /bin/kill -INT "$BASHPID" + cat -- "$TEST_WORK/owned-live-defaulted.json" + return 0 + fi + return 91 + } + OWNED_KIND=(); OWNED_NAMESPACE=(); OWNED_NAME=(); OWNED_UID=(); OWNED_RUN=() + create_owned_from_file "$TEST_WORK/owned-desired.json" Pod observability smoke-pod run-1 +) || signal_create_rc=$? +assert_eq 130 "$signal_create_rc" 'deferred create signal status' +assert_eq '1|uid-pod' "$(<"$TEST_WORK/signal-create.capture")" 'signal-safe UID registration' +rm -rf -- "$RAW_DELETE_WORK" +RAW_DELETE_WORK='' +pass 'create signal window closes only after exact UID registration' + +# Mutation caught: cleanup by name or without a matching UID could delete a +# replacement. The fixture exercises owned, absent, and foreign identities. +OWNED_KIND=(Pod NetworkPolicy PrometheusRule) +OWNED_NAMESPACE=(observability observability observability) +OWNED_NAME=(smoke-pod smoke-policy smoke-rule) +OWNED_UID=(uid-pod uid-policy uid-rule) +OWNED_RUN=(run-1 run-1 run-1) +declare -a cleanup_calls=() +owned_object_state() { + case "$3" in + smoke-pod) return 1 ;; + smoke-policy) printf '%s\n' 'uid-policy|run-1'; return 0 ;; + smoke-rule) printf '%s\n' 'foreign-uid|foreign-run'; return 0 ;; + esac +} +delete_owned_uid() { cleanup_calls+=("$1|$2|$3|$4"); [[ "$3" != smoke-rule ]]; } +cleanup_rc=0 +cleanup_owned_objects || cleanup_rc=$? +assert_eq 1 "$cleanup_rc" 'foreign cleanup status' +assert_eq 'NetworkPolicy|observability|smoke-policy|uid-policy' "${cleanup_calls[0]}" 'exact UID cleanup call' +assert_eq 1 "${#cleanup_calls[@]}" 'foreign object must not be deleted' +pass 'cleanup preserves absent and foreign objects and UID-deletes only owned state' + +# Regression RED: an execute preflight may create the private work/proxy and +# fail halfway. EXIT cleanup must already be armed before that first allocation. +preflight_cleanup_marker="$TEST_WORK/preflight-cleanup.marker" +preflight_rc=0 +( + TEST_MODE=true + require_execute_environment() { WORK='/tmp/platform-observability-smoke.fixture00'; return 1; } + cleanup_all() { printf 'cleanup\n' >"$preflight_cleanup_marker"; } + main --execute +) >/dev/null 2>&1 || preflight_rc=$? +assert_eq 1 "$preflight_rc" 'partial preflight failure status' +[[ -f "$preflight_cleanup_marker" ]] || fail 'partial execute preflight was not cleanup-armed' +pass 'execute cleanup is armed before preflight allocation' + +# The main execute orchestration is tested with all external work replaced at +# function boundaries. Wrong APPLY stops before mutation; missing external +# evidence returns PARTIAL=2; success performs cleanup and reaches final PASS. +run_orchestration_fixture() { + local confirmations=$1 + ( + TEST_MODE=true + PUBLIC_EDGE_IP='203.0.113.10' + TEST_CONFIRMATIONS="$confirmations" + mapfile -t CONFIRMATION_QUEUE <<<"$TEST_CONFIRMATIONS" + CONFIRMATION_INDEX=0 + require_execute_environment() { :; } + run_read_only_machine_checks() { printf 'machine\n' >>"${ORCHESTRATION_LOG:-/dev/null}"; printf 'READ_ONLY_FIXTURE=PASS\n'; } + repeat_commit_gate() { printf 'commit-gate\n' >>"${ORCHESTRATION_LOG:-/dev/null}"; } + run_network_boundary_transaction() { printf 'network\n' >>"${ORCHESTRATION_LOG:-/dev/null}"; OWNED_KIND+=(Pod); OWNED_NAMESPACE+=(observability); OWNED_NAME+=(fixture-pod); OWNED_UID+=(fixture-uid); OWNED_RUN+=(fixture-run); } + run_restart_persistence_transaction() { printf 'restart\n' >>"${ORCHESTRATION_LOG:-/dev/null}"; } + run_certbot_acceptance() { printf 'certbot\n' >>"${ORCHESTRATION_LOG:-/dev/null}"; } + run_oidc_acceptance() { require_confirmation OIDC_ADMIN_CONFIRMED; require_confirmation OIDC_VIEWER_CONFIRMED; require_confirmation OIDC_DENIED_CONFIRMED; require_confirmation BREAK_GLASS_CONFIRMED; require_confirmation REMOVAL_VIEWER_CONFIRMED; require_confirmation SESSION_REVOKED; require_confirmation RELOGIN_DENIED; printf 'OBSERVABILITY_OIDC_ACCEPTANCE=PASS\n'; } + run_slack_acceptance() { require_confirmation FIRING; require_confirmation RESOLVED; printf 'OBSERVABILITY_SLACK_ACCEPTANCE=PASS\n'; } + owned_object_state() { return 1; } + delete_owned_uid() { :; } + main --execute + ) +} + +wrong_rc=0 +wrong_output="$(run_orchestration_fixture 'WRONG' 2>&1)" || wrong_rc=$? +assert_eq 1 "$wrong_rc" 'wrong APPLY orchestration status' +assert_not_contains "$wrong_output" 'OBSERVABILITY_MACHINE_ACCEPTANCE=PASS' 'wrong APPLY reached mutation' + +partial_confirmations=$'APPLY default\nOIDC_ADMIN_CONFIRMED\nOIDC_VIEWER_CONFIRMED\nOIDC_DENIED_CONFIRMED\nBREAK_GLASS_CONFIRMED\nREMOVAL_VIEWER_CONFIRMED\nSESSION_REVOKED\nRELOGIN_DENIED\nFIRING\nRESOLVED' +partial_rc=0 +partial_output="$(run_orchestration_fixture "$partial_confirmations" 2>&1)" || partial_rc=$? +assert_eq 2 "$partial_rc" 'external partial orchestration status' +assert_contains "$partial_output" 'OBSERVABILITY_EXTERNAL_BOUNDARY=PARTIAL' 'external partial marker' +assert_not_contains "$partial_output" 'OBSERVABILITY_SMOKE=PASS' 'partial acceptance falsely passed' + +full_confirmations="$partial_confirmations"$'\nEXTERNAL_CLIENT_VERIFIED public_ip=203.0.113.10 status=403 source=outside-lan-tailscale proxy=disabled' +: >"$TEST_WORK/orchestration-order.log" +full_output="$(ORCHESTRATION_LOG="$TEST_WORK/orchestration-order.log" run_orchestration_fixture "$full_confirmations")" +assert_contains "$full_output" 'OBSERVABILITY_MACHINE_ACCEPTANCE=PASS' 'machine acceptance marker' +assert_contains "$full_output" 'OBSERVABILITY_SMOKE=PASS' 'full acceptance marker' +assert_eq $'machine\ncommit-gate\nnetwork\nrestart\ncertbot\ncommit-gate\nmachine' \ + "$(<"$TEST_WORK/orchestration-order.log")" 'post-mutation full machine revalidation order' +pass 'execute orchestration is gated and cannot turn partial into PASS' + +bash -n "$SMOKE" +bash -n "$TEST_ROOT/scripts/validate/test-observability-smoke.sh" +printf 'OBSERVABILITY SMOKE TEST PASS (%d assertions)\n' "$ASSERTIONS" diff --git a/scripts/validate/test-platform-observability-rules.sh b/scripts/validate/test-platform-observability-rules.sh new file mode 100644 index 0000000..4b7d0a3 --- /dev/null +++ b/scripts/validate/test-platform-observability-rules.sh @@ -0,0 +1,1070 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +readonly RULE_DIR="${ROOT_DIR}/services/observability/rules/platform" +readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b" +readonly EXPECTED_RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md" + +runbook_url_blocked=0 +temp_dir='' + +# Each raw metric in the three Task 7 rule resources is bound to the +# inventory phase and scrape pool that supplied it. Probe metrics select a +# pool from their explicit probe-group matcher below. +declare -A new_rule_metric_phase=( + [minio_cluster_usage_buckets_total_bytes]=target-initial + [minio_cluster_usage_buckets_quota_total_bytes]=target-initial + [cnpg_collector_up]=target-initial + [alloy_config_last_load_successful]=target-initial + [loki_write_batch_retries_total]=target-initial + [loki_write_dropped_entries_total]=target-initial + [loki_runtime_config_last_reload_successful]=target-initial + [loki_ingester_wal_disk_usage_percent]=target-initial + [probe_success]=post-substrate + [probe_ssl_earliest_cert_expiry]=post-substrate +) +declare -A new_rule_metric_pool=( + [minio_cluster_usage_buckets_total_bytes]=serviceMonitor/object-storage/aistor-bucket-usage/0 + [minio_cluster_usage_buckets_quota_total_bytes]=serviceMonitor/object-storage/aistor-bucket-usage/0 + [cnpg_collector_up]=podMonitor/platform-data/platform-postgres/0 + [alloy_config_last_load_successful]=serviceMonitor/observability-agent/alloy/0 + [loki_write_batch_retries_total]=serviceMonitor/observability-agent/alloy/0 + [loki_write_dropped_entries_total]=serviceMonitor/observability-agent/alloy/0 + [loki_runtime_config_last_reload_successful]=serviceMonitor/observability/loki/0 + [loki_ingester_wal_disk_usage_percent]=serviceMonitor/observability/loki/0 + [probe_success]=probe-group + [probe_ssl_earliest_cert_expiry]=probe-group +) +declare -A new_rule_probe_pool=( + ["probe_success|public-edge"]=probe/observability/platform-public-edge + ["probe_success|private-edge"]=probe/observability/platform-private-edge + ["probe_success|private-internal"]=probe/observability/platform-private-internal + ["probe_ssl_earliest_cert_expiry|public-edge"]=probe/observability/platform-public-edge + ["probe_ssl_earliest_cert_expiry|private-edge"]=probe/observability/platform-private-edge +) +declare -A new_rule_matcher_contract=( + [minio_cluster_usage_buckets_total_bytes]=$'bucket\t=~\tloki|tempo' + [minio_cluster_usage_buckets_quota_total_bytes]=$'bucket\t=~\tloki|tempo' + [cnpg_collector_up]=$'cluster\t=\tplatform-postgres\nnamespace\t=\tplatform-data' + [alloy_config_last_load_successful]=$'namespace\t=\tobservability-agent' + [loki_write_batch_retries_total]=$'namespace\t=\tobservability-agent' + [loki_write_dropped_entries_total]=$'namespace\t=\tobservability-agent' + [loki_runtime_config_last_reload_successful]=$'namespace\t=\tobservability' + [loki_ingester_wal_disk_usage_percent]=$'namespace\t=\tobservability' + ["probe_success|=|public-edge"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=\tpublic-edge' + ["probe_success|=|private-edge"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=\tprivate-edge' + ["probe_success|=|private-internal"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=\tprivate-internal' + ["probe_ssl_earliest_cert_expiry|=~|public-edge|private-edge"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=~\tpublic-edge|private-edge' +) +declare -A recording_rule_expression=() +declare -A recording_rule_state=() +declare -a selector_matcher_labels=() +declare -A selector_matcher_operator=() +declare -A selector_matcher_value=() + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +pass() { + printf 'PASS: %s\n' "$*" +} + +cleanup() { + if [[ -n "$temp_dir" && "$temp_dir" == /tmp/tmp.* && -d "$temp_dir" ]]; then + rm -rf -- "$temp_dir" + fi +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "required command is unavailable: $1" +} + +assert_inventory() { + local inventory_dir=$1 + local expected_phase=$2 + local inventory_file="${inventory_dir}/inventory.json" + local checksum_file="${inventory_dir}/inventory.sha256" + local expected_sha actual_sha + + [[ -f "$inventory_file" ]] || fail "missing inventory: ${inventory_file}" + [[ -f "$checksum_file" ]] || fail "missing inventory checksum: ${checksum_file}" + expected_sha="$(awk '$2 == "inventory.json" { print $1 }' "$checksum_file")" + [[ "$expected_sha" =~ ^[0-9a-f]{64}$ ]] || fail "invalid inventory checksum record: ${checksum_file}" + actual_sha="$(sha256sum "$inventory_file" | awk '{ print $1 }')" + [[ "$actual_sha" == "$expected_sha" ]] || fail "inventory checksum mismatch: ${expected_phase}" + jq -e --arg phase "$expected_phase" ' + .schema == "platform-observability-metric-inventory/v1" and + .phase == $phase and + (.targets | type == "array" and length > 0) and + all(.targets[]; + .health == "up" and + (.metrics | type == "array" and length > 0) + ) + ' "$inventory_file" >/dev/null || fail "inventory schema/phase mismatch: ${expected_phase}" +} + +assert_metric_label() { + local inventory_file=$1 + local scrape_pool=$2 + local metric_name=$3 + local label_name=$4 + + jq -e \ + --arg pool "$scrape_pool" \ + --arg metric "$metric_name" \ + --arg label "$label_name" ' + any(.targets[]; + .scrape_pool == $pool and + any(.metrics[]; + .name == $metric and + (.label_names | index($label)) != null + ) + ) + ' "$inventory_file" >/dev/null || + fail "inventory lacks ${scrape_pool} ${metric_name}{${label_name}}" +} + +assert_metric() { + local inventory_file=$1 + local scrape_pool=$2 + local metric_name=$3 + + jq -e \ + --arg pool "$scrape_pool" \ + --arg metric "$metric_name" ' + any(.targets[]; + .scrape_pool == $pool and + any(.metrics[]; .name == $metric) + ) + ' "$inventory_file" >/dev/null || + fail "inventory lacks ${scrape_pool} ${metric_name}" +} + +parse_explicit_matchers() { + local selector=$1 + local context=$2 + local matchers matcher_re match label operator value + + matchers=${selector#*\{} + matchers=${matchers%\}} + selector_matcher_labels=() + selector_matcher_operator=() + selector_matcher_value=() + matcher_re='^[[:space:],]*("[^"]+"|[A-Za-z_][A-Za-z0-9_]*)[[:space:]]*(=~|!~|!=|=)[[:space:]]*"(([^"\\]|\\.)*)"' + + while [[ -n "${matchers//[[:space:],]/}" ]]; do + [[ "$matchers" =~ $matcher_re ]] || + fail "unparseable explicit matcher in ${context}: ${matchers}" + match=${BASH_REMATCH[0]} + label=${BASH_REMATCH[1]} + operator=${BASH_REMATCH[2]} + value=${BASH_REMATCH[3]} + label=${label#\"} + label=${label%\"} + [[ -z ${selector_matcher_operator[$label]+x} ]] || + fail "duplicate explicit matcher label ${label} in ${context}" + selector_matcher_labels+=("$label") + selector_matcher_operator["$label"]=$operator + selector_matcher_value["$label"]=$value + matchers=${matchers:${#match}} + [[ -z "${matchers//[[:space:]]/}" || "$matchers" =~ ^[[:space:]]*, ]] || + fail "unparseable explicit matcher separator in ${context}: ${matchers}" + done +} + +assert_exact_matcher_contract() { + local metric=$1 + local has_selector=$2 + local pool_binding=$3 + local context=$4 + local contract_key=$metric + local group_label='observability.hyeonworks.com/probe-group' + local expected_signature actual_signature label + + (( has_selector == 1 )) || + fail "exact matcher contract requires a selector for ${metric} in ${context}" + if [[ "$pool_binding" == probe-group ]]; then + [[ -n ${selector_matcher_operator[$group_label]+x} ]] || + fail "exact matcher contract requires ${group_label} for ${metric} in ${context}" + contract_key="${metric}|${selector_matcher_operator[$group_label]}|${selector_matcher_value[$group_label]}" + fi + + expected_signature=${new_rule_matcher_contract[$contract_key]:-} + [[ -n "$expected_signature" ]] || + fail "exact matcher contract is not mapped for ${contract_key} in ${context}" + actual_signature="$({ + for label in "${selector_matcher_labels[@]}"; do + printf '%s\t%s\t%s\n' \ + "$label" \ + "${selector_matcher_operator[$label]}" \ + "${selector_matcher_value[$label]}" + done + } | LC_ALL=C sort)" + [[ "$actual_signature" == "$expected_signature" ]] || + fail "exact matcher contract mismatch for ${metric} in ${context}" +} + +assert_mapped_raw_metric() { + local metric=$1 + local has_selector=$2 + local target_inventory=$3 + local substrate_inventory=$4 + local context=$5 + local phase pool_binding inventory pool group_label operator value group probe_key + local -a pools groups + local -A seen_pools=() + + phase=${new_rule_metric_phase[$metric]:-} + pool_binding=${new_rule_metric_pool[$metric]:-} + [[ -n "$phase" && -n "$pool_binding" ]] || + fail "unmapped raw metric: ${metric} in ${context}" + case "$phase" in + target-initial) inventory=$target_inventory ;; + post-substrate) inventory=$substrate_inventory ;; + *) fail "unmapped inventory phase ${phase} for raw metric ${metric}" ;; + esac + + pools=() + if [[ "$pool_binding" == probe-group ]]; then + (( has_selector == 1 )) || + fail "unmapped probe scrape pool without an explicit probe-group matcher: ${metric} in ${context}" + group_label='observability.hyeonworks.com/probe-group' + [[ -n ${selector_matcher_operator[$group_label]+x} ]] || + fail "unmapped probe scrape pool without ${group_label}: ${metric} in ${context}" + operator=${selector_matcher_operator[$group_label]} + value=${selector_matcher_value[$group_label]} + case "$operator" in + '=') groups=("$value") ;; + '=~') IFS='|' read -r -a groups <<<"$value" ;; + *) fail "unmapped probe-group matcher ${group_label}${operator} in ${context}" ;; + esac + for group in "${groups[@]}"; do + probe_key="${metric}|${group}" + pool=${new_rule_probe_pool[$probe_key]:-} + [[ -n "$pool" ]] || + fail "unmapped probe scrape pool for ${metric} ${group_label}=${group} in ${context}" + if [[ -z ${seen_pools[$pool]+x} ]]; then + pools+=("$pool") + seen_pools["$pool"]=1 + fi + done + else + pools=("$pool_binding") + fi + + for pool in "${pools[@]}"; do + assert_metric "$inventory" "$pool" "$metric" + if (( has_selector == 1 )); then + for label in "${selector_matcher_labels[@]}"; do + assert_metric_label "$inventory" "$pool" "$metric" "$label" + done + fi + done + assert_exact_matcher_contract "$metric" "$has_selector" "$pool_binding" "$context" +} + +validate_raw_selector_provenance() { + local selector=$1 + local target_inventory=$2 + local substrate_inventory=$3 + local context=$4 + local metric + + metric=$(sed -E 's/[[:space:]]*\{.*$//' <<<"$selector") + [[ "$metric" =~ ^[A-Za-z_:][A-Za-z0-9_:]*$ ]] || + fail "unparseable raw metric selector in ${context}: ${selector}" + parse_explicit_matchers "$selector" "$context" + assert_mapped_raw_metric "$metric" 1 "$target_inventory" "$substrate_inventory" "$context" +} + +is_promql_syntax_token() { + case "$1" in + and|or|unless|bool|by|without|on|ignoring|group_left|group_right|offset|start|end|\ + min|max|sum|avg|group|stddev|stdvar|topk|bottomk|count|count_values|quantile|limitk|limit_ratio|\ + s|m|h|d|w|y|e|NaN|Inf) + return 0 + ;; + esac + return 1 +} + +is_promql_function_call() { + local token=$1 + local expression=$2 + + grep -Eq "(^|[^[:alnum:]_:])${token}[[:space:]]*\\(" <<<"$expression" +} + +load_recording_rule_expressions() { + local rule_json record encoded_expression expression + + recording_rule_expression=() + recording_rule_state=() + for rule_json in "$@"; do + while IFS=$'\t' read -r record encoded_expression; do + expression=$(printf '%s' "$encoded_expression" | base64 -d) + [[ -z ${recording_rule_expression[$record]+x} ]] || + fail "duplicate recording rule name ${record} in ${rule_json}" + recording_rule_expression["$record"]=$expression + done < <( + jq -r ' + .spec.groups[].rules[] + | select(has("record")) + | [.record, (.expr | @base64)] + | @tsv + ' "$rule_json" + ) + done +} + +validate_recording_rule_provenance() { + local record=$1 + local target_inventory=$2 + local substrate_inventory=$3 + local context=$4 + local state + + [[ -n ${recording_rule_expression[$record]+x} ]] || + fail "unresolved recording rule reference: ${record} in ${context}" + state=${recording_rule_state[$record]:-new} + case "$state" in + done) return 0 ;; + visiting) fail "recording rule dependency cycle: ${record} in ${context}" ;; + esac + recording_rule_state["$record"]=visiting + validate_expression_provenance \ + "${recording_rule_expression[$record]}" \ + "$target_inventory" "$substrate_inventory" \ + "${context} -> recording rule ${record}" + recording_rule_state["$record"]=done +} + +validate_expression_provenance() { + local expression=$1 + local target_inventory=$2 + local substrate_inventory=$3 + local context=$4 + local flattened selector without_selectors bare_expression token + + flattened=$(tr '\n' ' ' <<<"$expression") + while IFS= read -r selector; do + [[ -n "$selector" ]] || continue + validate_raw_selector_provenance \ + "$selector" "$target_inventory" "$substrate_inventory" "$context" + done < <( + grep -oE '[A-Za-z_:][A-Za-z0-9_:]*[[:space:]]*\{[^}]*\}' <<<"$flattened" || true + ) + + without_selectors=$(sed -E 's/[A-Za-z_:][A-Za-z0-9_:]*[[:space:]]*\{[^}]*\}//g' <<<"$flattened") + bare_expression=$(sed -E \ + -e 's/(by|without|on|ignoring|group_left|group_right)[[:space:]]*\([^)]*\)//g' \ + -e 's/"[^"]*"//g' <<<"$without_selectors") + while IFS= read -r token; do + [[ -n "$token" ]] || continue + if is_promql_syntax_token "$token" || is_promql_function_call "$token" "$bare_expression"; then + continue + fi + if [[ -n ${recording_rule_expression[$token]+x} ]]; then + validate_recording_rule_provenance \ + "$token" "$target_inventory" "$substrate_inventory" "$context" + elif [[ "$token" == *:* ]]; then + validate_recording_rule_provenance \ + "$token" "$target_inventory" "$substrate_inventory" "$context" + elif [[ -n ${new_rule_metric_phase[$token]+x} ]]; then + assert_mapped_raw_metric "$token" 0 "$target_inventory" "$substrate_inventory" "$context" + else + fail "unmapped raw metric or recording rule reference: ${token} in ${context}" + fi + done < <( + { grep -oE '[A-Za-z_:][A-Za-z0-9_:]*' <<<"$bare_expression" || true; } | sort -u + ) +} + +assert_new_rule_provenance() { + local target_inventory=$1 + local substrate_inventory=$2 + local rule_json rule_name encoded_expression expression + + shift 2 + load_recording_rule_expressions "$@" + for rule_json in "$@"; do + while IFS=$'\t' read -r rule_name encoded_expression; do + expression=$(printf '%s' "$encoded_expression" | base64 -d) + validate_expression_provenance \ + "$expression" "$target_inventory" "$substrate_inventory" \ + "${rule_json} rule ${rule_name}" + done < <( + jq -r ' + .spec.groups[].rules[] + | [(.record // .alert), (.expr | @base64)] + | @tsv + ' "$rule_json" + ) + done +} + +assert_provenance_rejects() { + local expected_reason=$1 + shift + local output + + if output="$( (assert_new_rule_provenance "$@") 2>&1 )"; then + fail "provenance regression unexpectedly accepted a typo: ${expected_reason}" + fi + grep -Fq "$expected_reason" <<<"$output" || + fail "provenance regression did not reject the typo for ${expected_reason}: ${output}" +} + +assert_new_rule_provenance_regressions() { + local target_inventory=$1 + local substrate_inventory=$2 + local quota_json=$3 + local certificate_json=$4 + local services_json=$5 + local metric_typo_json="${temp_dir}/provenance-metric-typo.rules.json" + local matcher_typo_json="${temp_dir}/provenance-matcher-typo.rules.json" + local quota_matcher_removed_json="${temp_dir}/provenance-quota-matcher-removed.rules.json" + local quota_matcher_value_typo_json="${temp_dir}/provenance-quota-matcher-value-typo.rules.json" + local cnpg_matcher_removed_json="${temp_dir}/provenance-cnpg-matcher-removed.rules.json" + local cnpg_matcher_value_typo_json="${temp_dir}/provenance-cnpg-matcher-value-typo.rules.json" + local record_typo_json="${temp_dir}/provenance-record-typo.rules.json" + + jq ' + (.spec.groups[].rules[] + | select(.record == "platform:aistor_bucket_quota_usage_percent") + | .expr) + |= sub( + "minio_cluster_usage_buckets_total_bytes"; + "minio_cluster_usage_buckets_total_typo_bytes" + ) + ' "$quota_json" >"$metric_typo_json" + assert_provenance_rejects \ + 'unmapped raw metric' \ + "$target_inventory" "$substrate_inventory" \ + "$metric_typo_json" "$certificate_json" "$services_json" + + jq ' + (.spec.groups[].rules[] + | select(.alert == "PlatformCNPGCollectorDown") + | .expr) + |= sub("cluster=\"platform-postgres\""; "clustr=\"platform-postgres\"") + ' "$services_json" >"$matcher_typo_json" + assert_provenance_rejects \ + 'inventory lacks' \ + "$target_inventory" "$substrate_inventory" \ + "$quota_json" "$certificate_json" "$matcher_typo_json" + + jq ' + (.spec.groups[].rules[] + | select(.record == "platform:aistor_bucket_quota_usage_percent") + | .expr) + |= gsub("\\{bucket=~\\\"loki\\|tempo\\\"\\}"; "") + ' "$quota_json" >"$quota_matcher_removed_json" + assert_provenance_rejects \ + 'exact matcher contract' \ + "$target_inventory" "$substrate_inventory" \ + "$quota_matcher_removed_json" "$certificate_json" "$services_json" + + jq ' + (.spec.groups[].rules[] + | select(.record == "platform:aistor_bucket_quota_usage_percent") + | .expr) + |= sub("loki\\|tempo"; "loki|temop") + ' "$quota_json" >"$quota_matcher_value_typo_json" + assert_provenance_rejects \ + 'exact matcher contract' \ + "$target_inventory" "$substrate_inventory" \ + "$quota_matcher_value_typo_json" "$certificate_json" "$services_json" + + jq ' + (.spec.groups[].rules[] + | select(.alert == "PlatformCNPGCollectorDown") + | .expr) + |= sub("namespace=\\\"platform-data\\\","; "") + ' "$services_json" >"$cnpg_matcher_removed_json" + assert_provenance_rejects \ + 'exact matcher contract' \ + "$target_inventory" "$substrate_inventory" \ + "$quota_json" "$certificate_json" "$cnpg_matcher_removed_json" + + jq ' + (.spec.groups[].rules[] + | select(.alert == "PlatformCNPGCollectorDown") + | .expr) + |= sub("platform-data"; "platform-dtaa") + ' "$services_json" >"$cnpg_matcher_value_typo_json" + assert_provenance_rejects \ + 'exact matcher contract' \ + "$target_inventory" "$substrate_inventory" \ + "$quota_json" "$certificate_json" "$cnpg_matcher_value_typo_json" + + jq ' + (.spec.groups[].rules[] + | select(.alert == "PlatformAIStorBucketQuotaUsage") + | .expr) + |= sub( + "platform:aistor_bucket_quota_usage_percent"; + "platform:aistor_bucket_quota_usage_typo_percent" + ) + ' "$quota_json" >"$record_typo_json" + assert_provenance_rejects \ + 'unresolved recording rule reference' \ + "$target_inventory" "$substrate_inventory" \ + "$record_typo_json" "$certificate_json" "$services_json" + + pass 'provenance rejects real new-rule metric, exact matcher, and recording-reference mutations' +} + +assert_new_rule_semantics() { + [[ $# -eq 3 ]] || fail 'new-rule semantic projection requires quota, certificate, and service rule JSON' + local actual expected canonical_expected + + expected='[ + ["platform-aistor-storage-quota", "platform.aistor-storage-quota", 0, "record", "platform:aistor_bucket_quota_usage_percent", "100 * max by (bucket) ( minio_cluster_usage_buckets_total_bytes{bucket=~\"loki|tempo\"} ) / max by (bucket) ( minio_cluster_usage_buckets_quota_total_bytes{bucket=~\"loki|tempo\"} > 0 )", null, null], + ["platform-aistor-storage-quota", "platform.aistor-storage-quota", 1, "alert", "PlatformAIStorBucketQuotaUsage", "platform:aistor_bucket_quota_usage_percent >= 70 and platform:aistor_bucket_quota_usage_percent < 85", "15m", "warning"], + ["platform-aistor-storage-quota", "platform.aistor-storage-quota", 2, "alert", "PlatformAIStorBucketQuotaUsage", "platform:aistor_bucket_quota_usage_percent >= 85 and platform:aistor_bucket_quota_usage_percent < 95", "10m", "critical"], + ["platform-aistor-storage-quota", "platform.aistor-storage-quota", 3, "alert", "PlatformAIStorBucketQuotaUsage", "platform:aistor_bucket_quota_usage_percent >= 95", "5m", "emergency"], + ["platform-certificate-probes", "platform.blackbox", 0, "alert", "PlatformPublicEdgeProbeFailed", "probe_success{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=\"public-edge\" } == 0", "5m", "critical"], + ["platform-certificate-probes", "platform.blackbox", 1, "alert", "PlatformPrivateEdgeProbeFailed", "probe_success{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=\"private-edge\" } == 0", "5m", "critical"], + ["platform-certificate-probes", "platform.blackbox", 2, "alert", "PlatformInternalHealthProbeFailed", "probe_success{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=\"private-internal\" } == 0", "5m", "critical"], + ["platform-certificate-probes", "platform.certificates", 0, "alert", "PlatformCertificateExpiry", "( probe_ssl_earliest_cert_expiry{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=~\"public-edge|private-edge\" } - time() ) >= 14 * 24 * 60 * 60 and ( probe_ssl_earliest_cert_expiry{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=~\"public-edge|private-edge\" } - time() ) < 30 * 24 * 60 * 60", null, "warning"], + ["platform-certificate-probes", "platform.certificates", 1, "alert", "PlatformCertificateExpiry", "probe_ssl_earliest_cert_expiry{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=~\"public-edge|private-edge\" } - time() < 14 * 24 * 60 * 60", null, "critical"], + ["platform-verified-services", "platform.verified-services", 0, "alert", "PlatformCNPGCollectorDown", "min by (cluster, namespace) ( cnpg_collector_up{ namespace=\"platform-data\", cluster=\"platform-postgres\" } ) == 0", "5m", "critical"], + ["platform-verified-services", "platform.verified-services", 1, "alert", "PlatformAlloyConfigLoadFailed", "min by (namespace) ( alloy_config_last_load_successful{namespace=\"observability-agent\"} ) == 0", "5m", "critical"], + ["platform-verified-services", "platform.verified-services", 2, "alert", "PlatformAlloyLogDeliveryRetries", "sum by (namespace) ( increase(loki_write_batch_retries_total{namespace=\"observability-agent\"}[15m]) ) > 0", "5m", "warning"], + ["platform-verified-services", "platform.verified-services", 3, "alert", "PlatformAlloyLogEntriesDropped", "sum by (namespace) ( increase(loki_write_dropped_entries_total{namespace=\"observability-agent\"}[15m]) ) > 0", "5m", "critical"], + ["platform-verified-services", "platform.verified-services", 4, "alert", "PlatformLokiRuntimeConfigReloadFailed", "min by (namespace) ( loki_runtime_config_last_reload_successful{namespace=\"observability\"} ) == 0", "5m", "critical"], + ["platform-verified-services", "platform.verified-services", 5, "alert", "PlatformLokiWALDiskUsageHigh", "max by (namespace) ( loki_ingester_wal_disk_usage_percent{namespace=\"observability\"} ) >= 0.8", "15m", "warning"] + ]' + + actual="$(jq -c -s ' + def normalized_expr: + gsub("[[:space:]]+"; " ") + | sub("^ "; "") + | sub(" $"; ""); + [.[] as $resource + | $resource.spec.groups[] as $group + | $group.rules + | to_entries[] + | [ + $resource.metadata.name, + $group.name, + .key, + (if .value.record then "record" else "alert" end), + (.value.record // .value.alert), + (.value.expr | normalized_expr), + (.value.for // null), + (.value.labels.severity // null) + ] + ] + ' "$@")" || fail 'could not project new-rule semantics' + canonical_expected="$(jq -c . <<<"$expected")" || + fail 'invalid expected new-rule semantic projection' + + [[ "$actual" == "$canonical_expected" ]] || + fail 'new-rule semantic projection mismatch' +} + +assert_semantics_rejects() { + local mutation=$1 + shift + local output + + declare -F assert_new_rule_semantics >/dev/null || + fail 'new-rule semantic validator is unavailable' + if output="$( (assert_new_rule_semantics "$@") 2>&1 )"; then + fail "semantic regression unexpectedly accepted ${mutation}" + fi + grep -Fq 'new-rule semantic projection mismatch' <<<"$output" || + fail "semantic regression rejected ${mutation} for the wrong reason: ${output}" +} + +assert_new_rule_semantic_regressions() { + local quota_json=$1 + local certificate_json=$2 + local services_json=$3 + local public_group_drift_json="${temp_dir}/semantic-public-group-drift.rules.json" + local selector_removed_json="${temp_dir}/semantic-selector-removed.rules.json" + local duplicate_selector_json="${temp_dir}/semantic-duplicate-selector.rules.json" + local cnpg_rebound_json="${temp_dir}/semantic-cnpg-rebound.rules.json" + + jq ' + (.spec.groups[].rules[] + | select(.alert == "PlatformPublicEdgeProbeFailed") + | .expr) + |= sub("public-edge"; "private-edge") + ' "$certificate_json" >"$public_group_drift_json" + assert_semantics_rejects \ + 'PlatformPublicEdgeProbeFailed rebound to the valid private-edge fixture' \ + "$quota_json" "$public_group_drift_json" "$services_json" + + jq ' + (.spec.groups[].rules[] + | select(.alert == "PlatformPublicEdgeProbeFailed") + | .expr) = "vector(0)" + ' "$certificate_json" >"$selector_removed_json" + assert_semantics_rejects \ + 'PlatformPublicEdgeProbeFailed with its selector removed' \ + "$quota_json" "$selector_removed_json" "$services_json" + + jq ' + (.spec.groups[].rules[] + | select(.alert == "PlatformPublicEdgeProbeFailed") + | .expr) + += " or probe_success{namespace=\"observability\",\"observability.hyeonworks.com/probe-group\"=\"private-edge\"} == 0" + ' "$certificate_json" >"$duplicate_selector_json" + assert_semantics_rejects \ + 'PlatformPublicEdgeProbeFailed with an extra valid probe selector' \ + "$quota_json" "$duplicate_selector_json" "$services_json" + + jq ' + (.spec.groups[].rules[] + | select(.alert == "PlatformCNPGCollectorDown") + | .expr) = "min by (namespace) (alloy_config_last_load_successful{namespace=\"observability-agent\"}) == 0" + ' "$services_json" >"$cnpg_rebound_json" + assert_semantics_rejects \ + 'PlatformCNPGCollectorDown rebound to the valid Alloy fixture' \ + "$quota_json" "$certificate_json" "$cnpg_rebound_json" + + pass 'semantic projection rejects rule-identity, selector-cardinality, and fixture-routing mutations' +} + +render_rule_json() { + local source_file=$1 + local output_file=$2 + + [[ -f "$source_file" ]] || fail "missing rule source: ${source_file}" + kubectl create --dry-run=client --validate=false -f "$source_file" -o json >"$output_file" + jq -e ' + .apiVersion == "monitoring.coreos.com/v1" and + .kind == "PrometheusRule" and + .metadata.namespace == "observability" and + .metadata.labels["observability.hyeonworks.com/instance"] == "home" and + (.spec.groups | type == "array" and length > 0) + ' "$output_file" >/dev/null || fail "invalid PrometheusRule envelope: ${source_file}" +} + +alert_runbooks_are_exact() { + local rule_json=$1 + + jq -e --arg expected "$EXPECTED_RUNBOOK_URL" ' + all(.spec.groups[].rules[] | select(has("alert")); + .annotations.runbook_url == $expected + ) + ' "$rule_json" >/dev/null +} + +assert_exact_runbook_regression() { + local rule_json=$1 mutated="${temp_dir}/runbook-url-mutated.rules.json" + + alert_runbooks_are_exact "$rule_json" || + fail 'the production rule fixture lacks the exact runbook URL' + jq ' + (.spec.groups[].rules[] | select(has("alert")) | .annotations.runbook_url) = + "https://git.learn.hyeonworks.com/wrong/repository/runbook.md" + ' "$rule_json" >"$mutated" + if alert_runbooks_are_exact "$mutated"; then + fail 'a different valid HTTPS runbook URL was accepted' + fi + pass 'runbook annotation rejects a different valid HTTPS URL' +} + +assert_alert_contract() { + local rule_json=$1 + + jq -e ' + all(.spec.groups[].rules[] | select(has("alert")); + (.labels | keys | sort) == ["severity"] and + (.labels.severity | IN("warning", "critical", "emergency")) and + (.annotations.summary | type == "string" and length > 0) and + (.annotations.description | type == "string" and length > 0) + ) + ' "$rule_json" >/dev/null || fail "alert label/summary/description contract failed: ${rule_json}" + + if ! alert_runbooks_are_exact "$rule_json"; then + printf 'BLOCKED: alert lacks the exact operator-reachable HTTPS runbook_url: %s\n' "$rule_json" >&2 + runbook_url_blocked=1 + fi +} + +assert_exact_alert_names() { + local rule_json=$1 + shift + local expected actual + + expected="$(printf '%s\n' "$@" | sort)" + actual="$(jq -r '.spec.groups[].rules[] | select(has("alert")) | .alert' "$rule_json" | sort)" + [[ "$actual" == "$expected" ]] || { + printf 'Expected alert names:\n%s\nActual alert names:\n%s\n' "$expected" "$actual" >&2 + fail "unexpected alert set: ${rule_json}" + } +} + +run_promtool() { + local work_dir=$1 + shift + + docker run --rm \ + --network none \ + --volume "${work_dir}:/work:ro" \ + --entrypoint /bin/promtool \ + "$PROMETHEUS_IMAGE" "$@" +} + +run_live_promql() { + local -a rule_jsons=("$@") + local proxy_path='/api/v1/namespaces/observability/services/http:observability-core-kube-pr-prometheus:9090/proxy/api/v1' + local now start rule_name encoded_expr expression encoded_query response query_status result_count + + now="$(date +%s)" + start="$((now - 300))" + + while IFS=$'\t' read -r rule_name encoded_expr; do + expression="$(printf '%s' "$encoded_expr" | base64 -d)" + encoded_query="$(jq -rn --arg query "$expression" '$query | @uri')" + + response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query?query=${encoded_query}")" || + fail "live instant query transport failed: ${rule_name}" + query_status="$(jq -r '.status' <<<"$response")" + [[ "$query_status" == success ]] || + fail "live instant query rejected: ${rule_name}" + result_count="$(jq -r '.data.result | length' <<<"$response")" + printf 'LIVE INSTANT PASS: %s (%s series)\n' "$rule_name" "$result_count" + + response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query_range?query=${encoded_query}&start=${start}&end=${now}&step=60")" || + fail "live range query transport failed: ${rule_name}" + query_status="$(jq -r '.status' <<<"$response")" + [[ "$query_status" == success ]] || + fail "live range query rejected: ${rule_name}" + result_count="$(jq -r '.data.result | length' <<<"$response")" + printf 'LIVE RANGE PASS: %s (%s series)\n' "$rule_name" "$result_count" + done < <( + jq -r ' + .spec.groups[].rules[] | + [(.alert // .record), (.expr | @base64)] | + @tsv + ' "${rule_jsons[@]}" + ) +} + +assert_live_private_403() { + local proxy_path='/api/v1/namespaces/observability/services/http:observability-core-kube-pr-prometheus:9090/proxy/api/v1' + local success_query status_query success_response status_response + local success_set status_set + + success_query="$(jq -rn --arg query 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"}' '$query | @uri')" + status_query="$(jq -rn --arg query 'probe_http_status_code{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"}' '$query | @uri')" + success_response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query?query=${success_query}")" || + fail 'live private-edge probe_success transport failed' + status_response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query?query=${status_query}")" || + fail 'live private-edge status transport failed' + + success_set="$(jq -cS '[.data.result[] | select(.value[1] == "1") | .metric.instance] | sort' <<<"$success_response")" + status_set="$(jq -cS '[.data.result[] | select(.value[1] == "403") | .metric.instance] | sort' <<<"$status_response")" + [[ "$success_set" == "$status_set" ]] || + fail 'live private-edge success and HTTP 403 instance sets differ' + [[ "$(jq 'length' <<<"$success_set")" -eq 3 ]] || + fail 'live private-edge contract must have exactly three successful HTTP 403 endpoints' + pass 'live private-edge three endpoints are probe_success=1 with HTTP 403' +} + +main() { + [[ $# -eq 1 || ( $# -eq 2 && "$2" == --live-prometheus ) ]] || + fail "usage: $0 VERIFIED_OUTPUT_DIR [--live-prometheus]" + local inventory_root=$1 + local live_prometheus=0 + local target_inventory="${inventory_root}/target-initial/inventory.json" + local substrate_inventory="${inventory_root}/post-substrate/inventory.json" + local quota_json certificate_json services_json core_json + local rendered_names duplicate_names + + [[ ${2:-} == --live-prometheus ]] && live_prometheus=1 + + require_command docker + require_command base64 + require_command jq + require_command kubectl + require_command sha256sum + + assert_inventory "${inventory_root}/target-initial" target-initial + assert_inventory "${inventory_root}/post-substrate" post-substrate + + assert_metric_label "$target_inventory" serviceMonitor/object-storage/aistor-bucket-usage/0 minio_cluster_usage_buckets_total_bytes bucket + assert_metric_label "$target_inventory" serviceMonitor/object-storage/aistor-bucket-usage/0 minio_cluster_usage_buckets_quota_total_bytes bucket + assert_metric_label "$target_inventory" podMonitor/platform-data/platform-postgres/0 cnpg_collector_up cluster + assert_metric_label "$target_inventory" podMonitor/platform-data/platform-postgres/0 cnpg_collector_up namespace + assert_metric_label "$target_inventory" serviceMonitor/observability-agent/alloy/0 alloy_config_last_load_successful namespace + assert_metric_label "$target_inventory" serviceMonitor/observability-agent/alloy/0 loki_write_batch_retries_total namespace + assert_metric_label "$target_inventory" serviceMonitor/observability-agent/alloy/0 loki_write_dropped_entries_total namespace + assert_metric_label "$target_inventory" serviceMonitor/observability/loki/0 loki_runtime_config_last_reload_successful namespace + assert_metric_label "$target_inventory" serviceMonitor/observability/loki/0 loki_ingester_wal_disk_usage_percent namespace + assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_success namespace + assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_success observability.hyeonworks.com/probe-group + assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_success instance + assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_ssl_earliest_cert_expiry namespace + assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_ssl_earliest_cert_expiry observability.hyeonworks.com/probe-group + assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_ssl_earliest_cert_expiry instance + assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_success namespace + assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_success observability.hyeonworks.com/probe-group + assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_success instance + assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_http_status_code observability.hyeonworks.com/probe-group + assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_ssl_earliest_cert_expiry namespace + assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_ssl_earliest_cert_expiry observability.hyeonworks.com/probe-group + assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_ssl_earliest_cert_expiry instance + assert_metric_label "$substrate_inventory" probe/observability/platform-private-internal probe_success namespace + assert_metric_label "$substrate_inventory" probe/observability/platform-private-internal probe_success observability.hyeonworks.com/probe-group + assert_metric_label "$substrate_inventory" probe/observability/platform-private-internal probe_success instance + pass "inventory hashes, phases, metrics, and labels" + + temp_dir="$(mktemp -d)" + chmod 0755 "$temp_dir" + trap cleanup EXIT + quota_json="${temp_dir}/storage-quota.rules.json" + certificate_json="${temp_dir}/certificate.rules.json" + services_json="${temp_dir}/verified-service.rules.json" + core_json="${temp_dir}/core.rules.json" + + render_rule_json "${RULE_DIR}/storage-quota-rules.yaml" "$quota_json" + render_rule_json "${RULE_DIR}/certificate-rules.yaml" "$certificate_json" + render_rule_json "${RULE_DIR}/verified-service-rules.yaml" "$services_json" + render_rule_json "${RULE_DIR}/core-rules.yaml" "$core_json" + assert_new_rule_provenance_regressions \ + "$target_inventory" "$substrate_inventory" \ + "$quota_json" "$certificate_json" "$services_json" + assert_new_rule_semantic_regressions \ + "$quota_json" "$certificate_json" "$services_json" + assert_exact_runbook_regression "$quota_json" + assert_new_rule_provenance \ + "$target_inventory" "$substrate_inventory" \ + "$quota_json" "$certificate_json" "$services_json" + pass 'new-rule raw metrics, matcher labels, and recording dependencies have inventory provenance' + assert_new_rule_semantics \ + "$quota_json" "$certificate_json" "$services_json" + pass 'new-rule identities, groups, expressions, durations, and severities have exact semantic binding' + assert_alert_contract "$quota_json" + assert_alert_contract "$certificate_json" + assert_alert_contract "$services_json" + if ! alert_runbooks_are_exact "$core_json"; then + printf 'BLOCKED: core alert lacks the exact operator-reachable HTTPS runbook_url: %s\n' "$core_json" >&2 + runbook_url_blocked=1 + fi + + jq -e ' + ([.spec.groups[].rules[] | select(.record == "platform:aistor_bucket_quota_usage_percent")] | length) == 1 and + ([.spec.groups[].rules[] | select(.alert == "PlatformAIStorBucketQuotaUsage")] | length) == 3 and + ([.spec.groups[].rules[] | select(.alert == "PlatformAIStorBucketQuotaUsage") | .for] | sort) == ["10m", "15m", "5m"] and + ([.spec.groups[].rules[] | select(.alert == "PlatformAIStorBucketQuotaUsage") | .labels.severity] | sort) == ["critical", "emergency", "warning"] + ' "$quota_json" >/dev/null || fail "AIStor quota rule shape is not exact" + + assert_exact_alert_names "$certificate_json" \ + PlatformCertificateExpiry \ + PlatformCertificateExpiry \ + PlatformInternalHealthProbeFailed \ + PlatformPrivateEdgeProbeFailed \ + PlatformPublicEdgeProbeFailed + + assert_exact_alert_names "$services_json" \ + PlatformAlloyConfigLoadFailed \ + PlatformAlloyLogDeliveryRetries \ + PlatformAlloyLogEntriesDropped \ + PlatformCNPGCollectorDown \ + PlatformLokiRuntimeConfigReloadFailed \ + PlatformLokiWALDiskUsageHigh + + duplicate_names="$( + comm -12 \ + <(jq -r '.spec.groups[].rules[] | select(has("alert")) | .alert' "$core_json" | sort -u) \ + <(jq -r '.spec.groups[].rules[] | select(has("alert")) | .alert' "$quota_json" "$certificate_json" "$services_json" | sort -u) + )" + [[ -z "$duplicate_names" ]] || fail "new rules duplicate core alert names: ${duplicate_names}" + pass "rule envelopes, bounded labels, annotations, and non-duplicate alert sets" + + rendered_names="$( + kubectl kustomize "$RULE_DIR" | + kubectl create --dry-run=client --validate=false -f - -o name | + sort + )" + [[ "$rendered_names" == $'prometheusrule.monitoring.coreos.com/platform-aistor-storage-quota\nprometheusrule.monitoring.coreos.com/platform-certificate-probes\nprometheusrule.monitoring.coreos.com/platform-observability-core\nprometheusrule.monitoring.coreos.com/platform-verified-services' ]] || { + printf 'Rendered resources:\n%s\n' "$rendered_names" >&2 + fail "platform rule kustomization resource set is not exact" + } + pass "platform rule kustomization" + + jq '.spec' "$quota_json" >"${temp_dir}/storage-quota.prometheus.json" + jq '.spec' "$certificate_json" >"${temp_dir}/certificate.prometheus.json" + jq '.spec' "$services_json" >"${temp_dir}/verified-service.prometheus.json" + jq '.spec' "$core_json" >"${temp_dir}/core.prometheus.json" + chmod 0444 "${temp_dir}"/*.prometheus.json + run_promtool "$temp_dir" check rules \ + /work/storage-quota.prometheus.json \ + /work/certificate.prometheus.json \ + /work/verified-service.prometheus.json \ + /work/core.prometheus.json + pass "promtool check rules" + + printf '%s\n' \ + 'rule_files:' \ + ' - /work/storage-quota.prometheus.json' \ + ' - /work/certificate.prometheus.json' \ + 'evaluation_interval: 1m' \ + 'tests:' \ + ' - name: quota warning starts at 70 and excludes 85' \ + ' interval: 1m' \ + ' input_series:' \ + ' - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="loki"}'\''' \ + ' values: '\''70+0x20'\''' \ + ' - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="loki"}'\''' \ + ' values: '\''100+0x20'\''' \ + ' alert_rule_test:' \ + ' - eval_time: 16m' \ + ' alertname: PlatformAIStorBucketQuotaUsage' \ + ' exp_alerts:' \ + ' - exp_labels:' \ + ' bucket: loki' \ + ' severity: warning' \ + ' exp_annotations:' \ + ' summary: AIStor bucket quota usage is at warning level' \ + ' description: Bucket loki has remained between 70 and 85 percent used for 15 minutes.' \ + ' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \ + ' - name: quota critical starts at 85 and excludes 95' \ + ' interval: 1m' \ + ' input_series:' \ + ' - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="tempo"}'\''' \ + ' values: '\''85+0x20'\''' \ + ' - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="tempo"}'\''' \ + ' values: '\''100+0x20'\''' \ + ' alert_rule_test:' \ + ' - eval_time: 11m' \ + ' alertname: PlatformAIStorBucketQuotaUsage' \ + ' exp_alerts:' \ + ' - exp_labels:' \ + ' bucket: tempo' \ + ' severity: critical' \ + ' exp_annotations:' \ + ' summary: AIStor bucket quota usage is at critical level' \ + ' description: Bucket tempo has remained between 85 and 95 percent used for 10 minutes.' \ + ' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \ + ' - name: quota emergency starts at 95' \ + ' interval: 1m' \ + ' input_series:' \ + ' - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="loki"}'\''' \ + ' values: '\''95+0x20'\''' \ + ' - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="loki"}'\''' \ + ' values: '\''100+0x20'\''' \ + ' alert_rule_test:' \ + ' - eval_time: 6m' \ + ' alertname: PlatformAIStorBucketQuotaUsage' \ + ' exp_alerts:' \ + ' - exp_labels:' \ + ' bucket: loki' \ + ' severity: emergency' \ + ' exp_annotations:' \ + ' summary: AIStor bucket quota usage is at emergency level' \ + ' description: Stop the loki ingest path according to the runbook; do not delete objects automatically.' \ + ' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \ + ' - name: zero quota cannot produce a percentage alert' \ + ' interval: 1m' \ + ' input_series:' \ + ' - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="loki"}'\''' \ + ' values: '\''100+0x20'\''' \ + ' - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="loki"}'\''' \ + ' values: '\''0+0x20'\''' \ + ' alert_rule_test:' \ + ' - eval_time: 20m' \ + ' alertname: PlatformAIStorBucketQuotaUsage' \ + ' exp_alerts: []' \ + ' - name: successful expected 403 private probe remains healthy' \ + ' interval: 1m' \ + ' input_series:' \ + ' - series: '\''probe_success{namespace="observability",job="blackbox-private-edge",instance="https://db-admin.learn.hyeonworks.com/","observability.hyeonworks.com/probe-group"="private-edge"}'\''' \ + ' values: '\''1+0x10'\''' \ + ' - series: '\''probe_http_status_code{namespace="observability",job="blackbox-private-edge",instance="https://db-admin.learn.hyeonworks.com/","observability.hyeonworks.com/probe-group"="private-edge"}'\''' \ + ' values: '\''403+0x10'\''' \ + ' alert_rule_test:' \ + ' - eval_time: 10m' \ + ' alertname: PlatformPrivateEdgeProbeFailed' \ + ' exp_alerts: []' \ + ' - name: failed private probe fires after five minutes' \ + ' interval: 1m' \ + ' input_series:' \ + ' - series: '\''probe_success{namespace="observability",job="blackbox-private-edge",instance="https://db-admin.learn.hyeonworks.com/","observability.hyeonworks.com/probe-group"="private-edge"}'\''' \ + ' values: '\''0+0x10'\''' \ + ' alert_rule_test:' \ + ' - eval_time: 6m' \ + ' alertname: PlatformPrivateEdgeProbeFailed' \ + ' exp_alerts:' \ + ' - exp_labels:' \ + ' namespace: observability' \ + ' job: blackbox-private-edge' \ + ' instance: https://db-admin.learn.hyeonworks.com/' \ + ' "observability.hyeonworks.com/probe-group": private-edge' \ + ' severity: critical' \ + ' exp_annotations:' \ + ' summary: Private edge boundary probe failed' \ + ' description: The expected private edge response for https://db-admin.learn.hyeonworks.com/ has failed for 5 minutes; an expected HTTP 403 with probe_success=1 is healthy.' \ + ' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \ + ' - name: TLS warning includes exactly fourteen days' \ + ' interval: 1m' \ + ' input_series:' \ + ' - series: '\''probe_ssl_earliest_cert_expiry{namespace="observability",instance="https://git.learn.hyeonworks.com/api/healthz","observability.hyeonworks.com/probe-group"="public-edge"}'\''' \ + ' values: '\''1209660+0x2'\''' \ + ' alert_rule_test:' \ + ' - eval_time: 1m' \ + ' alertname: PlatformCertificateExpiry' \ + ' exp_alerts:' \ + ' - exp_labels:' \ + ' namespace: observability' \ + ' instance: https://git.learn.hyeonworks.com/api/healthz' \ + ' "observability.hyeonworks.com/probe-group": public-edge' \ + ' severity: warning' \ + ' exp_annotations:' \ + ' summary: TLS certificate expires within 30 days' \ + ' description: The earliest certificate for https://git.learn.hyeonworks.com/api/healthz expires in fewer than 30 days but not fewer than 14 days.' \ + ' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \ + ' - name: TLS warning excludes exactly thirty days' \ + ' interval: 1m' \ + ' input_series:' \ + ' - series: '\''probe_ssl_earliest_cert_expiry{namespace="observability",instance="https://git.learn.hyeonworks.com/api/healthz","observability.hyeonworks.com/probe-group"="public-edge"}'\''' \ + ' values: '\''2592060+0x2'\''' \ + ' alert_rule_test:' \ + ' - eval_time: 1m' \ + ' alertname: PlatformCertificateExpiry' \ + ' exp_alerts: []' \ + ' - name: TLS critical covers fewer than fourteen days' \ + ' interval: 1m' \ + ' input_series:' \ + ' - series: '\''probe_ssl_earliest_cert_expiry{namespace="observability",instance="https://git.learn.hyeonworks.com/api/healthz","observability.hyeonworks.com/probe-group"="public-edge"}'\''' \ + ' values: '\''864060+0x2'\''' \ + ' alert_rule_test:' \ + ' - eval_time: 1m' \ + ' alertname: PlatformCertificateExpiry' \ + ' exp_alerts:' \ + ' - exp_labels:' \ + ' namespace: observability' \ + ' instance: https://git.learn.hyeonworks.com/api/healthz' \ + ' "observability.hyeonworks.com/probe-group": public-edge' \ + ' severity: critical' \ + ' exp_annotations:' \ + ' summary: TLS certificate expires within 14 days' \ + ' description: The earliest certificate for https://git.learn.hyeonworks.com/api/healthz expires in fewer than 14 days.' \ + ' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \ + >"${temp_dir}/rules.test.yaml" + chmod 0444 "${temp_dir}/rules.test.yaml" + + run_promtool "$temp_dir" test rules /work/rules.test.yaml + pass "promtool quota, probe, private-403, and TLS boundaries" + + if (( live_prometheus == 1 )); then + run_live_promql "$quota_json" "$certificate_json" "$services_json" "$core_json" + assert_live_private_403 + pass "live Prometheus instant and range queries" + fi + + (( runbook_url_blocked == 0 )) || + fail "operator-reachable HTTPS runbook_url contract is not defined" +} + +main "$@" diff --git a/scripts/validate/test-private-dns-observability.sh b/scripts/validate/test-private-dns-observability.sh new file mode 100644 index 0000000..408d571 --- /dev/null +++ b/scripts/validate/test-private-dns-observability.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)" +readonly APPLY="$ROOT/scripts/bootstrap/apply-private-dns.sh" +readonly LAN="$ROOT/infrastructure/networking/private-dns/host/dnsmasq-lan.conf" +readonly TAIL="$ROOT/infrastructure/networking/private-dns/host/dnsmasq-tailscale.conf" +readonly CORE="$ROOT/infrastructure/networking/private-dns/kubernetes/coredns-custom.yaml" +readonly BUSYBOX='docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0' +readonly -a HOSTS=( + git.learn.hyeonworks.com + id.learn.hyeonworks.com + storage-admin.learn.hyeonworks.com + db-admin.learn.hyeonworks.com + grafana.learn.hyeonworks.com +) + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +pass() { + printf 'PASS: %s\n' "$*" +} + +for file in "$APPLY" "$LAN" "$TAIL" "$CORE"; do + [[ -f "$file" && ! -L "$file" ]] || fail "missing regular source: $file" +done +bash -n "$APPLY" + +for host in "${HOSTS[@]}"; do + grep -Fqx "address=/$host/192.168.0.107" "$LAN" || fail "LAN source lacks $host" + grep -Fqx "address=/$host/100.92.240.34" "$TAIL" || fail "Tail source lacks $host" + grep -Fq "192.168.0.107 $host" "$CORE" || fail "CoreDNS source lacks $host" +done +pass 'all five private names are source-bound for LAN, Tail, and Pods' + +grep -Fq 'PUBLIC_PRIVATE_HOSTS=(storage-admin.learn.hyeonworks.com db-admin.learn.hyeonworks.com grafana.learn.hyeonworks.com)' "$APPLY" || + fail 'public A/AAAA absence set is not exact' +grep -Fq 'PRIVATE_HOSTS=(git.learn.hyeonworks.com id.learn.hyeonworks.com storage-admin.learn.hyeonworks.com db-admin.learn.hyeonworks.com grafana.learn.hyeonworks.com)' "$APPLY" || + fail 'private resolver verification set is not exact' +grep -Fq "readonly BUSYBOX_IMAGE=\"$BUSYBOX\"" "$APPLY" || fail 'BusyBox pin constant is not exact' +grep -Fq -- '--image="$BUSYBOX_IMAGE"' "$APPLY" || fail 'Pod smoke does not use the pinned BusyBox constant' +for host in "${HOSTS[@]}"; do + grep -Fq "nslookup $host" "$APPLY" || fail "Pod smoke omits $host" +done +pass 'apply source checks public absence and all five resolver paths with pinned BusyBox' + +grep -Fq 'managed_listeners_are_exact' "$APPLY" || fail 'repeat execution has no exact managed-listener classifier' +grep -Fq 'lan_was_active=' "$APPLY" || fail 'rollback does not snapshot LAN active state' +grep -Fq 'tail_was_active=' "$APPLY" || fail 'rollback does not snapshot Tail active state' +grep -Fq 'lan_was_enabled=' "$APPLY" || fail 'rollback does not snapshot LAN enabled state' +grep -Fq 'tail_was_enabled=' "$APPLY" || fail 'rollback does not snapshot Tail enabled state' +grep -Fq 'restore_service_state' "$APPLY" || fail 'rollback does not restore managed service state' +pass 'repeat execution and rollback service-state contracts are present' + +printf 'PRIVATE DNS OBSERVABILITY CONTRACT PASS\n' diff --git a/scripts/validate/test-render-observability-access.sh b/scripts/validate/test-render-observability-access.sh new file mode 100755 index 0000000..c37dd3d --- /dev/null +++ b/scripts/validate/test-render-observability-access.sh @@ -0,0 +1,1066 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +RENDERER="$ROOT/scripts/validate/render-observability-access.sh" + +# The renderer is intentionally sourced: these tests exercise the same parser and +# manifest assertions used by executable rendering, without downloading charts or +# contacting a Kubernetes API server. +source "$RENDERER" + +ASSERTIONS=0 +WORK='' +PUBLISH_ROOTS=() + +cleanup() { + local rc=$? + trap - EXIT HUP INT TERM + if [[ -n "$WORK" && "$WORK" == /tmp/platform-observability-access-test.* && + -d "$WORK" && ! -L "$WORK" ]]; then + rm -rf -- "$WORK" + fi + local publish_root + for publish_root in "${PUBLISH_ROOTS[@]}"; do + if [[ "$publish_root" == /tmp/platform-observability-metrics.?????? && + -d "$publish_root" && ! -L "$publish_root" ]]; then + rm -rf -- "$publish_root" + fi + done + exit "$rc" +} +trap cleanup EXIT HUP INT TERM + +pass() { + ASSERTIONS=$((ASSERTIONS + 1)) + printf 'PASS: %s\n' "$1" +} + +expect_pass() { + local label="$1" + shift + if "$@" >/dev/null 2>&1; then + pass "$label" + else + printf 'FAIL: expected success: %s\n' "$label" >&2 + exit 1 + fi +} + +expect_fail() { + local label="$1" + shift + if "$@" >/dev/null 2>&1; then + printf 'FAIL: expected rejection: %s\n' "$label" >&2 + exit 1 + fi + pass "$label" +} + +expect_output() { + local label="$1" expected="$2" + shift 2 + local actual + actual="$($@)" || { + printf 'FAIL: output command failed: %s\n' "$label" >&2 + exit 1 + } + if [[ "$actual" != "$expected" ]]; then + printf 'FAIL: unexpected output: %s\nexpected:\n%s\nactual:\n%s\n' \ + "$label" "$expected" "$actual" >&2 + exit 1 + fi + pass "$label" +} + +mutate_yaml() { + local source="$1" destination="$2" mutation="$3" + python3 - "$source" "$destination" "$mutation" <<'PY' +import sys +import yaml + +class FixtureLoader(yaml.SafeLoader): + pass + + +FixtureLoader.add_constructor( + "tag:yaml.org,2002:value", + lambda loader, node: loader.construct_scalar(node), +) + +source, destination, mutation = sys.argv[1:] +with open(source, "r", encoding="utf-8") as stream: + items = [item for item in yaml.load_all(stream, Loader=FixtureLoader) if item is not None] + + +def object_named(kind, name): + matches = [ + item for item in items + if item.get("kind") == kind and (item.get("metadata") or {}).get("name") == name + ] + if len(matches) != 1: + raise SystemExit(f"fixture object is not exact: {kind}/{name}") + return matches[0] + + +if mutation == "grafana-public-service": + object_named("Service", "grafana")["spec"]["type"] = "LoadBalancer" +elif mutation == "grafana-missing-class": + object_named("Ingress", "grafana")["spec"].pop("ingressClassName", None) +elif mutation == "grafana-main-token": + deployment = object_named("Deployment", "grafana") + main = next(c for c in deployment["spec"]["template"]["spec"]["containers"] if c["name"] == "grafana") + main.setdefault("volumeMounts", []).append({ + "name": "dashboard-sidecar-api-access", + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "readOnly": True, + }) +elif mutation == "grafana-cluster-role": + role = object_named("Role", "grafana-dashboard-sidecar") + role["kind"] = "ClusterRole" + role["metadata"].pop("namespace", None) +elif mutation == "oidc-viewer-fallback": + config = object_named("ConfigMap", "grafana") + config["data"]["grafana.ini"] = config["data"]["grafana.ini"].replace("|| null", "|| 'Viewer'") +elif mutation == "oidc-editor": + config = object_named("ConfigMap", "grafana") + config["data"]["grafana.ini"] = config["data"]["grafana.ini"].replace("&& 'Viewer'", "&& 'Editor'") +elif mutation == "oidc-grafana-admin": + config = object_named("ConfigMap", "grafana") + config["data"]["grafana.ini"] = config["data"]["grafana.ini"].replace( + "allow_assign_grafana_admin = false", "allow_assign_grafana_admin = true" + ) +elif mutation == "oidc-group-missing": + config = object_named("ConfigMap", "grafana") + config["data"]["grafana.ini"] = config["data"]["grafana.ini"].replace( + "/platform-observability-viewers", "/not-authorized" + ) +elif mutation == "blackbox-arbitrary-target": + probe = object_named("Probe", "platform-public-edge") + probe["spec"]["targets"]["staticConfig"]["static"].append("https://example.invalid/") +elif mutation == "blackbox-public-service": + object_named("Service", "blackbox-exporter")["spec"]["type"] = "NodePort" +elif mutation == "missing-instance-label": + object_named("Probe", "platform-public-edge")["metadata"]["labels"].pop( + "observability.hyeonworks.com/instance", None + ) +elif mutation == "outside-target": + items.append({ + "apiVersion": "monitoring.coreos.com/v1", + "kind": "ServiceMonitor", + "metadata": { + "name": "rabbitmq", + "namespace": "messaging", + "labels": {"observability.hyeonworks.com/instance": "home"}, + }, + "spec": {"selector": {"matchLabels": {"app": "rabbitmq"}}, "endpoints": [{"port": "metrics"}]}, + }) +elif mutation == "forbidden-product": + items.append({ + "apiVersion": "monitoring.coreos.com/v1", + "kind": "PrometheusRule", + "metadata": { + "name": "platform-jvm", + "namespace": "observability", + "labels": {"observability.hyeonworks.com/instance": "home"}, + }, + "spec": {"groups": [{"name": "jvm", "rules": [{"alert": "JVMHeapHigh", "expr": "vector(0)"}]}]}, + }) +elif mutation == "alertmanager-inline-secret": + items.append({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "inline-fixture", "namespace": "observability"}, + "type": "Opaque", + "stringData": {"password": "forbidden-fixture-value"}, + }) +elif mutation == "alertmanager-missing-title": + config = object_named("AlertmanagerConfig", "platform-alertmanager") + receiver = next( + receiver for receiver in config["spec"]["receivers"] + if receiver["name"] == "platform-slack" + ) + receiver["slackConfigs"][0].pop("title") +elif mutation == "dns-public-address": + config = object_named("ConfigMap", "coredns-custom") + config["data"]["learn-hyeonworks.server"] = config["data"]["learn-hyeonworks.server"].replace( + "192.168.0.107 grafana.learn.hyeonworks.com", + "203.0.113.10 grafana.learn.hyeonworks.com", + ) +elif mutation == "grafana-broad-policy": + policy = object_named("NetworkPolicy", "observability-allow-grafana-ingress") + policy["spec"] = {"podSelector": {}, "policyTypes": ["Ingress"], "ingress": [{}]} +elif mutation == "blackbox-broad-egress": + policy = object_named("NetworkPolicy", "observability-allow-blackbox-egress") + policy["spec"] = {"podSelector": {}, "policyTypes": ["Egress"], "egress": [{}]} +elif mutation == "targets-broad-egress": + policy = object_named("NetworkPolicy", "observability-allow-prometheus-platform-targets") + policy["spec"] = {"podSelector": {}, "policyTypes": ["Egress"], "egress": [{}]} +elif mutation == "grafana-extra-allow-all": + items.append({ + "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", + "metadata": {"name": "observability-extra-grafana-allow-all", "namespace": "observability"}, + "spec": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/name": "grafana", "app.kubernetes.io/instance": "grafana", + }}, + "policyTypes": ["Ingress", "Egress"], "ingress": [{}], "egress": [{}], + }, + }) +elif mutation == "blackbox-extra-allow-all": + items.append({ + "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", + "metadata": {"name": "observability-extra-blackbox-allow-all", "namespace": "observability"}, + "spec": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/name": "prometheus-blackbox-exporter", + "app.kubernetes.io/instance": "blackbox-exporter", + }}, + "policyTypes": ["Ingress", "Egress"], "ingress": [{}], "egress": [{}], + }, + }) +elif mutation == "targets-extra-allow-all": + items.append({ + "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", + "metadata": {"name": "observability-extra-prometheus-allow-all", "namespace": "observability"}, + "spec": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/name": "prometheus", + "app.kubernetes.io/instance": "observability-core-kube-pr-prometheus", + }}, + "policyTypes": ["Egress"], "egress": [{}], + }, + }) +elif mutation == "alertmanager-extra-allow-all": + items.append({ + "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", + "metadata": {"name": "observability-extra-alertmanager-allow-all", "namespace": "observability"}, + "spec": { + "podSelector": {"matchLabels": { + "app.kubernetes.io/name": "alertmanager", + "app.kubernetes.io/instance": "observability-core-kube-pr-alertmanager", + }}, + "policyTypes": ["Egress"], "egress": [{}], + }, + }) +else: + raise SystemExit(f"unknown mutation: {mutation}") + +with open(destination, "w", encoding="utf-8") as stream: + yaml.safe_dump_all(items, stream, explicit_start=True, sort_keys=False) +PY +} + +write_inventory_phase() { + local root="$1" phase="$2" pool="$3" provenance="$4" metric="$5" + local directory="$root/$phase" hash + mkdir -m 0700 -- "$directory" + printf '%s\n' \ + '{' \ + ' "schema": "platform-observability-metric-inventory/v1",' \ + " \"phase\": \"$phase\"," \ + ' "captured_at_utc": "2026-08-12T00:00:00Z",' \ + ' "context": "default",' \ + ' "api_server": "https://127.0.0.1:6443",' \ + ' "prometheus": "observability/observability-core-kube-pr-prometheus",' \ + ' "preexisting_verify_only": ["grafana"],' \ + ' "targets": [' \ + ' {' \ + " \"scrape_pool\": \"$pool\"," \ + ' "scrape_url": "http://127.0.0.1:9090/metrics",' \ + ' "health": "up",' \ + ' "last_error": "",' \ + " \"provenance\": \"$provenance\"," \ + " \"metrics\": [{\"name\": \"$metric\", \"label_names\": [\"__name__\", \"instance\", \"namespace\"]}]" \ + ' }' \ + ' ]' \ + '}' >"$directory/inventory.json" + chmod 0600 "$directory/inventory.json" + hash="$(sha256sum "$directory/inventory.json" | awk '{print $1}')" + printf '%s inventory.json\n' "$hash" >"$directory/inventory.sha256" + chmod 0600 "$directory/inventory.sha256" +} + +make_publish_root() { + local root + root="$(mktemp -d /tmp/platform-observability-metrics.XXXXXX)" + chmod 0700 "$root" + PUBLISH_ROOTS+=("$root") + write_inventory_phase "$root" target-initial \ + serviceMonitor/observability/grafana/0 preexisting-verify-only up + write_inventory_phase "$root" post-substrate \ + probe/observability/platform-public-edge substrate-owned probe_success + PUBLISH_ROOT=$root +} + +publication_tree_fingerprint() { + local root=$1 + python3 - "$root" <<'PY' +import hashlib +import os +import stat +import sys + + +root = sys.argv[1] +records = [] + + +def record(path, relative): + metadata = os.lstat(path) + content_hash = "-" + if stat.S_ISREG(metadata.st_mode): + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + try: + digest = hashlib.sha256() + while True: + chunk = os.read(descriptor, 1024 * 1024) + if not chunk: + break + digest.update(chunk) + content_hash = digest.hexdigest() + finally: + os.close(descriptor) + records.append("|".join(( + relative, + str(metadata.st_dev), + str(metadata.st_ino), + oct(stat.S_IFMT(metadata.st_mode)), + oct(stat.S_IMODE(metadata.st_mode)), + str(metadata.st_uid), + str(metadata.st_gid), + str(metadata.st_nlink), + str(metadata.st_size), + str(metadata.st_mtime_ns), + str(metadata.st_ctime_ns), + content_hash, + ))) + if stat.S_ISDIR(metadata.st_mode): + with os.scandir(path) as entries: + for entry in sorted(entries, key=lambda item: item.name): + child_relative = entry.name if relative == "." else f"{relative}/{entry.name}" + record(entry.path, child_relative) + + +record(root, ".") +print(hashlib.sha256("\0".join(records).encode()).hexdigest()) +PY +} + +make_inventory_only_clone() { + local source=$1 root phase name + root="$(mktemp -d /tmp/platform-observability-metrics.XXXXXX)" + chmod 0700 "$root" + PUBLISH_ROOTS+=("$root") + for phase in target-initial post-substrate; do + mkdir -m 0700 -- "$root/$phase" + for name in inventory.json inventory.sha256; do + cp --no-dereference --reflink=never -- \ + "$source/$phase/$name" "$root/$phase/$name" + chmod 0600 "$root/$phase/$name" + done + done + PUBLISH_ROOT=$root +} + +inventory_clone_matches_source() { + local source=$1 clone=$2 phase name uid + uid=$(id -u) + for phase in target-initial post-substrate; do + [[ -d "$source/$phase" && ! -L "$source/$phase" ]] || return 1 + [[ -d "$clone/$phase" && ! -L "$clone/$phase" ]] || return 1 + [[ "$(stat -c '%u|%a' -- "$source/$phase")" == "$uid|700" ]] || return 1 + [[ "$(stat -c '%u|%a' -- "$clone/$phase")" == "$uid|700" ]] || return 1 + for name in inventory.json inventory.sha256; do + [[ -f "$source/$phase/$name" && ! -L "$source/$phase/$name" ]] || return 1 + [[ -f "$clone/$phase/$name" && ! -L "$clone/$phase/$name" ]] || return 1 + [[ "$(stat -c '%u|%a|%h' -- "$source/$phase/$name")" == "$uid|600|1" ]] || return 1 + [[ "$(stat -c '%u|%a|%h' -- "$clone/$phase/$name")" == "$uid|600|1" ]] || return 1 + cmp -s "$source/$phase/$name" "$clone/$phase/$name" || return 1 + done + done +} + +published_complete_exact() { + local root=$1 source=$2 name uid + local expected_root expected_source + expected_root=$'alertmanager.yaml\nblackbox.yaml\ndashboards.yaml\ngrafana.yaml\npost-substrate\nprivate-dns.yaml\nrules.yaml\ntarget-initial\ntargets.yaml' + expected_source=$'alertmanager.yaml\nblackbox.yaml\ndashboards.yaml\ngrafana.yaml\nprivate-dns.yaml\nrules.yaml\ntargets.yaml' + [[ "$(find "$root" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" == "$expected_root" ]] || return 1 + [[ "$(find "$source" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" == "$expected_source" ]] || return 1 + uid=$(id -u) + for name in \ + grafana.yaml blackbox.yaml targets.yaml dashboards.yaml rules.yaml alertmanager.yaml private-dns.yaml; do + [[ -f "$source/$name" && ! -L "$source/$name" ]] || return 1 + [[ -f "$root/$name" && ! -L "$root/$name" ]] || return 1 + [[ "$(stat -c '%u|%a|%h' -- "$source/$name")" == "$uid|600|1" ]] || return 1 + [[ "$(stat -c '%u|%a|%h' -- "$root/$name")" == "$uid|600|1" ]] || return 1 + cmp -s "$source/$name" "$root/$name" || return 1 + done +} + +published_rules_alerts_exact() { + local root=$1 source=$2 name + local expected=$'alertmanager.yaml\ndashboards.yaml\npost-substrate\nrules.yaml\ntarget-initial' + [[ "$(find "$root" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" == "$expected" ]] || return 1 + for name in dashboards.yaml rules.yaml alertmanager.yaml; do + [[ -f "$root/$name" && ! -L "$root/$name" && "$(stat -c %a -- "$root/$name")" == 600 ]] || return 1 + cmp -s "$source/$name" "$root/$name" || return 1 + done +} + +collision_left_no_partial_publish() { + local root=$1 + [[ -f "$root/rules.yaml" && ! -e "$root/dashboards.yaml" && ! -e "$root/alertmanager.yaml" ]] || return 1 +} + +complete_collision_left_no_partial() { + local root=$1 uid expected + expected=$'post-substrate\nprivate-dns.yaml\ntarget-initial' + [[ "$(find "$root" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" == "$expected" ]] || return 1 + [[ -f "$root/private-dns.yaml" && ! -L "$root/private-dns.yaml" ]] || return 1 + uid=$(id -u) + [[ "$(stat -c '%u|%a|%h' -- "$root/private-dns.yaml")" == "$uid|600|1" ]] || return 1 + [[ "$(<"$root/private-dns.yaml")" == 'preexisting complete collision' ]] +} + +secure_render_context_creates_private_files() { + local directory=$1 + ( + access_secure_render_context + printf '%s\n' rendered >"$directory/rendered.yaml" + ) || return 1 + [[ "$(stat -c %a -- "$directory/rendered.yaml")" == 600 ]] +} + +WORK="$(mktemp -d /tmp/platform-observability-access-test.XXXXXX)" +chmod 0700 "$WORK" + +grafana="$WORK/grafana.yaml" +cat >"$grafana" <<'YAML' +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: grafana-dashboard-sidecar + namespace: observability +rules: + - apiGroups: [""] + resources: [configmaps] + verbs: [list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: grafana-dashboard-sidecar + namespace: observability +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: grafana-dashboard-sidecar +subjects: + - kind: ServiceAccount + name: grafana + namespace: observability +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana + namespace: observability +data: + grafana.ini: | + [auth] + login_maximum_lifetime_duration = 8h + login_maximum_inactive_lifetime_duration = 30m + [auth.anonymous] + enabled = false + [auth.generic_oauth] + enabled = true + scopes = openid profile email + groups_attribute_path = groups + allowed_groups = /platform-observability-admins /platform-observability-viewers + role_attribute_strict = true + allow_assign_grafana_admin = false + role_attribute_path = contains(groups[*], '/platform-observability-admins') && 'Admin' || contains(groups[*], '/platform-observability-viewers') && 'Viewer' || null + auth_url = https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/auth + token_url = https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/token + api_url = https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/userinfo + signout_redirect_url = https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/logout +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: observability +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + ports: + - name: service + port: 80 + targetPort: grafana +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana + namespace: observability +spec: + ingressClassName: traefik + rules: + - host: grafana.learn.hyeonworks.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana + port: + number: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: observability +spec: + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + spec: + automountServiceAccountToken: false + serviceAccountName: grafana + containers: + - name: grafana-sc-dashboard + image: quay.io/kiwigrid/k8s-sidecar:2.10.0@sha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3 + volumeMounts: + - name: dashboard-sidecar-api-access + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + - name: grafana + image: docker.io/grafana/grafana:13.1.1@sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b + env: + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-admin + key: admin-password + - name: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: grafana-keycloak-oidc + key: client-secret + volumes: + - name: dashboard-sidecar-api-access + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + expirationSeconds: 3600 + path: token + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt + - downwardAPI: + items: + - path: namespace + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: grafana + namespace: observability + labels: + observability.hyeonworks.com/instance: home +spec: + selector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + endpoints: + - port: service + path: /metrics +YAML + +kubectl kustomize "$ROOT/services/observability/network-policies" >"$WORK/core-policies.yaml" + +blackbox="$WORK/blackbox.yaml" +cat >"$blackbox" <<'YAML' +apiVersion: v1 +kind: Service +metadata: + name: blackbox-exporter + namespace: observability +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + ports: + - name: http + port: 9115 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: blackbox-exporter + namespace: observability +spec: + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + spec: + automountServiceAccountToken: false + containers: + - name: blackbox-exporter + image: quay.io/prometheus/blackbox-exporter:v0.28.0@sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68 +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: blackbox-exporter + namespace: observability + labels: + observability.hyeonworks.com/instance: home +spec: + selector: + matchLabels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + endpoints: + - port: http + path: /metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: platform-public-edge + namespace: observability + labels: + observability.hyeonworks.com/instance: home +spec: + jobName: blackbox-public-edge + module: http_2xx + prober: + url: blackbox-exporter.observability.svc.cluster.local:9115 + scheme: http + path: /probe + proxyFromEnvironment: false + targets: + staticConfig: + labels: + observability.hyeonworks.com/probe-group: public-edge + static: + - https://git.learn.hyeonworks.com/api/healthz + - https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: platform-private-edge + namespace: observability + labels: + observability.hyeonworks.com/instance: home +spec: + jobName: blackbox-private-edge + module: http_private_edge_403 + prober: + url: blackbox-exporter.observability.svc.cluster.local:9115 + scheme: http + path: /probe + proxyFromEnvironment: false + targets: + staticConfig: + labels: + observability.hyeonworks.com/probe-group: private-edge + static: + - https://grafana.learn.hyeonworks.com/ + - https://storage-admin.learn.hyeonworks.com/ + - https://db-admin.learn.hyeonworks.com/ +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: platform-private-internal + namespace: observability + labels: + observability.hyeonworks.com/instance: home +spec: + jobName: blackbox-private-internal + module: http_private_internal_200 + prober: + url: blackbox-exporter.observability.svc.cluster.local:9115 + scheme: http + path: /probe + proxyFromEnvironment: false + targets: + staticConfig: + labels: + observability.hyeonworks.com/probe-group: private-internal + static: + - http://grafana.observability.svc.cluster.local/api/health + - http://pgadmin.platform-admin.svc.cluster.local/misc/ping + - http://minio-aistor-console.object-storage.svc.cluster.local:9090/ + relabelingConfigs: + - sourceLabels: [__param_target] + regex: http://grafana[.]observability[.]svc[.]cluster[.]local/api/health + targetLabel: __param_hostname + replacement: grafana.learn.hyeonworks.com + - sourceLabels: [__param_target] + regex: http://pgadmin[.]platform-admin[.]svc[.]cluster[.]local/misc/ping + targetLabel: __param_hostname + replacement: db-admin.learn.hyeonworks.com + - sourceLabels: [__param_target] + regex: http://minio-aistor-console[.]object-storage[.]svc[.]cluster[.]local:9090/ + targetLabel: __param_hostname + replacement: storage-admin.learn.hyeonworks.com +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-prometheus-to-blackbox + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: observability-core-kube-pr-prometheus + ports: + - protocol: TCP + port: 9115 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-blackbox-egress + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + policyTypes: [Egress] + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - ipBlock: {cidr: 192.168.0.107/32} + ports: + - {protocol: TCP, port: 443} + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + ports: + - {protocol: TCP, port: 3000} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: platform-admin + podSelector: + matchLabels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/instance: pgadmin + ports: + - {protocol: TCP, port: 5050} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: object-storage + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + ports: + - {protocol: TCP, port: 9090} +YAML + +kubectl kustomize "$ROOT/services/observability/targets" >"$WORK/targets.yaml" +kubectl kustomize "$ROOT/services/observability/dashboards/platform" >"$WORK/dashboards.yaml" +kubectl kustomize "$ROOT/services/observability/rules/platform" >"$WORK/rules.yaml" + +cat >"$WORK/alertmanager-base.yaml" <<'YAML' +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: observability-core-kube-pr-alertmanager + namespace: observability +spec: + alertmanagerConfiguration: + name: platform-alertmanager +YAML +kubectl kustomize "$ROOT/services/observability/alerting" >"$WORK/alerting.yaml" +python3 - "$WORK/alertmanager-base.yaml" "$WORK/alerting.yaml" "$WORK/alertmanager.yaml" <<'PY' +import sys +import yaml + +class FixtureLoader(yaml.SafeLoader): + pass + +FixtureLoader.add_constructor( + "tag:yaml.org,2002:value", + lambda loader, node: loader.construct_scalar(node), +) + +items = [] +for path in sys.argv[1:3]: + with open(path, "r", encoding="utf-8") as stream: + items.extend(item for item in yaml.load_all(stream, Loader=FixtureLoader) if item is not None) +with open(sys.argv[3], "w", encoding="utf-8") as stream: + yaml.safe_dump_all(items, stream, explicit_start=True, sort_keys=False) +PY + +kubectl kustomize "$ROOT/infrastructure/networking/private-dns/kubernetes" >"$WORK/private-dns.yaml" +kubectl kustomize "$ROOT/infrastructure/networking/traefik/overlays/trust" >"$WORK/traefik-trust.yaml" +kubectl kustomize "$ROOT/infrastructure/networking/traefik/overlays/observe" >"$WORK/traefik-observe.yaml" + +expected_complete=$'grafana.yaml\nblackbox.yaml\ntargets.yaml\ndashboards.yaml\nrules.yaml\nalertmanager.yaml\nprivate-dns.yaml' +expect_output 'complete mode has the exact seven access artifacts' "$expected_complete" access_output_names complete +expect_output 'grafana mode has one artifact' 'grafana.yaml' access_output_names grafana +expect_output 'blackbox mode has one artifact' 'blackbox.yaml' access_output_names blackbox +expect_output 'targets mode has one artifact' 'targets.yaml' access_output_names targets +expect_output 'rules-alerts mode has three artifacts' $'dashboards.yaml\nrules.yaml\nalertmanager.yaml' access_output_names rules-alerts +expect_fail 'unknown component is rejected' access_output_names arbitrary + +expect_pass 'Grafana private access contract' assert_access_grafana_contract "$grafana" "$WORK/core-policies.yaml" +expect_pass 'Grafana and sidecar images are digest pinned' assert_pinned_images "$grafana" + +mutate_yaml "$WORK/core-policies.yaml" "$WORK/grafana-broad-policy.yaml" grafana-broad-policy +expect_fail 'broad Grafana ingress NetworkPolicy is rejected' \ + assert_access_grafana_contract "$grafana" "$WORK/grafana-broad-policy.yaml" +mutate_yaml "$WORK/core-policies.yaml" "$WORK/grafana-extra-policy.yaml" grafana-extra-allow-all +expect_fail 'an extra Grafana allow-all NetworkPolicy is rejected' \ + assert_access_grafana_contract "$grafana" "$WORK/grafana-extra-policy.yaml" + +mutate_yaml "$grafana" "$WORK/grafana-public-service.yaml" grafana-public-service +expect_fail 'Grafana public Service is rejected' assert_access_grafana_contract "$WORK/grafana-public-service.yaml" "$WORK/core-policies.yaml" +mutate_yaml "$grafana" "$WORK/grafana-missing-class.yaml" grafana-missing-class +expect_fail 'Grafana missing ingressClass is rejected' assert_access_grafana_contract "$WORK/grafana-missing-class.yaml" "$WORK/core-policies.yaml" +mutate_yaml "$grafana" "$WORK/grafana-main-token.yaml" grafana-main-token +expect_fail 'Grafana main container token mount is rejected' assert_access_grafana_contract "$WORK/grafana-main-token.yaml" "$WORK/core-policies.yaml" +mutate_yaml "$grafana" "$WORK/grafana-cluster-role.yaml" grafana-cluster-role +expect_fail 'cluster-wide dashboard ConfigMap RBAC is rejected' assert_access_grafana_contract "$WORK/grafana-cluster-role.yaml" "$WORK/core-policies.yaml" + +for mutation in oidc-viewer-fallback oidc-editor oidc-grafana-admin oidc-group-missing; do + mutate_yaml "$grafana" "$WORK/$mutation.yaml" "$mutation" + expect_fail "Grafana OIDC regression $mutation is rejected" \ + assert_access_grafana_contract "$WORK/$mutation.yaml" "$WORK/core-policies.yaml" +done + +sed 's/@sha256:[0-9a-f]\{64\}//g' "$grafana" >"$WORK/grafana-tag-only.yaml" +expect_fail 'tag-only Grafana and sidecar images are rejected' assert_pinned_images "$WORK/grafana-tag-only.yaml" + +expect_pass 'Blackbox fixed target and network contract' assert_access_blackbox_contract "$blackbox" +expect_pass 'Blackbox image is digest pinned' assert_pinned_images "$blackbox" +mutate_yaml "$blackbox" "$WORK/blackbox-broad-egress.yaml" blackbox-broad-egress +expect_fail 'broad Blackbox egress NetworkPolicy is rejected' \ + assert_access_blackbox_contract "$WORK/blackbox-broad-egress.yaml" +mutate_yaml "$blackbox" "$WORK/blackbox-extra-policy.yaml" blackbox-extra-allow-all +expect_fail 'an extra Blackbox allow-all NetworkPolicy is rejected' \ + assert_access_blackbox_contract "$WORK/blackbox-extra-policy.yaml" +mutate_yaml "$blackbox" "$WORK/blackbox-arbitrary.yaml" blackbox-arbitrary-target +expect_fail 'arbitrary Blackbox target is rejected' assert_access_blackbox_contract "$WORK/blackbox-arbitrary.yaml" +mutate_yaml "$blackbox" "$WORK/blackbox-public.yaml" blackbox-public-service +expect_fail 'public Blackbox Service is rejected' assert_access_blackbox_contract "$WORK/blackbox-public.yaml" +sed 's/@sha256:[0-9a-f]\{64\}//g' "$blackbox" >"$WORK/blackbox-tag-only.yaml" +expect_fail 'tag-only Blackbox image is rejected' assert_pinned_images "$WORK/blackbox-tag-only.yaml" + +expect_pass 'target object set and target scope contract' assert_access_targets_contract "$WORK/targets.yaml" +mutate_yaml "$WORK/targets.yaml" "$WORK/targets-broad-egress.yaml" targets-broad-egress +expect_fail 'broad Prometheus target egress NetworkPolicy is rejected' \ + assert_access_targets_contract "$WORK/targets-broad-egress.yaml" +mutate_yaml "$WORK/targets.yaml" "$WORK/targets-extra-policy.yaml" targets-extra-allow-all +expect_fail 'an extra Prometheus allow-all NetworkPolicy is rejected' \ + assert_access_targets_contract "$WORK/targets-extra-policy.yaml" +expect_pass 'dashboard/rule/alertmanager contract' assert_access_rules_alerts_contract \ + "$WORK/dashboards.yaml" "$WORK/rules.yaml" "$WORK/alertmanager.yaml" +alertmanager_source_before="$(publication_tree_fingerprint "$WORK/alertmanager.yaml")" +mutate_yaml "$WORK/alertmanager.yaml" "$WORK/alertmanager-missing-title.yaml" alertmanager-missing-title +expect_fail 'Alertmanager Slack config without title is rejected' \ + assert_access_rules_alerts_contract \ + "$WORK/dashboards.yaml" "$WORK/rules.yaml" "$WORK/alertmanager-missing-title.yaml" +expect_output 'rejected Alertmanager handoff preserves its canonical source exactly' \ + "$alertmanager_source_before" publication_tree_fingerprint "$WORK/alertmanager.yaml" +mutate_yaml "$WORK/alertmanager.yaml" "$WORK/alertmanager-extra-policy.yaml" alertmanager-extra-allow-all +expect_fail 'an extra Alertmanager allow-all NetworkPolicy is rejected' \ + assert_access_rules_alerts_contract \ + "$WORK/dashboards.yaml" "$WORK/rules.yaml" "$WORK/alertmanager-extra-policy.yaml" +expect_pass 'all monitor/rule instance labels are scoped' assert_access_scope_contract \ + "$blackbox" "$WORK/targets.yaml" "$WORK/rules.yaml" + +mutate_yaml "$blackbox" "$WORK/missing-instance.yaml" missing-instance-label +expect_fail 'missing Probe instance label is rejected' assert_access_scope_contract "$WORK/missing-instance.yaml" +mutate_yaml "$blackbox" "$WORK/outside-target.yaml" outside-target +expect_fail 'out-of-scope platform target is rejected' assert_access_scope_contract "$WORK/outside-target.yaml" +mutate_yaml "$WORK/rules.yaml" "$WORK/forbidden-product.yaml" forbidden-product +expect_fail 'Spring/JVM/Kafka/batch/backup scope is rejected' assert_access_scope_contract "$WORK/forbidden-product.yaml" +mutate_yaml "$WORK/alertmanager.yaml" "$WORK/inline-secret.yaml" alertmanager-inline-secret +expect_fail 'inline credential Secret is rejected' assert_no_credentials "$WORK/inline-secret.yaml" + +expect_pass 'private DNS is exact and private-only' assert_access_private_dns_contract \ + "$WORK/private-dns.yaml" \ + "$ROOT/infrastructure/networking/private-dns/host/dnsmasq-lan.conf" \ + "$ROOT/infrastructure/networking/private-dns/host/dnsmasq-tailscale.conf" +mutate_yaml "$WORK/private-dns.yaml" "$WORK/private-dns-public.yaml" dns-public-address +expect_fail 'public Grafana DNS address assumption is rejected' assert_access_private_dns_contract \ + "$WORK/private-dns-public.yaml" \ + "$ROOT/infrastructure/networking/private-dns/host/dnsmasq-lan.conf" \ + "$ROOT/infrastructure/networking/private-dns/host/dnsmasq-tailscale.conf" + +expect_pass 'Traefik trust overlay render is accepted' assert_access_traefik_trust_contract "$WORK/traefik-trust.yaml" +expect_fail 'Traefik observe overlay render is rejected' assert_access_traefik_trust_contract "$WORK/traefik-observe.yaml" + +make_publish_root +inventory_root=$PUBLISH_ROOT +expect_pass 'both verified inventory phases are accepted' validate_access_inventory_root "$inventory_root" + +make_publish_root +inventory_unsafe_root=$PUBLISH_ROOT +chmod 0777 "$inventory_unsafe_root" +expect_fail 'world-writable inventory root is rejected' validate_access_inventory_root "$inventory_unsafe_root" + +make_publish_root +inventory_unsafe_phase=$PUBLISH_ROOT +chmod 0755 "$inventory_unsafe_phase/post-substrate" +expect_fail 'group-readable inventory phase is rejected' validate_access_inventory_root "$inventory_unsafe_phase" + +make_publish_root +inventory_unsafe_file=$PUBLISH_ROOT +chmod 0644 "$inventory_unsafe_file/post-substrate/inventory.json" +expect_fail 'group-readable inventory file is rejected' validate_access_inventory_root "$inventory_unsafe_file" + +make_publish_root +inventory_hardlink_file=$PUBLISH_ROOT +mv -T "$inventory_hardlink_file/post-substrate/inventory.json" "$WORK/shared-inventory.json" +ln "$WORK/shared-inventory.json" "$inventory_hardlink_file/post-substrate/inventory.json" +expect_fail 'hard-linked inventory file is rejected' validate_access_inventory_root "$inventory_hardlink_file" + +make_publish_root +inventory_bad_checksum=$PUBLISH_ROOT +printf '%064d inventory.json\n' 0 >"$inventory_bad_checksum/post-substrate/inventory.sha256" +chmod 0600 "$inventory_bad_checksum/post-substrate/inventory.sha256" +expect_fail 'inventory checksum drift is rejected' validate_access_inventory_root "$inventory_bad_checksum" + +make_publish_root +inventory_missing_phase=$PUBLISH_ROOT +mv -T "$inventory_missing_phase/post-substrate" "$WORK/inventory-missing-phase-saved" +expect_fail 'missing post-substrate phase is rejected' validate_access_inventory_root "$inventory_missing_phase" + +expect_pass 'rules-alerts component requires inventories' access_component_requires_inventory rules-alerts +expect_pass 'complete component requires inventories' access_component_requires_inventory complete +expect_fail 'grafana component does not accept inventory as a substitute for its source' access_component_requires_inventory grafana + +expect_pass 'render context creates private files by default' \ + secure_render_context_creates_private_files "$WORK" + +publish_source="$WORK/publish-source" +mkdir -m 0700 -- "$publish_source" +printf '%s\n' 'kind: ConfigMap' >"$publish_source/dashboards.yaml" +printf '%s\n' 'kind: PrometheusRule' >"$publish_source/rules.yaml" +printf '%s\n' 'kind: AlertmanagerConfig' >"$publish_source/alertmanager.yaml" +chmod 0600 "$publish_source"/*.yaml + +make_publish_root +publish_success_root=$PUBLISH_ROOT +expect_pass 'rules-alerts publication is complete and exact' \ + publish_access_outputs "$publish_success_root" "$publish_source" rules-alerts +expect_pass 'rules-alerts publication preserves exact bytes and modes' \ + published_rules_alerts_exact "$publish_success_root" "$publish_source" + +complete_publish_source="$WORK/complete-publish-source" +mkdir -m 0700 -- "$complete_publish_source" +for complete_name in \ + grafana.yaml blackbox.yaml targets.yaml dashboards.yaml rules.yaml alertmanager.yaml private-dns.yaml; do + cp --no-dereference --reflink=never -- \ + "$WORK/$complete_name" "$complete_publish_source/$complete_name" +done +chmod 0600 "$complete_publish_source"/*.yaml + +published_root_before="$(publication_tree_fingerprint "$publish_success_root")" +complete_source_before="$(publication_tree_fingerprint "$complete_publish_source")" +expect_fail 'a rules-alerts-published root rejects later complete publication' \ + publish_access_outputs "$publish_success_root" "$complete_publish_source" complete +expect_output 'rejected complete publication preserves the published root exactly' \ + "$published_root_before" publication_tree_fingerprint "$publish_success_root" +expect_output 'rejected complete publication preserves all complete sources exactly' \ + "$complete_source_before" publication_tree_fingerprint "$complete_publish_source" + +make_inventory_only_clone "$publish_success_root" +complete_publish_root=$PUBLISH_ROOT +target_inventory_before="$(publication_tree_fingerprint "$complete_publish_root/target-initial")" +post_inventory_before="$(publication_tree_fingerprint "$complete_publish_root/post-substrate")" +expect_pass 'the inventory-only clone exactly copies the four published inventory files' \ + inventory_clone_matches_source "$publish_success_root" "$complete_publish_root" +expect_pass 'an inventory-only clone passes inventory validation' \ + validate_access_inventory_root "$complete_publish_root" +expect_pass 'an inventory-only clone accepts complete publication' \ + publish_access_outputs "$complete_publish_root" "$complete_publish_source" complete +expect_pass 'complete publication has the exact nine-entry topology and seven secure YAML files' \ + published_complete_exact "$complete_publish_root" "$complete_publish_source" +expect_output 'complete publication preserves target-initial inventory exactly' \ + "$target_inventory_before" publication_tree_fingerprint "$complete_publish_root/target-initial" +expect_output 'complete publication preserves post-substrate inventory exactly' \ + "$post_inventory_before" publication_tree_fingerprint "$complete_publish_root/post-substrate" +expect_output 'complete publication preserves the rules-alerts source root exactly' \ + "$published_root_before" publication_tree_fingerprint "$publish_success_root" +expect_output 'complete publication preserves all complete sources exactly' \ + "$complete_source_before" publication_tree_fingerprint "$complete_publish_source" + +make_inventory_only_clone "$publish_success_root" +complete_collision_root=$PUBLISH_ROOT +printf '%s\n' 'preexisting complete collision' >"$complete_collision_root/private-dns.yaml" +chmod 0600 "$complete_collision_root/private-dns.yaml" +complete_collision_before="$(publication_tree_fingerprint "$complete_collision_root")" +expect_fail 'a last-name collision rejects complete publication before any output is added' \ + publish_access_outputs "$complete_collision_root" "$complete_publish_source" complete +expect_pass 'a complete last-name collision leaves no partial publication' \ + complete_collision_left_no_partial "$complete_collision_root" +expect_output 'a rejected complete collision preserves its destination tree exactly' \ + "$complete_collision_before" publication_tree_fingerprint "$complete_collision_root" +expect_output 'a rejected complete collision preserves the rules-alerts source root exactly' \ + "$published_root_before" publication_tree_fingerprint "$publish_success_root" +expect_output 'a rejected complete collision preserves all complete sources exactly' \ + "$complete_source_before" publication_tree_fingerprint "$complete_publish_source" + +make_publish_root +publish_collision_root=$PUBLISH_ROOT +printf '%s\n' 'preexisting' >"$publish_collision_root/rules.yaml" +chmod 0600 "$publish_collision_root/rules.yaml" +expect_fail 'a later-name collision rejects the whole publication' \ + publish_access_outputs "$publish_collision_root" "$publish_source" rules-alerts +expect_pass 'a later-name collision leaves no partial publication' \ + collision_left_no_partial_publish "$publish_collision_root" + +printf 'Assertions: %d\n' "$ASSERTIONS" +printf 'OBSERVABILITY ACCESS RENDER ASSERTION TEST PASS\n' diff --git a/scripts/validate/test-render-observability-core.sh b/scripts/validate/test-render-observability-core.sh new file mode 100755 index 0000000..97dce8b --- /dev/null +++ b/scripts/validate/test-render-observability-core.sh @@ -0,0 +1,1093 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly RENDERER="$ROOT/scripts/validate/render-observability-core.sh" +readonly EXPECTED_REJECTION_STATUS=23 + +if [[ ! -f "$RENDERER" ]]; then + printf 'EXPECTED RED: renderer is missing: %s\n' "$RENDERER" >&2 + exit 1 +fi + +# Sourcing must define assertions without running the renderer or changing shell options. +source "$RENDERER" + +fixture_dir="$(mktemp -d /tmp/platform-observability-render-test.XXXXXX)" +handoff_dirs=() +cleanup() { + local rc=$? + local directory + for directory in "${handoff_dirs[@]}"; do + case "$directory" in + /tmp/platform-observability-core-apply.*) rm -rf -- "$directory" ;; + esac + done + case "$fixture_dir" in + /tmp/platform-observability-render-test.*) rm -rf -- "$fixture_dir" ;; + esac + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +pass_count=0 + +expect_pass() { + local name="$1" + local output + shift + if ! output="$({ "$@"; } 2>&1)"; then + printf 'FAIL (expected pass): %s\n%s\n' "$name" "$output" >&2 + exit 1 + fi + pass_count=$((pass_count + 1)) +} + +expect_stateful_pass() { + local name="$1" + local output_file="$fixture_dir/stateful-output" + local rc + shift + set +e + "$@" >"$output_file" 2>&1 + rc=$? + set -e + if [[ "$rc" != 0 ]]; then + printf 'FAIL (expected stateful pass): %s\n' "$name" >&2 + sed -n '1,20p' "$output_file" >&2 + exit 1 + fi + pass_count=$((pass_count + 1)) +} + +expect_fail() { + local name="$1" + local output rc + shift + set +e + output="$({ "$@"; } 2>&1)" + rc=$? + set -e + if [[ "$rc" != "$EXPECTED_REJECTION_STATUS" ]]; then + printf 'FAIL (expected rejection status %s, got %s): %s\n%s\n' \ + "$EXPECTED_REJECTION_STATUS" "$rc" "$name" "$output" >&2 + exit 1 + fi + if [[ "$output" != REJECT:* ]]; then + printf 'FAIL (expected pinned rejection message): %s\n%s\n' "$name" "$output" >&2 + exit 1 + fi + if [[ "$output" == *'DO-NOT-LEAK-FIXTURE'* ]]; then + printf 'FAIL (credential leaked): %s\n' "$name" >&2 + exit 1 + fi + pass_count=$((pass_count + 1)) +} + +write_file() { + local path="$1" + shift + printf '%s\n' "$@" >"$path" +} + +assert_chart_pin_row() { + local expected="$*" + local label="$1" + local actual + actual="$(chart_pin_rows | awk -F '\t' -v label="$label" '$1 == label { print $1, $2, $3, $4, $5, $6, $7 }')" + [[ "$actual" == "$expected" ]] +} + +assert_image_pin_row() { + local expected="$*" + local image="$1" + local actual + actual="$(image_pin_rows | awk -F '\t' -v image="$image" '$1 == image { print $1, $2 }')" + [[ "$actual" == "$expected" ]] +} + +expect_pass 'Helm version pin' test "$(observability_expected_helm_version)" = v3.19.4 +expect_pass 'Kustomize version pin' test "$(observability_expected_kustomize_version)" = v5.8.1 +expect_pass 'Kubernetes render target pin' test "$(observability_helm_kube_version)" = 1.36.2 +expected_render_roots=$'infrastructure/namespaces/observability\n' +expected_render_roots+=$'infrastructure/controllers/prometheus-operator-crds\n' +expected_render_roots+=$'infrastructure/storage/observability-local-pv\n' +expected_render_roots+=$'services/observability/kube-prometheus-stack\n' +expected_render_roots+=$'services/observability/loki\n' +expected_render_roots+=$'services/observability/tempo\n' +expected_render_roots+=$'services/observability/alloy\n' +expected_render_roots+=$'services/observability/node-exporter\n' +expected_render_roots+=$'services/observability/grafana\n' +expected_render_roots+=$'services/observability/blackbox-exporter\n' +expected_render_roots+=$'services/observability/network-policies\n' +expected_render_roots+=$'services/observability/targets\n' +expected_render_roots+=$'services/observability/rules/platform\n' +expected_render_roots+=$'services/observability/alerting\n' +expected_render_roots+=$'services/observability/dashboards/platform\n' +expected_render_roots+='services/observability' +expect_pass 'complete observability render root table' test "$(render_root_rows)" = "$expected_render_roots" +expected_chart_table=$'prometheus-operator-crds\thttps://prometheus-community.github.io/helm-charts\tprometheus-operator-crds\t31.0.0\tv0.93.0\t5d11cac50321bc7ae56b35d2701e56f21fe079ca3aea872abb28a3e2119d5cfd\tinfrastructure/controllers/prometheus-operator-crds\n' +expected_chart_table+=$'kube-prometheus-stack\thttps://prometheus-community.github.io/helm-charts\tkube-prometheus-stack\t88.0.1\tv0.93.0\t0998b3fae8660cd9ea224caa1c7c4b5d2040d5a02c64e3cdfd3f48d184b50117\tservices/observability/kube-prometheus-stack\n' +expected_chart_table+=$'loki\thttps://grafana-community.github.io/helm-charts\tloki\t18.7.1\t3.7.4\tde9ab59827f2ed24634e94db120ea2467f5d08b02e8d7c250e6bf88c17bfb48d\tservices/observability/loki\n' +expected_chart_table+=$'tempo\thttps://grafana-community.github.io/helm-charts\ttempo\t2.2.3\t2.10.7\t14c52efe5d0cad5456ffa5a8be1e5107be47d184be3e209f748be51a7b0316fe\tservices/observability/tempo\n' +expected_chart_table+=$'alloy\thttps://grafana.github.io/helm-charts\talloy\t1.11.0\tv1.18.0\t11d253b62e47beeacd89eb4283fc056962ecbf143984863c1998be13da0772dd\tservices/observability/alloy\n' +expected_chart_table+=$'prometheus-node-exporter\thttps://prometheus-community.github.io/helm-charts\tprometheus-node-exporter\t4.56.1\t1.12.1\t70fab291e00a34c58fd943bb075fecc2e2e73fa95db1bcbbe7cdfe008795354a\tservices/observability/node-exporter' +expected_chart_table+=$'\ngrafana\thttps://grafana-community.github.io/helm-charts\tgrafana\t12.10.1\t13.1.1\tff865a8b9d1d7563901b8e6aa653b4f345bd4d88344c149065bd23958e2beeb8\tservices/observability/grafana' +expected_chart_table+=$'\nprometheus-blackbox-exporter\thttps://prometheus-community.github.io/helm-charts\tprometheus-blackbox-exporter\t11.16.0\tv0.28.0\t932aa65df0538d9dc003c46bb663ebe44a06078af7a6a54577af5339668ac65b\tservices/observability/blackbox-exporter' +expect_pass 'exact eight-row chart table with no extras' test "$(chart_pin_rows)" = "$expected_chart_table" + +expected_image_table=$'quay.io/prometheus-operator/prometheus-operator:v0.93.0\tsha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2\n' +expected_image_table+=$'quay.io/prometheus-operator/prometheus-config-reloader:v0.93.0\tsha256:65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905\n' +expected_image_table+=$'quay.io/prometheus/prometheus:v3.13.2-distroless\tsha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b\n' +expected_image_table+=$'quay.io/prometheus/alertmanager:v0.33.1\tsha256:a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b\n' +expected_image_table+=$'registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1\tsha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4\n' +expected_image_table+=$'quay.io/prometheus/node-exporter:v1.12.1\tsha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798\n' +expected_image_table+=$'docker.io/grafana/loki:3.7.4\tsha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e\n' +expected_image_table+=$'docker.io/grafana/tempo:2.10.7\tsha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c\n' +expected_image_table+=$'docker.io/grafana/alloy:v1.18.0\tsha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17\n' +expected_image_table+=$'ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.153.0\tsha256:9708a1d33398c8725549104d1f6009be62569e47e094430543da0f66758cc522\n' +expected_image_table+=$'docker.io/library/busybox:1.37.0\tsha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0\n' +expected_image_table+=$'quay.io/minio/aistor/mc\tsha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b\n' +expected_image_table+=$'docker.io/grafana/grafana:13.1.1\tsha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b\n' +expected_image_table+=$'quay.io/kiwigrid/k8s-sidecar:2.10.0\tsha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3\n' +expected_image_table+=$'quay.io/prometheus/blackbox-exporter:v0.28.0\tsha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68' +expect_pass 'exact fifteen-row image table with no extras' test "$(image_pin_rows)" = "$expected_image_table" +expect_pass 'Prometheus CRD chart pin' assert_chart_pin_row prometheus-operator-crds https://prometheus-community.github.io/helm-charts prometheus-operator-crds 31.0.0 v0.93.0 5d11cac50321bc7ae56b35d2701e56f21fe079ca3aea872abb28a3e2119d5cfd infrastructure/controllers/prometheus-operator-crds +expect_pass 'KPS chart pin' assert_chart_pin_row kube-prometheus-stack https://prometheus-community.github.io/helm-charts kube-prometheus-stack 88.0.1 v0.93.0 0998b3fae8660cd9ea224caa1c7c4b5d2040d5a02c64e3cdfd3f48d184b50117 services/observability/kube-prometheus-stack +expect_pass 'Loki chart pin' assert_chart_pin_row loki https://grafana-community.github.io/helm-charts loki 18.7.1 3.7.4 de9ab59827f2ed24634e94db120ea2467f5d08b02e8d7c250e6bf88c17bfb48d services/observability/loki +expect_pass 'Tempo chart pin' assert_chart_pin_row tempo https://grafana-community.github.io/helm-charts tempo 2.2.3 2.10.7 14c52efe5d0cad5456ffa5a8be1e5107be47d184be3e209f748be51a7b0316fe services/observability/tempo +expect_pass 'Alloy chart pin' assert_chart_pin_row alloy https://grafana.github.io/helm-charts alloy 1.11.0 v1.18.0 11d253b62e47beeacd89eb4283fc056962ecbf143984863c1998be13da0772dd services/observability/alloy +expect_pass 'node-exporter chart pin' assert_chart_pin_row prometheus-node-exporter https://prometheus-community.github.io/helm-charts prometheus-node-exporter 4.56.1 1.12.1 70fab291e00a34c58fd943bb075fecc2e2e73fa95db1bcbbe7cdfe008795354a services/observability/node-exporter +expect_pass 'Grafana chart pin' assert_chart_pin_row grafana https://grafana-community.github.io/helm-charts grafana 12.10.1 13.1.1 ff865a8b9d1d7563901b8e6aa653b4f345bd4d88344c149065bd23958e2beeb8 services/observability/grafana +expect_pass 'Blackbox chart pin' assert_chart_pin_row prometheus-blackbox-exporter https://prometheus-community.github.io/helm-charts prometheus-blackbox-exporter 11.16.0 v0.28.0 932aa65df0538d9dc003c46bb663ebe44a06078af7a6a54577af5339668ac65b services/observability/blackbox-exporter + +expect_pass 'operator image pin' assert_image_pin_row quay.io/prometheus-operator/prometheus-operator:v0.93.0 sha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2 +expect_pass 'reloader image pin' assert_image_pin_row quay.io/prometheus-operator/prometheus-config-reloader:v0.93.0 sha256:65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905 +expect_pass 'Prometheus image pin' assert_image_pin_row quay.io/prometheus/prometheus:v3.13.2-distroless sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b +expect_pass 'Alertmanager image pin' assert_image_pin_row quay.io/prometheus/alertmanager:v0.33.1 sha256:a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b +expect_pass 'KSM image pin' assert_image_pin_row registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1 sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4 +expect_pass 'node-exporter image pin' assert_image_pin_row quay.io/prometheus/node-exporter:v1.12.1 sha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798 +expect_pass 'Loki image pin' assert_image_pin_row docker.io/grafana/loki:3.7.4 sha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e +expect_pass 'Tempo image pin' assert_image_pin_row docker.io/grafana/tempo:2.10.7 sha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c +expect_pass 'Alloy image pin' assert_image_pin_row docker.io/grafana/alloy:v1.18.0 sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17 +expect_pass 'telemetrygen image pin' assert_image_pin_row ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.153.0 sha256:9708a1d33398c8725549104d1f6009be62569e47e094430543da0f66758cc522 +expect_pass 'BusyBox image pin' assert_image_pin_row docker.io/library/busybox:1.37.0 sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0 +expect_pass 'AIStor mc image pin' assert_image_pin_row quay.io/minio/aistor/mc sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b +expect_pass 'Grafana image pin' assert_image_pin_row docker.io/grafana/grafana:13.1.1 sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b +expect_pass 'Grafana sidecar image pin' assert_image_pin_row quay.io/kiwigrid/k8s-sidecar:2.10.0 sha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3 +expect_pass 'Blackbox image pin' assert_image_pin_row quay.io/prometheus/blackbox-exporter:v0.28.0 sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68 +expect_pass 'tagged Grafana digest is in the rendered-image allowlist' grep -Fqx \ + docker.io/grafana/grafana:13.1.1@sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b \ + <(_observability_image_allowlist) +expect_pass 'tagged Blackbox digest is in the rendered-image allowlist' grep -Fqx \ + quay.io/prometheus/blackbox-exporter:v0.28.0@sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68 \ + <(_observability_image_allowlist) + +alertmanager_fixture="$fixture_dir/alertmanager.yaml" +write_file "$alertmanager_fixture" \ + 'apiVersion: monitoring.coreos.com/v1' \ + 'kind: Alertmanager' \ + 'metadata:' \ + ' name: observability-core-kube-pr-alertmanager' \ + ' namespace: observability' \ + 'spec:' \ + ' alertmanagerConfiguration:' \ + ' name: platform-alertmanager' +alerting_fixture="$fixture_dir/alerting.yaml" +write_file "$alerting_fixture" '{"apiVersion":"monitoring.coreos.com/v1alpha1","kind":"AlertmanagerConfig","metadata":{"name":"platform-alertmanager","namespace":"observability","labels":{"observability.hyeonworks.com/instance":"home"}},"spec":{"route":{"receiver":"platform-slack","groupBy":["cluster","namespace","alertname","severity"],"groupWait":"30s","groupInterval":"5m","repeatInterval":"4h","routes":[{"receiver":"platform-null","matchers":[{"name":"alertname","matchType":"=","value":"InfoInhibitor"}]}]},"inhibitRules":[{"sourceMatch":[{"name":"alertname","matchType":"=","value":"InfoInhibitor"}],"targetMatch":[{"name":"severity","matchType":"=","value":"info"}],"equal":["namespace"]}],"receivers":[{"name":"platform-null"},{"name":"platform-slack","slackConfigs":[{"apiURL":{"name":"alertmanager-slack-webhook","key":"url"},"sendResolved":true,"linkNames":false,"mrkdwnIn":["text","fields"],"fallback":"{{ if eq .Status \"firing\" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}","title":"{{ if eq .Status \"firing\" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}","titleLink":"https://grafana.learn.hyeonworks.com/","color":"{{ if eq .Status \"resolved\" }}good{{ else if or (eq .CommonLabels.severity \"emergency\") (eq .CommonLabels.severity \"critical\") }}danger{{ else if eq .CommonLabels.severity \"warning\" }}warning{{ else }}#439FE0{{ end }}","fields":[{"title":"Status","value":"{{ .Status | toUpper }}","short":true},{"title":"Severity","value":"{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}","short":true},{"title":"Location","value":"{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}","short":true},{"title":"Alert count","value":"{{ len .Alerts }}","short":true}],"text":"{{ range .Alerts }}\n*Alert status:* {{ .Status | toUpper }}\n*Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}\n*Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}\n*Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}\n*Started:* {{ .StartsAt.Format \"2006-01-02T15:04:05Z07:00\" }}\n{{ if eq .Status \"resolved\" }}*Ended:* {{ .EndsAt.Format \"2006-01-02T15:04:05Z07:00\" }}{{ end }}\n{{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}\n\n{{ end }}\n · ","footer":"hyeonworks observability · Alertmanager"}]}]}}' \ + '---' \ + 'apiVersion: networking.k8s.io/v1' \ + 'kind: NetworkPolicy' \ + 'metadata:' \ + ' name: observability-allow-alertmanager-public-https' \ + ' namespace: observability' \ + 'spec:' \ + ' podSelector:' \ + ' matchLabels:' \ + ' app.kubernetes.io/name: alertmanager' \ + ' app.kubernetes.io/instance: observability-core-kube-pr-alertmanager' \ + ' policyTypes: [Egress]' \ + ' egress:' \ + ' - to:' \ + ' - ipBlock:' \ + ' cidr: 0.0.0.0/0' \ + ' except: [10.0.0.0/8, 100.64.0.0/10, 172.16.0.0/12, 192.168.0.0/16]' \ + ' ports:' \ + ' - protocol: TCP' \ + ' port: 443' +expect_pass 'exact structured Alertmanager routing contract' \ + assert_alertmanager_routing_contract "$alertmanager_fixture" "$alerting_fixture" + +alertmanager_selector_fixture="$fixture_dir/alertmanager-selector.yaml" +sed '/^spec:$/a\ alertmanagerConfigSelector: {}' \ + "$alertmanager_fixture" >"$alertmanager_selector_fixture" +expect_fail 'Alertmanager discovery selector bypasses the global configuration' \ + assert_alertmanager_routing_contract "$alertmanager_selector_fixture" "$alerting_fixture" + +alerting_route_fixture="$fixture_dir/alerting-route-drift.yaml" +sed 's/"repeatInterval":"4h"/"repeatInterval":"12h"/' \ + "$alerting_fixture" >"$alerting_route_fixture" +expect_fail 'Alertmanager Slack route drift' \ + assert_alertmanager_routing_contract "$alertmanager_fixture" "$alerting_route_fixture" + +write_pod() { + local path="$1" + local namespace="$2" + local name="$3" + local pod_fields="$4" + local container_fields="${5:-}" + cat >"$path" <"$tag_image" +expect_fail 'tag-only image' assert_pinned_images "$tag_image" + +wrong_digest="$fixture_dir/wrong-digest.yaml" +sed 's/ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/' \ + "$valid_image" >"$wrong_digest" +expect_fail 'unpinned digest' assert_pinned_images "$wrong_digest" + +secret="$fixture_dir/secret.yaml" +write_file "$secret" \ + 'apiVersion: v1' 'kind: Secret' 'metadata:' ' name: forbidden' \ + ' namespace: observability' 'stringData:' ' password: DO-NOT-LEAK-FIXTURE' +expect_fail 'credential Secret' assert_no_credentials "$secret" + +literal_credential="$fixture_dir/literal-credential.yaml" +write_file "$literal_credential" \ + 'apiVersion: v1' 'kind: Pod' 'metadata:' ' name: forbidden' \ + ' namespace: observability' 'spec:' ' automountServiceAccountToken: false' \ + ' containers:' ' - name: app' ' image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b' \ + ' env:' ' - name: ACCESS_KEY' ' value: DO-NOT-LEAK-FIXTURE' +expect_fail 'literal credential payload' assert_no_credentials "$literal_credential" + +for field in \ + ' privileged: true' \ + ' hostNetwork: true' \ + ' hostPID: true' \ + ' hostIPC: true'; do + slug="$(printf '%s' "$field" | tr -cd '[:alpha:]' | tr '[:upper:]' '[:lower:]')" + pod="$fixture_dir/observability-$slug.yaml" + if [[ "$field" == *privileged* ]]; then + container_security="$(printf ' securityContext:\n privileged: true')" + write_pod "$pod" observability core '' "$container_security" + else + write_pod "$pod" observability core "$field" '' + fi + expect_fail "observability forbids $slug" assert_namespace_pod_security "$pod" +done + +obs_hostpath="$fixture_dir/observability-hostpath.yaml" +write_pod "$obs_hostpath" observability core \ + $' volumes:\n - name: host\n hostPath:\n path: /var/log/pods' \ + $' volumeMounts:\n - name: host\n mountPath: /logs\n readOnly: true' +expect_fail 'observability hostPath' assert_namespace_pod_security "$obs_hostpath" + +controller_namespace="$fixture_dir/controller-namespace.yaml" +write_file "$controller_namespace" 'apiVersion: apps/v1' 'kind: Deployment' \ + 'metadata:' ' name: controller-security' ' namespace: observability' 'spec:' \ + ' selector:' ' matchLabels:' ' app: controller-security' ' template:' \ + ' metadata:' ' labels:' ' app: controller-security' ' spec:' \ + ' automountServiceAccountToken: false' ' hostPID: true' ' containers:' \ + ' - name: app' \ + ' image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b' +expect_fail 'controller template inherits namespace security' assert_namespace_pod_security "$controller_namespace" +controller_valid="$fixture_dir/controller-valid.yaml" +sed '/hostPID: true/d' "$controller_namespace" >"$controller_valid" +expect_pass 'controller observability namespace positive control' \ + assert_namespace_pod_security "$controller_valid" +controller_missing_namespace="$fixture_dir/controller-missing-namespace.yaml" +sed '/namespace: observability/d' "$controller_valid" >"$controller_missing_namespace" +expect_fail 'controller namespace missing' \ + assert_namespace_pod_security "$controller_missing_namespace" +controller_foreign_namespace="$fixture_dir/controller-foreign-namespace.yaml" +sed 's/namespace: observability/namespace: default/' \ + "$controller_valid" >"$controller_foreign_namespace" +expect_fail 'controller namespace foreign' \ + assert_namespace_pod_security "$controller_foreign_namespace" + +agent_allowed="$fixture_dir/agent-allowed.yaml" +write_pod "$agent_allowed" observability-agent alloy \ + $' volumes:\n - name: pod-logs\n hostPath:\n path: /var/log/pods' \ + $' volumeMounts:\n - name: pod-logs\n mountPath: /var/log/pods\n readOnly: true' +expect_pass 'Alloy read-only allowlisted hostPath' assert_namespace_pod_security "$agent_allowed" + +missing_namespace="$fixture_dir/missing-namespace.yaml" +sed '/namespace: observability-agent/d' "$agent_allowed" >"$missing_namespace" +expect_fail 'Pod namespace missing' assert_namespace_pod_security "$missing_namespace" + +foreign_namespace="$fixture_dir/foreign-namespace.yaml" +sed 's/namespace: observability-agent/namespace: default/' "$agent_allowed" >"$foreign_namespace" +expect_fail 'Pod namespace foreign' assert_namespace_pod_security "$foreign_namespace" + +agent_bad_path="$fixture_dir/agent-bad-path.yaml" +sed 's|/var/log/pods|/etc|g' "$agent_allowed" >"$agent_bad_path" +expect_fail 'agent hostPath allowlist' assert_namespace_pod_security "$agent_bad_path" + +agent_rw="$fixture_dir/agent-rw.yaml" +sed 's/readOnly: true/readOnly: false/' "$agent_allowed" >"$agent_rw" +expect_fail 'agent hostPath readOnly' assert_namespace_pod_security "$agent_rw" + +automount_missing="$fixture_dir/automount-missing.yaml" +sed '/automountServiceAccountToken:/d' "$valid_image" >"$automount_missing" +expect_fail 'automount missing' assert_namespace_pod_security "$automount_missing" + +automount_true="$fixture_dir/automount-true.yaml" +sed 's/automountServiceAccountToken: false/automountServiceAccountToken: true/' \ + "$valid_image" >"$automount_true" +expect_fail 'automount true' assert_namespace_pod_security "$automount_true" + +for exposure in ingress NodePort LoadBalancer; do + public="$fixture_dir/public-$exposure.yaml" + if [[ "$exposure" == ingress ]]; then + write_file "$public" 'apiVersion: networking.k8s.io/v1' 'kind: Ingress' \ + 'metadata:' ' name: public' ' namespace: observability' 'spec: {}' + else + write_file "$public" 'apiVersion: v1' 'kind: Service' 'metadata:' \ + ' name: public' ' namespace: observability' 'spec:' " type: $exposure" \ + ' ports:' ' - port: 80' + fi + expect_fail "public $exposure" assert_no_public_exposure "$public" +done +grafana_ingress="$fixture_dir/grafana-private-ingress.yaml" +write_file "$grafana_ingress" \ + 'apiVersion: networking.k8s.io/v1' 'kind: Ingress' \ + 'metadata:' ' name: grafana' ' namespace: observability' \ + 'spec:' ' ingressClassName: traefik' ' rules:' \ + ' - host: grafana.learn.hyeonworks.com' ' http:' ' paths:' \ + ' - path: /' ' pathType: Prefix' ' backend:' \ + ' service:' ' name: grafana' ' port:' \ + ' number: 80' +expect_pass 'exact Host-Nginx-fronted Grafana ingress' \ + assert_no_public_exposure "$grafana_ingress" +grafana_public_ingress="$fixture_dir/grafana-public-ingress.yaml" +sed 's/grafana.learn.hyeonworks.com/public.example.com/' \ + "$grafana_ingress" >"$grafana_public_ingress" +expect_fail 'Grafana ingress host drift' \ + assert_no_public_exposure "$grafana_public_ingress" +cluster_ip="$fixture_dir/cluster-ip.yaml" +write_file "$cluster_ip" 'apiVersion: v1' 'kind: Service' 'metadata:' \ + ' name: internal' ' namespace: observability' 'spec:' ' type: ClusterIP' \ + ' ports:' ' - port: 9090' +expect_pass 'public exposure assertion positive control' assert_no_public_exposure "$cluster_ip" + +for auxiliary in gateway canary chunks-cache results-cache test minio; do + loki_aux="$fixture_dir/loki-$auxiliary.yaml" + write_file "$loki_aux" 'apiVersion: apps/v1' 'kind: Deployment' 'metadata:' \ + " name: loki-$auxiliary" ' namespace: observability' 'spec: {}' + expect_fail "Loki auxiliary $auxiliary" assert_no_loki_auxiliary "$loki_aux" +done +loki_main="$fixture_dir/loki-main.yaml" +write_file "$loki_main" 'apiVersion: apps/v1' 'kind: StatefulSet' 'metadata:' \ + ' name: loki' ' namespace: observability' 'spec: {}' +expect_pass 'Loki auxiliary assertion positive control' assert_no_loki_auxiliary "$loki_main" + +for auxiliary in metrics-generator tempo-query; do + tempo_aux="$fixture_dir/tempo-$auxiliary.yaml" + write_file "$tempo_aux" 'apiVersion: apps/v1' 'kind: Deployment' 'metadata:' \ + " name: $auxiliary" ' namespace: observability' 'spec: {}' + expect_fail "Tempo auxiliary $auxiliary" assert_tempo_topology "$tempo_aux" +done + +tempo_http="$fixture_dir/tempo-http.yaml" +write_file "$tempo_http" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \ + ' name: tempo' ' namespace: observability' 'data:' ' tempo.yaml: |' \ + ' receivers:' ' otlp:' ' protocols:' ' grpc:' \ + ' endpoint: 0.0.0.0:4317' ' http:' \ + ' endpoint: 0.0.0.0:4318' +expect_fail 'Tempo OTLP HTTP receiver' assert_tempo_topology "$tempo_http" +tempo_grpc="$fixture_dir/tempo-grpc.yaml" +write_file "$tempo_grpc" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \ + ' name: tempo' ' namespace: observability' 'data:' ' tempo.yaml: |' \ + ' receivers:' ' otlp:' ' protocols:' ' grpc:' \ + ' endpoint: 0.0.0.0:4317' +expect_pass 'Tempo topology assertion positive control' assert_tempo_topology "$tempo_grpc" + +selector_values="$fixture_dir/selector-values.yaml" +write_file "$selector_values" 'prometheus:' ' prometheusSpec:' \ + ' serviceMonitorSelectorNilUsesHelmValues: false' \ + ' podMonitorSelectorNilUsesHelmValues: false' \ + ' probeSelectorNilUsesHelmValues: false' \ + ' ruleSelectorNilUsesHelmValues: false' +expect_pass 'selector values false' assert_selector_values "$selector_values" + +selector_true="$fixture_dir/selector-true.yaml" +sed '0,/false/s//true/' "$selector_values" >"$selector_true" +expect_fail 'nilUsesHelmValues true' assert_selector_values "$selector_true" + +prometheus_missing="$fixture_dir/prometheus-missing-selector.yaml" +write_file "$prometheus_missing" 'apiVersion: monitoring.coreos.com/v1' \ + 'kind: Prometheus' 'metadata:' ' name: home' ' namespace: observability' \ + 'spec:' ' serviceMonitorSelector: {}' ' podMonitorSelector: {}' \ + ' probeSelector: {}' +expect_fail 'Prometheus selector missing' assert_prometheus_selectors "$prometheus_missing" +prometheus_valid="$fixture_dir/prometheus-valid.yaml" +write_file "$prometheus_valid" 'apiVersion: monitoring.coreos.com/v1' \ + 'kind: Prometheus' 'metadata:' ' name: home' ' namespace: observability' \ + 'spec:' ' serviceMonitorSelector: {}' ' podMonitorSelector: {}' \ + ' probeSelector: {}' ' ruleSelector: {}' +expect_pass 'Prometheus selector assertion positive control' assert_prometheus_selectors "$prometheus_valid" + +crd="$fixture_dir/kps-crd.yaml" +write_file "$crd" 'apiVersion: apiextensions.k8s.io/v1' \ + 'kind: CustomResourceDefinition' 'metadata:' ' name: prometheuses.monitoring.coreos.com' +expect_fail 'CRD in workload render' assert_no_crds "$crd" +expect_pass 'CRD assertion positive control' assert_no_crds "$valid_image" + +shared_pv="$fixture_dir/shared-pv.yaml" +write_file "$shared_pv" 'apiVersion: v1' 'kind: PersistentVolume' 'metadata:' \ + ' name: prometheus' 'spec:' ' storageClassName: shared-local' ' local:' \ + ' path: /srv/prometheus' '---' 'apiVersion: v1' 'kind: PersistentVolume' \ + 'metadata:' ' name: alertmanager' 'spec:' ' storageClassName: shared-local' \ + ' local:' ' path: /srv/alertmanager' +expect_fail 'shared Local PV StorageClass' assert_unique_local_pv_storage_classes "$shared_pv" +single_pv="$fixture_dir/single-pv.yaml" +write_file "$single_pv" 'apiVersion: v1' 'kind: PersistentVolume' 'metadata:' \ + ' name: prometheus' 'spec:' ' storageClassName: prometheus-local' ' local:' \ + ' path: /srv/prometheus' +expect_pass 'Local PV assertion positive control' assert_unique_local_pv_storage_classes "$single_pv" + +for fake in spring kafka batch backup; do + fake_monitor="$fixture_dir/fake-$fake.yaml" + write_file "$fake_monitor" 'apiVersion: monitoring.coreos.com/v1' \ + 'kind: ServiceMonitor' 'metadata:' " name: $fake-monitor" \ + ' namespace: observability' 'spec: {}' + expect_fail "fake $fake monitor or rule" assert_no_unscoped_monitors_rules "$fake_monitor" +done +scoped_monitor="$fixture_dir/scoped-monitor.yaml" +write_file "$scoped_monitor" 'apiVersion: monitoring.coreos.com/v1' \ + 'kind: ServiceMonitor' 'metadata:' ' name: prometheus' \ + ' namespace: observability' 'spec: {}' +expect_pass 'monitor/rule assertion positive control' assert_no_unscoped_monitors_rules "$scoped_monitor" + +chart="$fixture_dir/Chart.yaml" +write_file "$chart" 'apiVersion: v2' 'name: loki' 'version: 18.7.1' 'appVersion: 3.7.4' +expect_pass 'chart metadata pin accepted' validate_chart_metadata "$chart" loki 18.7.1 3.7.4 +expect_fail 'chart metadata name mismatch' validate_chart_metadata "$chart" tempo 18.7.1 3.7.4 +expect_fail 'chart metadata version mismatch' validate_chart_metadata "$chart" loki 18.7.2 3.7.4 +expect_fail 'chart metadata appVersion mismatch' validate_chart_metadata "$chart" loki 18.7.1 3.7.5 + +handoff="$fixture_dir/platform-observability-core-apply.fixture" +mkdir -m 0700 "$handoff" +expect_fail 'handoff prefix restricted to /tmp' validate_handoff_directory "$handoff" +safe_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)" +handoff_dirs+=("$safe_handoff") +chmod 0700 "$safe_handoff" +expect_pass 'safe empty handoff' validate_handoff_directory "$safe_handoff" +touch "$safe_handoff/unexpected" +expect_fail 'handoff must be empty' validate_handoff_directory "$safe_handoff" +rm -f -- "$safe_handoff/unexpected" +rmdir -- "$safe_handoff" + +empty_suffix='/tmp/platform-observability-core-apply.' +expect_fail 'handoff suffix must be nonempty' validate_handoff_directory "$empty_suffix" + +root_fixture="$fixture_dir/render-roots" +mkdir -p "$root_fixture/existing/child" +expect_pass 'physical render roots accepted' validate_render_roots "$root_fixture" existing/child +expect_fail 'missing render root rejected' validate_render_roots "$root_fixture" missing/child +expect_pass 'missing root preflight creates nothing' test ! -e "$root_fixture/missing" +mkdir -p "$root_fixture/physical/child" +ln -s physical "$root_fixture/symlinked" +expect_fail 'symlinked render root lineage rejected' validate_render_roots "$root_fixture" symlinked/child + +cache_root="$fixture_dir/cache-root" +mkdir -p "$cache_root/services/component/.helm/charts" +_OBS_ROOT="$cache_root" +_OBS_CREATED_CACHE_DIRS=() +expect_stateful_pass 'exclusive cache directory creation' _observability_create_cache_directory \ + services/component chart 1.0.0 +expect_fail 'existing exact cache is no-clobber' _observability_create_cache_directory \ + services/component chart 1.0.0 +expect_stateful_pass 'tracked cache cleanup' _observability_cleanup_cache_dirs +expect_pass 'pre-existing .helm preserved' test -d "$cache_root/services/component/.helm" +expect_pass 'pre-existing charts preserved' test -d "$cache_root/services/component/.helm/charts" +expect_pass 'exact created cache removed' test ! -e "$cache_root/services/component/.helm/charts/chart-1.0.0" +mkdir -p "$cache_root/services/missing-parent" +expect_fail 'missing component root rejected before cache mkdir' _observability_create_cache_directory \ + services/missing-parent/component chart 1.0.0 +expect_pass 'missing component root leaves no .helm' test ! -e "$cache_root/services/missing-parent/component" +mkdir -p "$cache_root/services/new-component" +_OBS_CREATED_CACHE_DIRS=() +_OBS_CREATED_CACHE_IDENTITIES=() +_OBS_CREATED_CACHE_TYPES=() +expect_stateful_pass 'new cache parents tracked' _observability_create_cache_directory \ + services/new-component chart 1.0.0 +expect_stateful_pass 'new cache parents cleaned' _observability_cleanup_cache_dirs +expect_pass 'created .helm parent removed' test ! -e "$cache_root/services/new-component/.helm" + +mkdir -p "$cache_root/services/race-component/.helm/charts" +_OBS_CREATED_CACHE_DIRS=() +_OBS_CREATED_CACHE_IDENTITIES=() +_OBS_CREATED_CACHE_TYPES=() +expect_stateful_pass 'race cache created' _observability_create_cache_directory \ + services/race-component chart 1.0.0 +race_cache="${_OBS_CREATED_CACHE_DIRS[0]}" +race_identity="${_OBS_CREATED_CACHE_IDENTITIES[0]}" +mkdir "$race_cache/owned-subdirectory" +write_file "$race_cache/owned-subdirectory/owned-marker" 'OWNED-CACHE-CONTENT' +cleanup_race_hook="$fixture_dir/cache-cleanup-race-hook.sh" +write_file "$cleanup_race_hook" '#!/usr/bin/env bash' 'set -Eeuo pipefail' \ + 'replacement="$1"' 'mkdir -m 0700 -- "$replacement"' \ + 'printf "%s\n" REPLACEMENT-MUST-SURVIVE >"$replacement/replacement-marker"' +chmod 0700 "$cleanup_race_hook" +expect_stateful_pass 'owned cache cleanup is identity-bound across replacement race' \ + _observability_cleanup_owned_cache_directory \ + "$race_cache" "$race_identity" tree "$cleanup_race_hook" +expect_pass 'replacement directory survives cleanup race' test -d "$race_cache" +expect_pass 'replacement content survives cleanup race' test \ + "$(cat "$race_cache/replacement-marker")" = REPLACEMENT-MUST-SURVIVE +expect_pass 'owned quarantine is completely cleaned' test -z \ + "$(find "$cache_root/services/race-component/.helm/charts" \ + -mindepth 1 -maxdepth 1 ! -name chart-1.0.0 -print -quit)" + +run_cleanup_exit_gate() ( + _OBS_WORK='' + trap _observability_cleanup EXIT + : +) +expect_fail 'cleanup identity mismatch fails the successful exit gate' run_cleanup_exit_gate +expect_pass 'exit-gate mismatch still preserves replacement' test \ + "$(cat "$race_cache/replacement-marker")" = REPLACEMENT-MUST-SURVIVE +_OBS_CREATED_CACHE_DIRS=() +_OBS_CREATED_CACHE_IDENTITIES=() +_OBS_CREATED_CACHE_TYPES=() + +handoff_work="$fixture_dir/handoff-work" +mkdir -m 0700 "$handoff_work" +for handoff_name in namespaces crds storage kps loki tempo alloy node-exporter grafana blackbox core-policies targets core-rules alerting dashboards aggregate; do + write_file "$handoff_work/$handoff_name.yaml" "fixture-$handoff_name" + chmod 0600 "$handoff_work/$handoff_name.yaml" +done +write_file "$handoff_work/resource-index.tsv" 'fixture-index' +chmod 0600 "$handoff_work/resource-index.tsv" + +published_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)" +handoff_dirs+=("$published_handoff") +chmod 0700 "$published_handoff" +published_identity="$(pin_handoff_directory "$published_handoff")" +expect_pass 'atomic no-clobber handoff publication' publish_verified_handoff \ + "$published_handoff" "$published_identity" "$handoff_work" +expect_pass 'exact handoff entry set accepted' validate_handoff_entry_set \ + "$published_handoff" "$published_identity" + +ln -s /dev/null "$published_handoff/extra-symlink" +expect_fail 'extra handoff symlink rejected' validate_handoff_entry_set \ + "$published_handoff" "$published_identity" +rm -f -- "$published_handoff/extra-symlink" +mkdir "$published_handoff/extra-directory" +expect_fail 'extra handoff directory rejected' validate_handoff_entry_set \ + "$published_handoff" "$published_identity" +rmdir "$published_handoff/extra-directory" +mv "$published_handoff/aggregate.yaml" "$fixture_dir/aggregate-handoff-saved" +ln -s /dev/null "$published_handoff/aggregate.yaml" +expect_fail 'exact handoff entry symlink rejected' validate_handoff_entry_set \ + "$published_handoff" "$published_identity" +rm -f -- "$published_handoff/aggregate.yaml" +mv "$fixture_dir/aggregate-handoff-saved" "$published_handoff/aggregate.yaml" +mv "$published_handoff/core-rules.yaml" "$fixture_dir/core-rules-handoff-saved" +mkdir "$published_handoff/core-rules.yaml" +expect_fail 'exact handoff entry directory rejected' validate_handoff_entry_set \ + "$published_handoff" "$published_identity" +rmdir "$published_handoff/core-rules.yaml" +mv "$fixture_dir/core-rules-handoff-saved" "$published_handoff/core-rules.yaml" + +replaced_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)" +handoff_dirs+=("$replaced_handoff") +chmod 0700 "$replaced_handoff" +replaced_identity="$(pin_handoff_directory "$replaced_handoff")" +replaced_original="$replaced_handoff.original" +mv "$replaced_handoff" "$replaced_original" +handoff_dirs+=("$replaced_original") +mkdir -m 0700 "$replaced_handoff" +expect_fail 'replaced handoff identity rejected before writes' publish_verified_handoff \ + "$replaced_handoff" "$replaced_identity" "$handoff_work" +expect_pass 'replacement remains empty' test -z "$(find "$replaced_handoff" -mindepth 1 -maxdepth 1 -print -quit)" + +occupied_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)" +handoff_dirs+=("$occupied_handoff") +chmod 0700 "$occupied_handoff" +occupied_identity="$(pin_handoff_directory "$occupied_handoff")" +write_file "$occupied_handoff/namespaces.yaml" 'DO-NOT-CLOBBER' +expect_fail 'nonempty handoff rejected immediately before writes' publish_verified_handoff \ + "$occupied_handoff" "$occupied_identity" "$handoff_work" +expect_pass 'existing handoff file is not clobbered' test \ + "$(cat "$occupied_handoff/namespaces.yaml")" = DO-NOT-CLOBBER + +child_a="$fixture_dir/child-a.yaml" +child_b="$fixture_dir/child-b.yaml" +aggregate="$fixture_dir/aggregate.yaml" +write_file "$child_a" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \ + ' name: a' ' namespace: observability' 'data:' ' value: one' +write_file "$child_b" 'apiVersion: v1' 'kind: Service' 'metadata:' \ + ' name: b' ' namespace: observability' 'spec:' ' type: ClusterIP' \ + ' ports:' ' - port: 80' +{ cat "$child_a"; printf '%s\n' '---'; cat "$child_b"; } >"$aggregate" +index="$fixture_dir/resource-index.tsv" +expect_pass 'ordered resource index' build_resource_index "$index" \ + "kps:$child_a" "loki:$child_b" +expected_index=$'1\tkps\tv1\tConfigMap\tobservability\ta\t' +if [[ "$(sed -n '2p' "$index")" != "$expected_index"* ]]; then + printf 'FAIL: resource index order or schema differs\n' >&2 + exit 1 +fi +pass_count=$((pass_count + 1)) +expect_pass 'child aggregate equivalence' assert_child_aggregate_equivalence \ + "$aggregate" "kps:$child_a" "loki:$child_b" + +aggregate_missing="$fixture_dir/aggregate-missing.yaml" +cp "$child_a" "$aggregate_missing" +expect_fail 'aggregate missing resource' assert_child_aggregate_equivalence \ + "$aggregate_missing" "kps:$child_a" "loki:$child_b" + +aggregate_drift="$fixture_dir/aggregate-drift.yaml" +sed 's/value: one/value: changed/' "$aggregate" >"$aggregate_drift" +expect_fail 'aggregate canonical hash drift' assert_child_aggregate_equivalence \ + "$aggregate_drift" "kps:$child_a" "loki:$child_b" + +duplicate_child="$fixture_dir/duplicate-child.yaml" +cp "$child_a" "$duplicate_child" +expect_fail 'duplicate child tuple' assert_child_aggregate_equivalence \ + "$aggregate" "kps:$child_a" "loki:$child_b" "tempo:$duplicate_child" + +assert_task2_manifests() { + local manifest_root="${1:-$ROOT}" + python3 - "$manifest_root" <<'PY' +import pathlib +import sys + +import yaml + + +root = pathlib.Path(sys.argv[1]) +namespace_root = root / "infrastructure/namespaces/observability" +storage_root = root / "infrastructure/storage/observability-local-pv" + + +def load_documents(path): + with path.open(encoding="utf-8") as stream: + return [value for value in yaml.safe_load_all(stream) if value is not None] + + +namespace_kustomization = yaml.safe_load( + (namespace_root / "kustomization.yaml").read_text(encoding="utf-8") +) +assert namespace_kustomization["resources"] == [ + "observability.yaml", + "observability-agent.yaml", +] +namespaces = [] +for resource in namespace_kustomization["resources"]: + namespaces.extend(load_documents(namespace_root / resource)) +assert len(namespaces) == 2 +assert [item["kind"] for item in namespaces].count("Namespace") == 2 +assert len({item["metadata"]["name"] for item in namespaces}) == 2 +by_name = {item["metadata"]["name"]: item for item in namespaces} +assert set(by_name) == {"observability", "observability-agent"} +for item in by_name.values(): + assert item["metadata"]["labels"]["observability.hyeonworks.com/enabled"] == "true" + +for level in ("enforce", "audit", "warn"): + labels = by_name["observability"]["metadata"]["labels"] + assert labels[f"pod-security.kubernetes.io/{level}"] == "restricted" + assert labels[f"pod-security.kubernetes.io/{level}-version"] == "latest" +agent_labels = by_name["observability-agent"]["metadata"]["labels"] +assert agent_labels["pod-security.kubernetes.io/enforce"] == "privileged" +assert agent_labels["pod-security.kubernetes.io/enforce-version"] == "latest" +for level in ("audit", "warn"): + assert agent_labels[f"pod-security.kubernetes.io/{level}"] == "restricted" + assert agent_labels[f"pod-security.kubernetes.io/{level}-version"] == "latest" + +assert (namespace_root / "discovery-namespaces.txt").read_text(encoding="utf-8") == ( + "kube-system\ngitea\nkeycloak\nplatform-data\nobject-storage\n" +) + +storage_kustomization = yaml.safe_load( + (storage_root / "kustomization.yaml").read_text(encoding="utf-8") +) +assert storage_kustomization["resources"] == [ + "storage-classes.yaml", + "persistent-volumes.yaml", +] +storage_items = [] +for resource in storage_kustomization["resources"]: + storage_items.extend(load_documents(storage_root / resource)) +assert len(storage_items) == 12 +assert [item["kind"] for item in storage_items].count("StorageClass") == 6 +assert [item["kind"] for item in storage_items].count("PersistentVolume") == 6 +assert {item["kind"] for item in storage_items} == {"StorageClass", "PersistentVolume"} +class_items = [item for item in storage_items if item["kind"] == "StorageClass"] +volume_items = [item for item in storage_items if item["kind"] == "PersistentVolume"] +assert len({item["metadata"]["name"] for item in class_items}) == 6 +assert len({item["metadata"]["name"] for item in volume_items}) == 6 +assert len({item["metadata"]["labels"]["app.kubernetes.io/component"] for item in volume_items}) == 6 +classes = { + item["metadata"]["name"]: item + for item in class_items +} +volumes = { + item["metadata"]["labels"]["app.kubernetes.io/component"]: item + for item in volume_items +} +expected = { + "prometheus": ("/srv/k3s/ssd/observability/prometheus", "20Gi", "ssd-local-observability-prometheus-retain"), + "grafana": ("/srv/k3s/ssd/observability/grafana", "2Gi", "ssd-local-observability-grafana-retain"), + "alertmanager": ("/srv/k3s/ssd/observability/alertmanager", "1Gi", "ssd-local-observability-alertmanager-retain"), + "alloy": ("/srv/k3s/ssd/observability/alloy", "2Gi", "ssd-local-observability-alloy-retain"), + "loki": ("/srv/k3s/ssd/observability/loki", "5Gi", "ssd-local-observability-loki-retain"), + "tempo": ("/srv/k3s/ssd/observability/tempo", "5Gi", "ssd-local-observability-tempo-retain"), +} +assert set(classes) == {row[2] for row in expected.values()} +assert set(volumes) == set(expected) +for owner, (path, capacity, class_name) in expected.items(): + storage_class = classes[class_name] + assert storage_class["provisioner"] == "kubernetes.io/no-provisioner" + assert storage_class["reclaimPolicy"] == "Retain" + assert storage_class["volumeBindingMode"] == "WaitForFirstConsumer" + assert storage_class["allowVolumeExpansion"] is False + volume = volumes[owner] + spec = volume["spec"] + assert spec["capacity"]["storage"] == capacity + assert spec["accessModes"] == ["ReadWriteOnce"] + assert spec["persistentVolumeReclaimPolicy"] == "Retain" + assert spec["storageClassName"] == class_name + assert spec["local"]["path"] == path + expression = spec["nodeAffinity"]["required"]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert expression == { + "key": "kubernetes.io/hostname", + "operator": "In", + "values": ["donghyeon-system-product-name"], + } +actual_capacity = 0 +for volume in volume_items: + value = volume["spec"]["capacity"]["storage"] + assert isinstance(value, str) and value.endswith("Gi") and value[:-2].isdigit() + actual_capacity += int(value[:-2]) +assert actual_capacity == 35 +PY +} + +expect_pass 'Task 2 namespace and 35Gi Local PV matrix' assert_task2_manifests + +expect_any_failure() { + local name="$1" + shift + if "$@" >"$fixture_dir/expected-any-failure.out" 2>&1; then + printf 'FAIL (expected any failure): %s\n' "$name" >&2 + exit 1 + fi + pass_count=$((pass_count + 1)) +} + +task2_manifest_fixture() { + local destination="$1" + mkdir -p "$destination/infrastructure/namespaces" "$destination/infrastructure/storage" + cp -R "$ROOT/infrastructure/namespaces/observability" \ + "$destination/infrastructure/namespaces/observability" + cp -R "$ROOT/infrastructure/storage/observability-local-pv" \ + "$destination/infrastructure/storage/observability-local-pv" +} + +duplicate_manifest_root="$fixture_dir/task2-manifest-duplicate" +task2_manifest_fixture "$duplicate_manifest_root" +{ + printf '%s\n' '---' + sed -n '1,/^---$/p' \ + "$ROOT/infrastructure/storage/observability-local-pv/storage-classes.yaml" | sed '$d' +} >>"$duplicate_manifest_root/infrastructure/storage/observability-local-pv/storage-classes.yaml" +expect_any_failure 'duplicate StorageClass object is rejected' \ + assert_task2_manifests "$duplicate_manifest_root" + +extra_kind_root="$fixture_dir/task2-manifest-extra-kind" +task2_manifest_fixture "$extra_kind_root" +write_file "$extra_kind_root/infrastructure/storage/observability-local-pv/extra.yaml" \ + 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' ' name: unexpected' +printf '%s\n' ' - extra.yaml' >> \ + "$extra_kind_root/infrastructure/storage/observability-local-pv/kustomization.yaml" +expect_any_failure 'extra manifest kind is rejected' \ + assert_task2_manifests "$extra_kind_root" + +capacity_drift_root="$fixture_dir/task2-manifest-capacity-drift" +task2_manifest_fixture "$capacity_drift_root" +sed -i '0,/storage: 20Gi/s//storage: 19Gi/' \ + "$capacity_drift_root/infrastructure/storage/observability-local-pv/persistent-volumes.yaml" +expect_any_failure 'actual rendered capacity drift is rejected' \ + assert_task2_manifests "$capacity_drift_root" + +readonly PREPARE_OBSERVABILITY_PATHS="$ROOT/scripts/bootstrap/prepare-observability-local-paths.sh" +source_options_before="$-" +# shellcheck source=/dev/null +source "$PREPARE_OBSERVABILITY_PATHS" +expect_pass 'path preparation script is source-safe' test "$-" = "$source_options_before" + +olp_fixture="$fixture_dir/observability-local-paths" +olp_root="$olp_fixture/root" +olp_ssd="$olp_fixture/srv/k3s/ssd" +olp_aistor="$olp_fixture/srv/k3s/aistor" +olp_external="$olp_fixture/external" +mkdir -p "$olp_root" "$olp_ssd/observability" "$olp_aistor" "$olp_external" + +olp_available='64424509440' +olp_uses=('0' '0' '0' '0' '0' '0') +olp_context='default' +olp_node='donghyeon-system-product-name' +olp_confirmation='APPLY default' +olp_helper_audit="$olp_fixture/helper-audit.log" +olp_helper_race='' +olp_helper_signal='' +olp_helper_signal_target='' + +_olp_root_facts() { printf '%s\t%s\n' '/dev/root' 'ext4'; } +_olp_available_bytes() { printf '%s\n' "$olp_available"; } +_olp_current_context() { printf '%s\n' "$olp_context"; } +_olp_node_names() { printf '%s\n' "$olp_node"; } +_olp_read_confirmation() { printf '%s\n' "$olp_confirmation"; } + +_olp_fixture_privileged_helper() { + if (( $# != 4 )) || [[ "$1" != preflight && "$1" != apply ]] || \ + [[ "$2" != "$olp_root" || "$3" != "$olp_ssd" || "$4" != "$olp_aistor" ]]; then + return 97 + fi + local program + program="$(_olp_helper_program)" || return + OLP_HELPER_FIXTURE_ROOT="$olp_fixture" \ + OLP_HELPER_FIXTURE_VIRTUAL_AISTOR=1 \ + OLP_HELPER_FIXTURE_OWNERS=1 \ + OLP_HELPER_FIXTURE_AUDIT="$olp_helper_audit" \ + OLP_HELPER_FIXTURE_RACE="$olp_helper_race" \ + OLP_HELPER_FIXTURE_EXTERNAL="$olp_external" \ + OLP_HELPER_FIXTURE_SIGNAL="$olp_helper_signal" \ + OLP_HELPER_FIXTURE_SIGNAL_TARGET="$olp_helper_signal_target" \ + /usr/bin/python3 - "$@" <<<"$program" +} +_olp_privileged_helper() { _olp_fixture_privileged_helper "$@"; } + +dry_output="$fixture_dir/observability-dry-run.out" +expect_stateful_pass 'default dry-run emits facts without mutation' \ + observability_local_paths_dry_run "$olp_root" "$olp_ssd" "$olp_aistor" +observability_local_paths_dry_run "$olp_root" "$olp_ssd" "$olp_aistor" >"$dry_output" +expect_pass 'dry-run leaves every planned path absent' test -z \ + "$(find "$olp_ssd/observability" -mindepth 1 -maxdepth 1 -print -quit)" +expect_pass 'dry-run prints sanitized root filesystem facts' test "$(sed -n '1,3p' "$dry_output")" = \ + $'Root SSD device: /dev/root\nRoot SSD filesystem: ext4\nRoot SSD available bytes: 64424509440' +expect_pass 'dry-run prints all six exact planned fixture paths' test \ + "$(sed -n '4,9p' "$dry_output")" = \ + "$(for owner in prometheus grafana alertmanager alloy loki tempo; do + printf 'Planned path: %s/observability/%s\n' "$olp_ssd" "$owner" + done)" + +assert_olp_execute_fails() { + local output rc + set +e + output="$(observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" 2>&1)" + rc=$? + set -e + [[ "$rc" != 0 && "$output" == ERROR:* ]] +} + +assert_olp_execute_rejected() { + assert_olp_execute_fails + [[ -z "$(find "$olp_ssd/observability" -mindepth 1 -maxdepth 1 -print -quit)" ]] +} + +exact_numeric_reply=$'PREFLIGHT\t53687091200\t1073741824\t0\t0\t0\t0\t0' +expect_pass '50GiB available and 1GiB aggregate are inclusive' \ + _olp_validate_preflight_reply "$exact_numeric_reply" +expect_any_failure 'one byte below 50GiB is rejected' \ + _olp_validate_preflight_reply $'PREFLIGHT\t53687091199\t0\t0\t0\t0\t0\t0' +expect_any_failure 'one target byte above 1GiB is rejected before summing' \ + _olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t1073741825\t0\t0\t0\t0\t0' +expect_any_failure 'aggregate one byte above 1GiB is rejected' \ + _olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t536870913\t536870912\t0\t0\t0\t0' +expect_any_failure 'signed decimal is rejected' \ + _olp_validate_preflight_reply $'PREFLIGHT\t+53687091200\t0\t0\t0\t0\t0\t0' +expect_any_failure 'leading-zero decimal is rejected' \ + _olp_validate_preflight_reply $'PREFLIGHT\t053687091200\t0\t0\t0\t0\t0\t0' +expect_any_failure 'multiline numeric protocol is rejected' \ + _olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t0\t0\t0\t0\t0\t0\n0' +expect_any_failure '2^63 available value is rejected' \ + _olp_validate_preflight_reply $'PREFLIGHT\t9223372036854775808\t0\t0\t0\t0\t0\t0' +expect_any_failure 'repeated 2^63 target values fail before mutation' \ + _olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808' + +write_file "$olp_external/command-marker" DO-NOT-EXECUTE +for unexpected in sudo install chown rm curl; do + set +e + _olp_fixture_privileged_helper "$unexpected" "$olp_root" "$olp_ssd" "$olp_aistor" \ + >"$fixture_dir/unexpected-helper-$unexpected.out" 2>&1 + unexpected_rc=$? + set -e + expect_pass "unexpected helper action $unexpected returns 97" test "$unexpected_rc" = 97 + expect_pass "unexpected helper action $unexpected executes nothing" test \ + "$(cat "$olp_external/command-marker")" = DO-NOT-EXECUTE +done +set +e +_olp_fixture_privileged_helper apply "$olp_root" "$olp_ssd" \ + >"$fixture_dir/unexpected-helper-argc.out" 2>&1 +unexpected_rc=$? +set -e +expect_pass 'unexpected helper argc returns 97' test "$unexpected_rc" = 97 + +olp_context='other' +expect_pass 'execute rejects non-default context' assert_olp_execute_rejected +olp_context='default' +olp_node='other-node' +expect_pass 'execute rejects any other node set' assert_olp_execute_rejected +olp_node='donghyeon-system-product-name' +olp_confirmation='apply default' +expect_pass 'execute rejects inexact TTY confirmation' assert_olp_execute_rejected +olp_confirmation='APPLY default' + +olp_context_calls="$fixture_dir/observability-context-calls" +write_file "$olp_context_calls" 0 +_olp_current_context() { + local calls + calls="$(cat "$olp_context_calls")" + calls=$((calls + 1)) + printf '%s\n' "$calls" >"$olp_context_calls" + if (( calls == 1 )); then + printf '%s\n' default + else + printf '%s\n' changed-during-confirmation + fi +} +expect_pass 'execute rechecks context after TTY confirmation' assert_olp_execute_rejected +olp_context='default' +_olp_current_context() { printf '%s\n' "$olp_context"; } + +mkdir "$olp_ssd/observability/prometheus" +write_file "$olp_ssd/observability/prometheus/existing" DO-NOT-CLOBBER +expect_pass 'execute rejects nonempty existing target' assert_olp_execute_fails +expect_pass 'nonempty rejection preserves content' test \ + "$(cat "$olp_ssd/observability/prometheus/existing")" = DO-NOT-CLOBBER +rm -f -- "$olp_ssd/observability/prometheus/existing" +rmdir "$olp_ssd/observability/prometheus" +ln -s "$olp_aistor" "$olp_ssd/observability/prometheus" +expect_pass 'execute rejects symlink target' assert_olp_execute_fails +expect_pass 'symlink rejection preserves link' test -L "$olp_ssd/observability/prometheus" +unlink "$olp_ssd/observability/prometheus" + +write_file "$olp_helper_audit" START +expect_stateful_pass 'execute prepares all six paths' \ + observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" +expected_owner_audit="$(printf '%s\n' \ + 'set-owner-mode prometheus 1000 2000 0770' \ + 'set-owner-mode grafana 472 472 0770' \ + 'set-owner-mode alertmanager 1000 2000 0770' \ + 'set-owner-mode alloy 473 473 0770' \ + 'set-owner-mode loki 10001 10001 0770' \ + 'set-owner-mode tempo 10001 10001 0770')" +expect_pass 'descriptor helper uses exact owner group mode matrix' test \ + "$(sed -n '/^set-owner-mode /p' "$olp_helper_audit")" = "$expected_owner_audit" +for owner in prometheus grafana alertmanager alloy loki tempo; do + expect_pass "execute creates $owner path" test -d "$olp_ssd/observability/$owner" + expect_pass "execute sets $owner mode" test "$(stat -c %a "$olp_ssd/observability/$owner")" = 770 + rmdir "$olp_ssd/observability/$owner" +done + +chmod 0751 "$olp_external" +write_file "$olp_external/race-marker" EXTERNAL-MUST-SURVIVE +mkdir -m 0755 "$olp_ssd/observability/prometheus" +olp_helper_race='ancestor' +set +e +observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \ + >"$fixture_dir/observability-ancestor-race.out" 2>&1 +race_rc=$? +set -e +expect_pass 'ancestor replacement race is rejected' test "$race_rc" != 0 +expect_pass 'ancestor replacement leaves external marker untouched' test \ + "$(cat "$olp_external/race-marker")" = EXTERNAL-MUST-SURVIVE +expect_pass 'ancestor replacement leaves external metadata untouched' test \ + "$(stat -c %a "$olp_external")" = 751 +expect_pass 'ancestor replacement restores pinned pre-existing metadata' test \ + "$(stat -c %a "$olp_ssd/observability.pinned/prometheus")" = 755 +unlink "$olp_ssd/observability" +mv "$olp_ssd/observability.pinned" "$olp_ssd/observability" + +olp_helper_race='target:prometheus' +set +e +observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \ + >"$fixture_dir/observability-target-race.out" 2>&1 +race_rc=$? +set -e +expect_pass 'target replacement race is rejected' test "$race_rc" != 0 +expect_pass 'target replacement leaves external marker untouched' test \ + "$(cat "$olp_external/race-marker")" = EXTERNAL-MUST-SURVIVE +expect_pass 'target replacement leaves external metadata untouched' test \ + "$(stat -c %a "$olp_external")" = 751 +expect_pass 'target replacement restores pinned target metadata' test \ + "$(stat -c %a "$olp_ssd/observability/prometheus.pinned")" = 755 +unlink "$olp_ssd/observability/prometheus" +mv "$olp_ssd/observability/prometheus.pinned" "$olp_ssd/observability/prometheus" +rmdir "$olp_ssd/observability/prometheus" + +for signal_case in HUP:prometheus INT:alloy TERM:alloy; do + olp_helper_race='' + olp_helper_signal="${signal_case%%:*}" + olp_helper_signal_target="${signal_case#*:}" + write_file "$olp_helper_audit" START + if [[ "$olp_helper_signal_target" == prometheus ]]; then + rmdir "$olp_ssd/observability" + else + mkdir -m 0755 "$olp_ssd/observability/prometheus" + fi + set +e + observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \ + >"$fixture_dir/observability-signal-${olp_helper_signal}.out" 2>&1 + signal_rc=$? + set -e + expect_pass "$olp_helper_signal post-create signal is rejected" test "$signal_rc" != 0 + for owner in grafana alertmanager alloy loki tempo; do + expect_pass "$olp_helper_signal leaves no created $owner residue" test \ + ! -e "$olp_ssd/observability/$owner" + done + if [[ "$olp_helper_signal_target" == prometheus ]]; then + expect_pass "$olp_helper_signal leaves no first-target residue" test \ + ! -e "$olp_ssd/observability/prometheus" + expect_pass "$olp_helper_signal leaves no created parent residue" test \ + ! -e "$olp_ssd/observability" + mkdir "$olp_ssd/observability" + else + expect_pass "$olp_helper_signal restores pre-existing mode" test \ + "$(stat -c %a "$olp_ssd/observability/prometheus")" = 755 + expected_restore="$(printf 'restore-chown prometheus %s %s\nrestore-chmod prometheus 0755' \ + "$(id -u)" "$(id -g)")" + expect_pass "$olp_helper_signal restores chown then chmod in exact order" test \ + "$(tail -n 2 "$olp_helper_audit")" = "$expected_restore" + rmdir "$olp_ssd/observability/prometheus" + fi +done +olp_helper_signal='' +olp_helper_signal_target='' + +printf 'Assertions: %s\n' "$pass_count" +printf 'OBSERVABILITY CORE RENDER ASSERTION TEST PASS\n' + diff --git a/scripts/validate/test-scan-platform-sensitive-source.sh b/scripts/validate/test-scan-platform-sensitive-source.sh new file mode 100644 index 0000000..6a61395 --- /dev/null +++ b/scripts/validate/test-scan-platform-sensitive-source.sh @@ -0,0 +1,1485 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +set +x +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly SCANNER="$ROOT/scripts/validate/scan-platform-sensitive-source.sh" +# shellcheck source=scan-platform-sensitive-source.sh +source "$SCANNER" + +WORK='' +ASSERTIONS=0 + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +pass() { + ASSERTIONS=$((ASSERTIONS + 1)) + printf 'PASS: %s\n' "$1" +} + +cleanup() { + trap - EXIT HUP INT TERM + case "$WORK" in + /tmp/platform-sensitive-source-test.??????) + rm -rf -- "$WORK" + ;; + esac +} +install_cleanup_traps() { + trap cleanup EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM +} +install_cleanup_traps + +new_fixture() { + cleanup + install_cleanup_traps + WORK="$(mktemp -d /tmp/platform-sensitive-source-test.XXXXXX)" + mkdir -p -- \ + "$WORK/platform/scripts/validate" \ + "$WORK/platform/services/observability" \ + "$WORK/platform/infrastructure" \ + "$WORK/platform/bootstrap" \ + "$WORK/platform/clusters" \ + "$WORK/platform/components" \ + "$WORK/docs/plans" + chmod 0700 "$WORK" "$WORK/platform" "$WORK/docs" +} + +if [[ "${PLATFORM_SENSITIVE_SOURCE_CLEANUP_PROBE:-0}" == 1 ]]; then + new_fixture + new_fixture + exit 0 +fi + +run_scanner() { + env -i HOME="$HOME" PATH="${PLATFORM_SENSITIVE_SOURCE_TEST_PATH:-/usr/local/bin:/usr/bin:/bin}" \ + PLATFORM_SENSITIVE_SOURCE_LIBRARY="$SCANNER" \ + PLATFORM_SENSITIVE_SOURCE_PLATFORM_ROOT="$WORK/platform" \ + PLATFORM_SENSITIVE_SOURCE_DOCS_ROOT="$WORK/docs" \ + PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS="${PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS:-}" \ + bash -s -- "$@" <<'RUNNER' +set -Eeuo pipefail +source "$PLATFORM_SENSITIVE_SOURCE_LIBRARY" +scanner_args=("$@") +if [[ -n "$PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS" ]]; then + scanner_args+=(--rendered-manifest "$PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS") +fi +scan_platform_sensitive_roots \ + "$PLATFORM_SENSITIVE_SOURCE_PLATFORM_ROOT" \ + "$PLATFORM_SENSITIVE_SOURCE_DOCS_ROOT" \ + "${scanner_args[@]}" +RUNNER +} + +expect_clean() { + local description=$1 output + if ! output="$(run_scanner 2>&1)"; then + printf '%s\n' "$output" >&2 + fail "$description" + fi + [[ "$output" == *'PLATFORM SENSITIVE SOURCE SCAN PASS'* ]] || { + printf '%s\n' "$output" >&2 + fail "$description did not emit the PASS marker" + } + pass "$description" +} + +expect_reject() { + local description=$1 expected_file=$2 output rc=0 + shift 2 + output="$(run_scanner "$@" 2>&1)" || rc=$? + (( rc != 0 )) || { + printf '%s\n' "$output" >&2 + fail "$description was accepted" + } + [[ "$output" == *"$expected_file"* ]] || { + printf '%s\n' "$output" >&2 + fail "$description did not report the filename" + } + [[ "$output" != *'should-never-be-printed'* ]] || { + printf '%s\n' "$output" >&2 + fail "$description leaked matched content" + } + [[ "$output" != *'hooks.slack.com/services/'* ]] || { + printf '%s\n' "$output" >&2 + fail "$description leaked a Slack URL" + } + pass "$description" +} + +expect_scanner_error() { + local description=$1 output rc=0 + output="$(run_scanner 2>&1)" || rc=$? + (( rc != 0 )) || { + printf '%s\n' "$output" >&2 + fail "$description was accepted" + } + [[ "$output" == *'SCANNER_ERROR:'* ]] || { + printf '%s\n' "$output" >&2 + fail "$description did not fail as a scanner error" + } + [[ "$output" != *'ActualProductionSecretValue9283'* ]] || { + fail "$description leaked matched content" + } + pass "$description" +} + +expect_canonical_test_source_clean() { + local finding + finding="$(_sensitive_source_python_scan "$ROOT/scripts/validate/test-scan-platform-sensitive-source.sh" \ + 1 "$SCANNER")" || fail 'canonical scanner test content leaf returned an execution error' + [[ -z "$finding" ]] || { + [[ "$finding" == CREDENTIAL_LITERAL ]] || \ + fail 'canonical scanner test content leaf returned an unexpected finding class' + fail 'canonical scanner test content leaf found a credential literal' + } + pass 'canonical scanner test source has no content-scan finding' +} + +expect_canonical_test_source_clean + +new_fixture +expect_clean 'empty full-root fixture is accepted' + +mkdir -p "$WORK/platform/docs/superpowers/plans" +printf 'UNRESOLVED_%s\n' 'PLACEHOLDER' >"$WORK/platform/docs/superpowers/plans/pending.md" +expect_reject 'platform/docs remains in the platform-root scan' 'platform/docs/superpowers/plans/pending.md' + +new_fixture +newline_file=$'odd\nname.md' +printf 'TO%s should-never-be-printed\n' 'DO' >"$WORK/docs/plans/$newline_file" +expect_reject 'NUL-safe traversal reports a newline filename without content' 'docs/plans/odd' + +new_fixture +mkdir -p "$WORK/platform/.git" "$WORK/platform/vendor/.helm" +printf 'TO%s should-never-be-printed\n' 'DO' >"$WORK/platform/.git/ignored" +printf 'TB%s should-never-be-printed\n' 'D' >"$WORK/platform/vendor/.helm/ignored" +expect_clean 'only .git and .helm directory contents are excluded' + +new_fixture +mkdir -p "$WORK/platform/.superpowers/sdd/test-scratch" +printf 'password = should-never-be-printed-value\n' > \ + "$WORK/platform/.superpowers/sdd/test-scratch/snapshot.env" +expect_clean 'the exact platform-root SDD scratch subtree is excluded' + +new_fixture +mkdir -p "$WORK/platform/.superpowers/not-sdd" +printf 'password = should-never-be-printed-value\n' > \ + "$WORK/platform/.superpowers/not-sdd/source.env" +expect_reject 'a platform-root .superpowers non-SDD subtree remains scanned' \ + 'platform/.superpowers/not-sdd/source.env' + +new_fixture +mkdir -p "$WORK/docs/.superpowers/sdd/test-scratch" +printf 'password = should-never-be-printed-value\n' > \ + "$WORK/docs/.superpowers/sdd/test-scratch/snapshot.env" +expect_reject 'a .superpowers subtree under another scanned root remains scanned' \ + 'docs/.superpowers/sdd/test-scratch/snapshot.env' + +new_fixture +printf 'TO%s should-never-be-printed\n' 'DO' >"$WORK/docs/plans/unresolved.md" +expect_reject 'unfinished English marker is rejected' 'docs/plans/unresolved.md' +printf '%s%s should-never-be-printed\n' '나중에' '채움' >"$WORK/docs/plans/unresolved.md" +expect_reject 'unfinished Korean marker is rejected' 'docs/plans/unresolved.md' + +new_fixture +printf 'pattern | TO%s real work | TB%s should-never-be-printed\n' 'DO' 'D' > \ + "$WORK/docs/plans/detector-context-bypass.md" +expect_reject 'detector prose cannot exempt real unfinished markers' \ + 'docs/plans/detector-context-bypass.md' + +new_fixture +mkdir -p "$WORK/docs/mirror/scripts/validate" +printf '%s\n' \ + 'unfinished = ("TO" + "DO", "TB" + "D", "나중에" + "채움", "UNRESOLVED_" + "PLACEHOLDER")' > \ + "$WORK/docs/mirror/scripts/validate/scan-platform-sensitive-source.sh" +expect_reject 'only the canonical production scanner may hold the exact detector definition' \ + 'docs/mirror/scripts/validate/scan-platform-sensitive-source.sh' + +new_fixture +printf '%s\n' 'readonly SLACK_URL='"'"'https://hooks.slack.com/services/T123/B456/S789'"'" > \ + "$WORK/platform/scripts/validate/test-example.sh" +printf '%s\n' 'readonly ADMIN_PASSWORD='"'"'SyntheticFixturePassword-123456789'"'" >> \ + "$WORK/platform/scripts/validate/test-example.sh" +expect_clean 'clearly synthetic focused-test credential literals are contextually allowed' +printf 'pass%sword = Actual%s\n' '' 'ProductionSecretValue9283' > \ + "$WORK/platform/scripts/validate/test-real-credential.sh" +expect_reject 'test-source context does not exempt a non-synthetic credential literal' \ + 'platform/scripts/validate/test-real-credential.sh' +rm -f -- "$WORK/platform/scripts/validate/test-real-credential.sh" +cat >"$WORK/platform/services/reference-only.yaml" <<'YAML' +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: example + env: + - name: CLIENT_SECRET + valueFrom: + secretKeyRef: + name: runtime-only + key: client-secret + - name: PASSWORD_TEMPLATE + value: ${RUNTIME_PASSWORD} + - name: ACCESS_KEY_PLACEHOLDER + value: +YAML +printf '%s\n' 'OBSERVABILITY_SLACK_WEBHOOK=CREATE_CONFIRMED' > \ + "$WORK/platform/bootstrap/status.env" +printf '%s\n' \ + "check_pattern PASSWORD_NONPLACEHOLDER 'Password:[[:space:]]*[^<[:space:]][^[:space:]]*'" > \ + "$WORK/docs/plans/scanner-pattern.md" +expect_clean 'references, bare key names, templates, and explicit placeholders are accepted' +printf '%s\n' 'webhook = https://hooks.slack.com/services/should-never-be-printed/B456/S789' > \ + "$WORK/platform/bootstrap/live.env" +expect_reject 'Slack-shaped literal outside test-source context is rejected' 'platform/bootstrap/live.env' +printf '%s\n' 'password = should-never-be-printed-value' >"$WORK/platform/bootstrap/live.env" +expect_reject 'credential literal outside test-source context is rejected' 'platform/bootstrap/live.env' + +new_fixture +printf '%s\n' '{"password": "should-never-be-printed-quoted-literal"}' > \ + "$WORK/platform/bootstrap/quoted.json" +expect_reject 'quoted JSON credential key with a literal RHS is rejected' \ + 'platform/bootstrap/quoted.json' +printf '%s\n' 'password=should-never-be-printed-$-literal' > \ + "$WORK/platform/bootstrap/dollar.env" +expect_reject 'a literal dollar does not exempt a credential RHS' \ + 'platform/bootstrap/dollar.env' + +new_fixture +printf '%s\n' 'password = " should-never-be-printed-spaced-literal "' > \ + "$WORK/platform/bootstrap/spaced.env" +expect_reject 'quoted whitespace cannot hide a credential literal' \ + 'platform/bootstrap/spaced.env' +rm -f -- "$WORK/platform/bootstrap/spaced.env" +printf 'pass%sword="${RUNTIME_PASSWORD}"-Actual%s\n' '' \ + 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/quoted-tail.env" +expect_reject 'a quoted runtime reference cannot hide a literal suffix' \ + 'platform/bootstrap/quoted-tail.env' +rm -f -- "$WORK/platform/bootstrap/quoted-tail.env" + +for separator in space comma semicolon; do + case "$separator" in + space) tail=' ActualProductionSecretValue9283' ;; + comma) tail=',ActualProductionSecretValue9283' ;; + semicolon) tail=';ActualProductionSecretValue9283' ;; + esac + printf 'pass%sword=${RUNTIME_PASSWORD}%s\n' '' "$tail" > \ + "$WORK/platform/bootstrap/unquoted-$separator-tail.env" + expect_reject "an unquoted runtime reference cannot hide a $separator literal suffix" \ + "platform/bootstrap/unquoted-$separator-tail.env" + rm -f -- "$WORK/platform/bootstrap/unquoted-$separator-tail.env" +done + +printf 'pass%sword=pass%sword_Actual%s\n' \ + '' '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/bare-identifier.env" +expect_reject 'a credential-key-prefixed bare identifier RHS is still a literal' \ + 'platform/bootstrap/bare-identifier.env' +rm -f -- "$WORK/platform/bootstrap/bare-identifier.env" +printf 'client_%ssecret=client_%ssecret_Actual%s\n' \ + '' '' 'ProductionValue9283' >"$WORK/platform/bootstrap/bare-client-identifier.env" +expect_reject 'a client-secret-prefixed bare identifier RHS is still a literal' \ + 'platform/bootstrap/bare-client-identifier.env' + +new_fixture +printf '%s; pass%sword=Actual%s\n' \ + "check_pattern X 'Password:[[:space:]]*[^<[:space:]]+'" \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/mixed-regex.env" +expect_reject 'a detector regex cannot exempt a later credential assignment' \ + 'platform/bootstrap/mixed-regex.env' +rm -f -- "$WORK/platform/bootstrap/mixed-regex.env" + +printf '%s pass%sword=Actual%s\n' \ + "check_pattern X 'Password:[[:space:]]*[^<[:space:]]+'" \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/mixed-regex-space.env" +expect_reject 'a detector regex cannot consume a same-statement credential assignment' \ + 'platform/bootstrap/mixed-regex-space.env' + +new_fixture +printf 'config: {"pass%sword": Actual%s}\n' \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/mapping-literal.yaml" +expect_reject 'a YAML mapping credential key cannot use a bare literal scalar' \ + 'platform/bootstrap/mapping-literal.yaml' +printf 'pass%sword: Actual%s SecretValue9283\n' \ + '' 'Production' >"$WORK/platform/bootstrap/spaced-scalar.yaml" +expect_reject 'YAML credential scalar whitespace is not prose' \ + 'platform/bootstrap/spaced-scalar.yaml' +rm -f -- "$WORK/platform/bootstrap/spaced-scalar.yaml" +printf 'pass%sword: Actual%s,tail\n' \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/comma-scalar.yaml" +expect_reject 'YAML credential scalar punctuation is not prose' \ + 'platform/bootstrap/comma-scalar.yaml' + +new_fixture +printf 'admin_pass%sword: Actual%s\n' \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/admin-password.yaml" +expect_reject 'a normalized YAML admin password key is credential-bearing' \ + 'platform/bootstrap/admin-password.yaml' +printf 'oidc_client_%ssecret: Actual%s\n' \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/oidc-client-secret.yaml" +expect_reject 'a normalized YAML OIDC client-secret key is credential-bearing' \ + 'platform/bootstrap/oidc-client-secret.yaml' + +new_fixture +printf 'OIDCClient%sSecret: Actual%s\n' \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/acronym-client-secret.yaml" +expect_reject 'an acronym CamelCase OIDC client-secret key is credential-bearing' \ + 'platform/bootstrap/acronym-client-secret.yaml' +printf 'DBPass%sword: Actual%s\n' \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/acronym-password.yaml" +expect_reject 'an acronym CamelCase DB password key is credential-bearing' \ + 'platform/bootstrap/acronym-password.yaml' + +new_fixture +printf '{"pass\\u0077ord":"Actual%s","pass%sword":"${RUNTIME_PASSWORD}"}\n' \ + 'ProductionSecretValue9283' '' >"$WORK/platform/bootstrap/duplicate-key.json" +expect_reject 'JSON rejects a Unicode-decoded duplicate credential key before collapse' \ + 'platform/bootstrap/duplicate-key.json' + +for value_kind in numeric array object; do + case "$value_kind" in + numeric) value='12345678901234567890' ;; + array) value='["ActualProductionSecretValue9283"]' ;; + object) value='{"value":"ActualProductionSecretValue9283"}' ;; + esac + printf '{"admin_pass%sword":%s}\n' '' "$value" > \ + "$WORK/platform/bootstrap/$value_kind-credential.json" + expect_reject "a credential-key JSON $value_kind value fails closed" \ + "platform/bootstrap/$value_kind-credential.json" + rm -f -- "$WORK/platform/bootstrap/$value_kind-credential.json" +done +rm -f -- "$WORK/platform/bootstrap/duplicate-key.json" + +cat >"$WORK/platform/bootstrap/reference-container.yaml" <<'YAML' +envValueFrom: + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: + secretKeyRef: + name: grafana-keycloak-oidc + key: client-secret +YAML +expect_clean 'an exact Kubernetes secretKeyRef container remains a reference' +cat >"$WORK/platform/bootstrap/reference-container.yaml" <<'YAML' +envValueFrom: + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: + secretKeyRef: + name: grafana-keycloak-oidc + key: client-secret + value: ActualProductionSecretValue9283 +YAML +expect_reject 'a secretKeyRef container with a literal child fails closed' \ + 'platform/bootstrap/reference-container.yaml' + +new_fixture +cat >"$WORK/platform/bootstrap/webhook-feature.yaml" <<'YAML' +operators: + object-store: + webhook: + enabled: true + replicas: 1 +YAML +expect_clean 'the exact chart webhook feature block is not a credential value' +cat >"$WORK/platform/bootstrap/webhook-feature.yaml" <<'YAML' +operators: + object-store: + webhook: + enabled: true + replicas: 1 + value: ActualProductionSecretValue9283 +YAML +expect_reject 'a webhook feature block with a literal child fails closed' \ + 'platform/bootstrap/webhook-feature.yaml' + +new_fixture +printf 'pass%sword: Actual%s\n' \ + '' 'ProductionSecretValue9283' >"$WORK/docs/plans/literal-password.md" +expect_reject 'Markdown assignment-shaped password literals are rejected' \ + 'docs/plans/literal-password.md' +printf 'client-%ssecret: Actual%s\n' \ + '' 'ProductionSecretValue9283' >"$WORK/docs/plans/literal-client-secret.md" +expect_reject 'Markdown assignment-shaped client-secret literals are rejected' \ + 'docs/plans/literal-client-secret.md' +rm -f -- "$WORK/docs/plans/literal-password.md" "$WORK/docs/plans/literal-client-secret.md" +printf ' pass%sword: Actual%s\n' \ + '' 'ProductionSecretValue9283' >"$WORK/docs/plans/indented-literal-password.md" +expect_reject 'indented Markdown assignment-shaped password literals are rejected' \ + 'docs/plans/indented-literal-password.md' + +new_fixture +printf 'pass%sword:Actual%s\n' \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/literal.conf" +expect_reject 'a colon credential in configuration syntax is not prose' \ + 'platform/bootstrap/literal.conf' + +new_fixture +printf 'cmd --from-file="pass%sword=${password_file}" pass%sword=Actual%s\n' \ + '' '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/from-file-tail.sh" +expect_reject 'one safe from-file argv cannot exempt a later credential assignment' \ + 'platform/bootstrap/from-file-tail.sh' + +new_fixture +cat >"$WORK/platform/bootstrap/references.env" <<'EOF' +password=${RUNTIME_PASSWORD} +client_secret="${OIDC_CLIENT_SECRET}" +access_key=$(runtime_access_key) +webhook='$(runtime_webhook)' +password=" ${SPACED_RUNTIME_PASSWORD} " +--from-file="password=${password_file}" +EOF +cat >"$WORK/platform/bootstrap/references.py" <<'PY' +sources = {"admin-password": password_file} +item = {"client-secret": base64.b64encode(payload).decode()} +PY +expect_clean 'exact shell interpolation and command references remain accepted' + +new_fixture +cat >"$WORK/platform/bootstrap/typed-none-reference.py" <<'PY' +def retain_required( + webhook: RetainedWebhook, +): + return webhook + +def retain_optional( + webhook: RetainedWebhook | None = None, +): + return webhook +PY +printf 'web%shook = runtime / "input-webhook"\n' '' > \ + "$WORK/platform/scripts/validate/test-runtime-path-reference.py" +printf 'web%shook = open_validated_webhook(webhook_file)\nself.web%shook = self.vault.open_validated_webhook(os.fspath(self.webhook_path))\n' \ + '' '' >"$WORK/platform/bootstrap/runtime-call-reference.py" +expect_clean 'Python typed defaults, fixed test paths, and literal-free calls are references' + +printf 'web%shook: RetainedWebhook | None = "Actual%s"\n' \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/typed-literal.py" +expect_reject 'a Python typed credential parameter cannot hide a literal default' \ + 'platform/bootstrap/typed-literal.py' +printf 'web%shook = runtime / "Actual%s"\n' \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/path-literal.py" +expect_reject 'a Python credential runtime path cannot hide a literal basename' \ + 'platform/bootstrap/path-literal.py' +for call_kind in string bytes nested star kwargs; do + case "$call_kind" in + string) expression='open_validated_webhook("ActualProductionSecretValue9283")' ;; + bytes) expression='open_validated_webhook(b"ActualProductionSecretValue9283")' ;; + nested) expression='open_validated_webhook(os.fspath("ActualProductionSecretValue9283"))' ;; + star) expression='open_validated_webhook(*webhook_file)' ;; + kwargs) expression='open_validated_webhook(**webhook_file)' ;; + esac + printf 'web%shook = %s\n' '' "$expression" > \ + "$WORK/platform/bootstrap/call-$call_kind-literal.py" + expect_reject "a Python credential call rejects $call_kind literal or expansion input" \ + "platform/bootstrap/call-$call_kind-literal.py" +done + +printf 'pass%sword=base64.b64encode(b"Actual%s").decode()\n' \ + '' 'ProductionSecretValue9283' >"$WORK/platform/bootstrap/base64-literal.py" +expect_reject 'base64 syntax does not exempt a literal credential' \ + 'platform/bootstrap/base64-literal.py' + +new_fixture +prompt_expected="$WORK/prompt-fixture.expected" +printf '%s%s\n%s\n' \ + 'while b"KeePassXC master pass' \ + 'word: " not in observed:' \ + ' observed += read_prompt()' >"$prompt_expected" +printf '%s%s\n%s\n' \ + 'while b"KeePassXC master pass' \ + 'word: " not in observed:' \ + ' observed += read_prompt()' > \ + "$WORK/platform/scripts/validate/test-python-prompt.py" +cmp -s -- "$prompt_expected" "$WORK/platform/scripts/validate/test-python-prompt.py" || \ + fail 'fixed-format prompt fixture bytes changed' +[[ "$(sha256sum "$WORK/platform/scripts/validate/test-python-prompt.py" | awk '{print $1}')" == \ + 3c3cec8dc3f1d273be0e5a4760f2c89a01b11f851e28f0d2bbf960ff37fb1c56 ]] || \ + fail 'fixed-format prompt fixture hash changed' +rm -f -- "$prompt_expected" +expect_clean 'a Python test prompt ending at a credential label is not a credential literal' + +new_fixture +cat >"$WORK/platform/scripts/validate/test-python-assert-prompt.py" <<'PY' +self.assertEqual(prompt, b"KeePassXC master password: ") +PY +expect_clean 'a sole Python assertEqual prompt constant remains accepted' + +new_fixture +python3 - "$WORK/platform/scripts/validate/test-python-unknown-encoding.py" <<'PY' +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +path.write_bytes( + b"# coding: unknown-scanner-encoding\n" + b"pass" b"word = ActualProductionSecretValue9283\n" +) +PY +[[ "$(sha256sum "$WORK/platform/scripts/validate/test-python-unknown-encoding.py" | awk '{print $1}')" == \ + 2daee060681860677b2bffc8f7a086505299e2d321c1201cfa187fb24f99a3f4 ]] || \ + fail 'unknown-encoding fixture bytes changed' +expect_scanner_error 'an unknown Python source encoding fails closed as a scanner error' + +new_fixture +python3 - "$WORK/platform/scripts/validate/test-python-latin1-prompt.py" <<'PY' +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +path.write_bytes( + b"# coding: latin-1\n" + b"marker = 'caf\xe9'\n" + b"while b'KeePassXC master pass" b"word: ' not in observed:\n" + b" observed += read_prompt()\n" +) +PY +python3 -m py_compile "$WORK/platform/scripts/validate/test-python-latin1-prompt.py" +find "$WORK/platform/scripts/validate" -type d -name __pycache__ -exec rm -rf -- {} + +[[ "$(sha256sum "$WORK/platform/scripts/validate/test-python-latin1-prompt.py" | awk '{print $1}')" == \ + 06b3e6067415d4241e65b704e861707cfdff0372319075d69cf32f7178f55577 ]] || \ + fail 'Latin-1 prompt fixture bytes changed' +expect_clean 'a valid Latin-1 Python prompt-only context remains accepted' + +new_fixture +python3 - "$WORK/platform/scripts/validate/test-python-latin1-literal.py" <<'PY' +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +path.write_bytes( + b"# coding: latin-1\n" + b"marker = 'caf\xe9'\n" + b"pass" b"word = ActualProductionSecretValue9283\n" +) +PY +python3 -m py_compile "$WORK/platform/scripts/validate/test-python-latin1-literal.py" +find "$WORK/platform/scripts/validate" -type d -name __pycache__ -exec rm -rf -- {} + +[[ "$(sha256sum "$WORK/platform/scripts/validate/test-python-latin1-literal.py" | awk '{print $1}')" == \ + f7cf7c400e901e17d8f460ec1aa087cd31c21208e01050d2a81e777b9066ec16 ]] || \ + fail 'Latin-1 literal fixture bytes changed' +expect_reject 'a valid Latin-1 Python credential literal is rejected' \ + 'platform/scripts/validate/test-python-latin1-literal.py' + +new_fixture +printf '%s%s\n%s\n' \ + 'while b"KeePassXC master pass' \ + 'word: " not in observed' \ + ' observed += read_prompt()' > \ + "$WORK/platform/scripts/validate/test-python-malformed-prompt.py" +[[ "$(sha256sum "$WORK/platform/scripts/validate/test-python-malformed-prompt.py" | awk '{print $1}')" == \ + 05e824e262eec08a1be21c4c3488c9a5d2c32418ee2ab6b1e8a325a898eaed3d ]] || \ + fail 'malformed prompt fixture bytes changed' +expect_scanner_error 'malformed Python containing a credential prompt fails closed as a scanner error' + +new_fixture +cat >"$WORK/platform/scripts/validate/test-python-triple-prompt-bypass.py" <<'PY' +message = b"""KeePassXC master password: +ActualProductionSecretValue9283""" +PY +expect_reject 'a triple-quoted Python prompt token cannot contain a credential literal' \ + 'platform/scripts/validate/test-python-triple-prompt-bypass.py' + +new_fixture +cat >"$WORK/platform/scripts/validate/test-python-multiline-prompt-bypass.py" <<'PY' +message = ( + b"KeePassXC master password: " + b"ActualProductionSecretValue9283" +) +PY +expect_reject 'a multiline adjacent Python literal cannot continue an empty credential prompt' \ + 'platform/scripts/validate/test-python-multiline-prompt-bypass.py' + +new_fixture +printf 'message = b"".join((b"KeePassXC master pass%sword: ", b"Actual%s"))\n' \ + '' 'ProductionSecretValue9283' > \ + "$WORK/platform/scripts/validate/test-python-join-prompt-bypass.py" +expect_reject 'a join expression cannot continue an empty credential prompt with a literal' \ + 'platform/scripts/validate/test-python-join-prompt-bypass.py' + +new_fixture +printf 'message = b"KeePassXC master pass%sword: " + b"Actual%s"\n' \ + '' 'ProductionSecretValue9283' > \ + "$WORK/platform/scripts/validate/test-python-plus-prompt-bypass.py" +expect_reject 'a plus-joined Python string literal cannot continue an empty credential prompt' \ + 'platform/scripts/validate/test-python-plus-prompt-bypass.py' + +new_fixture +printf 'message = b"KeePassXC master pass%sword: " b"Actual%s"\n' \ + '' 'ProductionSecretValue9283' > \ + "$WORK/platform/scripts/validate/test-python-adjacent-prompt-bypass.py" +expect_reject 'an adjacent Python string literal cannot continue an empty credential prompt' \ + 'platform/scripts/validate/test-python-adjacent-prompt-bypass.py' + +new_fixture +printf 'message = "pass%sword: Actual%s; pass%sword: "\n' \ + '' 'ProductionSecretValue9283' '' > \ + "$WORK/platform/scripts/validate/test-python-prompt-bypass.py" +expect_reject 'a later empty Python prompt cannot exempt an earlier quoted credential literal' \ + 'platform/scripts/validate/test-python-prompt-bypass.py' +printf 'pass%sword = "Actual%s"\n' '' 'ProductionSecretValue9283' > \ + "$WORK/platform/scripts/validate/test-python-quoted-literal.py" +expect_reject 'a Python test file rejects a quoted credential literal' \ + 'platform/scripts/validate/test-python-quoted-literal.py' + +new_fixture +cp -- "$ROOT/scripts/validate/test-slack-webhook-recovery-vault.py" \ + "$WORK/platform/scripts/validate/test-slack-webhook-recovery-vault.py" +expect_clean 'a test-named Python vault fixture permits safe test context and nonliteral references' +printf 'web%shook = Actual%s\n' '' 'ProductionSecretValue9283' > \ + "$WORK/platform/scripts/validate/test-python-explicit-literal.py" +expect_reject 'a Python test file rejects an explicit credential literal' \ + 'platform/scripts/validate/test-python-explicit-literal.py' + +new_fixture +printf '%s\n' '-----BEGIN PRI''VATE KEY-----' 'should-never-be-printed' >"$WORK/docs/private.pem" +expect_reject 'private key header is rejected' 'docs/private.pem' + +new_fixture +cat >"$WORK/platform/services/credential-secret.yaml" <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: forbidden-source +stringData: + password: should-never-be-printed +YAML +expect_reject 'source-controlled credential Secret payload is rejected' 'platform/services/credential-secret.yaml' + +new_fixture +cat >"$WORK/platform/services/custom-tag-stream.yaml" <<'YAML' +apiVersion: example.test/v1 +kind: Synthetic +spec: + tagged: !!value fixture +--- +apiVersion: v1 +kind: Secret +metadata: + name: forbidden-source +stringData: + opaque: c2hvdWxkLW5ldmVyLWJlLXByaW50ZWQ= +YAML +expect_reject 'custom-tag streams still expose later source Secret payloads' \ + 'platform/services/custom-tag-stream.yaml' + +new_fixture +cat >"$WORK/platform/services/duplicate-payload.yaml" <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: duplicate-payload +stringData: + opaque: c2hvdWxkLW5ldmVyLWJlLXByaW50ZWQ= +stringData: {} +YAML +expect_reject 'duplicate source Secret keys fail closed' \ + 'platform/services/duplicate-payload.yaml' + +new_fixture +cat >"$WORK/platform/services/credential-secret.YAML" <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: uppercase-extension +stringData: + opaque: c2hvdWxkLW5ldmVyLWJlLXByaW50ZWQ= +YAML +expect_reject 'uppercase YAML source Secret payload is rejected' \ + 'platform/services/credential-secret.YAML' + +new_fixture +printf '%s\n' 'kind: [unterminated' >"$WORK/platform/services/malformed.yaml" +expect_reject 'a source YAML parser error fails closed without content' \ + 'platform/services/malformed.yaml' + +new_fixture +stream_fifo="$WORK/source-stream.yaml" +stream_output="$WORK/source-stream.finding" +mkfifo -m 0600 "$stream_fifo" +PYTHONUNBUFFERED=1 _sensitive_source_yaml_secret_scan "$stream_fifo" >"$stream_output" & +stream_pid=$! +exec {stream_fd}>"$stream_fifo" +printf '%s\n' \ + 'apiVersion: v1' \ + 'kind: Secret' \ + 'metadata:' \ + ' name: first-streamed-secret' \ + 'stringData:' \ + ' opaque: c2hvdWxkLW5ldmVyLWJlLXByaW50ZWQ=' >&"$stream_fd" +for stream_document in {1..200}; do + printf '%s\n' \ + '---' \ + 'apiVersion: v1' \ + 'kind: ConfigMap' \ + 'metadata:' \ + " name: streamed-padding-$stream_document" >&"$stream_fd" +done +stream_seen=false +for _ in {1..50}; do + if [[ -s "$stream_output" ]]; then + stream_seen=true + break + fi + sleep 0.02 +done +exec {stream_fd}>&- +wait "$stream_pid" +[[ "$stream_seen" == true ]] || \ + fail 'source YAML scan materialized the stream before reporting the first Secret' +[[ "$(<"$stream_output")" == SECRET_SOURCE_PAYLOAD ]] || \ + fail 'streamed source YAML scan returned an unexpected finding class' +pass 'source YAML documents are inspected incrementally before end-of-stream' + +new_fixture +mkdir -p "$WORK/render-handoff" +cat >"$WORK/render-handoff/rendered.yaml" <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: alertmanager-observability-core-kube-pr-alertmanager + namespace: observability +type: Opaque +data: + alertmanager.yaml: cmVjZWl2ZXJzOgotIG5hbWU6ICJudWxsIgpyb3V0ZToKICByZWNlaXZlcjogIm51bGwiCiAgcm91dGVzOiBbXQo= +YAML +PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS="$WORK/render-handoff/rendered.yaml" \ + run_scanner >/dev/null || \ + fail 'credential-free allowlisted rendered Secret was rejected' +pass 'credential-free allowlisted rendered Secret is accepted' +source_secret_rc=0 +cp -- "$WORK/render-handoff/rendered.yaml" "$WORK/platform/rendered.yaml" +source_secret_output="$(run_scanner 2>&1)" || source_secret_rc=$? +(( source_secret_rc != 0 )) || fail 'an allowlisted-looking Secret in source was accepted' +[[ "$source_secret_output" == *'platform/rendered.yaml'* ]] || \ + fail 'allowlisted-looking source Secret did not report its filename' +pass 'an allowlisted-looking Secret remains forbidden in source' +python3 - "$WORK/platform/rendered.yaml" <<'PY' +import base64 +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +encoded = base64.b64encode(b"password: should-never-be-printed\n").decode() +text = path.read_text() +head, _ = text.rsplit(" alertmanager.yaml: ", 1) +path.write_text(head + " alertmanager.yaml: " + encoded + "\n") +PY +cp -- "$WORK/platform/rendered.yaml" "$WORK/render-handoff/rendered.yaml" +rm -f -- "$WORK/platform/rendered.yaml" +PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS="$WORK/render-handoff/rendered.yaml" \ + expect_reject 'decoded rendered Secret credential payload is rejected' 'rendered.yaml' +cat >"$WORK/render-handoff/rendered.yaml" <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: gitea + namespace: gitea +type: Opaque +stringData: + assertions: "" + config_environment.sh: synthetic-fixture +--- +apiVersion: v1 +kind: Secret +metadata: + name: gitea-init + namespace: gitea +type: Opaque +stringData: + configure_gitea.sh: synthetic-fixture + configure_gpg_environment.sh: synthetic-fixture + init_directory_structure.sh: synthetic-fixture +--- +apiVersion: v1 +kind: Secret +metadata: + name: gitea-inline-config + namespace: gitea +type: Opaque +stringData: + _generals_: "" + actions: synthetic-fixture + cache: synthetic-fixture + database: synthetic-fixture + indexer: synthetic-fixture + metrics: synthetic-fixture + packages: synthetic-fixture + queue: synthetic-fixture + repository: synthetic-fixture + security: synthetic-fixture + server: synthetic-fixture + service: synthetic-fixture + session: synthetic-fixture +YAML +if PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS="$WORK/render-handoff/rendered.yaml" \ + run_scanner >/dev/null; then + pass 'expected credential-free Gitea renderer Secrets are accepted' +else + fail 'expected credential-free Gitea renderer Secrets were rejected' +fi +python3 - "$WORK/render-handoff/rendered.yaml" <<'PY' +import sys +import yaml + +path = sys.argv[1] +with open(path, encoding="utf-8") as stream: + documents = list(yaml.safe_load_all(stream)) +documents[0]["stringData"]["config_environment.sh"] = ( + "pass" + "word = Actual" + "ProductionSecretValue9283" +) +with open(path, "w", encoding="utf-8") as stream: + yaml.safe_dump_all(documents, stream, explicit_start=True, sort_keys=False) +PY +PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS="$WORK/render-handoff/rendered.yaml" \ + expect_reject 'literal credential inside an allowlisted Gitea Secret is rejected' 'rendered.yaml' +python3 - "$WORK/render-handoff/rendered.yaml" <<'PY' +import sys +import yaml + +path = sys.argv[1] +with open(path, encoding="utf-8") as stream: + documents = list(yaml.safe_load_all(stream)) +documents[0]["stringData"]["config_environment.sh"] = ( + "password = should-never-be-printed-${RUNTIME_SUFFIX}" +) +with open(path, "w", encoding="utf-8") as stream: + yaml.safe_dump_all(documents, stream, explicit_start=True, sort_keys=False) +PY +PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS="$WORK/render-handoff/rendered.yaml" \ + expect_reject 'a decoded literal prefix plus interpolation is rejected' 'rendered.yaml' +python3 - "$WORK/render-handoff/rendered.yaml" <<'PY' +import sys +import yaml + +path = sys.argv[1] +with open(path, encoding="utf-8") as stream: + documents = list(yaml.safe_load_all(stream)) +documents[0]["stringData"]["config_environment.sh"] = ( + 'pass' + 'word = " ${RUNTIME_PASSWORD} "' +) +with open(path, "w", encoding="utf-8") as stream: + yaml.safe_dump_all(documents, stream, explicit_start=True, sort_keys=False) +PY +if PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS="$WORK/render-handoff/rendered.yaml" \ + run_scanner >/dev/null; then + pass 'a decoded quoted and spaced exact runtime reference remains accepted' +else + fail 'a decoded quoted and spaced exact runtime reference was rejected' +fi +python3 - "$WORK/render-handoff/rendered.yaml" <<'PY' +import sys +import yaml + +path = sys.argv[1] +with open(path, encoding="utf-8") as stream: + documents = list(yaml.safe_load_all(stream)) +documents[0]["stringData"]["config_environment.sh"] = ( + 'pass' + 'word="${RUNTIME_PASSWORD}"-ActualProductionSecretValue9283' +) +with open(path, "w", encoding="utf-8") as stream: + yaml.safe_dump_all(documents, stream, explicit_start=True, sort_keys=False) +PY +PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS="$WORK/render-handoff/rendered.yaml" \ + expect_reject 'a decoded quoted runtime reference cannot hide a literal suffix' \ + 'rendered.yaml' +cat >"$WORK/render-handoff/rendered.yaml" <<'YAML' +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: synthetic.example.test +spec: + synthetic: !!value fixture +--- +apiVersion: v1 +kind: Secret +metadata: + name: alertmanager-observability-core-kube-pr-alertmanager + namespace: observability +type: Opaque +data: + alertmanager.yaml: cmVjZWl2ZXJzOgotIG5hbWU6ICJudWxsIgpyb3V0ZToKICByZWNlaXZlcjogIm51bGwiCiAgcm91dGVzOiBbXQo= +YAML +if PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS="$WORK/render-handoff/rendered.yaml" \ + run_scanner >/dev/null; then + pass 'non-Secret CRD custom tags do not block streamed Secret validation' +else + fail 'non-Secret CRD custom tag blocked streamed Secret validation' +fi +cat >"$WORK/render-handoff/rendered.yaml" <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: alertmanager-observability-core-kube-pr-alertmanager + namespace: observability +type: Opaque +data: + alertmanager.yaml: cmVjZWl2ZXJzOgotIG5hbWU6ICJudWxsIgpyb3V0ZToKICByZWNlaXZlcjogIm51bGwiCiAgcm91dGVzOiBbXQo= + alertmanager.yaml: cmVjZWl2ZXJzOgotIG5hbWU6ICJudWxsIgpyb3V0ZToKICByZWNlaXZlcjogIm51bGwiCiAgcm91dGVzOiBbXQo= +YAML +PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS="$WORK/render-handoff/rendered.yaml" \ + expect_reject 'duplicate rendered Secret keys fail closed' 'rendered.yaml' +cat >"$WORK/render-handoff/rendered.yaml" <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: unexpected-rendered-secret + namespace: observability +type: Opaque +data: + allowed-looking-key: bm9uY3JlZGVudGlhbAo= +YAML +PLATFORM_SENSITIVE_SOURCE_RENDERED_MANIFESTS="$WORK/render-handoff/rendered.yaml" \ + expect_reject 'a rendered Secret outside the exact allowlist is rejected' 'rendered.yaml' + +new_fixture +printf 'target: spring-boot\n' >"$WORK/platform/services/observability/future-target.yaml" +expect_reject 'future observability target is rejected' 'platform/services/observability/future-target.yaml' +printf 'monitor-target: batch\n' >"$WORK/platform/services/observability/future-target.yaml" +expect_reject 'future batch target is rejected' 'platform/services/observability/future-target.yaml' +printf 'alert-target: backup\n' >"$WORK/platform/services/observability/future-target.yaml" +expect_reject 'future backup target is rejected' 'platform/services/observability/future-target.yaml' +rm -f -- "$WORK/platform/services/observability/future-target.yaml" +printf 'target: spring-boot\n' >"$WORK/platform/services/observability/future-target.md" +expect_reject 'future observability target in Markdown is rejected' \ + 'platform/services/observability/future-target.md' + +new_fixture +cat >"$WORK/platform/scripts/validate/fake-renderer" <<'SH' +#!/usr/bin/env bash +set -Eeuo pipefail +out='' +component='' +while (( $# > 0 )); do + case "$1" in + --verified-output-dir) out=$2; shift 2 ;; + --component) component=$2; shift 2 ;; + *) shift ;; + esac +done +label="$(basename -- "$0")" +handoff=no +[[ -z "$out" ]] || handoff=yes +printf '%s\n' "$label${component:+:$component}:handoff=$handoff" >>"$PLATFORM_SENSITIVE_SOURCE_RENDER_LOG" + +emit_phase1_log() { + local publish=$1 name + for name in namespaces ssd-local-pv; do + printf 'Rendered %-20s %8d bytes\n' "$name" 123 + done + printf 'Verified %-20s SHA-256 %064d\n' cloudnative-pg-chart 1 + for name in cnpg-operator platform-postgres; do + printf 'Rendered %-20s %8d bytes\n' "$name" 123 + done + printf 'Verified %-20s SHA-256 %064d\n' gitea-chart 2 + for name in gitea gitea-oidc; do + printf 'Rendered %-20s %8d bytes\n' "$name" 123 + done + if [[ "$publish" == yes ]]; then + printf 'Preserved six verified manifests for the apply handoff.\n' + fi + printf 'Phase 1 baseline and Gitea OIDC desired rendering invariants passed.\n' + printf 'Temporary rendered manifests and generated chart caches will be removed on exit.\n' +} + +emit_core_log() { + local name count=10 + for name in prometheus-operator-crds kube-prometheus-stack loki tempo alloy \ + prometheus-node-exporter grafana prometheus-blackbox-exporter; do + printf 'Verified %-28s SHA-256 %064d\n' "$name" "$count" + count=$((count + 1)) + done + for name in namespaces crds storage kps loki tempo alloy node-exporter grafana \ + blackbox core-policies targets core-rules alerting dashboards aggregate; do + printf 'Rendered %-18s %8d bytes SHA-256 %064d\n' "$name" 123 "$count" + count=$((count + 1)) + done + printf 'OBSERVABILITY CORE STATIC RENDER PASS\n' +} + +emit_access_log() { + local name count=40 + for name in prometheus-operator-crds kube-prometheus-stack loki tempo alloy \ + prometheus-node-exporter grafana prometheus-blackbox-exporter; do + if [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_MISSING_VERIFIED:-0}" != 0 && \ + "$name" == grafana ]]; then + continue + fi + printf 'Verified %-28s SHA-256 %064d\n' "$name" "$count" + count=$((count + 1)) + done + access_rendered=( + namespaces crds storage kps loki tempo alloy node-exporter grafana + blackbox core-policies targets core-rules alerting dashboards aggregate + ) + if [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_SWAP_RENDERED:-0}" != 0 ]]; then + access_rendered=(crds namespaces "${access_rendered[@]:2}") + fi + for name in "${access_rendered[@]}"; do + printf 'Rendered %-18s %8d bytes SHA-256 %064d\n' "$name" 123 "$count" + count=$((count + 1)) + done + printf 'OBSERVABILITY CORE STATIC RENDER PASS\n' + printf 'OBSERVABILITY ACCESS STATIC RENDER PASS\n' +} + +emit_admin_metadata_log() { + case "${PLATFORM_SENSITIVE_SOURCE_FAKE_ADMIN_LOG_VARIANT:-safe}" in + safe) + printf 'Pulled: docker.io/dpage/pgadmin4-helm:9.16.0\n' + printf 'Digest: sha256:%064d\n' 8 + ;; + wrong-registry) + printf 'Pulled: registry.example.test/dpage/pgadmin4-helm:9.16.0\n' + printf 'Digest: sha256:%064d\n' 8 + ;; + wrong-chart) + printf 'Pulled: docker.io/dpage/other-chart:9.16.0\n' + printf 'Digest: sha256:%064d\n' 8 + ;; + wrong-version) + printf 'Pulled: docker.io/dpage/pgadmin4-helm:9.16.1\n' + printf 'Digest: sha256:%064d\n' 8 + ;; + bad-digest) + printf 'Pulled: docker.io/dpage/pgadmin4-helm:9.16.0\n' + printf 'Digest: sha256:%063d\n' 8 + ;; + wrong-order) + printf 'Digest: sha256:%064d\n' 8 + printf 'Pulled: docker.io/dpage/pgadmin4-helm:9.16.0\n' + ;; + extra) + printf 'Pulled: docker.io/dpage/pgadmin4-helm:9.16.0\n' + printf 'Digest: sha256:%064d\n' 8 + printf 'Fetched chart metadata.\n' + ;; + *) exit 44 ;; + esac +} + +if [[ "$label" == render-observability-access.sh ]]; then + [[ -z "$out" ]] || exit 42 + [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_FAIL:-0}" == 0 ]] || exit 41 + emit_access_log + if [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_PATH:-0}" != 0 ]]; then + printf 'Rendered /tmp/should-never-be-printed %d bytes SHA-256 %064d\n' 123 2 + fi + if [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_DUPLICATE_CORE:-0}" != 0 ]]; then + printf 'OBSERVABILITY CORE STATIC RENDER PASS\n' + fi + if [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_SECRET:-0}" != 0 ]]; then + printf '%s\n' 'apiVersion: v1' 'kind: Secret' 'stringData:' ' password: should-never-be-printed' + fi + exit 0 +fi +[[ -n "$out" && -d "$out" ]] +case "$label" in + render-phase1.sh) + names=(namespaces ssd-local-pv cnpg-operator platform-postgres gitea gitea-oidc) + ;; + render-phase2.sh) + names=(phase2-namespaces aistor-local-pv keycloak-operator platform-postgres-keycloak keycloak aistor-operator minio-aistor aistor-network-policies) + ;; + render-admin-services.sh) + names=(admin-namespace pgadmin-local-pv coredns-custom aistor-admin-oidc pgadmin) + ;; + render-observability-core.sh) + names=(namespaces crds storage kps loki tempo alloy node-exporter grafana blackbox core-policies targets core-rules alerting dashboards aggregate) + ;; + *) exit 43 ;; +esac +for name in "${names[@]}"; do + cat >"$out/$name.yaml" <"$out/$artifact.yaml" <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: gitea + namespace: gitea +type: Opaque +stringData: + assertions: "" + config_environment.sh: synthetic-fixture +--- +apiVersion: v1 +kind: Secret +metadata: + name: gitea-init + namespace: gitea +type: Opaque +stringData: + configure_gitea.sh: synthetic-fixture + configure_gpg_environment.sh: synthetic-fixture + init_directory_structure.sh: synthetic-fixture +--- +apiVersion: v1 +kind: Secret +metadata: + name: gitea-inline-config + namespace: gitea +type: Opaque +stringData: + _generals_: "" + actions: synthetic-fixture + cache: synthetic-fixture + database: synthetic-fixture + indexer: synthetic-fixture + metrics: synthetic-fixture + packages: synthetic-fixture + queue: synthetic-fixture + repository: synthetic-fixture + security: synthetic-fixture + server: synthetic-fixture + service: synthetic-fixture + session: synthetic-fixture +YAML + if [[ "$artifact" == gitea && \ + "${PLATFORM_SENSITIVE_SOURCE_FAKE_BASELINE_OIDC_KEYS:-0}" != 0 ]]; then + printf ' oauth2_client: synthetic-fixture\n' >>"$out/$artifact.yaml" + fi + if [[ "$artifact" == gitea-oidc && \ + "${PLATFORM_SENSITIVE_SOURCE_FAKE_OIDC_BASELINE_KEYS:-0}" == 0 ]]; then + printf ' oauth2_client: synthetic-fixture\n' >>"$out/$artifact.yaml" + fi + done +fi +if [[ "$label" == render-observability-core.sh ]]; then + if [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_CORE_MISSING:-0}" == 0 ]]; then + secret_name=alertmanager-observability-core-kube-pr-alertmanager + [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_UNKNOWN:-0}" == 0 ]] || secret_name=unexpected-rendered-secret + cat >"$out/kps.yaml" <>"$out/aggregate.yaml" <<'YAML' +--- +apiVersion: v1 +kind: Secret +metadata: + name: alertmanager-observability-core-kube-pr-alertmanager + namespace: observability +type: Opaque +data: + alertmanager.yaml: cmVjZWl2ZXJzOgotIG5hbWU6ICJudWxsIgpyb3V0ZToKICByZWNlaXZlcjogIm51bGwiCiAgcm91dGVzOiBbXQo= +YAML + fi + python3 - "$out" <<'PY' +import hashlib +import json +import os +import sys +import yaml + +directory = sys.argv[1] +artifacts = ( + "namespaces", "crds", "storage", "kps", "loki", "tempo", "alloy", + "node-exporter", "grafana", "blackbox", "core-policies", "targets", + "core-rules", "alerting", "dashboards", +) +with open(os.path.join(directory, "resource-index.tsv"), "w", encoding="utf-8") as target: + target.write("order\tartifact\tapiVersion\tkind\tnamespace\tname\tcanonicalSha256\n") + order = 0 + for artifact in artifacts: + with open(os.path.join(directory, artifact + ".yaml"), encoding="utf-8") as source: + for item in yaml.safe_load_all(source): + if item is None: + continue + order += 1 + metadata = item.get("metadata") or {} + identity = ( + str(item.get("apiVersion", "")), str(item.get("kind", "")), + str(metadata.get("namespace", "")), str(metadata.get("name", "")), + ) + canonical = json.dumps(item, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + target.write("\t".join((str(order), artifact, *identity, digest)) + "\n") +PY + if [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_INDEX_BAD:-0}" != 0 ]]; then + sed -i '2s/[0-9a-f]$/0/' "$out/resource-index.tsv" + fi +fi +if [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_GITEA_WRONG_ARTIFACT:-0}" != 0 && \ + "$label" == render-phase1.sh ]]; then + cat >"$out/namespaces.yaml" <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: gitea + namespace: gitea +type: Opaque +stringData: + assertions: synthetic-fixture + config_environment.sh: synthetic-fixture +YAML +fi +if [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_CORE_MISSING_GITEA_CROSSOVER:-0}" != 0 && \ + "$label" == render-phase1.sh ]]; then + cat >"$out/gitea.yaml" <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: gitea + namespace: gitea +type: Opaque +stringData: + assertions: synthetic-fixture + config_environment.sh: synthetic-fixture +YAML +fi +if [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_EXTRA_FILE:-0}" != 0 ]]; then + printf 'unexpected\n' >"$out/unexpected.bin" +fi +case "$label" in + render-phase1.sh) + emit_phase1_log yes + ;; + render-phase2.sh) + printf 'Validating the Phase 1 baseline and Gitea OIDC desired profile first.\n' + emit_phase1_log no + for name in phase2-namespaces aistor-local-pv aistor-network-policies \ + keycloak-operator platform-postgres-keycloak keycloak; do + printf 'Rendered %-24s %8d bytes\n' "$name" 123 + done + printf 'Verified %-24s SHA-256 %064d\n' aistor-operator-chart 3 + printf 'Rendered %-24s %8d bytes\n' aistor-operator 123 + printf 'Verified %-24s SHA-256 %064d\n' aistor-objectstore-chart 4 + printf 'Rendered %-24s %8d bytes\n' minio-aistor 123 + printf 'Preserved eight verified manifests for the AIStor apply handoff.\n' + printf 'Phase 2 rendering and source invariants passed.\n' + printf 'No live ObjectStore CRD or Kubernetes cluster access was required.\n' + printf 'Temporary Phase 2 rendered manifests and generated chart caches will be removed on exit.\n' + ;; + render-admin-services.sh) + emit_admin_metadata_log + printf '검증 %-18s SHA-256 %064d\n' pgadmin 5 + printf '검증 %-18s SHA-256 %064d\n' aistor-objectstore 6 + for name in admin-namespace pgadmin-local-pv coredns-custom aistor-admin-oidc pgadmin; do + printf '렌더 %-20s %8d bytes SHA-256 %064d\n' "$name" 123 7 + done + printf 'ADMIN SERVICES STATIC RENDER PASS\n' + ;; + render-observability-core.sh) + emit_core_log + ;; +esac +if [[ "${PLATFORM_SENSITIVE_SOURCE_FAKE_LOG_INJECTION_LABEL:-}" == "$label" ]]; then + printf 'password: should-never-be-printed-renderer-log\n' +fi +SH +chmod 0755 "$WORK/platform/scripts/validate/fake-renderer" +for renderer in \ + render-phase1.sh render-phase2.sh render-admin-services.sh \ + render-observability-core.sh render-observability-access.sh; do + cp -- "$WORK/platform/scripts/validate/fake-renderer" \ + "$WORK/platform/scripts/validate/$renderer" +done +: >"$WORK/renderer.log" +export PLATFORM_SENSITIVE_SOURCE_RENDER_LOG="$WORK/renderer.log" +scan_platform_renderer_secrets "$WORK/platform" >/dev/null || \ + fail 'private renderer orchestration rejected safe exact outputs' +expected_renderers=$'render-phase1.sh:handoff=yes\nrender-phase2.sh:handoff=yes\nrender-admin-services.sh:handoff=yes\nrender-observability-core.sh:handoff=yes\nrender-observability-access.sh:grafana:handoff=no' +[[ "$(cat "$WORK/renderer.log")" == "$expected_renderers" ]] || \ + fail 'private renderer orchestration did not run the exact renderer set' +pass 'private renderer orchestration scans the exact renderer set' +pass 'access renderer receives no verified output handoff' +pass 'core metadata is validated separately from exact YAML Secret inputs' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_INDEX_BAD=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'mismatched core canonical resource index was accepted' +[[ "$renderer_output" == *'observability-core resource index contract failed'* ]] || \ + fail 'mismatched core canonical resource index did not fail at its contract' +pass 'core canonical resource index mismatch fails closed' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_EXTRA_FILE=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'unknown renderer handoff entry was accepted' +[[ "$renderer_output" == *'renderer handoff entry contract failed'* ]] || \ + fail 'unknown renderer handoff entry did not fail at its entry contract' +pass 'unknown renderer handoff entries fail closed' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_UNKNOWN=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'non-allowlisted Secret from renderer orchestration was accepted' +[[ "$renderer_output" != *'alertmanager.yaml:'* ]] || fail 'renderer failure leaked manifest content' +pass 'private renderer orchestration rejects non-allowlisted Secret output' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_CORE_MISSING=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'missing allowlisted core Secret was accepted' +[[ "$renderer_output" == *'RENDERED_REQUIRED_SECRET_MISSING'* ]] || \ + fail 'missing allowlisted core Secret did not produce its filename-free finding' +pass 'private renderer orchestration requires the core allowlisted Secret' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_AGGREGATE_MISSING=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'core aggregate without its Alertmanager Secret was accepted' +[[ "$renderer_output" == *'RENDERED_REQUIRED_SECRET_MISSING'* ]] || \ + fail 'missing aggregate Alertmanager Secret did not produce its filename-free finding' +pass 'core aggregate requires its own exact Alertmanager Secret' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_GITEA_WRONG_ARTIFACT=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'allowlisted Gitea Secret in the wrong artifact was accepted' +pass 'Gitea rendered Secrets are pinned to their exact handoff artifacts' +: >"$WORK/renderer.log" +baseline_key_rc=0 +baseline_key_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_BASELINE_OIDC_KEYS=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || baseline_key_rc=$? +: >"$WORK/renderer.log" +oidc_key_rc=0 +oidc_key_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_OIDC_BASELINE_KEYS=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || oidc_key_rc=$? +(( baseline_key_rc != 0 )) || fail 'OIDC-only inline config keys were accepted in baseline Gitea' +(( oidc_key_rc != 0 )) || fail 'baseline inline config keys were accepted in OIDC Gitea' +[[ "$baseline_key_output$oidc_key_output" != *'should-never-be-printed'* ]] || \ + fail 'Gitea artifact-specific key rejection leaked content' +pass 'Gitea inline config keys are exact for baseline and OIDC artifacts' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_CORE_MISSING=1 \ + PLATFORM_SENSITIVE_SOURCE_FAKE_CORE_MISSING_GITEA_CROSSOVER=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'a Gitea Secret satisfied the missing core Secret requirement' +[[ "$renderer_output" == *'RENDERED_REQUIRED_SECRET_MISSING'* ]] || \ + fail 'missing exact core Secret did not produce its filename-free finding' +pass 'Gitea allowlist results cannot satisfy the core Secret requirement' +for admin_log_variant in \ + wrong-registry wrong-chart wrong-version bad-digest wrong-order extra; do + : >"$WORK/renderer.log" + renderer_rc=0 + renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_ADMIN_LOG_VARIANT="$admin_log_variant" \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? + (( renderer_rc != 0 )) || fail "admin $admin_log_variant metadata output was accepted" + [[ "$renderer_output" == *'unexpected renderer output contract: admin'* ]] || \ + fail "admin $admin_log_variant metadata did not fail at its output contract" + pass "admin metadata rejects $admin_log_variant output" +done +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_FAIL=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'nonzero access renderer was accepted' +[[ "$renderer_output" == *'renderer failed: observability-access RC=41'* ]] || \ + fail 'nonzero access renderer did not fail closed at the access boundary' +pass 'private renderer orchestration fails closed on access renderer failure' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_DUPLICATE_CORE=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'duplicate access CORE PASS marker was accepted' +pass 'access output requires exactly one ordered CORE PASS marker' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_MISSING_VERIFIED=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'access output with only seven Verified lines was accepted' +pass 'access output requires the exact eight ordered Verified lines' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_SWAP_RENDERED=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'access output with reordered Rendered lines was accepted' +pass 'access output requires the exact sixteen ordered Rendered lines' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_PATH=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'access path-bearing progress output was accepted' +[[ "$renderer_output" != *'should-never-be-printed'* ]] || \ + fail 'access path-bearing output leaked its path' +pass 'access output rejects path-bearing progress diagnostics' +: >"$WORK/renderer.log" +renderer_rc=0 +renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_ACCESS_SECRET=1 \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? +(( renderer_rc != 0 )) || fail 'unexpected access renderer Secret output was accepted' +[[ "$renderer_output" != *'should-never-be-printed'* ]] || \ + fail 'unexpected access output leaked content' +pass 'access renderer cannot smuggle an independently unscanned Secret' +for injected_renderer in \ + render-phase1.sh render-phase2.sh render-admin-services.sh \ + render-observability-core.sh; do + : >"$WORK/renderer.log" + renderer_rc=0 + renderer_output="$(PLATFORM_SENSITIVE_SOURCE_FAKE_LOG_INJECTION_LABEL="$injected_renderer" \ + scan_platform_renderer_secrets "$WORK/platform" 2>&1)" || renderer_rc=$? + (( renderer_rc != 0 )) || fail "$injected_renderer unexpected stdout was accepted" + [[ "$renderer_output" != *'should-never-be-printed'* ]] || \ + fail "$injected_renderer unexpected stdout leaked content" + pass "$injected_renderer stdout is restricted to its bounded grammar" +done +unset PLATFORM_SENSITIVE_SOURCE_RENDER_LOG + +new_fixture +mkdir -p "$WORK/fake-bin" +printf '#!/usr/bin/env bash\nexit 2\n' >"$WORK/fake-bin/find" +chmod 0755 "$WORK/fake-bin/find" +PLATFORM_SENSITIVE_SOURCE_TEST_PATH="$WORK/fake-bin:/usr/local/bin:/usr/bin:/bin" +find_error_rc=0 +output="$(run_scanner 2>&1)" || find_error_rc=$? +(( find_error_rc != 0 )) || fail 'find error was accepted' +[[ "$output" == *'SCANNER_ERROR'* ]] || { + printf '%s\n' "$output" >&2 + fail 'find error did not fail closed' +} +pass 'find errors fail closed' + +new_fixture +mkdir -p "$WORK/fake-bin" +printf '#!/usr/bin/env bash\nexit 2\n' >"$WORK/fake-bin/rg" +chmod 0755 "$WORK/fake-bin/rg" +printf 'ordinary input\n' >"$WORK/docs/plans/scanner-error.md" +PLATFORM_SENSITIVE_SOURCE_TEST_PATH="$WORK/fake-bin:/usr/local/bin:/usr/bin:/bin" +scanner_error_rc=0 +output="$(run_scanner 2>&1)" || scanner_error_rc=$? +(( scanner_error_rc != 0 )) || fail 'scanner error was accepted' +[[ "$output" == *'SCANNER_ERROR'* ]] || { + printf '%s\n' "$output" >&2 + fail 'scanner error did not fail closed' +} +pass 'rg errors fail closed' + +cleanup_before="$( + find /tmp -maxdepth 1 -type d -name 'platform-sensitive-source-test.??????' \ + -printf '%f\n' | LC_ALL=C sort +)" +PLATFORM_SENSITIVE_SOURCE_CLEANUP_PROBE=1 bash "$BASH_SOURCE" +cleanup_after="$( + find /tmp -maxdepth 1 -type d -name 'platform-sensitive-source-test.??????' \ + -printf '%f\n' | LC_ALL=C sort +)" +[[ "$cleanup_after" == "$cleanup_before" ]] || \ + fail 'focused cleanup probe left a platform-sensitive-source-test directory' +pass 'focused test cleanup leaves no temporary fixture residue' + +printf 'PLATFORM SENSITIVE SOURCE ASSERTION TEST PASS (%d assertions)\n' "$ASSERTIONS" diff --git a/scripts/validate/test-slack-webhook-recovery-dirfd.py b/scripts/validate/test-slack-webhook-recovery-dirfd.py new file mode 100644 index 0000000..a5ba6e9 --- /dev/null +++ b/scripts/validate/test-slack-webhook-recovery-dirfd.py @@ -0,0 +1,1397 @@ +#!/usr/bin/env python3 + +import importlib.util +import json +import os +import pathlib +import signal +import stat +import subprocess +import sys +import tempfile +import argparse +import unittest +from unittest import mock + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +PROD = ROOT / "scripts/libexec/slack-webhook-recovery-dirfd.py" +KDBX_BYTES = ( + b"\x03\xd9\xa2\x9a\x67\xfb\x4b\xb5" + b"\x01\x00\x04\x00" + b"synthetic-keepass-payload-for-task-2\n" +) +MAIN_RELATIVE = ("HyeonworksRecovery", "vault", "hyeonworks-recovery.kdbx") +BACKUP_NAME = "hyeonworks-recovery.pre-slack-20260813T060000Z.kdbx" + + +def load_helper(): + spec = importlib.util.spec_from_file_location("swr_dirfd_helper", PROD) + if spec is None or spec.loader is None: + raise AssertionError("production dirfd helper is absent") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class HelperPresenceTest(unittest.TestCase): + def test_production_helper_exists_and_exports_open_directory_chain(self): + self.assertTrue(PROD.is_file(), "production dirfd helper is absent") + helper = load_helper() + self.assertTrue( + hasattr(helper, "open_directory_chain"), + "open_directory_chain is absent", + ) + + +class DirfdHelperTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not PROD.is_file(): + raise unittest.SkipTest("production dirfd helper is absent") + cls.helper = load_helper() + if not hasattr(cls.helper, "open_directory_chain"): + raise unittest.SkipTest("open_directory_chain is absent") + + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + self.addCleanup(self.tempdir.cleanup) + self.root = pathlib.Path(self.tempdir.name) + self.mount_root = self.root / "mount-root" + self.runtime = self.root / "runtime" + self.destination = self.runtime / "baseline.kdbx" + self.baseline_path = self.runtime / "baseline.json" + self.candidate_path = self.runtime / "candidate.kdbx" + self.snapshot_current_path = self.runtime / "current.kdbx" + self.result_path = self.runtime / "result.json" + self.runtime.mkdir(mode=0o700) + self._build_mount_root(self.mount_root, KDBX_BYTES) + + def _build_mount_root(self, mount_root: pathlib.Path, main_bytes: bytes): + backups = mount_root / "HyeonworksRecovery" / "vault" / "backups" + backups.mkdir(parents=True, mode=0o700) + for directory in ( + mount_root, + mount_root / "HyeonworksRecovery", + mount_root / "HyeonworksRecovery" / "vault", + backups, + ): + directory.chmod(0o700) + main_path = mount_root.joinpath(*MAIN_RELATIVE) + main_path.write_bytes(main_bytes) + main_path.chmod(0o600) + return main_path + + def _main_path(self, mount_root: pathlib.Path | None = None) -> pathlib.Path: + base = self.mount_root if mount_root is None else mount_root + return base.joinpath(*MAIN_RELATIVE) + + def _snapshot_state(self): + state = self.helper.snapshot_database( + os.fspath(self.mount_root), + os.fspath(self.destination), + ) + self.helper.write_result(os.fspath(self.baseline_path), state) + return state + + def _commit(self, backup_name: str = BACKUP_NAME): + return self.helper.commit_database( + os.fspath(self.mount_root), + os.fspath(self.baseline_path), + os.fspath(self.candidate_path), + backup_name, + ) + + def _supplementary_gid(self) -> int: + gids = [gid for gid in os.getgroups() if gid != 1000] + if not gids: + self.skipTest("no alternate supplementary gid is available") + return gids[0] + + def test_snapshot_copies_exact_database_and_private_state(self): + state = self._snapshot_state() + + self.assertEqual(self.destination.read_bytes(), KDBX_BYTES) + self.assertEqual(state["schema"], "platform-slack-keepass-fs-v1") + self.assertEqual(state["operation"], "snapshot") + self.assertEqual(state["source"]["relative_path"], "/".join(MAIN_RELATIVE)) + self.assertEqual(state["source"]["size"], len(KDBX_BYTES)) + self.assertEqual(state["source"]["mode"], "0600") + self.assertEqual(state["mount"]["relative_vault"], "HyeonworksRecovery/vault") + + reloaded = json.loads(self.baseline_path.read_text(encoding="utf-8")) + self.assertEqual(reloaded["schema"], state["schema"]) + self.assertEqual(reloaded["source"]["sha256"], state["source"]["sha256"]) + + def test_commit_command_prearms_and_persists_rename_attempt_status(self): + # Production break caught: the coordinator loses the helper after the + # rename attempt and sees no durable state with which to prohibit retry. + args = argparse.Namespace( + mount_root=os.fspath(self.mount_root), + baseline=os.fspath(self.baseline_path), + candidate=os.fspath(self.candidate_path), + backup_name=BACKUP_NAME, + result=os.fspath(self.result_path), + ) + + def lost_after_rename(*_args, status_hook=None): + self.assertEqual( + json.loads(self.result_path.read_text(encoding="utf-8"))["fs_commit"], + "precommit-failure", + ) + self.assertIsNotNone(status_hook) + status_hook("rename-attempted") + raise RuntimeError("synthetic lost helper response") + + with mock.patch.object(self.helper, "commit_database", side_effect=lost_after_rename): + with self.assertRaises(RuntimeError): + self.helper._cmd_commit(args) + + persisted = json.loads(self.result_path.read_text(encoding="utf-8")) + self.assertEqual( + persisted, + { + "fs_commit": "rename-attempted", + "operation": "commit-status", + "schema": "platform-slack-keepass-fs-v1", + }, + ) + + def test_snapshot_rejects_symlinks_in_every_path_component(self): + for index, part in enumerate(("HyeonworksRecovery", "vault", "hyeonworks-recovery.kdbx")): + with self.subTest(component=part): + mount_root = self.root / f"mount-symlink-{index}" + mount_root.mkdir(mode=0o700) + (mount_root / "real-target").mkdir(mode=0o700) + (mount_root / "real-target" / "vault").mkdir(mode=0o700) + (mount_root / "real-target" / "vault" / "backups").mkdir(mode=0o700) + (mount_root / "real-target" / "vault" / "hyeonworks-recovery.kdbx").write_bytes(KDBX_BYTES) + (mount_root / "real-target" / "vault" / "hyeonworks-recovery.kdbx").chmod(0o600) + target = mount_root / "real-target" + if part == "HyeonworksRecovery": + (mount_root / "HyeonworksRecovery").symlink_to(target) + elif part == "vault": + (mount_root / "HyeonworksRecovery").mkdir(mode=0o700) + (mount_root / "HyeonworksRecovery" / "vault").symlink_to(target / "vault") + else: + (mount_root / "HyeonworksRecovery").mkdir(mode=0o700) + (mount_root / "HyeonworksRecovery" / "vault").mkdir(mode=0o700) + (mount_root / "HyeonworksRecovery" / "vault" / "backups").mkdir(mode=0o700) + (mount_root / "HyeonworksRecovery" / "vault" / "hyeonworks-recovery.kdbx").symlink_to( + target / "vault" / "hyeonworks-recovery.kdbx" + ) + with self.assertRaises(Exception): + self.helper.snapshot_database( + os.fspath(mount_root), + os.fspath(self.runtime / f"symlink-{index}.kdbx"), + ) + + def test_snapshot_rejects_symlink_mount_root(self): + real_mount = self.root / "real-mount" + self._build_mount_root(real_mount, KDBX_BYTES) + symlink_mount = self.root / "symlink-mount" + symlink_mount.symlink_to(real_mount, target_is_directory=True) + + with self.assertRaises(Exception): + self.helper.snapshot_database( + os.fspath(symlink_mount), + os.fspath(self.destination), + ) + + def test_snapshot_rejects_nonprivate_mode_on_each_traversed_mount_directory(self): + directories = ( + self.mount_root, + self.mount_root / "HyeonworksRecovery", + self.mount_root / "HyeonworksRecovery" / "vault", + ) + for directory in directories: + with self.subTest(directory=directory.name): + self.setUp() + directory = { + "mount-root": self.mount_root, + "HyeonworksRecovery": self.mount_root / "HyeonworksRecovery", + "vault": self.mount_root / "HyeonworksRecovery" / "vault", + }[directory.name] + directory.chmod(0o777) + with self.assertRaises(Exception): + self.helper.snapshot_database( + os.fspath(self.mount_root), + os.fspath(self.destination), + ) + + def test_snapshot_rejects_alternate_gid_on_each_traversed_mount_directory(self): + alternate_gid = self._supplementary_gid() + relative_directories = ((), ("HyeonworksRecovery",), ("HyeonworksRecovery", "vault")) + for parts in relative_directories: + with self.subTest(parts=parts): + self.setUp() + directory = self.mount_root.joinpath(*parts) + os.chown(directory, -1, alternate_gid) + with self.assertRaises(Exception): + self.helper.snapshot_database( + os.fspath(self.mount_root), + os.fspath(self.destination), + ) + + def test_commit_rejects_nonprivate_or_alternate_gid_backups_directory(self): + alternate_gid = self._supplementary_gid() + for case in ("mode", "gid"): + with self.subTest(case=case): + self.setUp() + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + backups = self.mount_root / "HyeonworksRecovery" / "vault" / "backups" + if case == "mode": + backups.chmod(0o777) + else: + os.chown(backups, -1, alternate_gid) + with self.assertRaises(Exception): + self._commit() + + def test_snapshot_rejects_main_kdbx_owned_by_supplementary_gid(self): + os.chown(self._main_path(), -1, self._supplementary_gid()) + + with self.assertRaises(Exception): + self.helper.snapshot_database( + os.fspath(self.mount_root), + os.fspath(self.destination), + ) + + def test_snapshot_rejects_kdbx_symlink_and_hardlink(self): + other = self.mount_root / "other.kdbx" + other.write_bytes(KDBX_BYTES) + other.chmod(0o600) + main_path = self._main_path() + main_path.unlink() + main_path.symlink_to(other) + with self.assertRaises(Exception): + self.helper.snapshot_database(os.fspath(self.mount_root), os.fspath(self.destination)) + + self.setUp() + main_path = self._main_path() + hardlink = self.mount_root / "hardlink.kdbx" + os.link(main_path, hardlink) + with self.assertRaises(Exception): + self.helper.snapshot_database(os.fspath(self.mount_root), os.fspath(self.destination)) + + def test_snapshot_rejects_wrong_type_and_mode(self): + with self.subTest(case="wrong-type"): + main_path = self._main_path() + main_path.unlink() + main_path.mkdir(mode=0o700) + with self.assertRaises(Exception): + self.helper.snapshot_database(os.fspath(self.mount_root), os.fspath(self.destination)) + + self.setUp() + with self.subTest(case="wrong-mode"): + self._main_path().chmod(0o644) + with self.assertRaises(Exception): + self.helper.snapshot_database(os.fspath(self.mount_root), os.fspath(self.destination)) + + def test_snapshot_staging_requires_exclusive_nofollow_destination(self): + self.destination.write_bytes(b"occupied") + self.destination.chmod(0o600) + with self.assertRaises(Exception): + self.helper.snapshot_database(os.fspath(self.mount_root), os.fspath(self.destination)) + + self.destination.unlink() + target = self.runtime / "target.kdbx" + target.write_bytes(b"target") + target.chmod(0o600) + self.destination.symlink_to(target) + with self.assertRaises(Exception): + self.helper.snapshot_database(os.fspath(self.mount_root), os.fspath(self.destination)) + + def test_snapshot_commands_reject_main_replacement_after_source_open(self): + # Production break caught: a retained main FD is copied after its + # canonical pathname has been replaced, publishing a stale snapshot. + for operation in ("snapshot", "snapshot-current"): + with self.subTest(operation=operation): + self.setUp() + retained_main = self.runtime / f"{operation}.retained-main.kdbx" + drift_bytes = KDBX_BYTES + b"canonical-drift\n" + original_open_regular_at = self.helper.open_regular_at + replaced = False + + if operation == "snapshot-current": + self.candidate_path.write_bytes(KDBX_BYTES) + self.candidate_path.chmod(0o600) + + def replace_after_source_open(dir_fd, name, **kwargs): + nonlocal replaced + fd = original_open_regular_at(dir_fd, name, **kwargs) + if not replaced and name == MAIN_RELATIVE[-1]: + replaced = True + self._main_path().rename(retained_main) + self._main_path().write_bytes(drift_bytes) + self._main_path().chmod(0o600) + return fd + + with mock.patch.object( + self.helper, + "open_regular_at", + side_effect=replace_after_source_open, + ): + with self.assertRaises(self.helper.FsContractError): + if operation == "snapshot": + self.helper.snapshot_database( + os.fspath(self.mount_root), + os.fspath(self.destination), + ) + else: + self.helper.snapshot_committed_database( + os.fspath(self.mount_root), + os.fspath(self.candidate_path), + os.fspath(self.snapshot_current_path), + ) + + self.assertTrue(replaced) + destination = ( + self.destination + if operation == "snapshot" + else self.snapshot_current_path + ) + self.assertFalse(destination.exists()) + self.assertEqual(self._main_path().read_bytes(), drift_bytes) + + def test_snapshot_commands_remove_destination_on_canonical_tree_rebind_after_copy(self): + # Production break caught: copying from a retained old tree succeeds + # even though the canonical mount tree is rebound before publication. + for operation in ("snapshot", "snapshot-current"): + with self.subTest(operation=operation): + self.setUp() + detached_recovery = self.mount_root / f"HyeonworksRecovery.{operation}.detached" + drift_bytes = KDBX_BYTES + b"canonical-tree-drift\n" + real_stable_copy = self.helper.stable_copy_fd + rebound = False + + if operation == "snapshot-current": + self.candidate_path.write_bytes(KDBX_BYTES) + self.candidate_path.chmod(0o600) + + def rebind_after_copy(src_fd, dst_fd, **kwargs): + nonlocal rebound + copied = real_stable_copy(src_fd, dst_fd, **kwargs) + if not rebound: + rebound = True + recovery = self.mount_root / "HyeonworksRecovery" + recovery.rename(detached_recovery) + self._build_mount_root(self.mount_root, drift_bytes) + return copied + + with mock.patch.object( + self.helper, + "stable_copy_fd", + side_effect=rebind_after_copy, + ): + with self.assertRaises(self.helper.FsContractError): + if operation == "snapshot": + self.helper.snapshot_database( + os.fspath(self.mount_root), + os.fspath(self.destination), + ) + else: + self.helper.snapshot_committed_database( + os.fspath(self.mount_root), + os.fspath(self.candidate_path), + os.fspath(self.snapshot_current_path), + ) + + self.assertTrue(rebound) + destination = ( + self.destination + if operation == "snapshot" + else self.snapshot_current_path + ) + self.assertFalse(destination.exists()) + self.assertEqual(self._main_path().read_bytes(), drift_bytes) + + def test_snapshot_commands_recheck_canonical_tree_immediately_before_publication(self): + # Production break caught: the canonical tree is rebound while the + # post-copy canonical traversal is already retained but before return. + for operation in ("snapshot", "snapshot-current"): + with self.subTest(operation=operation): + self.setUp() + detached_recovery = self.mount_root / f"HyeonworksRecovery.{operation}.late-detached" + drift_bytes = KDBX_BYTES + b"late-canonical-tree-drift\n" + original_open_mount_tree = self.helper._open_mount_tree + open_count = 0 + rebound = False + + if operation == "snapshot-current": + self.candidate_path.write_bytes(KDBX_BYTES) + self.candidate_path.chmod(0o600) + + def rebind_during_postcopy_reopen(stack, mount_root, *, include_backups): + nonlocal open_count, rebound + opened = original_open_mount_tree( + stack, + mount_root, + include_backups=include_backups, + ) + open_count += 1 + if open_count == 3 and not rebound: + rebound = True + recovery = self.mount_root / "HyeonworksRecovery" + recovery.rename(detached_recovery) + self._build_mount_root(self.mount_root, drift_bytes) + return opened + + with mock.patch.object( + self.helper, + "_open_mount_tree", + side_effect=rebind_during_postcopy_reopen, + ): + with self.assertRaises(self.helper.FsContractError): + if operation == "snapshot": + self.helper.snapshot_database( + os.fspath(self.mount_root), + os.fspath(self.destination), + ) + else: + self.helper.snapshot_committed_database( + os.fspath(self.mount_root), + os.fspath(self.candidate_path), + os.fspath(self.snapshot_current_path), + ) + + self.assertTrue(rebound) + destination = ( + self.destination + if operation == "snapshot" + else self.snapshot_current_path + ) + self.assertFalse(destination.exists()) + self.assertEqual(self._main_path().read_bytes(), drift_bytes) + + def test_snapshot_rejects_generated_destination_with_wrong_gid(self): + alternate_gid = self._supplementary_gid() + real_create = self.helper._create_private_file + + def create_with_wrong_snapshot_gid(dir_fd, name, **kwargs): + fd = real_create(dir_fd, name, **kwargs) + if name == self.destination.name: + os.fchown(fd, -1, alternate_gid) + return fd + + with mock.patch.object( + self.helper, + "_create_private_file", + side_effect=create_with_wrong_snapshot_gid, + ): + with self.assertRaises(Exception): + self.helper.snapshot_database( + os.fspath(self.mount_root), + os.fspath(self.destination), + ) + + def test_write_result_rejects_generated_file_with_wrong_gid(self): + alternate_gid = self._supplementary_gid() + real_create = self.helper._create_private_file + + def create_with_wrong_result_gid(dir_fd, name, **kwargs): + fd = real_create(dir_fd, name, **kwargs) + if name.startswith(f".{self.result_path.name}.tmp."): + os.fchown(fd, -1, alternate_gid) + return fd + + with mock.patch.object( + self.helper, + "_create_private_file", + side_effect=create_with_wrong_result_gid, + ): + with self.assertRaises(Exception): + self.helper.write_result( + os.fspath(self.result_path), + {"schema": "synthetic"}, + ) + + self.assertFalse(self.result_path.exists()) + + def test_commit_rejects_baseline_drift_and_backup_collision(self): + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + self._main_path().write_bytes(KDBX_BYTES + b"drift\n") + self._main_path().chmod(0o600) + with self.assertRaises(Exception): + self._commit() + + self.setUp() + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + backup = self.mount_root / "HyeonworksRecovery" / "vault" / "backups" / BACKUP_NAME + backup.write_bytes(b"collision") + backup.chmod(0o600) + with self.assertRaises(Exception): + self._commit() + + def test_commit_rejects_candidate_owned_by_supplementary_gid(self): + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + os.chown(self.candidate_path, -1, self._supplementary_gid()) + + with self.assertRaises(Exception): + self._commit() + + def test_commit_rejects_baseline_owned_by_supplementary_gid(self): + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + os.chown(self.baseline_path, -1, self._supplementary_gid()) + + with self.assertRaises(Exception): + self._commit() + + def test_status_rejects_result_owned_by_supplementary_gid(self): + self.helper.write_result( + os.fspath(self.result_path), + { + "schema": "platform-slack-keepass-fs-v1", + "operation": "commit-status", + "fs_commit": "precommit-failure", + }, + ) + os.chown(self.result_path, -1, self._supplementary_gid()) + + with self.assertRaises(Exception): + self.helper.read_commit_status(os.fspath(self.result_path)) + + def test_commit_rejects_generated_backup_with_wrong_gid_before_replace(self): + self._snapshot_state() + original_bytes = self._main_path().read_bytes() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + alternate_gid = self._supplementary_gid() + real_create = self.helper._create_private_file + + def create_with_wrong_backup_gid(dir_fd, name, **kwargs): + fd = real_create(dir_fd, name, **kwargs) + if name == BACKUP_NAME: + os.fchown(fd, -1, alternate_gid) + return fd + + with mock.patch.object( + self.helper, + "_create_private_file", + side_effect=create_with_wrong_backup_gid, + ): + with self.assertRaises(Exception): + self._commit() + + self.assertEqual(self._main_path().read_bytes(), original_bytes) + + def test_regular_file_uid_and_gid_attestation_targets_each_exact_role(self): + # Production break caught: a role-specific ownership check is absent, + # while unrelated runtime-parent metadata can remain fully valid. + roles = ( + "main", + "baseline", + "candidate", + "generated-snapshot", + "generated-result", + "generated-backup", + "reopened-backup", + "stage", + "installed-main", + ) + for identity_field in ("uid", "gid"): + for role in roles: + with self.subTest(identity_field=identity_field, role=role): + self.setUp() + if role not in {"main", "generated-snapshot", "generated-result"}: + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate-role-attestation\n") + self.candidate_path.chmod(0o600) + + real_fstat = self.helper.os.fstat + real_replace = self.helper.os.replace + path_counts = {} + published = False + targeted = False + + def recording_replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None): + nonlocal published + result = real_replace( + src, + dst, + src_dir_fd=src_dir_fd, + dst_dir_fd=dst_dir_fd, + ) + if dst == MAIN_RELATIVE[-1]: + published = True + return result + + def fstat_with_targeted_identity(fd): + nonlocal targeted + observed = real_fstat(fd) + try: + fd_path = os.readlink(f"/proc/self/fd/{fd}") + except OSError: + return observed + path_counts[fd_path] = path_counts.get(fd_path, 0) + 1 + count = path_counts[fd_path] + basename = pathlib.Path(fd_path).name + is_target = False + if role == "main": + is_target = fd_path == os.fspath(self._main_path()) + elif role == "baseline": + is_target = fd_path == os.fspath(self.baseline_path) + elif role == "candidate": + is_target = fd_path == os.fspath(self.candidate_path) + elif role == "generated-snapshot": + is_target = fd_path == os.fspath(self.destination) and count == 3 + elif role == "generated-result": + is_target = basename.startswith(f".{self.result_path.name}.tmp.") and count == 3 + elif role == "generated-backup": + is_target = basename == BACKUP_NAME and count == 2 + elif role == "reopened-backup": + is_target = basename == BACKUP_NAME and count == 4 + elif role == "stage": + is_target = basename.startswith(f".{MAIN_RELATIVE[-1]}.stage.") and count == 3 + elif role == "installed-main": + is_target = published and fd_path.endswith(f"/vault/{MAIN_RELATIVE[-1]}") + if not is_target or targeted: + return observed + targeted = True + values = list(observed) + index = 4 if identity_field == "uid" else 5 + values[index] = values[index] + 1 + return os.stat_result(values) + + with mock.patch.object( + self.helper.os, + "fstat", + side_effect=fstat_with_targeted_identity, + ), mock.patch.object( + self.helper.os, + "replace", + side_effect=recording_replace, + ): + if role == "main": + with self.assertRaises(self.helper.FsContractError): + self.helper.snapshot_database( + os.fspath(self.mount_root), + os.fspath(self.destination), + ) + elif role == "generated-snapshot": + with self.assertRaises(self.helper.FsContractError): + self.helper.snapshot_database( + os.fspath(self.mount_root), + os.fspath(self.destination), + ) + self.assertFalse(self.destination.exists()) + elif role == "generated-result": + with self.assertRaises(self.helper.FsContractError): + self.helper.write_result( + os.fspath(self.result_path), + {"schema": "synthetic"}, + ) + self.assertFalse(self.result_path.exists()) + elif role == "installed-main": + result = self._commit() + self.assertEqual(result["fs_commit"], "committed-but-uncertain") + else: + with self.assertRaises(self.helper.FsContractError): + self._commit() + + self.assertTrue(targeted) + + def test_commit_rejects_backup_path_replacement_with_identical_bytes(self): + self._snapshot_state() + original_bytes = self._main_path().read_bytes() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + backups = self.mount_root / "HyeonworksRecovery" / "vault" / "backups" + backup_path = backups / BACKUP_NAME + retained_name = backups / f"{BACKUP_NAME}.retained" + real_fsync = self.helper.os.fsync + replaced = False + + def replace_after_backup_copy(fd): + nonlocal replaced + fd_path = os.readlink(f"/proc/self/fd/{fd}") + result = real_fsync(fd) + if not replaced and fd_path.endswith("/backups"): + replaced = True + backup_path.rename(retained_name) + backup_path.write_bytes(original_bytes) + backup_path.chmod(0o600) + return result + + with mock.patch.object(self.helper.os, "fsync", side_effect=replace_after_backup_copy): + with self.assertRaises(Exception): + self._commit() + + self.assertTrue(replaced) + self.assertEqual(self._main_path().read_bytes(), original_bytes) + self.assertEqual(backup_path.read_bytes(), original_bytes) + self.assertEqual(retained_name.read_bytes(), original_bytes) + + def test_commit_rejects_detached_vault_tree_before_any_write(self): + self._snapshot_state() + original_bytes = self._main_path().read_bytes() + candidate_bytes = KDBX_BYTES + b"candidate\n" + self.candidate_path.write_bytes(candidate_bytes) + self.candidate_path.chmod(0o600) + original_open_chain = self.helper.open_directory_chain + detached_recovery = self.mount_root / "HyeonworksRecovery.detached" + renamed = False + + def rename_before_backups_open(root_fd, parts): + nonlocal renamed + root_path = pathlib.Path(os.readlink(f"/proc/self/fd/{root_fd}")) + if not renamed and tuple(parts) == ("backups",) and root_path.name == "vault": + renamed = True + recovery = self.mount_root / "HyeonworksRecovery" + recovery.rename(detached_recovery) + self._build_mount_root(self.mount_root, original_bytes) + return original_open_chain(root_fd, parts) + + with mock.patch.object( + self.helper, + "open_directory_chain", + side_effect=rename_before_backups_open, + ): + with self.assertRaises(Exception): + self._commit() + + self.assertTrue(renamed) + canonical_main = self._main_path() + detached_vault = detached_recovery / "vault" + detached_main = detached_vault / MAIN_RELATIVE[-1] + self.assertEqual(canonical_main.read_bytes(), original_bytes) + self.assertEqual(detached_main.read_bytes(), original_bytes) + self.assertEqual(list((self.mount_root / "HyeonworksRecovery" / "vault" / "backups").iterdir()), []) + self.assertEqual(list((detached_vault / "backups").iterdir()), []) + self.assertEqual( + [path for path in detached_vault.iterdir() if path.name.startswith(f".{MAIN_RELATIVE[-1]}.stage.")], + [], + ) + + def test_commit_backup_proof_failure_stops_before_replace(self): + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + + replace_calls = [] + with mock.patch.object(self.helper, "files_equal_fd", return_value=False), mock.patch.object( + self.helper.os, + "replace", + side_effect=lambda *args, **kwargs: replace_calls.append((args, kwargs)), + ): + with self.assertRaises(Exception): + self._commit() + self.assertEqual(replace_calls, []) + + def test_commit_never_verifies_when_recovery_tree_is_rebound_inside_replace(self): + # Production break caught: stage-to-main replace runs on a newly + # detached retained vault and is falsely reported as verified. + self._snapshot_state() + original_bytes = self._main_path().read_bytes() + candidate_bytes = KDBX_BYTES + b"candidate-inside-replace\n" + self.candidate_path.write_bytes(candidate_bytes) + self.candidate_path.chmod(0o600) + detached_recovery = self.mount_root / "HyeonworksRecovery.detached-during-replace" + real_replace = self.helper.os.replace + replace_calls = [] + rebound = False + + def rebind_inside_replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None): + nonlocal rebound + replace_calls.append((src, dst, src_dir_fd, dst_dir_fd)) + if not rebound and dst == MAIN_RELATIVE[-1] and src.startswith(f".{MAIN_RELATIVE[-1]}.stage."): + rebound = True + recovery = self.mount_root / "HyeonworksRecovery" + recovery.rename(detached_recovery) + self._build_mount_root(self.mount_root, original_bytes) + return real_replace(src, dst, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd) + + with mock.patch.object( + self.helper.os, + "replace", + side_effect=rebind_inside_replace, + ): + result = self._commit() + + self.assertTrue(rebound) + self.assertEqual(result["fs_commit"], "committed-but-uncertain") + self.assertEqual(len(replace_calls), 1) + self.assertEqual(self._main_path().read_bytes(), original_bytes) + detached_vault = detached_recovery / "vault" + self.assertEqual((detached_vault / MAIN_RELATIVE[-1]).read_bytes(), candidate_bytes) + self.assertEqual((detached_vault / "backups" / BACKUP_NAME).read_bytes(), original_bytes) + self.assertEqual( + [path for path in detached_vault.iterdir() if path.name.startswith(f".{MAIN_RELATIVE[-1]}.stage.")], + [], + ) + + def test_commit_rejects_canonical_main_replacement_after_final_pre_replace_open(self): + # Production break caught: the final baseline check uses a retained FD + # but never proves MAIN_NAME still names it immediately before replace. + self._snapshot_state() + original_bytes = self._main_path().read_bytes() + drift_bytes = KDBX_BYTES + b"pre-replace-drift\n" + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + retained_main = self.mount_root / "HyeonworksRecovery" / "vault" / "pre-replace-retained.kdbx" + original_open_regular_at = self.helper.open_regular_at + real_replace = self.helper.os.replace + main_open_count = 0 + replace_calls = [] + + def replace_after_final_main_open(dir_fd, name, **kwargs): + nonlocal main_open_count + fd = original_open_regular_at(dir_fd, name, **kwargs) + if name == MAIN_RELATIVE[-1]: + main_open_count += 1 + if main_open_count == 2: + self._main_path().rename(retained_main) + self._main_path().write_bytes(drift_bytes) + self._main_path().chmod(0o600) + return fd + + def recording_replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None): + replace_calls.append((src, dst, src_dir_fd, dst_dir_fd)) + return real_replace(src, dst, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd) + + with mock.patch.object( + self.helper, + "open_regular_at", + side_effect=replace_after_final_main_open, + ), mock.patch.object(self.helper.os, "replace", side_effect=recording_replace): + with self.assertRaises(self.helper.FsContractError): + self._commit() + + self.assertEqual(main_open_count, 2) + self.assertEqual(replace_calls, []) + self.assertEqual(self._main_path().read_bytes(), drift_bytes) + self.assertEqual(retained_main.read_bytes(), original_bytes) + backup_path = self.mount_root / "HyeonworksRecovery" / "vault" / "backups" / BACKUP_NAME + self.assertEqual(backup_path.read_bytes(), original_bytes) + vault = self.mount_root / "HyeonworksRecovery" / "vault" + self.assertEqual( + [path for path in vault.iterdir() if path.name.startswith(f".{MAIN_RELATIVE[-1]}.stage.")], + [], + ) + + def test_commit_never_verifies_canonical_rebind_after_postpublication_sync(self): + # Production break caught: canonical ancestry changes after the stage + # has been published and synced but before verified status is returned. + self._snapshot_state() + original_bytes = self._main_path().read_bytes() + candidate_bytes = KDBX_BYTES + b"candidate-postpublication\n" + self.candidate_path.write_bytes(candidate_bytes) + self.candidate_path.chmod(0o600) + detached_recovery = self.mount_root / "HyeonworksRecovery.detached-postpublication" + real_replace = self.helper.os.replace + real_syncfs = self.helper.sync_filesystem_fd + replace_calls = [] + published = False + rebound = False + + def recording_replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None): + nonlocal published + replace_calls.append((src, dst, src_dir_fd, dst_dir_fd)) + result = real_replace(src, dst, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd) + if dst == MAIN_RELATIVE[-1]: + published = True + return result + + def rebind_after_postpublication_sync(fd): + nonlocal rebound + result = real_syncfs(fd) + if published and not rebound: + rebound = True + recovery = self.mount_root / "HyeonworksRecovery" + recovery.rename(detached_recovery) + self._build_mount_root(self.mount_root, original_bytes) + return result + + with mock.patch.object( + self.helper.os, + "replace", + side_effect=recording_replace, + ), mock.patch.object( + self.helper, + "sync_filesystem_fd", + side_effect=rebind_after_postpublication_sync, + ): + result = self._commit() + + self.assertTrue(rebound) + self.assertEqual(result["fs_commit"], "committed-but-uncertain") + self.assertEqual(len(replace_calls), 1) + self.assertEqual(self._main_path().read_bytes(), original_bytes) + detached_vault = detached_recovery / "vault" + self.assertEqual((detached_vault / MAIN_RELATIVE[-1]).read_bytes(), candidate_bytes) + self.assertEqual((detached_vault / "backups" / BACKUP_NAME).read_bytes(), original_bytes) + + def test_commit_rechecks_canonical_tree_immediately_before_verified_status(self): + # Production break caught: a canonical traversal retained after sync + # is detached before verified status is selected. + self._snapshot_state() + original_bytes = self._main_path().read_bytes() + candidate_bytes = KDBX_BYTES + b"candidate-late-status\n" + self.candidate_path.write_bytes(candidate_bytes) + self.candidate_path.chmod(0o600) + detached_recovery = self.mount_root / "HyeonworksRecovery.detached-late-status" + original_open_mount_tree = self.helper._open_mount_tree + real_replace = self.helper.os.replace + published = False + rebound = False + replace_calls = [] + + def recording_replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None): + nonlocal published + replace_calls.append((src, dst, src_dir_fd, dst_dir_fd)) + result = real_replace(src, dst, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd) + if dst == MAIN_RELATIVE[-1]: + published = True + return result + + def rebind_during_first_postpublication_reopen(stack, mount_root, *, include_backups): + nonlocal rebound + opened = original_open_mount_tree( + stack, + mount_root, + include_backups=include_backups, + ) + if published and not rebound: + rebound = True + recovery = self.mount_root / "HyeonworksRecovery" + recovery.rename(detached_recovery) + self._build_mount_root(self.mount_root, original_bytes) + return opened + + with mock.patch.object( + self.helper.os, + "replace", + side_effect=recording_replace, + ), mock.patch.object( + self.helper, + "_open_mount_tree", + side_effect=rebind_during_first_postpublication_reopen, + ): + result = self._commit() + + self.assertTrue(rebound) + self.assertEqual(result["fs_commit"], "committed-but-uncertain") + self.assertEqual(len(replace_calls), 1) + self.assertEqual(self._main_path().read_bytes(), original_bytes) + detached_vault = detached_recovery / "vault" + self.assertEqual((detached_vault / MAIN_RELATIVE[-1]).read_bytes(), candidate_bytes) + self.assertEqual((detached_vault / "backups" / BACKUP_NAME).read_bytes(), original_bytes) + + def test_commit_uses_dirfd_replace_and_defers_signals(self): + self._snapshot_state() + candidate_bytes = KDBX_BYTES + b"candidate\n" + self.candidate_path.write_bytes(candidate_bytes) + self.candidate_path.chmod(0o600) + + replace_calls = [] + mask_calls = [] + real_replace = self.helper.os.replace + real_pthread_sigmask = getattr(signal, "pthread_sigmask") + + def recording_replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None): + replace_calls.append((src, dst, src_dir_fd, dst_dir_fd)) + return real_replace(src, dst, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd) + + def recording_sigmask(how, mask): + mask_calls.append((how, tuple(sorted(mask)))) + if len(mask_calls) == 1: + return set() + return set(mask) + + with mock.patch.object(self.helper.os, "replace", side_effect=recording_replace), mock.patch.object( + self.helper.signal, + "pthread_sigmask", + side_effect=recording_sigmask, + ): + result = self._commit() + + self.assertEqual(result["fs_commit"], "verified-commit") + self.assertEqual(self._main_path().read_bytes(), candidate_bytes) + self.assertEqual(len(replace_calls), 1) + src, dst, src_dir_fd, dst_dir_fd = replace_calls[0] + self.assertEqual(dst, MAIN_RELATIVE[-1]) + self.assertNotEqual(src, os.fspath(self.candidate_path)) + self.assertIsInstance(src_dir_fd, int) + self.assertIsInstance(dst_dir_fd, int) + self.assertEqual(len(mask_calls), 2) + self.assertEqual(mask_calls[0][0], signal.SIG_BLOCK) + self.assertEqual(set(mask_calls[0][1]), {signal.SIGHUP, signal.SIGINT, signal.SIGTERM}) + self.assertEqual(mask_calls[1][0], signal.SIG_SETMASK) + self.assertEqual( + self.helper.snapshot_committed_database( + os.fspath(self.mount_root), + os.fspath(self.candidate_path), + os.fspath(self.snapshot_current_path), + )["operation"], + "snapshot-current", + ) + self.assertEqual(self.snapshot_current_path.read_bytes(), candidate_bytes) + + def test_commit_classifies_pre_replace_sync_failures(self): + cases = [] + + def backup_file_failure(fd): + return "backups/" in os.readlink(f"/proc/self/fd/{fd}") and os.path.basename( + os.readlink(f"/proc/self/fd/{fd}") + ) == BACKUP_NAME + + def backup_dir_failure(fd): + return os.readlink(f"/proc/self/fd/{fd}").endswith("/backups") + + cases.append(("backup-file-fsync", backup_file_failure, None)) + cases.append(("backup-dir-fsync", backup_dir_failure, None)) + cases.append(("filesystem-sync", None, "syncfs")) + + for name, fsync_matcher, helper_failure in cases: + with self.subTest(case=name): + self.setUp() + self._snapshot_state() + original_bytes = self._main_path().read_bytes() + candidate_bytes = KDBX_BYTES + name.encode("utf-8") + self.candidate_path.write_bytes(candidate_bytes) + self.candidate_path.chmod(0o600) + replace_calls = [] + real_fsync = self.helper.os.fsync + + def failing_fsync(fd): + if fsync_matcher is not None and fsync_matcher(fd): + raise OSError("synthetic fsync failure") + return real_fsync(fd) + + patches = [ + mock.patch.object( + self.helper.os, + "replace", + side_effect=lambda *args, **kwargs: replace_calls.append((args, kwargs)), + ) + ] + if fsync_matcher is not None: + patches.append(mock.patch.object(self.helper.os, "fsync", side_effect=failing_fsync)) + if helper_failure == "syncfs": + patches.append( + mock.patch.object( + self.helper, + "sync_filesystem_fd", + side_effect=OSError("synthetic syncfs failure"), + ) + ) + with patches[0]: + with patches[1] if len(patches) > 1 else mock.patch.object(self.helper, "write_result", self.helper.write_result): + with patches[2] if len(patches) > 2 else mock.patch.object(self.helper, "write_result", self.helper.write_result): + with self.assertRaises(Exception): + self._commit() + + self.assertEqual(replace_calls, []) + self.assertEqual(self._main_path().read_bytes(), original_bytes) + + def test_commit_cleans_only_owned_stage_after_pre_replace_vault_sync_failure(self): + self._snapshot_state() + original_bytes = self._main_path().read_bytes() + candidate_bytes = KDBX_BYTES + b"candidate\n" + self.candidate_path.write_bytes(candidate_bytes) + self.candidate_path.chmod(0o600) + + vault_dir = self.mount_root / "HyeonworksRecovery" / "vault" + backup_path = vault_dir / "backups" / BACKUP_NAME + foreign_stage = vault_dir / f".{MAIN_RELATIVE[-1]}.stage.foreign" + foreign_stage.write_bytes(b"foreign-stage") + foreign_stage.chmod(0o600) + + real_syncfs = self.helper.sync_filesystem_fd + + def fail_vault_syncfs(fd): + fd_path = os.readlink(f"/proc/self/fd/{fd}") + if fd_path.endswith("/vault"): + raise OSError("synthetic vault syncfs failure") + return real_syncfs(fd) + + with mock.patch.object(self.helper, "sync_filesystem_fd", side_effect=fail_vault_syncfs): + with self.assertRaises(Exception): + self._commit() + + owned_stage_prefix = f".{MAIN_RELATIVE[-1]}.stage.{os.getpid()}." + self.assertEqual(self._main_path().read_bytes(), original_bytes) + self.assertTrue(backup_path.is_file()) + self.assertEqual(backup_path.read_bytes(), original_bytes) + self.assertTrue(foreign_stage.is_file()) + self.assertEqual( + sorted(path.name for path in vault_dir.iterdir() if path.name.startswith(owned_stage_prefix)), + [], + ) + + def test_commit_classifies_replace_and_post_replace_failures_without_second_replace(self): + scenarios = ("replace-failure", "post-file-fsync", "post-dir-fsync", "post-syncfs") + for scenario in scenarios: + with self.subTest(case=scenario): + self.setUp() + self._snapshot_state() + candidate_bytes = KDBX_BYTES + scenario.encode("utf-8") + self.candidate_path.write_bytes(candidate_bytes) + self.candidate_path.chmod(0o600) + replace_calls = [] + real_replace = self.helper.os.replace + real_fsync = self.helper.os.fsync + real_syncfs = self.helper.sync_filesystem_fd + vault_fsync_calls = 0 + syncfs_calls = 0 + + def maybe_fail_replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None): + replace_calls.append((src, dst, src_dir_fd, dst_dir_fd)) + if scenario == "replace-failure": + raise OSError("synthetic replace failure") + return real_replace(src, dst, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd) + + def maybe_fail_fsync(fd): + nonlocal vault_fsync_calls + fd_path = os.readlink(f"/proc/self/fd/{fd}") + if scenario == "post-file-fsync" and fd_path.endswith(MAIN_RELATIVE[-1]): + raise OSError("synthetic installed-main fsync failure") + if fd_path.endswith("/vault"): + vault_fsync_calls += 1 + if scenario == "post-dir-fsync" and fd_path.endswith("/vault") and vault_fsync_calls == 2: + raise OSError("synthetic vault-dir fsync failure") + return real_fsync(fd) + + def maybe_fail_syncfs(fd): + nonlocal syncfs_calls + syncfs_calls += 1 + if scenario == "post-syncfs" and syncfs_calls == 3: + raise OSError("synthetic post-rename syncfs failure") + return real_syncfs(fd) + + with mock.patch.object(self.helper.os, "replace", side_effect=maybe_fail_replace), mock.patch.object( + self.helper.os, "fsync", side_effect=maybe_fail_fsync + ), mock.patch.object(self.helper, "sync_filesystem_fd", side_effect=maybe_fail_syncfs): + result = self._commit() + + self.assertEqual(result["fs_commit"], "committed-but-uncertain") + self.assertEqual(len(replace_calls), 1) + + def test_commit_blocks_signals_through_post_replace_classification(self): + self._snapshot_state() + candidate_bytes = KDBX_BYTES + b"candidate\n" + self.candidate_path.write_bytes(candidate_bytes) + self.candidate_path.chmod(0o600) + + real_replace = self.helper.os.replace + real_fsync = self.helper.os.fsync + real_syncfs = self.helper.sync_filesystem_fd + real_files_equal = self.helper.files_equal_fd + current_mask = set() + replaced = False + checkpoints = [] + + def recording_sigmask(how, mask): + nonlocal current_mask + mask_set = set(mask) + if how == signal.SIG_BLOCK: + previous = set(current_mask) + current_mask.update(mask_set) + return previous + if how == signal.SIG_SETMASK: + current_mask = set(mask_set) + return set() + raise AssertionError(f"unexpected sigmask mode: {how}") + + def recording_replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None): + nonlocal replaced + replaced = True + return real_replace(src, dst, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd) + + def recording_files_equal(left_fd, right_fd): + if replaced: + checkpoints.append(("post-compare", set(current_mask))) + return real_files_equal(left_fd, right_fd) + + def recording_fsync(fd): + if replaced: + checkpoints.append((os.readlink(f"/proc/self/fd/{fd}"), set(current_mask))) + return real_fsync(fd) + + def recording_syncfs(fd): + if replaced: + checkpoints.append((f"syncfs:{os.readlink(f'/proc/self/fd/{fd}')}", set(current_mask))) + return real_syncfs(fd) + + with mock.patch.object(self.helper.signal, "pthread_sigmask", side_effect=recording_sigmask), mock.patch.object( + self.helper.os, "replace", side_effect=recording_replace + ), mock.patch.object(self.helper, "files_equal_fd", side_effect=recording_files_equal), mock.patch.object( + self.helper.os, "fsync", side_effect=recording_fsync + ), mock.patch.object(self.helper, "sync_filesystem_fd", side_effect=recording_syncfs): + result = self._commit() + + self.assertEqual(result["fs_commit"], "verified-commit") + self.assertGreaterEqual(len(checkpoints), 3) + for _, observed_mask in checkpoints: + self.assertTrue( + {signal.SIGHUP, signal.SIGINT, signal.SIGTERM}.issubset(observed_mask), + f"post-replace checkpoint observed unblocked mask: {observed_mask!r}", + ) + self.assertEqual(current_mask, set()) + + def test_commit_rejects_malformed_extra_and_foreign_mount_state(self): + state = self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + + malformed = dict(state) + malformed.pop("source") + self.baseline_path.write_text(json.dumps(malformed), encoding="utf-8") + with self.assertRaises(Exception): + self._commit() + + extra = dict(state) + extra["unexpected"] = "value" + self.helper.write_result(os.fspath(self.baseline_path), extra) + with self.assertRaises(Exception): + self._commit() + + foreign_mount = dict(state) + foreign_mount["mount"] = dict(foreign_mount["mount"]) + foreign_mount["mount"]["realpath"] = os.fspath(self.root / "foreign-mount") + self.helper.write_result(os.fspath(self.baseline_path), foreign_mount) + with self.assertRaises(Exception): + self._commit() + + def test_commit_rejects_unsafe_baseline_path_metadata(self): + with self.subTest(case="symlink-replacement"): + self.setUp() + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + baseline_bytes = self.baseline_path.read_bytes() + target = self.runtime / "baseline-target.json" + target.write_bytes(baseline_bytes) + target.chmod(0o600) + self.baseline_path.unlink() + self.baseline_path.symlink_to(target) + with self.assertRaises(Exception): + self._commit() + + with self.subTest(case="hardlink-replacement"): + self.setUp() + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + baseline_bytes = self.baseline_path.read_bytes() + target = self.runtime / "baseline-target.json" + target.write_bytes(baseline_bytes) + target.chmod(0o600) + self.baseline_path.unlink() + os.link(target, self.baseline_path) + with self.assertRaises(Exception): + self._commit() + + with self.subTest(case="wrong-mode"): + self.setUp() + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + self.baseline_path.chmod(0o644) + with self.assertRaises(Exception): + self._commit() + + with self.subTest(case="wrong-type"): + self.setUp() + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + self.baseline_path.unlink() + self.baseline_path.mkdir(mode=0o700) + with self.assertRaises(Exception): + self._commit() + + def test_commit_uses_retained_baseline_fd_if_path_is_replaced_after_open(self): + self._snapshot_state() + self.candidate_path.write_bytes(KDBX_BYTES + b"candidate\n") + self.candidate_path.chmod(0o600) + + original_open_regular_at = self.helper.open_regular_at + baseline_parent = self.baseline_path.parent.resolve() + baseline_name = self.baseline_path.name + replacement_done = False + + def replacing_open_regular_at(dir_fd, name, **kwargs): + nonlocal replacement_done + fd = original_open_regular_at(dir_fd, name, **kwargs) + opened_path = pathlib.Path(os.readlink(f"/proc/self/fd/{fd}")).resolve() + parent_path = pathlib.Path(os.readlink(f"/proc/self/fd/{dir_fd}")).resolve() + if ( + not replacement_done + and name == baseline_name + and parent_path == baseline_parent + and opened_path == self.baseline_path.resolve() + ): + replacement_done = True + replacement = self.runtime / "baseline-replacement.json" + replacement.write_text('{"schema":"tampered"}', encoding="utf-8") + replacement.chmod(0o600) + self.baseline_path.unlink() + replacement.rename(self.baseline_path) + return fd + + with mock.patch.object(self.helper, "open_regular_at", side_effect=replacing_open_regular_at): + result = self._commit() + + self.assertTrue(replacement_done) + self.assertEqual(result["fs_commit"], "verified-commit") + self.assertEqual(self._main_path().read_bytes(), self.candidate_path.read_bytes()) + self.assertEqual(self.baseline_path.read_text(encoding="utf-8"), '{"schema":"tampered"}') + + with self.assertRaises(Exception): + self._commit() + + def test_cli_expected_failures_are_payload_free_for_all_mutating_commands(self): + marker = f"synthetic-unique-path-{os.getpid()}" + missing = os.fspath(self.root / marker) + commands = ( + ["snapshot", "--mount-root", missing, "--destination", missing, + "--result", missing], + ["commit", "--mount-root", missing, "--baseline", missing, + "--candidate", missing, "--backup-name", BACKUP_NAME, + "--result", missing], + ["snapshot-current", "--mount-root", missing, "--candidate", missing, + "--destination", missing, "--result", missing], + ) + for arguments in commands: + with self.subTest(command=arguments[0]): + completed = subprocess.run( + [sys.executable, os.fspath(PROD), *arguments], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=5, check=False, + ) + self.assertEqual(completed.returncode, 1) + self.assertEqual(completed.stdout, b"") + self.assertEqual(completed.stderr, b"dirfd helper failed\n") + self.assertNotIn(marker.encode(), completed.stderr) + self.assertNotIn(b"Traceback", completed.stderr) + + def test_process_audit_detects_retained_proc_fd_runtime_reference(self): + runtime_fd = os.open(self.runtime, os.O_RDONLY | os.O_DIRECTORY) + process = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)", + "keepassxc-cli", f"/proc/{os.getpid()}/fd/{runtime_fd}"], + pass_fds=(runtime_fd,), stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + with self.assertRaises(self.helper.FsContractError): + self.helper.audit_runtime_processes(os.fspath(self.runtime)) + self.assertIsNone(process.poll(), "audit killed a process it does not own") + finally: + process.terminate() + process.wait(timeout=5) + os.close(runtime_fd) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/validate/test-slack-webhook-recovery-vault.py b/scripts/validate/test-slack-webhook-recovery-vault.py new file mode 100644 index 0000000..36608f7 --- /dev/null +++ b/scripts/validate/test-slack-webhook-recovery-vault.py @@ -0,0 +1,1179 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import importlib.util +import hashlib +import os +import pathlib +import secrets +import shutil +import signal +import socket +import stat +import subprocess +import tempfile +import termios +import threading +import time +import unittest +from unittest import mock + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +PROD = ROOT / "scripts/libexec/slack-webhook-recovery-vault.py" +KEEPASSXC_CLI = "/usr/bin/keepassxc-cli" + + +def load_vault_module(): + spec = importlib.util.spec_from_file_location("slack_webhook_recovery_vault", PROD) + if spec is None or spec.loader is None: + raise AssertionError("KeePass database classifier is unavailable") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def synthetic_webhook(suffix: bytes = b"A") -> bytes: + prefix = b"https://" + b"hooks." + b"slack.com/services/" + return prefix + b"T01234567/B01234567/" + (b"X" * 23) + suffix + + +def write_private(path: pathlib.Path, payload: bytes) -> None: + path.write_bytes(payload) + os.chmod(path, 0o600) + + +def private_keepass_environment() -> dict[str, str]: + return { + "HOME": os.devnull, + "XDG_CONFIG_HOME": os.devnull, + "LC_ALL": "C.UTF-8", + "PATH": "/usr/bin:/bin", + } + + +def create_database(path: pathlib.Path, master: bytearray) -> bytes: + child_input = bytearray(master) + child_input.extend(b"\n") + child_input.extend(master) + child_input.extend(b"\n") + try: + completed = subprocess.run( + [KEEPASSXC_CLI, "db-create", "-q", "-p", os.fspath(path)], + input=bytes(child_input), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=private_keepass_environment(), + timeout=20, + check=False, + ) + finally: + for index in range(len(child_input)): + child_input[index] = 0 + if completed.returncode != 0: + raise AssertionError("synthetic KeePass database creation failed") + if bytes(master) in completed.stdout + completed.stderr: + raise AssertionError("synthetic master appeared in captured KeePassXC output") + os.chmod(path, 0o600) + return completed.stdout + completed.stderr + + +class VaultClassifierRedTest(unittest.TestCase): + """Initial Task 3 gate using a real database with synthetic credentials.""" + + @classmethod + def setUpClass(cls) -> None: + cls.fixture = tempfile.TemporaryDirectory( + prefix="slack-webhook-recovery-vault-test." + ) + cls.fixture_path = pathlib.Path(cls.fixture.name) + os.chmod(cls.fixture_path, 0o700) + cls.database = cls.fixture_path / "test.kdbx" + cls.master = bytearray(secrets.token_bytes(32).hex().encode("ascii")) + cls.db_create_output = create_database(cls.database, cls.master) + + @classmethod + def tearDownClass(cls) -> None: + for index in range(len(cls.master)): + cls.master[index] = 0 + cls.fixture.cleanup() + + def test_01_real_synthetic_kdbx_fixture_is_private_and_output_is_safe(self) -> None: + # Production break caught: a test harness that substitutes fake database + # bytes or leaks the synthetic master through KeePassXC output. + database_stat = self.database.stat() + self.assertTrue(stat.S_ISREG(database_stat.st_mode)) + self.assertEqual(stat.S_IMODE(database_stat.st_mode), 0o600) + if bytes(self.master) in self.db_create_output: + self.fail("synthetic master appeared in captured KeePassXC output") + + def test_02_vault_classifier_api_is_available(self) -> None: + # Production break caught: Task 1 can reach execute mode without the + # long-lived vault classifier boundary being implemented. + if not PROD.is_file(): + self.fail("KeePass database classifier is unavailable") + module = load_vault_module() + required = ( + "open_validated_webhook", + "read_operator_secrets", + "run_keepass", + "classify_database", + "build_candidate", + "verify_database", + "serve_private_socket", + ) + if any(not callable(getattr(module, name, None)) for name in required): + self.fail("KeePass database classifier is unavailable") + + +@unittest.skipUnless(PROD.is_file(), "production vault helper is not implemented yet") +class WebhookAndInputContractTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.vault = load_vault_module() + + def setUp(self) -> None: + self.fixture = tempfile.TemporaryDirectory( + prefix="slack-webhook-recovery-webhook-test." + ) + self.root = pathlib.Path(self.fixture.name) + os.chmod(self.root, 0o700) + + def tearDown(self) -> None: + self.fixture.cleanup() + + def test_valid_webhook_retains_one_fd_and_rejects_content_drift(self) -> None: + # Production break caught: reopening the path or accepting changed bytes + # after startup instead of binding the retained descriptor and content. + path = self.root / "webhook" + write_private(path, synthetic_webhook()) + retained = self.vault.open_validated_webhook(os.fspath(path)) + try: + self.assertEqual(retained.read_bytes(), synthetic_webhook()) + with path.open("r+b", buffering=0) as stream: + stream.seek(-1, os.SEEK_END) + stream.write(b"B") + with self.assertRaises(self.vault.VaultContractError): + retained.read_bytes() + finally: + retained.close() + + def test_retained_webhook_uses_mutable_storage_and_wipes_on_close(self) -> None: + path = self.root / "webhook-wipe" + write_private(path, synthetic_webhook()) + retained = self.vault.open_validated_webhook(os.fspath(path)) + content_reference = retained._content + self.assertIsInstance(content_reference, bytearray) + retained.close() + self.assertEqual(content_reference, bytearray(len(content_reference))) + + def test_startup_failure_wipes_retained_webhook_storage(self) -> None: + path = self.root / "webhook-startup-wipe" + write_private(path, synthetic_webhook()) + captured = [] + real_open = self.vault.open_validated_webhook + + def capture_open(webhook_path: str): + retained = real_open(webhook_path) + captured.append(retained._content) + return retained + + output_fd = os.open(os.devnull, os.O_WRONLY | os.O_CLOEXEC) + try: + with mock.patch.object( + self.vault, "_attest_runtime_root", return_value=os.fspath(self.root) + ), mock.patch.object( + self.vault, "open_validated_webhook", side_effect=capture_open + ), mock.patch.object( + self.vault, "read_operator_secrets", + side_effect=self.vault.VaultContractError("synthetic startup failure"), + ): + with self.assertRaises(self.vault.VaultContractError): + self.vault.serve_private_socket( + os.fspath(self.root), os.fspath(path), + input_fd=0, output_fd=output_fd, + ) + finally: + os.close(output_fd) + self.assertEqual(len(captured), 1) + self.assertEqual(captured[0], bytearray(len(captured[0]))) + + def test_socket_constructor_failure_finalizes_retained_resources(self) -> None: + path = self.root / "webhook-socket-constructor-failure" + write_private(path, synthetic_webhook()) + captured = [] + wiped_master = [] + real_open = self.vault.open_validated_webhook + + def capture_open(webhook_path: str): + retained = real_open(webhook_path) + captured.append(retained) + return retained + + with mock.patch.object( + self.vault, "_attest_runtime_root", return_value=os.fspath(self.root) + ), mock.patch.object( + self.vault, "open_validated_webhook", side_effect=capture_open + ), mock.patch.object( + self.vault.socket, "socket", side_effect=OSError("synthetic socket failure") + ): + with self.assertRaises(OSError): + self.vault.serve_private_socket( + os.fspath(self.root), os.fspath(path), + wipe_hook=lambda value: wiped_master.append(bytes(value)), + ) + self.assertEqual(len(captured), 1) + self.assertTrue(captured[0]._closed) + self.assertEqual( + captured[0]._content, bytearray(len(captured[0]._content)) + ) + self.assertEqual(wiped_master, [b""]) + + def test_retained_webhook_rejects_same_size_drift_during_read(self) -> None: + # Production break caught: checking only size/identity around a read, + # which misses an in-place same-size credential change during that read. + path = self.root / "webhook-concurrent" + write_private(path, synthetic_webhook(b"A")) + retained = self.vault.open_validated_webhook(os.fspath(path)) + real_read = self.vault.os.read + mutated = False + + def drifting_read(fd: int, count: int) -> bytes: + nonlocal mutated + chunk = real_read(fd, count) + if chunk and not mutated: + mutated = True + with path.open("r+b", buffering=0) as stream: + stream.seek(-1, os.SEEK_END) + stream.write(b"B") + return chunk + + try: + with mock.patch.object(self.vault.os, "read", side_effect=drifting_read): + with self.assertRaises(self.vault.VaultContractError): + retained.read_bytes() + finally: + retained.close() + + def test_open_webhook_rejects_same_size_drift_during_startup_binding(self) -> None: + # Production break caught: startup content binding that performs only + # one read and misses an in-place same-size change during that read. + path = self.root / "webhook-open-concurrent" + write_private(path, synthetic_webhook(b"A")) + real_read = self.vault.os.read + mutated = False + + def drifting_read(fd: int, count: int) -> bytes: + nonlocal mutated + chunk = real_read(fd, count) + if chunk and not mutated: + mutated = True + with path.open("r+b", buffering=0) as stream: + stream.seek(-1, os.SEEK_END) + stream.write(b"B") + return chunk + + with mock.patch.object(self.vault.os, "read", side_effect=drifting_read): + with self.assertRaises(self.vault.VaultContractError): + self.vault.open_validated_webhook(os.fspath(path)) + + def test_path_replacement_never_switches_the_retained_webhook(self) -> None: + # Production break caught: a consumer reopening a replaced pathname and + # silently consuming the replacement credential. + path = self.root / "webhook" + replacement = self.root / "replacement" + write_private(path, synthetic_webhook(b"A")) + write_private(replacement, synthetic_webhook(b"B")) + retained = self.vault.open_validated_webhook(os.fspath(path)) + try: + os.replace(replacement, path) + with self.assertRaises(self.vault.VaultContractError): + retained.read_bytes() + finally: + retained.close() + + def test_webhook_metadata_and_content_rejection_matrix(self) -> None: + # Production breaks caught: weakening any member of the exact metadata, + # one-line, bounded-content, or Slack URL grammar matrix. + valid = synthetic_webhook() + cases = { + "empty": b"", + "oversized": b"A" * 4097, + "carriage-return": valid + b"\r", + "line-feed": valid + b"\n", + "nul": valid + b"\0", + "leading-space": b" " + valid, + "trailing-space": valid + b" ", + "wrong-host": valid.replace(b"hooks.slack.com", b"example.invalid"), + "two-components": (b"https://" + b"hooks." + b"slack.com/services/T/B"), + "extra-component": valid + b"/extra", + "query": valid + b"?x=1", + } + for name, payload in cases.items(): + with self.subTest(case=name): + path = self.root / name + write_private(path, payload) + with self.assertRaises(self.vault.VaultContractError): + self.vault.open_validated_webhook(os.fspath(path)) + + wrong_mode = self.root / "wrong-mode" + write_private(wrong_mode, valid) + os.chmod(wrong_mode, 0o640) + with self.assertRaises(self.vault.VaultContractError): + self.vault.open_validated_webhook(os.fspath(wrong_mode)) + + original = self.root / "hardlink-original" + linked = self.root / "hardlink" + write_private(original, valid) + os.link(original, linked) + with self.assertRaises(self.vault.VaultContractError): + self.vault.open_validated_webhook(os.fspath(linked)) + + target = self.root / "symlink-target" + symlink = self.root / "symlink" + write_private(target, valid) + symlink.symlink_to(target) + with self.assertRaises((self.vault.VaultContractError, OSError)): + self.vault.open_validated_webhook(os.fspath(symlink)) + + directory = self.root / "directory" + directory.mkdir(mode=0o700) + with self.assertRaises((self.vault.VaultContractError, OSError)): + self.vault.open_validated_webhook(os.fspath(directory)) + + foreign = self.root / "foreign-owner" + write_private(foreign, valid) + with mock.patch.object(self.vault.os, "getuid", return_value=os.getuid() + 1): + with self.assertRaises(self.vault.VaultContractError): + self.vault.open_validated_webhook(os.fspath(foreign)) + + def test_operator_input_validates_app_and_keeps_master_mutable(self) -> None: + # Production break caught: accepting notes delimiters or returning an + # immutable master value that cannot be best-effort overwritten. + read_fd, write_fd = os.pipe() + output_read_fd, output_fd = os.pipe() + master = secrets.token_bytes(24).hex().encode("ascii") + os.write(write_fd, b"Synthetic Slack App\n" + master + b"\n") + os.close(write_fd) + try: + app_name, master_buffer = self.vault.read_operator_secrets( + input_fd=read_fd, output_fd=output_fd + ) + finally: + os.close(read_fd) + os.close(output_fd) + prompts = os.read(output_read_fd, 4096) + os.close(output_read_fd) + self.assertEqual( + prompts, + b"Slack app name: KeePassXC master password: ", + ) + self.assertEqual(app_name, "Synthetic Slack App") + self.assertIsInstance(master_buffer, bytearray) + self.assertEqual(bytes(master_buffer), master) + self.vault.wipe_mutable_buffer(master_buffer) + self.assertEqual(bytes(master_buffer), b"\0" * len(master)) + + def test_signal_during_hidden_input_restores_terminal_echo(self) -> None: + # Production break caught: the coordinator's bounded startup abort + # terminates the vault while ECHO is disabled and leaves the operator's + # terminal in a secret-hostile state. + master_fd, slave_fd = os.openpty() + child_pid = os.fork() + if child_pid == 0: + try: + os.close(master_fd) + self.vault.install_process_signal_handlers() + self.vault.read_operator_secrets( + input_fd=slave_fd, output_fd=slave_fd + ) + except BaseException: + os._exit(0) + os._exit(80) + os.close(slave_fd) + try: + deadline = time.monotonic() + 5.0 + observed = b"" + while b"Slack app name: " not in observed and time.monotonic() < deadline: + observed += os.read(master_fd, 256) + os.write(master_fd, b"Synthetic Slack App\n") + while b"KeePassXC master password: " not in observed and time.monotonic() < deadline: + observed += os.read(master_fd, 256) + attrs = termios.tcgetattr(master_fd) + self.assertFalse(attrs[3] & termios.ECHO) + os.kill(child_pid, signal.SIGTERM) + waited_pid, status = os.waitpid(child_pid, 0) + child_pid = -1 + self.assertEqual(waited_pid > 0, True) + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(os.WEXITSTATUS(status), 0) + restored = termios.tcgetattr(master_fd) + self.assertTrue(restored[3] & termios.ECHO) + finally: + if child_pid > 0: + os.kill(child_pid, signal.SIGKILL) + os.waitpid(child_pid, 0) + os.close(master_fd) + + def test_socket_is_not_published_until_operator_inputs_are_retained(self) -> None: + # Production break caught: sudo and the vault racing each other for + # /dev/tty because socket publication precedes secret retention. + read_fd, write_fd = os.pipe() + output_fd = os.open(os.devnull, os.O_WRONLY | os.O_CLOEXEC) + failures: list[str] = [] + + def target() -> None: + try: + self.vault.serve_private_socket( + os.fspath(self.root), + os.fspath(self.root / "webhook"), + input_fd=read_fd, + output_fd=output_fd, + accept_timeout=5.0, + io_timeout=2.0, + child_timeout=5.0, + ) + except BaseException as exc: + failures.append(f"{type(exc).__name__}:{exc}") + finally: + os.close(read_fd) + os.close(output_fd) + + write_private(self.root / "webhook", synthetic_webhook()) + attest_patch = mock.patch.object( + self.vault, "_attest_runtime_root", return_value=os.fspath(self.root) + ) + attest_patch.start() + thread = threading.Thread(target=target, daemon=True) + thread.start() + try: + time.sleep(0.1) + self.assertFalse((self.root / "vault.sock").exists()) + self.assertFalse( + (self.root / "vault-home").exists(), + "writable KeePass private home was created before input retention", + ) + os.write(write_fd, b"Synthetic Slack App\nsynthetic-master\n") + os.close(write_fd) + write_fd = -1 + deadline = time.monotonic() + 5 + while not (self.root / "vault.sock").exists() and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue((self.root / "vault.sock").exists()) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.connect(os.fspath(self.root / "vault.sock")) + client.sendall(b"SHUTDOWN\n") + self.assertEqual(client.recv(32), b"stopped\n") + thread.join(timeout=5) + self.assertFalse(thread.is_alive()) + self.assertEqual(failures, []) + finally: + attest_patch.stop() + if write_fd >= 0: + os.close(write_fd) + + def test_keepass_child_arms_linux_parent_death_signal(self) -> None: + # Production break caught: a vault killed during communicate leaves its + # separate-session KeePass child holding secret stdin alive. + self.assertTrue(callable(getattr(self.vault, "arm_parent_death_signal", None))) + fake_prctl = mock.Mock(return_value=0) + fake_libc = mock.Mock(prctl=fake_prctl) + with mock.patch.object(self.vault.os, "getppid", side_effect=[1234, 1234]), \ + mock.patch.object(self.vault.ctypes, "CDLL", return_value=fake_libc): + self.vault.arm_parent_death_signal(1234) + fake_prctl.assert_called_once_with(1, 9, 0, 0, 0) + with mock.patch.object(self.vault.os, "getppid", return_value=1), \ + mock.patch.object(self.vault.os, "kill") as kill_parent_race: + self.vault.arm_parent_death_signal(1234) + kill_parent_race.assert_called_once_with(os.getpid(), 9) + + def test_parent_death_signal_kills_actual_fake_child(self) -> None: + script = r''' +import importlib.util, os, subprocess, sys, time +spec = importlib.util.spec_from_file_location("vault", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +expected = os.getpid() +child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)"], + preexec_fn=lambda: module.arm_parent_death_signal(expected), +) +print(child.pid, flush=True) +''' + parent = subprocess.Popen( + [os.fspath(pathlib.Path(os.sys.executable)), "-c", script, os.fspath(PROD)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ) + child_pid = int(parent.stdout.readline().strip()) + parent.communicate(timeout=5) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + try: + status = pathlib.Path(f"/proc/{child_pid}/status").read_text() + except (FileNotFoundError, ProcessLookupError): + break + if "State:\tZ" in status: + break + time.sleep(0.02) + else: + os.kill(child_pid, 9) + self.fail("PDEATHSIG did not terminate the actual fake child") + + def test_operator_input_rejects_app_note_delimiters_and_whitespace(self) -> None: + # Production break caught: app text escaping its single-line Notes key. + invalid_apps = (b"", b" leading", b"trailing ", b"a;b", b"a=b", b"a\\b", b"a\rb") + for app in invalid_apps: + with self.subTest(app_case=repr(app)): + read_fd, write_fd = os.pipe() + output_fd = os.open(os.devnull, os.O_WRONLY | os.O_CLOEXEC) + os.write(write_fd, app + b"\nsynthetic-master\n") + os.close(write_fd) + try: + with self.assertRaises(self.vault.VaultContractError): + self.vault.read_operator_secrets( + input_fd=read_fd, output_fd=output_fd + ) + finally: + os.close(read_fd) + os.close(output_fd) + + def test_protocol_line_rejects_cr_nul_oversize_partial_and_pipeline(self) -> None: + # Production breaks caught: request framing ambiguity or unbounded reads. + cases = ( + b"PREPARE\r\n", + b"PREPARE\0\n", + b"A" * 4097 + b"\n", + b"PREPARE", + b"PREPARE\nSHUTDOWN\n", + ) + for payload in cases: + with self.subTest(payload_length=len(payload)): + server, client = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + try: + client.sendall(payload) + client.shutdown(socket.SHUT_WR) + with self.assertRaises(self.vault.VaultProtocolError): + self.vault.read_protocol_line(server, timeout=0.5) + finally: + server.close() + client.close() + + def test_parser_accepts_only_exact_commands_and_state(self) -> None: + # Production break caught: protocol fields selecting candidate/baseline + # names or bypassing the BUILD state gate. + valid = { + "PREPARE": ("PREPARE", None), + "BUILD ADD": ("BUILD", "ADD"), + "BUILD UPDATE": ("BUILD", "UPDATE"), + "VERIFY_CANDIDATE": ("VERIFY_CANDIDATE", None), + "SHUTDOWN": ("SHUTDOWN", None), + } + for line, expected in valid.items(): + self.assertEqual(self.vault.parse_protocol_command(line), expected) + for line in ( + "", + "PREPARE extra", + "BUILD", + "BUILD ADD extra", + "BUILD DELETE", + "SHUTDOWN extra", + "baseline.kdbx", + ): + with self.subTest(line=line): + with self.assertRaises(self.vault.VaultProtocolError): + self.vault.parse_protocol_command(line) + + def test_direct_cli_is_closed_and_has_no_environment_input_seam(self) -> None: + # Production break caught: adding a direct classifier/debug form or an + # environment-activated replacement for the mandatory /dev/tty input. + runtime_parent = pathlib.Path(f"/run/user/{os.getuid()}") + with tempfile.TemporaryDirectory( + prefix="slack-webhook-recovery-cli.", dir=runtime_parent + ) as raw_runtime: + runtime = pathlib.Path(raw_runtime) + os.chmod(runtime, 0o700) + webhook = runtime / "input-webhook" + write_private(webhook, synthetic_webhook()) + marker = b"synthetic-stdin-must-not-be-an-input-seam" + environment = { + "HOME": "/hostile-home", + "LC_ALL": "C.UTF-8", + "PATH": "/usr/bin:/bin", + "SWR_VAULT_INPUT_FD": "0", + } + invalid = subprocess.run( + [PYTHON := "/usr/bin/python3", os.fspath(PROD), "--serve"], + input=marker + b"\n", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + timeout=5, + check=False, + ) + self.assertEqual(invalid.returncode, 2) + self.assertEqual(invalid.stdout + invalid.stderr, b"") + + direct = subprocess.run( + [ + PYTHON, os.fspath(PROD), "--serve", "--runtime-root", + os.fspath(runtime), "--webhook-file", os.fspath(webhook), + ], + input=b"Synthetic App\n" + marker + b"\n", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + timeout=5, + check=False, + ) + self.assertEqual(direct.returncode, 1) + self.assertEqual(direct.stdout, b"") + self.assertEqual(direct.stderr, b"vault helper failed\n") + self.assertNotIn(marker, direct.stdout + direct.stderr) + + def test_direct_cli_preserves_first_signal_status_after_cleanup(self) -> None: + # Production break caught: translating the coordinator's forwarded + # HUP/INT/TERM into generic failure after terminal/resource cleanup. + for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM): + with self.subTest(signum=signum), mock.patch.object( + self.vault, "install_process_signal_handlers" + ), mock.patch.object( + self.vault, "serve_private_socket", + side_effect=self.vault.VaultSignal(signum), + ): + rc = self.vault._main( + ["--serve", "--runtime-root", "/synthetic/runtime", + "--webhook-file", "/synthetic/webhook"] + ) + self.assertEqual(rc, 128 + signum) + + def test_committed_snapshot_validation_matrix(self) -> None: + # Production breaks caught: verifying a symlink, hardlink, wrong-mode, + # foreign-owner, or out-of-runtime snapshot. + runtime_parent = pathlib.Path(f"/run/user/{os.getuid()}") + with tempfile.TemporaryDirectory( + prefix="slack-webhook-recovery-snapshot.", dir=runtime_parent + ) as raw_runtime: + runtime = pathlib.Path(raw_runtime) + os.chmod(runtime, 0o700) + valid = runtime / "candidate.kdbx" + write_private(valid, b"synthetic-encrypted-database") + self.vault._attest_snapshot(os.fspath(valid), os.fspath(runtime)) + + wrong_mode = runtime / "wrong-mode.kdbx" + write_private(wrong_mode, b"synthetic") + os.chmod(wrong_mode, 0o640) + with self.assertRaises(self.vault.VaultContractError): + self.vault._attest_snapshot(os.fspath(wrong_mode), os.fspath(runtime)) + + original = runtime / "linked-original.kdbx" + hardlink = runtime / "hardlink.kdbx" + write_private(original, b"synthetic") + os.link(original, hardlink) + with self.assertRaises(self.vault.VaultContractError): + self.vault._attest_snapshot(os.fspath(hardlink), os.fspath(runtime)) + + symlink = runtime / "symlink.kdbx" + symlink.symlink_to(valid) + with self.assertRaises(self.vault.VaultContractError): + self.vault._attest_snapshot(os.fspath(symlink), os.fspath(runtime)) + + with mock.patch.object(self.vault.os, "getuid", return_value=os.getuid() + 1): + with self.assertRaises(self.vault.VaultContractError): + self.vault._attest_snapshot(os.fspath(valid), os.fspath(runtime)) + + outside = self.root / "outside.kdbx" + write_private(outside, b"synthetic") + with self.assertRaises(self.vault.VaultProtocolError): + self.vault._attest_snapshot(os.fspath(outside), os.fspath(runtime)) + + def test_keepass_child_timeout_status_and_environment_are_sanitized(self) -> None: + # Production breaks caught: inherited hostile environment, unchecked + # child status, or an unbounded KeePass child. + script = self.root / "fake-keepass" + script.write_text( + "#!/usr/bin/python3\n" + "import os,sys,time\n" + "data=sys.stdin.buffer.read()\n" + "mode=sys.argv[1]\n" + "if mode=='timeout': time.sleep(2)\n" + "if mode=='status': sys.stderr.write('synthetic failure\\n'); raise SystemExit(7)\n" + "sys.stdout.write('\\n'.join(sorted(os.environ)))\n", + encoding="utf-8", + ) + os.chmod(script, 0o700) + home = self.root / "home" + home.mkdir(mode=0o700) + master = bytearray(b"synthetic-master") + with mock.patch.object(self.vault, "KEEPASSXC_CLI", os.fspath(script)): + with self.assertRaises(self.vault.VaultContractError): + self.vault.run_keepass( + ["timeout"], master, os.fspath(home), timeout=0.05 + ) + with self.assertRaises(self.vault.VaultContractError): + self.vault.run_keepass(["status"], master, os.fspath(home)) + result = self.vault.run_keepass( + ["environment"], master, os.fspath(home), check=True + ) + self.assertEqual(result.stdout, b"HOME\nLC_ALL\nPATH\nXDG_CONFIG_HOME") + self.assertNotIn(bytes(master), result.stdout + result.stderr) + + +@unittest.skipUnless(PROD.is_file(), "production vault helper is not implemented yet") +class RealSyntheticDatabaseContractTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.vault = load_vault_module() + + def setUp(self) -> None: + runtime_parent = pathlib.Path(f"/run/user/{os.getuid()}") + self.fixture = tempfile.TemporaryDirectory( + prefix="slack-webhook-recovery-vault-kdbx.", dir=runtime_parent + ) + self.root = pathlib.Path(self.fixture.name) + os.chmod(self.root, 0o700) + self.home = pathlib.Path(os.devnull) + self.master = bytearray(secrets.token_bytes(32).hex().encode("ascii")) + self.baseline = self.root / "baseline.kdbx" + create_database(self.baseline, self.master) + self.webhook_path = self.root / "input-webhook" + write_private(self.webhook_path, synthetic_webhook()) + self.webhook = self.vault.open_validated_webhook(os.fspath(self.webhook_path)) + self.app = "Synthetic Slack App" + + def tearDown(self) -> None: + self.webhook.close() + self.vault.wipe_mutable_buffer(self.master) + self.fixture.cleanup() + + def _classify(self, database: pathlib.Path, app: str | None = None) -> str: + return self.vault.classify_database( + os.fspath(database), + self.master, + self.webhook, + self.app if app is None else app, + os.fspath(self.home), + ) + + def _build(self, baseline: pathlib.Path, candidate: pathlib.Path, mode: str) -> None: + self.vault.build_candidate( + os.fspath(baseline), + os.fspath(candidate), + mode, + self.master, + self.webhook, + self.app, + os.fspath(self.home), + ) + + def test_keepass_cli_operates_without_a_writable_private_home(self) -> None: + # Architecture characterization: the installed KeePassXC CLI must not + # require a writable HOME/XDG config tree for real synthetic KDBX work. + before = hashlib.sha256(self.baseline.read_bytes()).digest() + candidate = self.root / "configless-candidate.kdbx" + shutil.copyfile(self.baseline, candidate) + os.chmod(candidate, 0o600) + environment = { + "HOME": os.devnull, + "XDG_CONFIG_HOME": os.devnull, + "LC_ALL": "C.UTF-8", + "PATH": "/usr/bin:/bin", + } + + listed = subprocess.run( + [KEEPASSXC_CLI, "ls", "-q", "-R", "-f", os.fspath(candidate)], + input=bytes(self.master) + b"\n", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + timeout=20, + check=False, + ) + added = subprocess.run( + [ + KEEPASSXC_CLI, "add", "-q", "-p", "-u", "synthetic-user", + "--notes", "synthetic-notes", os.fspath(candidate), + "/SyntheticConfiglessEntry", + ], + input=bytes(self.master) + b"\nsynthetic-password\n", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + timeout=20, + check=False, + ) + edited = subprocess.run( + [ + KEEPASSXC_CLI, "edit", "-q", "-t", "SyntheticConfiglessEdited", + os.fspath(candidate), "/SyntheticConfiglessEntry", + ], + input=bytes(self.master) + b"\n", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + timeout=20, + check=False, + ) + shown = subprocess.run( + [ + KEEPASSXC_CLI, "show", "-q", "-a", "Title", + os.fspath(candidate), "/SyntheticConfiglessEdited", + ], + input=bytes(self.master) + b"\n", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + timeout=20, + check=False, + ) + for result in (listed, added, edited, shown): + self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", "replace")) + self.assertEqual(result.stderr, b"") + self.assertNotIn(bytes(self.master), result.stdout + result.stderr) + self.assertEqual(hashlib.sha256(self.baseline.read_bytes()).digest(), before) + self.assertEqual(shown.stdout, b"SyntheticConfiglessEdited\n") + self.assertFalse((self.root / "vault-home").exists()) + + def test_absent_add_exact_noop_and_wrong_master_zero_change(self) -> None: + # Production breaks caught: incorrect absent/exact classification, + # mutating the baseline, or treating invalid credentials as absence. + before = hashlib.sha256(self.baseline.read_bytes()).digest() + self.assertEqual(self._classify(self.baseline), "absent") + candidate = self.root / "candidate.kdbx" + self._build(self.baseline, candidate, "ADD") + self.assertEqual(hashlib.sha256(self.baseline.read_bytes()).digest(), before) + self.assertTrue( + self.vault.verify_database( + os.fspath(candidate), + self.master, + self.webhook, + self.app, + os.fspath(self.home), + ) + ) + self.assertEqual(self._classify(candidate), "exact-noop") + wrong_master = bytearray(b"wrong-synthetic-master") + candidate_before = hashlib.sha256(candidate.read_bytes()).digest() + self.assertEqual( + self.vault.classify_database( + os.fspath(candidate), + wrong_master, + self.webhook, + self.app, + os.fspath(self.home), + ), + "failed", + ) + self.assertEqual(hashlib.sha256(candidate.read_bytes()).digest(), candidate_before) + self.vault.wipe_mutable_buffer(wrong_master) + + def test_metadata_password_app_workspace_channel_and_title_drift(self) -> None: + # Production break caught: comparing only the password or silently + # accepting any visible field drift. + exact = self.root / "exact.kdbx" + self._build(self.baseline, exact, "ADD") + self.assertEqual(self._classify(exact, app="Different App"), "mismatch") + + entry = self.vault.ENTRY_PATH + drift_cases = ( + ("username", ["edit", "-q", "-u", "other-workspace", os.fspath(exact), entry]), + ( + "notes", + ["edit", "-q", "--notes", "channel=other;app=Synthetic Slack App;recovery=revoke-and-reissue-in-slack", os.fspath(exact), entry], + ), + ("url", ["edit", "-q", "--url", "https://example.invalid", os.fspath(exact), entry]), + ) + for name, args in drift_cases: + with self.subTest(field=name): + drifted = self.root / f"{name}.kdbx" + shutil.copyfile(exact, drifted) + os.chmod(drifted, 0o600) + args = [value.replace(os.fspath(exact), os.fspath(drifted)) for value in args] + self.vault.run_keepass(args, self.master, os.fspath(self.home)) + self.assertEqual(self._classify(drifted), "mismatch") + + password_drift = self.root / "password.kdbx" + shutil.copyfile(exact, password_drift) + os.chmod(password_drift, 0o600) + other_path = self.root / "other-webhook" + write_private(other_path, synthetic_webhook(b"B")) + other = self.vault.open_validated_webhook(os.fspath(other_path)) + try: + self.vault.run_keepass( + ["edit", "-q", "-p", os.fspath(password_drift), entry], + self.master, + os.fspath(self.home), + webhook=other, + ) + finally: + other.close() + self.assertEqual(self._classify(password_drift), "mismatch") + + title_drift = self.root / "title.kdbx" + shutil.copyfile(exact, title_drift) + os.chmod(title_drift, 0o600) + self.vault.run_keepass( + ["edit", "-q", "-t", "Different title", os.fspath(title_drift), entry], + self.master, + os.fspath(self.home), + ) + self.assertEqual(self._classify(title_drift), "absent") + + def test_duplicate_title_is_ambiguous_and_update_is_explicit(self) -> None: + # Production breaks caught: destructive duplicate reconciliation or + # editing a mismatch before the explicit BUILD UPDATE command. + exact = self.root / "exact.kdbx" + self._build(self.baseline, exact, "ADD") + duplicate = self.root / "duplicate.kdbx" + shutil.copyfile(exact, duplicate) + os.chmod(duplicate, 0o600) + independently_created = self.root / "independent.kdbx" + self._build(self.baseline, independently_created, "ADD") + self.vault.run_keepass( + ["merge", "-q", "-s", os.fspath(duplicate), os.fspath(independently_created)], + self.master, + os.fspath(self.home), + ) + self.assertEqual(self._classify(duplicate), "ambiguous") + + mismatch = self.root / "mismatch.kdbx" + shutil.copyfile(exact, mismatch) + os.chmod(mismatch, 0o600) + self.vault.run_keepass( + ["edit", "-q", "--notes", "channel=wrong;app=wrong;recovery=wrong", os.fspath(mismatch), self.vault.ENTRY_PATH], + self.master, + os.fspath(self.home), + ) + before = hashlib.sha256(mismatch.read_bytes()).digest() + self.assertEqual(self._classify(mismatch), "mismatch") + self.assertEqual(hashlib.sha256(mismatch.read_bytes()).digest(), before) + updated = self.root / "updated.kdbx" + self._build(mismatch, updated, "UPDATE") + self.assertEqual(self._classify(updated), "exact-noop") + + def test_protected_comparison_requires_producer_success_and_exact_final_lf(self) -> None: + # Production break caught: trimming KeePass output or ignoring producer + # status before protected byte comparison. + expected = synthetic_webhook() + self.assertTrue(self.vault.protected_value_matches(0, expected + b"\n", expected)) + self.assertFalse(self.vault.protected_value_matches(1, expected + b"\n", expected)) + self.assertFalse(self.vault.protected_value_matches(0, expected, expected)) + self.assertFalse(self.vault.protected_value_matches(0, expected + b"\n\n", expected)) + + def test_private_socket_real_database_lifecycle_and_wipe(self) -> None: + # Production breaks caught: wrong socket mode/peer/protocol, secret-bearing + # replies, illegal state changes, or failure to wipe on shutdown. + read_fd, write_fd = os.pipe() + output_fd = os.open(os.devnull, os.O_WRONLY | os.O_CLOEXEC) + os.write(write_fd, self.app.encode("utf-8") + b"\n" + bytes(self.master) + b"\n") + os.close(write_fd) + wiped: list[bytes] = [] + webhook_storage: list[bytearray] = [] + failures: list[str] = [] + + def wipe_hook(buffer: bytearray) -> None: + wiped.append(bytes(buffer)) + + def target() -> None: + try: + real_open = self.vault.open_validated_webhook + def capture_open(path: str): + retained = real_open(path) + webhook_storage.append(retained._content) + return retained + with mock.patch.object( + self.vault, "open_validated_webhook", side_effect=capture_open + ): + self.vault.serve_private_socket( + os.fspath(self.root), + os.fspath(self.webhook_path), + input_fd=read_fd, + output_fd=output_fd, + accept_timeout=5.0, + io_timeout=5.0, + child_timeout=20.0, + wipe_hook=wipe_hook, + ) + except BaseException as exc: + failures.append(f"{type(exc).__name__}:{exc}") + finally: + os.close(read_fd) + os.close(output_fd) + + thread = threading.Thread(target=target, daemon=True) + thread.start() + socket_path = self.root / "vault.sock" + deadline = time.monotonic() + 5 + while not socket_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue(socket_path.exists()) + self.assertEqual(stat.S_IMODE(socket_path.stat().st_mode), 0o600) + + responses = bytearray() + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.settimeout(10) + client.connect(os.fspath(socket_path)) + try: + for command, expected in ( + (b"PREPARE\n", b"absent\n"), + (b"BUILD ADD\n", b"candidate-ready\n"), + (b"VERIFY_CANDIDATE\n", b"verified\n"), + (b"VERIFY_COMMITTED " + os.fsencode(self.root / "candidate.kdbx") + b"\n", b"verified\n"), + (b"SHUTDOWN\n", b"stopped\n"), + ): + client.sendall(command) + reply = b"" + while not reply.endswith(b"\n"): + reply += client.recv(128) + responses.extend(reply) + self.assertEqual(reply, expected) + finally: + client.close() + thread.join(timeout=10) + self.assertFalse(thread.is_alive()) + self.assertEqual(failures, []) + self.assertTrue(wiped) + self.assertEqual(wiped[-1], b"\0" * len(self.master)) + self.assertEqual(len(webhook_storage), 1) + self.assertEqual( + webhook_storage[0], bytearray(len(webhook_storage[0])) + ) + self.assertNotIn(bytes(self.master), responses) + self.assertNotIn(synthetic_webhook(), responses) + + def test_foreign_peer_is_rejected(self) -> None: + # Production break caught: accepting a local AF_UNIX client owned by a + # UID other than the invoking unprivileged user. + read_fd, write_fd = os.pipe() + output_fd = os.open(os.devnull, os.O_WRONLY | os.O_CLOEXEC) + os.write(write_fd, self.app.encode() + b"\n" + bytes(self.master) + b"\n") + os.close(write_fd) + failures: list[str] = [] + + def target() -> None: + try: + with mock.patch.object(self.vault, "peer_uid", return_value=os.getuid() + 1): + self.vault.serve_private_socket( + os.fspath(self.root), + os.fspath(self.webhook_path), + input_fd=read_fd, + output_fd=output_fd, + accept_timeout=5.0, + io_timeout=1.0, + child_timeout=5.0, + ) + except self.vault.VaultContractError: + pass + except BaseException as exc: + failures.append(type(exc).__name__) + finally: + os.close(read_fd) + os.close(output_fd) + + thread = threading.Thread(target=target, daemon=True) + thread.start() + socket_path = self.root / "vault.sock" + deadline = time.monotonic() + 5 + while not socket_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.settimeout(2) + try: + client.connect(os.fspath(socket_path)) + client.sendall(b"PREPARE\n") + try: + self.assertEqual(client.recv(32), b"") + except ConnectionResetError: + pass + finally: + client.close() + thread.join(timeout=5) + self.assertFalse(thread.is_alive()) + self.assertEqual(failures, []) + + def test_server_fixed_objects_remain_below_retained_runtime_after_path_replacement(self) -> None: + # Production break caught: resolving fixed private names through the + # startup pathname after its directory has been replaced by the same UID. + runtime = self.root / "server-runtime" + runtime.mkdir(mode=0o700) + baseline = runtime / "baseline.kdbx" + create_database(baseline, self.master) + webhook_path = runtime / "input-webhook" + write_private(webhook_path, synthetic_webhook()) + read_fd, write_fd = os.pipe() + output_fd = os.open(os.devnull, os.O_WRONLY | os.O_CLOEXEC) + os.write(write_fd, self.app.encode() + b"\n" + bytes(self.master) + b"\n") + os.close(write_fd) + failures: list[str] = [] + + def target() -> None: + try: + self.vault.serve_private_socket( + os.fspath(runtime), + os.fspath(webhook_path), + input_fd=read_fd, + output_fd=output_fd, + accept_timeout=5.0, + io_timeout=5.0, + child_timeout=20.0, + ) + except BaseException as exc: + failures.append(type(exc).__name__) + finally: + os.close(read_fd) + os.close(output_fd) + + thread = threading.Thread(target=target, daemon=True) + thread.start() + socket_path = runtime / "vault.sock" + deadline = time.monotonic() + 5 + while not socket_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue(socket_path.exists()) + + anchored = self.root / "server-runtime-anchored" + runtime.rename(anchored) + runtime.mkdir(mode=0o700) + foreign_candidate = runtime / "candidate.kdbx" + foreign_socket = runtime / "vault.sock" + write_private(foreign_candidate, b"foreign-candidate-must-survive") + write_private(foreign_socket, b"foreign-socket-name-must-survive") + + responses = bytearray() + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.settimeout(10) + client.connect(os.fspath(anchored / "vault.sock")) + try: + for command, expected in ( + (b"PREPARE\n", b"absent\n"), + (b"BUILD ADD\n", b"candidate-ready\n"), + (b"VERIFY_CANDIDATE\n", b"verified\n"), + ( + b"VERIFY_COMMITTED " + + os.fsencode(runtime / "candidate.kdbx") + + b"\n", + b"verified\n", + ), + (b"SHUTDOWN\n", b"stopped\n"), + ): + client.sendall(command) + reply = b"" + while not reply.endswith(b"\n"): + reply += client.recv(128) + responses.extend(reply) + self.assertEqual(reply, expected) + finally: + client.close() + thread.join(timeout=10) + self.assertFalse(thread.is_alive()) + self.assertEqual(failures, []) + self.assertTrue((anchored / "candidate.kdbx").is_file()) + self.assertFalse((anchored / "vault.sock").exists()) + self.assertEqual(foreign_candidate.read_bytes(), b"foreign-candidate-must-survive") + self.assertEqual(foreign_socket.read_bytes(), b"foreign-socket-name-must-survive") + self.assertNotIn(bytes(self.master), responses) + shutil.rmtree(anchored) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate/test-validate-blackbox-edge-source.sh b/scripts/validate/test-validate-blackbox-edge-source.sh new file mode 100644 index 0000000..e7f55b4 --- /dev/null +++ b/scripts/validate/test-validate-blackbox-edge-source.sh @@ -0,0 +1,562 @@ +#!/usr/bin/env bash + +# Production-boundary test for the temporary blackbox source proof. Every +# external side effect is replaced only below the Kubernetes/root boundaries; +# the validator CLI and its production control flow remain real. +set -Eeuo pipefail +set +x +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly VALIDATOR="$ROOT/scripts/validate/validate-blackbox-edge-source.sh" + +fail() { printf 'BLACKBOX EDGE SOURCE TEST FAILURE: %s\n' "$*" >&2; exit 1; } +assert_eq() { [[ "$1" == "$2" ]] || fail "$3: expected=$1 actual=$2"; } +assert_contains() { [[ "$1" == *"$2"* ]] || fail "$3"; } +assert_not_contains() { [[ "$1" != *"$2"* ]] || fail "$3"; } +assert_no_object_create() { + [[ "$1" != *' create --dry-run=server '* && "$1" != *' create -f '* ]] || fail "$2" +} +assert_no_name_delete() { + [[ "$1" != *' delete pod '* && "$1" != *' delete networkpolicy '* ]] || fail "$2" +} + +[[ -f "$VALIDATOR" && ! -L "$VALIDATOR" ]] || fail 'production validator is missing (RED: create validate-blackbox-edge-source.sh)' + +# RED 8: production uses / as its rollback anchor. A canonical absolute +# descendant of that anchor must remain valid without invoking sudo or any +# Kubernetes boundary; non-root fixture-anchor checks are exercised below. +production_root_anchor_rc=0 +PLATFORM_BLACKBOX_EDGE_TEST_MODE=0 bash -c ' + source "$1" + root_run() { "$@"; } + validate_root_chain /var/lib / +' bash "$VALIDATOR" || production_root_anchor_rc=$? +assert_eq 0 "$production_root_anchor_rc" 'production root anchor must accept a canonical descendant' + +fixture_root="$(mktemp -d /tmp/platform-blackbox-edge-test.XXXXXX)" +chmod 0700 "$fixture_root" +trap 'rm -rf -- "$fixture_root"' EXIT +mkdir -p "$fixture_root/bin" "$fixture_root/state" "$fixture_root/active" \ + "$fixture_root/rollbacks/observability-20260812T000000Z" +chmod 0700 "$fixture_root/bin" "$fixture_root/state" "$fixture_root/active" \ + "$fixture_root/rollbacks" "$fixture_root/rollbacks/observability-20260812T000000Z" +cp -- "$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf" \ + "$fixture_root/active/learn-services" +chmod 0600 "$fixture_root/active/learn-services" + +cat >"$fixture_root/bin/kubectl-proxy.py" <<'PY' +import json +import os +import pathlib +import socket +import sys + +sock_path = pathlib.Path(sys.argv[1]) +state = pathlib.Path(sys.argv[2]) +if sock_path.exists() or sock_path.is_symlink(): + raise SystemExit(91) +server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +server.bind(str(sock_path)) +os.chmod(sock_path, 0o600) +server.listen(1) +connection, _ = server.accept() +request = b"" +while b"\r\n\r\n" not in request: + piece = connection.recv(4096) + if not piece: + break + request += piece +head, _, body = request.partition(b"\r\n\r\n") +lines = head.decode("ascii", "replace").split("\r\n") +method, path, _ = lines[0].split(" ", 2) +length = 0 +for line in lines[1:]: + if line.lower().startswith("content-length:"): + length = int(line.split(":", 1)[1].strip()) +while len(body) < length: + piece = connection.recv(4096) + if not piece: + break + body += piece +try: + expected = json.loads(body.decode("utf-8"))["preconditions"]["uid"] +except Exception: + connection.sendall(b"HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\n\r\n") + connection.close(); server.close(); raise SystemExit(0) +kind = "pod" if "/pods/" in path else "networkpolicy" if "/networkpolicies/" in path else "" +item_path = state / f"{kind}.json" +log = pathlib.Path(os.environ["BB_TEST_PROXY_LOG"]) +with log.open("a", encoding="utf-8") as stream: + stream.write(f"{method} {path} uid={expected}\n") +if not kind or not item_path.exists(): + connection.sendall(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") +elif os.environ.get("BB_TEST_DELETE_REPLACE_KIND") == kind or os.environ.get("BB_TEST_FOREIGN_UID") == "1": + item = json.loads(item_path.read_text(encoding="utf-8")) + item["metadata"]["uid"] = "uid-external-replacement" + item["metadata"]["labels"]["platform.hyeonworks.com/source-proof-run"] = "foreign-run" + item_path.write_text(json.dumps(item), encoding="utf-8") + connection.sendall(b"HTTP/1.1 409 Conflict\r\nContent-Length: 0\r\n\r\n") +elif json.loads(item_path.read_text(encoding="utf-8"))["metadata"]["uid"] != expected: + connection.sendall(b"HTTP/1.1 409 Conflict\r\nContent-Length: 0\r\n\r\n") +elif os.environ.get("BB_TEST_DELETE_AMBIGUOUS_KIND") == kind: + item_path.unlink() + connection.close(); server.close(); raise SystemExit(0) +elif os.environ.get("BB_TEST_DELETE_READ_AMBIGUOUS_KIND") == kind: + (state / f"{kind}.delete-read-ambiguous").touch() + connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nContent-Type: application/json\r\n\r\n{\"status\":\"Success\"}") +elif os.environ.get("BB_TEST_DELETE_NEVER_DISAPPEARS_KIND") == kind: + connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nContent-Type: application/json\r\n\r\n{\"status\":\"Success\"}") +elif os.environ.get("BB_TEST_DELETE_TERMINATING_KIND") == kind: + pending_polls = int(os.environ.get("BB_TEST_DELETE_PENDING_POLLS", "1")) + if pending_polls < 1: + raise SystemExit(92) + (state / f"{kind}.delete-pending").write_text(str(pending_polls), encoding="utf-8") + connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nContent-Type: application/json\r\n\r\n{\"status\":\"Success\"}") +else: + item_path.unlink() + connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nContent-Type: application/json\r\n\r\n{\"status\":\"Success\"}") +connection.close() +server.close() +PY + +cat >"$fixture_root/bin/kubectl" <<'KUBECTL' +#!/usr/bin/env bash +set -Eeuo pipefail + +log_command() { printf '%q ' "$@" >>"${BB_TEST_LOG:?}"; printf '\n' >>"${BB_TEST_LOG:?}"; } +state_file() { printf '%s/%s.json\n' "${BB_TEST_STATE:?}" "$1"; } + +render_object() { + local kind=$1 name=$2 run=$3 uid=$4 bad=${5:-} live_defaults=${6:-0} mutate=${7:-} policy_mutate=${8:-} + /usr/bin/python3 - "$kind" "$name" "$run" "$uid" "$bad" "$live_defaults" "$mutate" "$policy_mutate" <<'PY' +import json +import sys +kind, name, run, uid, bad, live_defaults, mutate, policy_mutate = sys.argv[1:] +labels = { + "app.kubernetes.io/managed-by": "platform-blackbox-edge-source", + "platform.hyeonworks.com/source-proof-run": run, +} +metadata = {"namespace": "observability", "name": name, "uid": uid, "labels": labels} +if kind == "networkpolicy": + spec = { + "podSelector": {"matchLabels": {"platform.hyeonworks.com/source-proof-run": run}}, + "policyTypes": ["Egress"], + "egress": [ + {"to": [{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}, "podSelector": {"matchLabels": {"k8s-app": "kube-dns"}}}], "ports": [{"protocol": "UDP", "port": 53}, {"protocol": "TCP", "port": 53}]}, + {"to": [{"ipBlock": {"cidr": "192.168.0.107/32"}}], "ports": [{"protocol": "TCP", "port": 443}]}, + ], + } + item = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": metadata, "spec": spec} +else: + spec = { + "automountServiceAccountToken": False, + "restartPolicy": "Never", + "securityContext": {"runAsNonRoot": True, "runAsUser": 65534, "runAsGroup": 65534, "seccompProfile": {"type": "RuntimeDefault"}}, + "containers": [{"name": "probe", "image": "docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0", "imagePullPolicy": "IfNotPresent", "command": ["sh", "-c", "sleep 120"], "securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}}}], + } + item = {"apiVersion": "v1", "kind": "Pod", "metadata": metadata, "spec": spec} + if live_defaults == "1": + spec.update({ + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": True, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "schedulerName": "default-scheduler", + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + {"effect": "NoExecute", "key": "node.kubernetes.io/not-ready", "operator": "Exists", "tolerationSeconds": 300}, + {"effect": "NoExecute", "key": "node.kubernetes.io/unreachable", "operator": "Exists", "tolerationSeconds": 300}, + ], + }) + spec["containers"][0].update({ + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + }) + metadata.update({"creationTimestamp": "2026-08-12T17:20:51Z", "generation": 1}) + item["status"] = {"phase": "Pending", "qosClass": "BestEffort"} + if mutate == "security": + spec["containers"][0]["securityContext"]["privileged"] = True + elif mutate == "command": + spec["containers"][0]["command"] = ["sh", "-c", "sleep 120; id"] + elif mutate == "image": + spec["containers"][0]["image"] = "docker.io/library/busybox:latest" +if (kind == "pod" and mutate == "label") or (kind == "networkpolicy" and policy_mutate == "label"): + labels["platform.hyeonworks.com/unowned"] = "unexpected" +if bad == kind: + item["metadata"]["labels"]["platform.hyeonworks.com/source-proof-run"] = "foreign-run" +print(json.dumps(item)) +PY +} + +read_manifest() { + local body kind name run + body="$(/bin/cat)" + kind="$(printf '%s\n' "$body" | /usr/bin/awk '/^kind: / { print $2; exit }')" + name="$(printf '%s\n' "$body" | /usr/bin/awk '/^ name: / { print $2; exit }')" + run="$(printf '%s\n' "$body" | /usr/bin/awk '/source-proof-run:/ { gsub(/"/, "", $2); print $2; exit }')" + case "$kind" in NetworkPolicy) printf 'networkpolicy|%s|%s\n' "$name" "$run" ;; Pod) printf 'pod|%s|%s\n' "$name" "$run" ;; *) exit 95 ;; esac +} + +log_command "$@" +args=" $* " +if [[ "$args" == *' config current-context '* ]]; then printf 'fixture-context\n'; exit 0; fi +if [[ "$args" == *' get --raw=/readyz '* ]]; then printf 'ok\n'; exit 0; fi +if [[ "$args" == *' auth can-i '* ]]; then printf '%s\n' "${BB_TEST_AUTH:-yes}"; exit 0; fi + +if [[ "$args" == *' create '* ]]; then + IFS='|' read -r kind name run <<<"$(read_manifest)" + mode=create + [[ "$args" == *' --dry-run=server '* ]] && mode=dry-run + printf '%s %s\n' "$kind" "$mode" >>"${BB_TEST_CREATE_LOG:?}" + uid="uid-${kind}-${run}" + bad="${BB_TEST_BAD_SPEC_KIND:-}" + if [[ "$mode" == dry-run ]]; then render_object "$kind" "$name" "$run" dry-run "$bad" "${BB_TEST_LIVE_DEFAULTED_POD_DRY_RUN:-0}" "${BB_TEST_MUTATE_POD_FIELD:-}" "${BB_TEST_MUTATE_NETWORKPOLICY_FIELD:-}"; exit 0; fi + path="$(state_file "$kind")" + [[ ! -e "$path" && ! -L "$path" ]] || exit 1 + render_object "$kind" "$name" "$run" "$uid" "$bad" >"$path" + if [[ "${BB_TEST_CREATE_AMBIGUOUS_KIND:-}" == "$kind" || "${BB_TEST_CREATE_FAILURE_KIND:-}" == "$kind" ]]; then exit 1; fi + if [[ "${BB_TEST_BAD_RESPONSE_KIND:-}" == "$kind" ]]; then + render_object "$kind" "$name" "$run" "$uid" "$kind" + exit 0 + fi + /bin/cat -- "$path" + exit 0 +fi + +if [[ "$args" == *' get pod '* || "$args" == *' get networkpolicy '* ]]; then + if [[ "$args" =~ get\ (pod|networkpolicy)\ ([^[:space:]]+) ]]; then kind=${BASH_REMATCH[1]}; name=${BASH_REMATCH[2]}; else exit 96; fi + path="$(state_file "$kind")" + [[ ! -e "${BB_TEST_STATE:?}/${kind}.delete-read-ambiguous" ]] || exit 1 + if [[ -e "$path" && ! -L "$path" ]]; then + if [[ "${BB_TEST_FOREIGN_UID:-0}" == 1 ]]; then + /usr/bin/python3 - "$path" <<'PY' +import json, pathlib, sys +p = pathlib.Path(sys.argv[1]); item = json.loads(p.read_text()); item["metadata"]["uid"] = "foreign-uid"; item["metadata"]["labels"]["platform.hyeonworks.com/source-proof-run"] = "foreign-run"; p.write_text(json.dumps(item)) +PY + fi + if [[ -e "${BB_TEST_STATE:?}/${kind}.delete-pending" ]]; then + remaining="$(<"${BB_TEST_STATE:?}/${kind}.delete-pending")" + [[ "$remaining" =~ ^[1-9][0-9]*$ ]] || exit 99 + remaining=$((remaining - 1)) + /bin/cat -- "$path" + if (( remaining == 0 )); then + /bin/rm -f -- "${BB_TEST_STATE:?}/${kind}.delete-pending" "$path" + else + printf '%s\n' "$remaining" >"${BB_TEST_STATE:?}/${kind}.delete-pending" + fi + exit 0 + fi + /bin/cat -- "$path"; exit 0 + fi + [[ "$args" == *'--ignore-not-found'* ]] && exit 0 + exit 1 +fi + +if [[ "$args" == *' wait --for=condition=Ready '* && "$args" == *' pod/'* ]]; then + if [[ -n "${BB_TEST_SCHEDULED_POD_NODE:-}" && -e "$(state_file pod)" ]]; then + /usr/bin/python3 - "$(state_file pod)" "${BB_TEST_SCHEDULED_POD_NODE}" <<'PY' +import json, pathlib, sys +p = pathlib.Path(sys.argv[1]); item = json.loads(p.read_text()); item["spec"]["nodeName"] = sys.argv[2]; p.write_text(json.dumps(item)) +PY + fi + [[ -e "$(state_file pod)" ]]; exit $? +fi +if [[ "$args" == *' exec '* ]]; then + [[ "${BB_TEST_EXEC_SLEEP:-0}" == 1 ]] && /usr/bin/sleep 20 + case "${BB_TEST_CLIENT_STATUS:-403}" in + 403) printf 'HTTP/1.1 403 Forbidden\n' >&2 ;; + 200) printf 'HTTP/1.1 200 OK\n' >&2 ;; + 302) printf 'HTTP/1.1 302 Found\n' >&2 ;; + *) printf 'HTTP/1.1 500 Error\n' >&2 ;; + esac + exit "${BB_TEST_CLIENT_RC:-1}" +fi +if [[ "$args" == *' proxy '* ]]; then + socket='' + for argument in "$@"; do [[ "$argument" == --unix-socket=* ]] && socket=${argument#--unix-socket=}; done + [[ "$socket" == /tmp/platform-blackbox-edge-source.*/*.sock ]] || exit 97 + exec /usr/bin/python3 "${BB_TEST_PROXY_HELPER:?}" "$socket" "${BB_TEST_STATE:?}" +fi +if [[ "$args" == *' delete '* || "$args" == *' apply '* || "$args" == *' replace '* || "$args" == *' patch '* ]]; then exit 98; fi +exit 0 +KUBECTL + +cat >"$fixture_root/bin/sudo" <<'SUDO' +#!/usr/bin/env bash +set -Eeuo pipefail +while [[ "${1:-}" == -n || "${1:-}" == -- ]]; do shift; done +command_path=${1:-} +[[ "$command_path" == /* ]] || { printf 'non-absolute sudo child: %s\n' "$command_path" >&2; exit 97; } +printf '%q ' "$@" >>"${BB_TEST_ROOT_LOG:?}"; printf '\n' >>"${BB_TEST_ROOT_LOG:?}" +if [[ "$command_path" == /usr/bin/awk ]]; then + printf '%q ' "$@" >>"${BB_TEST_SUDO_LOG:?}"; printf '\n' >>"${BB_TEST_SUDO_LOG:?}" + case "${BB_TEST_LOG_MATCHES:-1}" in + 0) exit 0 ;; + 1) printf '10.42.0.55 403\n10.42.0.55 403\n10.42.0.55 403\n' ;; + 2) printf '10.42.0.55 403\n10.42.0.55 403\n10.42.0.55 403\n10.42.0.55 403\n' ;; + *) exit 98 ;; + esac + exit 0 +fi +if [[ "$command_path" == /usr/bin/install ]]; then + shift + filtered=() + while (( $# )); do case "$1" in -o|-g) shift 2 ;; *) filtered+=("$1"); shift ;; esac; done + exec /usr/bin/install "${filtered[@]}" +fi +if [[ "$command_path" == /bin/ln && "${BB_TEST_PUBLISH_RACE:-0}" == 1 ]]; then + /bin/mkdir -p -- "${BB_TEST_PROOF:?}.race-dir" + /bin/ln -s -- "${BB_TEST_PROOF}.race-dir" "${BB_TEST_PROOF:?}" +fi +exec "$@" +SUDO + +cat >"$fixture_root/bin/date" <<'DATE' +#!/usr/bin/env bash +set -Eeuo pipefail +if [[ ( "${BB_TEST_POST_PROMPT_ACTIVE_DRIFT:-0}" == 1 || "${BB_TEST_POST_PROMPT_PROOF_APPEARS:-0}" == 1 ) && ! -e "${BB_TEST_POST_PROMPT_MARKER:?}" ]]; then + : >"${BB_TEST_POST_PROMPT_MARKER:?}" + [[ "${BB_TEST_POST_PROMPT_ACTIVE_DRIFT:-0}" == 1 ]] && printf 'active drift\n' >"${BB_TEST_ACTIVE:?}" + [[ "${BB_TEST_POST_PROMPT_PROOF_APPEARS:-0}" == 1 ]] && printf 'unsafe existing proof\n' >"${BB_TEST_PROOF:?}" +fi +if [[ "$*" == *'+%Y-%m-%dT%H:%M:%SZ'* ]]; then printf '2026-08-12T00:00:00Z\n'; else /usr/bin/date "$@"; fi +DATE + +cat >"$fixture_root/bin/sleep" <<'SLEEP' +#!/usr/bin/env bash +set -Eeuo pipefail +[[ "$#" == 1 && "$1" == 0.05 ]] || exit 97 +exec /usr/bin/sleep 0.001 +SLEEP + +chmod 0700 "$fixture_root/bin"/* + +proof="$fixture_root/rollbacks/observability-20260812T000000Z/blackbox-source-proof.env" + +run_validator() { + local output_file="$fixture_root/out" rc=0 + /bin/rm -f -- "$fixture_root/state"/* "$fixture_root/commands.log" "$fixture_root/creates.log" \ + "$fixture_root/deletes.log" "$fixture_root/sudo.log" "$fixture_root/root.log" \ + "$fixture_root/proxy.log" "$fixture_root/post-prompt" + : >"$fixture_root/commands.log"; : >"$fixture_root/creates.log"; : >"$fixture_root/deletes.log" + : >"$fixture_root/sudo.log"; : >"$fixture_root/root.log"; : >"$fixture_root/proxy.log" + PLATFORM_BLACKBOX_EDGE_TEST_MODE=1 \ + PLATFORM_BLACKBOX_EDGE_KUBECTL="$fixture_root/bin/kubectl" \ + PLATFORM_BLACKBOX_EDGE_SUDO="$fixture_root/bin/sudo" \ + PLATFORM_BLACKBOX_EDGE_DATE="$fixture_root/bin/date" \ + PLATFORM_BLACKBOX_EDGE_SLEEP="$fixture_root/bin/sleep" \ + PLATFORM_BLACKBOX_EDGE_ACTIVE="$fixture_root/active/learn-services" \ + PLATFORM_BLACKBOX_EDGE_ROLLBACK_BASE="$fixture_root/rollbacks" \ + PLATFORM_BLACKBOX_EDGE_CONFIRMATION='PROVE BLACKBOX PRIVATE EDGE fixture-context' \ + PLATFORM_OBSERVABILITY_ROLLBACK_ID=20260812T000000Z \ + BB_TEST_LOG="$fixture_root/commands.log" BB_TEST_CREATE_LOG="$fixture_root/creates.log" \ + BB_TEST_DELETE_LOG="$fixture_root/deletes.log" BB_TEST_SUDO_LOG="$fixture_root/sudo.log" \ + BB_TEST_ROOT_LOG="$fixture_root/root.log" BB_TEST_PROXY_LOG="$fixture_root/proxy.log" \ + BB_TEST_PROXY_HELPER="$fixture_root/bin/kubectl-proxy.py" BB_TEST_STATE="$fixture_root/state" \ + BB_TEST_TMP="$fixture_root/state" BB_TEST_ACTIVE="$fixture_root/active/learn-services" \ + BB_TEST_PROOF="$proof" BB_TEST_POST_PROMPT_MARKER="$fixture_root/post-prompt" \ + BB_TEST_SCHEDULED_POD_NODE="${BB_TEST_SCHEDULED_POD_NODE:-}" \ + BB_TEST_DELETE_TERMINATING_KIND="${BB_TEST_DELETE_TERMINATING_KIND:-}" \ + BB_TEST_DELETE_PENDING_POLLS="${BB_TEST_DELETE_PENDING_POLLS:-1}" \ + BB_TEST_DELETE_READ_AMBIGUOUS_KIND="${BB_TEST_DELETE_READ_AMBIGUOUS_KIND:-}" \ + BB_TEST_DELETE_NEVER_DISAPPEARS_KIND="${BB_TEST_DELETE_NEVER_DISAPPEARS_KIND:-}" \ + bash "$VALIDATOR" --execute --context fixture-context >"$output_file" 2>&1 || rc=$? + RUN_OUTPUT="$(<"$output_file")"; RUN_RC=$rc +} + +# Production must reject escape hatches before a dry-run can contact a fake. +override_rc=0 +PLATFORM_BLACKBOX_EDGE_ACTIVE=/tmp/override bash "$VALIDATOR" >"$fixture_root/override.out" 2>&1 || override_rc=$? +assert_eq 1 "$override_rc" 'production override boundary' + +# Dry-run is read-only and contains neither token nor log payload. +: >"$fixture_root/commands.log" +dry_output="$(PLATFORM_BLACKBOX_EDGE_TEST_MODE=1 PLATFORM_BLACKBOX_EDGE_KUBECTL="$fixture_root/bin/kubectl" PLATFORM_BLACKBOX_EDGE_SUDO="$fixture_root/bin/sudo" PLATFORM_BLACKBOX_EDGE_DATE="$fixture_root/bin/date" PLATFORM_BLACKBOX_EDGE_SLEEP="$fixture_root/bin/sleep" PLATFORM_BLACKBOX_EDGE_ACTIVE="$fixture_root/active/learn-services" PLATFORM_BLACKBOX_EDGE_ROLLBACK_BASE="$fixture_root/rollbacks" BB_TEST_LOG="$fixture_root/commands.log" bash "$VALIDATOR")" +assert_contains "$dry_output" 'BLACKBOX_PRIVATE_EDGE_SOURCE_DRY_RUN=PASS' 'dry-run result label' +[[ ! -s "$fixture_root/commands.log" ]] || fail 'dry-run contacted Kubernetes' +assert_not_contains "$dry_output" 'hyeonworks_probe=' 'dry-run disclosed a token' + +# Bad active SHA and authorization fail before any create. +printf 'wrong active source\n' >"$fixture_root/active/learn-services" +run_validator +assert_eq 1 "$RUN_RC" 'wrong active guard must use fixed failure status' +assert_no_object_create "$(<"$fixture_root/commands.log")" 'wrong active guard created an object' +cp -- "$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf" "$fixture_root/active/learn-services" +BB_TEST_AUTH=no run_validator +assert_eq 1 "$RUN_RC" 'authorization denial must use fixed failure status' +assert_no_object_create "$(<"$fixture_root/commands.log")" 'authorization denial created an object' + +# RED 1: BusyBox wget returns 1 for HTTP 403. The pinned client must pass only +# for exactly that result and an exact final parsed 403—not GNU wget's rc 8. +# RED 1a: Kubernetes server-side Pod dry-run adds its default fields. They must +# not make the validator reject an otherwise exact restricted probe Pod. +BB_TEST_LIVE_DEFAULTED_POD_DRY_RUN=1 BB_TEST_CLIENT_STATUS=403 BB_TEST_CLIENT_RC=1 run_validator +assert_eq 0 "$RUN_RC" 'live-defaulted Pod server dry-run must pass' +/bin/rm -f -- "$proof" +BB_TEST_MUTATE_NETWORKPOLICY_FIELD=label run_validator +assert_eq 1 "$RUN_RC" 'NetworkPolicy extra metadata label must fail closed' +unset BB_TEST_MUTATE_NETWORKPOLICY_FIELD +for mutated_field in label security command image; do + BB_TEST_LIVE_DEFAULTED_POD_DRY_RUN=1 BB_TEST_MUTATE_POD_FIELD="$mutated_field" run_validator + assert_eq 1 "$RUN_RC" "live-defaulted Pod $mutated_field mutation must fail closed" +done +unset BB_TEST_LIVE_DEFAULTED_POD_DRY_RUN BB_TEST_MUTATE_POD_FIELD +BB_TEST_CLIENT_STATUS=403 BB_TEST_CLIENT_RC=1 run_validator +assert_eq 0 "$RUN_RC" 'BusyBox 403 exit 1 with final parsed 403 must pass' +assert_contains "$RUN_OUTPUT" 'BLACKBOX PRIVATE EDGE SOURCE PASS' 'BusyBox success label' +assert_not_contains "$RUN_OUTPUT" 'hyeonworks_probe=' 'BusyBox success disclosed token' +assert_eq 1 "$(wc -l <"$fixture_root/sudo.log")" 'exactly one sudo awk invocation' +assert_eq $'networkpolicy dry-run\nnetworkpolicy create\npod dry-run\npod create' "$(<"$fixture_root/creates.log")" 'create-only safe order' +assert_contains "$(<"$fixture_root/commands.log")" ' proxy ' 'cleanup must use bounded raw-delete proxy' +assert_no_name_delete "$(<"$fixture_root/commands.log")" 'cleanup must never name-delete' +assert_contains "$(<"$fixture_root/root.log")" '/usr/bin/mktemp' 'proof must stage under root mktemp' +assert_contains "$(<"$fixture_root/root.log")" "--tmpdir=$fixture_root/rollbacks/observability-20260812T000000Z" 'proof staging directory' +assert_contains "$(<"$fixture_root/root.log")" '/bin/ln -nT --' 'proof publication must not dereference a raced destination' +assert_eq $'schema=platform-blackbox-source-v1\nrollback_id=20260812T000000Z\nnginx_sha256='"$(sha256sum "$fixture_root/active/learn-services" | awk '{print $1}')"$'\ntested_at_utc=2026-08-12T00:00:00Z\ngrafana_remote_addr=10.42.0.55\ngrafana_status=403\nstorage_admin_remote_addr=10.42.0.55\nstorage_admin_status=403\ndb_admin_remote_addr=10.42.0.55\ndb_admin_status=403' "$(<"$proof")" 'exact proof schema and order' +[[ "$(stat -c %a "$proof")" == 600 && "$(stat -c %h "$proof")" == 1 ]] || fail 'proof must be unlinked 0600 evidence' +/bin/rm -f -- "$proof" + +BB_TEST_CLIENT_STATUS=403 BB_TEST_CLIENT_RC=0 run_validator +assert_eq 1 "$RUN_RC" '403 with non-BusyBox success rc must fail' +BB_TEST_CLIENT_STATUS=403 BB_TEST_CLIENT_RC=8 run_validator +assert_eq 1 "$RUN_RC" '403 with GNU-wget rc must fail' +BB_TEST_CLIENT_STATUS=200 BB_TEST_CLIENT_RC=1 run_validator +assert_eq 1 "$RUN_RC" '200 source masquerade must fail' +assert_not_contains "$RUN_OUTPUT" 'hyeonworks_probe=' 'masquerade failure disclosed token' + +# RED 2: values that drift after confirmation but before the first create are +# rejected by a repeated commit gate. +BB_TEST_POST_PROMPT_ACTIVE_DRIFT=1 run_validator +assert_eq 1 "$RUN_RC" 'post-prompt active SHA drift must fail' +assert_no_object_create "$(<"$fixture_root/commands.log")" 'active drift reached create' +cp -- "$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf" "$fixture_root/active/learn-services" +/bin/rm -f -- "$proof" +BB_TEST_POST_PROMPT_PROOF_APPEARS=1 run_validator +assert_eq 1 "$RUN_RC" 'post-prompt proof appearance must fail' +assert_no_object_create "$(<"$fixture_root/commands.log")" 'proof appearance reached create' +/bin/rm -f -- "$proof" + +# RED 3/4: a nonzero create may have committed. Exact owned state is cleaned +# through UID-preconditioned raw DELETE; never apply/update or name-delete. +BB_TEST_CREATE_AMBIGUOUS_KIND=networkpolicy run_validator +assert_eq 1 "$RUN_RC" 'ambiguous NetworkPolicy create must fail closed' +[[ ! -e "$fixture_root/state/networkpolicy.json" && ! -e "$fixture_root/state/pod.json" ]] || fail 'owned ambiguous NetworkPolicy was not cleaned' +assert_contains "$(<"$fixture_root/proxy.log")" '/networkpolicies/' 'NetworkPolicy raw UID delete' +BB_TEST_CREATE_AMBIGUOUS_KIND=pod run_validator +assert_eq 1 "$RUN_RC" 'ambiguous Pod create must fail closed' +[[ ! -e "$fixture_root/state/networkpolicy.json" && ! -e "$fixture_root/state/pod.json" ]] || fail 'owned partial create was not cleaned' +assert_contains "$(<"$fixture_root/proxy.log")" '/networkpolicies/' 'partial cleanup NetworkPolicy raw UID delete' +assert_contains "$(<"$fixture_root/proxy.log")" '/pods/' 'partial cleanup Pod raw UID delete' +BB_TEST_BAD_RESPONSE_KIND=pod run_validator +assert_eq 1 "$RUN_RC" 'malformed successful Pod create response must reclassify and fail closed' +[[ ! -e "$fixture_root/state/networkpolicy.json" && ! -e "$fixture_root/state/pod.json" ]] || fail 'owned malformed-response objects were not cleaned' + +# RED 4a: catch a cleanup wait that gives up while a normally terminating, +# exact-owned scheduled Pod remains present beyond 640 cleanup observations, +# then disappears while still requiring two consecutive absence reads. +export BB_TEST_SCHEDULED_POD_NODE=donghyeon-system-product-name \ + BB_TEST_DELETE_TERMINATING_KIND=pod BB_TEST_DELETE_PENDING_POLLS=700 +run_validator +assert_eq 0 "$RUN_RC" 'expected scheduled Pod node during terminating cleanup must pass' +/bin/rm -f -- "$proof" +export BB_TEST_SCHEDULED_POD_NODE=foreign-node BB_TEST_DELETE_TERMINATING_KIND=pod \ + BB_TEST_DELETE_PENDING_POLLS=2 +run_validator +assert_eq 1 "$RUN_RC" 'foreign scheduled Pod node must fail closed' +assert_not_contains "$RUN_OUTPUT" 'BLACKBOX PRIVATE EDGE SOURCE PASS' 'foreign scheduled Pod node became proof-command PASS' +[[ -e "$fixture_root/state/pod.json" ]] || fail 'foreign scheduled Pod disappeared' +unset BB_TEST_SCHEDULED_POD_NODE BB_TEST_DELETE_TERMINATING_KIND BB_TEST_DELETE_PENDING_POLLS +/bin/rm -f -- "$proof" + +# Cleanup GET ambiguity must fail closed with the exact-owned object retained. +BB_TEST_DELETE_READ_AMBIGUOUS_KIND=pod run_validator +assert_eq 1 "$RUN_RC" 'cleanup API/read ambiguity must fail closed' +assert_not_contains "$RUN_OUTPUT" 'BLACKBOX PRIVATE EDGE SOURCE PASS' 'cleanup API/read ambiguity became proof-command PASS' +[[ -e "$fixture_root/state/pod.json" ]] || fail 'read-ambiguous exact-owned Pod disappeared' +/bin/rm -f -- "$proof" + +# An exact-owned object that never disappears must exhaust the bounded wait, +# fail cleanup, and never become a proof-command PASS. +BB_TEST_DELETE_NEVER_DISAPPEARS_KIND=pod run_validator +assert_eq 1 "$RUN_RC" 'never-disappearing exact-owned Pod must fail bounded cleanup' +assert_not_contains "$RUN_OUTPUT" 'BLACKBOX PRIVATE EDGE SOURCE PASS' 'never-disappearing exact-owned Pod became proof-command PASS' +[[ -e "$fixture_root/state/pod.json" ]] || fail 'never-disappearing exact-owned Pod disappeared' +/bin/rm -f -- "$proof" + +# A transport-ambiguous raw delete is accepted only after bounded stable +# absence; a UID/label replacement remains untouched and fails safely. +BB_TEST_DELETE_AMBIGUOUS_KIND=pod run_validator +assert_eq 0 "$RUN_RC" 'ambiguous raw delete with stable absence must pass' +/bin/rm -f -- "$proof" +BB_TEST_DELETE_REPLACE_KIND=pod run_validator +assert_eq 1 "$RUN_RC" 'replacement during raw delete must fail safely' +[[ -e "$fixture_root/state/pod.json" ]] || fail 'replacement was deleted' +/bin/rm -f -- "$proof" "$fixture_root/state/pod.json" "$fixture_root/state/networkpolicy.json" + +# RED 5: a proof target that becomes a directory symlink only at publication +# must fail without allowing ln to create a hard link below that foreign path. +BB_TEST_PUBLISH_RACE=1 run_validator +assert_eq 1 "$RUN_RC" 'publication symlink race must fail safely' +[[ -L "$proof" ]] || fail 'publication race did not preserve the unsafe target' +[[ ! -e "$proof.race-dir"/* && ! -L "$proof.race-dir"/* ]] || fail 'publication followed a raced destination symlink' +/bin/rm -f -- "$proof"; /bin/rm -rf -- "$proof.race-dir" + +# Existing proof symlink/hardlink and any rollback-root symlink ancestor are +# unsafe. The validator must leave all temporary objects uncreated. +ln -s /dev/null "$proof" +run_validator +assert_eq 1 "$RUN_RC" 'symlink proof target must fail safely' +/bin/rm -f -- "$proof"; printf 'existing\n' >"$proof"; ln "$proof" "$proof.link" +run_validator +assert_eq 1 "$RUN_RC" 'hard-linked proof target must fail safely' +/bin/rm -f -- "$proof" "$proof.link" +/bin/mv -- "$fixture_root/rollbacks" "$fixture_root/rollbacks-real" +/bin/ln -s -- "$fixture_root/rollbacks-real" "$fixture_root/rollbacks" +run_validator +assert_eq 1 "$RUN_RC" 'rollback root symlink ancestor must fail safely' +assert_no_object_create "$(<"$fixture_root/commands.log")" 'symlink ancestor reached create' +/bin/rm -f -- "$fixture_root/rollbacks"; /bin/mv -- "$fixture_root/rollbacks-real" "$fixture_root/rollbacks" + +# A foreign object is never deleted, including on a signal path. +BB_TEST_FOREIGN_UID=1 run_validator +assert_eq 1 "$RUN_RC" 'foreign UID must fail safely' +assert_not_contains "$RUN_OUTPUT" 'BLACKBOX PRIVATE EDGE SOURCE PASS' 'foreign UID became proof-command PASS' +[[ -e "$fixture_root/state/networkpolicy.json" || -e "$fixture_root/state/pod.json" ]] || fail 'foreign object disappeared' +/bin/rm -f -- "$fixture_root/state"/*.json "$proof" + +: >"$fixture_root/commands.log"; : >"$fixture_root/creates.log"; : >"$fixture_root/root.log"; : >"$fixture_root/proxy.log" +setsid env \ + PLATFORM_BLACKBOX_EDGE_TEST_MODE=1 \ + PLATFORM_BLACKBOX_EDGE_KUBECTL="$fixture_root/bin/kubectl" \ + PLATFORM_BLACKBOX_EDGE_SUDO="$fixture_root/bin/sudo" \ + PLATFORM_BLACKBOX_EDGE_DATE="$fixture_root/bin/date" \ + PLATFORM_BLACKBOX_EDGE_SLEEP="$fixture_root/bin/sleep" \ + PLATFORM_BLACKBOX_EDGE_ACTIVE="$fixture_root/active/learn-services" \ + PLATFORM_BLACKBOX_EDGE_ROLLBACK_BASE="$fixture_root/rollbacks" \ + PLATFORM_BLACKBOX_EDGE_CONFIRMATION='PROVE BLACKBOX PRIVATE EDGE fixture-context' \ + PLATFORM_OBSERVABILITY_ROLLBACK_ID=20260812T000000Z \ + BB_TEST_LOG="$fixture_root/commands.log" BB_TEST_CREATE_LOG="$fixture_root/creates.log" \ + BB_TEST_DELETE_LOG="$fixture_root/deletes.log" BB_TEST_SUDO_LOG="$fixture_root/sudo.log" \ + BB_TEST_ROOT_LOG="$fixture_root/root.log" BB_TEST_PROXY_LOG="$fixture_root/proxy.log" \ + BB_TEST_PROXY_HELPER="$fixture_root/bin/kubectl-proxy.py" BB_TEST_STATE="$fixture_root/state" \ + BB_TEST_TMP="$fixture_root/state" BB_TEST_ACTIVE="$fixture_root/active/learn-services" \ + BB_TEST_PROOF="$proof" BB_TEST_POST_PROMPT_MARKER="$fixture_root/post-prompt" \ + BB_TEST_CLIENT_STATUS=403 BB_TEST_CLIENT_RC=1 BB_TEST_EXEC_SLEEP=1 \ + bash "$VALIDATOR" --execute --context fixture-context >"$fixture_root/signal.out" 2>&1 & +signal_pid=$! +for ((attempt = 0; attempt < 100; attempt++)); do + [[ "$(<"$fixture_root/commands.log")" == *' exec '* ]] && break + /usr/bin/sleep 0.02 +done +[[ "$(<"$fixture_root/commands.log")" == *' exec '* ]] || fail 'signal fixture did not reach client request' +/bin/kill -TERM -- "-$signal_pid" +signal_rc=0; wait "$signal_pid" || signal_rc=$? +assert_eq 143 "$signal_rc" 'SIGTERM must retain fixed status' +assert_contains "$(<"$fixture_root/proxy.log")" '/pods/' 'signal raw Pod cleanup' +assert_contains "$(<"$fixture_root/proxy.log")" '/networkpolicies/' 'signal raw NetworkPolicy cleanup' + +printf 'BLACKBOX EDGE SOURCE PRODUCTION-BOUNDARY TEST PASS\n' diff --git a/scripts/validate/validate-blackbox-edge-source.sh b/scripts/validate/validate-blackbox-edge-source.sh new file mode 100644 index 0000000..558b7ff --- /dev/null +++ b/scripts/validate/validate-blackbox-edge-source.sh @@ -0,0 +1,728 @@ +#!/usr/bin/env bash + +# Prove that a restricted in-cluster source is denied by the active Grafana +# Nginx guard. This is deliberately a short-lived prerequisite transaction, +# not a reusable HTTP probing tool. +set -Eeuo pipefail +set +x +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly DENY_GUARD="$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf" +readonly NAMESPACE=observability +readonly ROLLBACK_BASE_DEFAULT=/var/lib/hyeonworks/platform-rollbacks +readonly ACTIVE_DEFAULT=/etc/nginx/sites-available/learn-services +readonly KUBECTL_DEFAULT=/usr/local/bin/kubectl +readonly SUDO_DEFAULT=/usr/bin/sudo +readonly DATE_DEFAULT=/usr/bin/date +readonly SLEEP_DEFAULT=/usr/bin/sleep +readonly AWK_BIN=/usr/bin/awk +readonly CHMOD_BIN=/usr/bin/chmod +readonly CMP_BIN=/usr/bin/cmp +readonly CURL_BIN=/usr/bin/curl +readonly ID_BIN=/usr/bin/id +readonly INSTALL_BIN=/usr/bin/install +readonly JQ_BIN=/usr/bin/jq +readonly KILL_BIN=/bin/kill +readonly LN_BIN=/bin/ln +readonly MKTEMP_BIN=/usr/bin/mktemp +readonly REALPATH_BIN=/usr/bin/realpath +readonly RM_BIN=/bin/rm +readonly SHA256SUM_BIN=/usr/bin/sha256sum +readonly STAT_BIN=/usr/bin/stat +readonly TEST_BIN=/usr/bin/test +readonly TRUE_BIN=/usr/bin/true +readonly UNLINK_BIN=/usr/bin/unlink +readonly PROOF_SCHEMA=platform-blackbox-source-v1 +readonly BUSYBOX_IMAGE='docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0' +readonly EXPECTED_NODE='donghyeon-system-product-name' +readonly REQUEST_TIMEOUT=10s +readonly READY_TIMEOUT=30s +readonly DELETE_WAIT_POLLS=1200 +readonly PROXY_WAIT_POLLS=50 +readonly GRAFANA_HOST=grafana.learn.hyeonworks.com +readonly STORAGE_HOST=storage-admin.learn.hyeonworks.com +readonly DB_HOST=db-admin.learn.hyeonworks.com + +readonly TEST_MODE="${PLATFORM_BLACKBOX_EDGE_TEST_MODE:-0}" +readonly KUBECTL="${PLATFORM_BLACKBOX_EDGE_KUBECTL:-$KUBECTL_DEFAULT}" +readonly SUDO="${PLATFORM_BLACKBOX_EDGE_SUDO:-$SUDO_DEFAULT}" +readonly DATE_BIN="${PLATFORM_BLACKBOX_EDGE_DATE:-$DATE_DEFAULT}" +readonly SLEEP_BIN="${PLATFORM_BLACKBOX_EDGE_SLEEP:-$SLEEP_DEFAULT}" +readonly ACTIVE="${PLATFORM_BLACKBOX_EDGE_ACTIVE:-$ACTIVE_DEFAULT}" +readonly ROLLBACK_BASE="${PLATFORM_BLACKBOX_EDGE_ROLLBACK_BASE:-$ROLLBACK_BASE_DEFAULT}" + +execute=false +context='' +fixture_root='' +rollback_anchor='' +rollback_root='' +proof_path='' +run_temp='' +run_label='' +pod_name='' +policy_name='' +pod_uid='' +policy_uid='' +cleanup_started=false +declare -a LOG_RESULTS=() + +fail() { printf 'ERROR: %s\n' "$*" >&2; return 1; } + +usage() { + cat <<'USAGE' +Usage: + bash scripts/validate/validate-blackbox-edge-source.sh + bash scripts/validate/validate-blackbox-edge-source.sh --execute --context CONTEXT + +The default is read-only. Execute requires an exact context-qualified +confirmation, creates only a temporary restricted NetworkPolicy and Pod, and +never prints request tokens or Nginx log lines. +USAGE +} + +sha_of() { "$SHA256SUM_BIN" -- "$1" | "$AWK_BIN" '{print $1}'; } +is_sha256() { [[ "$1" =~ ^[0-9a-f]{64}$ ]]; } +require_regular_source() { [[ -f "$1" && ! -L "$1" ]] || fail "unsafe or missing source: $1"; } + +safe_fixture_command() { + local path=$1 fixture_bin=$2 resolved metadata type owner mode mode_value + [[ "$path" != /usr/bin/sudo ]] || return 1 + resolved="$("$REALPATH_BIN" --canonicalize-existing -- "$path" 2>/dev/null)" || return 1 + [[ "$resolved" == "$path" && "${path%/*}" == "$fixture_bin" && -f "$path" && ! -L "$path" && -x "$path" ]] || return 1 + metadata="$("$STAT_BIN" -c '%F|%u|%a' -- "$path")" || return 1 + IFS='|' read -r type owner mode <<<"$metadata" + mode_value=$((8#$mode)) + [[ "$type" == 'regular file' && "$owner" == "$EUID" && $((mode_value & 0022)) == 0 && $((mode_value & 07000)) == 0 ]] +} + +validate_test_mode() { + local active_parent resolved key + [[ "$TEST_MODE" == 0 || "$TEST_MODE" == 1 ]] || { fail 'invalid test-mode flag'; return 1; } + if [[ "$TEST_MODE" == 0 ]]; then + for key in ${!PLATFORM_BLACKBOX_EDGE_@}; do + fail "production rejects Blackbox edge override: $key" + return 1 + done + [[ "$KUBECTL" == "$KUBECTL_DEFAULT" && "$SUDO" == "$SUDO_DEFAULT" && "$DATE_BIN" == "$DATE_DEFAULT" && "$SLEEP_BIN" == "$SLEEP_DEFAULT" && "$ACTIVE" == "$ACTIVE_DEFAULT" && "$ROLLBACK_BASE" == "$ROLLBACK_BASE_DEFAULT" ]] || { + fail 'production Blackbox edge constants are not exact'; return 1; + } + rollback_anchor=/ + return 0 + fi + active_parent=${ACTIVE%/*} + resolved="$("$REALPATH_BIN" --canonicalize-existing -- "$active_parent/.." 2>/dev/null)" || { + fail 'test fixture root does not resolve'; return 1; + } + fixture_root=$resolved + [[ "$fixture_root" == /tmp/platform-blackbox-edge-test.* && "$ACTIVE" == "$fixture_root/active/learn-services" && "$ROLLBACK_BASE" == "$fixture_root/rollbacks" && -d "$fixture_root/bin" && ! -L "$fixture_root/bin" ]] || { + fail 'test mode is restricted to one isolated fixture tree'; return 1; + } + safe_fixture_command "$KUBECTL" "$fixture_root/bin" && + safe_fixture_command "$SUDO" "$fixture_root/bin" && + safe_fixture_command "$DATE_BIN" "$fixture_root/bin" && + safe_fixture_command "$SLEEP_BIN" "$fixture_root/bin" || { + fail 'test command boundary is unsafe'; return 1; + } + rollback_anchor=$fixture_root +} + +parse_args() { + local execute_count=0 context_count=0 + while (( $# > 0 )); do + case "$1" in + --execute) execute=true; execute_count=$((execute_count + 1)); shift ;; + --context) + (( $# >= 2 )) || { fail '--context requires a value'; return 1; } + context=$2; context_count=$((context_count + 1)); shift 2 + ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; fail "unsupported argument: $1"; return 1 ;; + esac + done + (( execute_count <= 1 && context_count <= 1 )) || { fail 'duplicate execute argument'; return 1; } + if [[ "$execute" == true ]]; then + [[ "$context" =~ ^[A-Za-z0-9._:-]+$ ]] || { fail '--execute requires an exact --context'; return 1; } + else + [[ -z "$context" ]] || { fail '--context is execute-only'; return 1; } + fi +} + +root_run() { "$SUDO" -n "$@"; } +root_metadata() { root_run "$STAT_BIN" -c '%F|%u:%g|%a|%h' -- "$1"; } +root_owner_expected() { + if [[ "$TEST_MODE" == 0 ]]; then [[ "$1" == 0:0 ]]; else [[ "$1" == "$EUID:$($ID_BIN -g)" ]]; fi +} + +root_safe_ancestor_dir() { + local path=$1 metadata type owner mode links mode_value + root_run "$TEST_BIN" ! -L "$path" || return 1 + root_run "$TEST_BIN" -d "$path" || return 1 + metadata="$(root_metadata "$path")" || return 1 + IFS='|' read -r type owner mode links <<<"$metadata" + mode_value=$((8#$mode)) + [[ "$type" == directory && "$links" =~ ^[1-9][0-9]*$ ]] || return 1 + root_owner_expected "$owner" || return 1 + (( (mode_value & 0022) == 0 )) +} + +root_safe_dir() { + local path=$1 expected_mode=$2 metadata type owner mode links + root_safe_ancestor_dir "$path" || return 1 + metadata="$(root_metadata "$path")" || return 1 + IFS='|' read -r type owner mode links <<<"$metadata" + [[ "$mode" == "$expected_mode" && "$links" =~ ^[1-9][0-9]*$ ]] +} + +root_safe_file() { + local path=$1 expected_mode=$2 metadata type owner mode links + root_run "$TEST_BIN" ! -L "$path" || return 1 + root_run "$TEST_BIN" -f "$path" || return 1 + metadata="$(root_metadata "$path")" || return 1 + IFS='|' read -r type owner mode links <<<"$metadata" + [[ ( "$type" == 'regular file' || "$type" == 'regular empty file' ) && "$mode" == "$expected_mode" && "$links" == 1 ]] || return 1 + root_owner_expected "$owner" +} + +validate_root_chain() { + local target=$1 anchor=$2 canonical relative current component + [[ "$target" == /* && "$anchor" == /* && "$target" != *'//' && "$target" != */./* && "$target" != */../* ]] || return 1 + canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$target")" || return 1 + [[ "$canonical" == "$target" && "$canonical" != *$'\n'* ]] || return 1 + canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$anchor")" || return 1 + [[ "$canonical" == "$anchor" && "$canonical" != *$'\n'* ]] || return 1 + if [[ "$anchor" == / ]]; then + # Root has no separator to append: "$anchor/" would become "//" and + # reject every normal absolute descendant after canonicalization. + [[ "$target" == /* ]] || return 1 + else + [[ "$target" == "$anchor" || "$target" == "$anchor/"* ]] || return 1 + fi + current=$anchor + root_safe_ancestor_dir "$current" || return 1 + relative=${target#"$anchor"} + relative=${relative#/} + [[ -z "$relative" ]] && return 0 + IFS='/' read -r -a _blackbox_path_parts <<<"$relative" + for component in "${_blackbox_path_parts[@]}"; do + [[ -n "$component" && "$component" != . && "$component" != .. ]] || return 1 + if [[ "$current" == / ]]; then current="/$component"; else current="$current/$component"; fi + root_safe_ancestor_dir "$current" || return 1 + done +} + +validate_rollback_root() { + local id=${PLATFORM_OBSERVABILITY_ROLLBACK_ID:-} + [[ "$id" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || { fail 'PLATFORM_OBSERVABILITY_ROLLBACK_ID format is invalid'; return 1; } + if [[ "$TEST_MODE" == 0 ]]; then [[ "$ROLLBACK_BASE" == "$ROLLBACK_BASE_DEFAULT" ]] || { fail 'production rollback base is not exact'; return 1; }; fi + rollback_root="$ROLLBACK_BASE/observability-$id" + proof_path="$rollback_root/blackbox-source-proof.env" + validate_root_chain "$ROLLBACK_BASE" "$rollback_anchor" || { fail 'rollback base lineage is unsafe'; return 1; } + validate_root_chain "$rollback_root" "$rollback_anchor" || { fail 'rollback root lineage is unsafe'; return 1; } + root_safe_dir "$rollback_root" 700 || { fail 'rollback root metadata is unsafe'; return 1; } +} + +validate_proof_target_absent() { + validate_rollback_root || return 1 + [[ "$proof_path" == "$rollback_root/blackbox-source-proof.env" ]] || { fail 'source-proof path is not exact'; return 1; } + root_run "$TEST_BIN" ! -e "$proof_path" && root_run "$TEST_BIN" ! -L "$proof_path" || { + fail 'source-proof evidence already exists or is unsafe'; return 1; + } +} + +validate_active_guard_expected() { + local expected=$1 deny_sha metadata type owner mode links active_sha + require_regular_source "$DENY_GUARD" || return 1 + is_sha256 "$expected" || { fail 'expected deny guard hash is invalid'; return 1; } + deny_sha="$(sha_of "$DENY_GUARD")" || { fail 'cannot hash deny guard source'; return 1; } + [[ "$deny_sha" == "$expected" ]] || { fail 'deny guard source changed during proof'; return 1; } + root_run "$TEST_BIN" ! -L "$ACTIVE" && root_run "$TEST_BIN" -f "$ACTIVE" || { + fail 'active Nginx config is missing or symlinked'; return 1; + } + metadata="$(root_metadata "$ACTIVE")" || { fail 'cannot stat active Nginx config'; return 1; } + IFS='|' read -r type owner mode links <<<"$metadata" + [[ "$type" == 'regular file' && "$links" == 1 ]] || { fail 'active Nginx config metadata is unsafe'; return 1; } + root_owner_expected "$owner" || { fail 'active Nginx config owner is unsafe'; return 1; } + if [[ "$TEST_MODE" == 0 ]]; then [[ "$mode" == 644 ]] || { fail 'active Nginx config mode is unsafe'; return 1; }; fi + active_sha="$(root_run "$SHA256SUM_BIN" -- "$ACTIVE" | "$AWK_BIN" '{print $1}')" || { + fail 'cannot hash active Nginx config'; return 1; + } + [[ "$active_sha" == "$expected" ]] || { fail 'active Nginx config is not the exact Grafana deny guard'; return 1; } +} + +validate_active_guard() { + local deny_sha + require_regular_source "$DENY_GUARD" || return 1 + deny_sha="$(sha_of "$DENY_GUARD")" || return 1 + is_sha256 "$deny_sha" || { fail 'deny guard hash is invalid'; return 1; } + validate_active_guard_expected "$deny_sha" || return 1 + printf '%s\n' "$deny_sha" +} + +confirm() { + local expected="PROVE BLACKBOX PRIVATE EDGE $context" answer + printf 'Type %s: ' "$expected" >&2 + if [[ "$TEST_MODE" == 1 ]]; then + answer=${PLATFORM_BLACKBOX_EDGE_CONFIRMATION:-} + printf '%s\n' "$answer" >&2 + else + [[ -t 0 ]] || { fail '--execute requires an interactive terminal'; return 1; } + IFS= read -r answer + fi + [[ "$answer" == "$expected" ]] || { fail 'cancelled'; return 1; } +} + +require_context_api_auth() { + local current answer + current="$("$KUBECTL" --request-timeout="$REQUEST_TIMEOUT" config current-context)" || { fail 'cannot read Kubernetes context'; return 1; } + [[ "$current" == "$context" && "$current" != *$'\n'* ]] || { fail 'current Kubernetes context does not match --context'; return 1; } + "$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" get --raw=/readyz >/dev/null || { fail 'Kubernetes API is not ready'; return 1; } + for answer in \ + "$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i create pods -n "$NAMESPACE")" \ + "$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i create networkpolicies.networking.k8s.io -n "$NAMESPACE")" \ + "$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i delete pods -n "$NAMESPACE")" \ + "$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i delete networkpolicies.networking.k8s.io -n "$NAMESPACE")"; do + [[ "$answer" == yes ]] || { fail 'Kubernetes authorization is insufficient'; return 1; } + done + root_run "$TRUE_BIN" || { fail 'sudo non-interactive authentication failed'; return 1; } +} + +prepare_local_temp() { + run_temp="$("$MKTEMP_BIN" -d /tmp/platform-blackbox-edge-source.XXXXXXXX)" || return 1 + [[ "$run_temp" == /tmp/platform-blackbox-edge-source.???????? && -d "$run_temp" && ! -L "$run_temp" ]] || return 1 + "$CHMOD_BIN" 0700 -- "$run_temp" +} + +cleanup_local_temp() { + [[ -n "$run_temp" && "$run_temp" == /tmp/platform-blackbox-edge-source.???????? && -d "$run_temp" && ! -L "$run_temp" ]] || return 0 + "$RM_BIN" -rf -- "$run_temp" + run_temp='' +} + +render_network_policy() { + /bin/cat </dev/null + ;; + pod) + "$JQ_BIN" -e --arg namespace "$NAMESPACE" --arg name "$name" --arg run "$run" --arg image "$BUSYBOX_IMAGE" --arg node "$EXPECTED_NODE" --argjson uid_required "$([[ "$require_uid" == true ]] && printf true || printf false)" --argjson scheduled_node_allowed "$([[ "$allow_scheduled_node" == true ]] && printf true || printf false)" ' + def optional_default($key; $value): + (has($key) | not) or .[$key] == $value; + .apiVersion == "v1" and .kind == "Pod" and + .metadata.namespace == $namespace and .metadata.name == $name and + .metadata.labels == { + "app.kubernetes.io/managed-by":"platform-blackbox-edge-source", + "platform.hyeonworks.com/source-proof-run":$run + } and + (($uid_required | not) or (.metadata.uid | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$"))) and + (.spec | + type == "object" and + optional_default("dnsPolicy"; "ClusterFirst") and + optional_default("enableServiceLinks"; true) and + optional_default("preemptionPolicy"; "PreemptLowerPriority") and + optional_default("priority"; 0) and + optional_default("schedulerName"; "default-scheduler") and + optional_default("serviceAccount"; "default") and + optional_default("serviceAccountName"; "default") and + optional_default("terminationGracePeriodSeconds"; 30) and + ((has("nodeName") | not) or ($scheduled_node_allowed and .nodeName == $node)) and + optional_default("tolerations"; [ + {"effect":"NoExecute","key":"node.kubernetes.io/not-ready","operator":"Exists","tolerationSeconds":300}, + {"effect":"NoExecute","key":"node.kubernetes.io/unreachable","operator":"Exists","tolerationSeconds":300} + ]) and + (.containers | type == "array" and length == 1) and + (.containers[0] | + optional_default("resources"; {}) and + optional_default("terminationMessagePath"; "/dev/termination-log") and + optional_default("terminationMessagePolicy"; "File")) and + (del( + .dnsPolicy, .enableServiceLinks, .preemptionPolicy, .priority, + .schedulerName, .serviceAccount, .serviceAccountName, + .terminationGracePeriodSeconds, .tolerations, .nodeName, + .containers[0].resources, .containers[0].terminationMessagePath, + .containers[0].terminationMessagePolicy + ) == { + "automountServiceAccountToken":false, + "restartPolicy":"Never", + "securityContext":{"runAsNonRoot":true,"runAsUser":65534,"runAsGroup":65534,"seccompProfile":{"type":"RuntimeDefault"}}, + "containers":[{"name":"probe","image":$image,"imagePullPolicy":"IfNotPresent","command":["sh","-c","sleep 120"],"securityContext":{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"capabilities":{"drop":["ALL"]}}}] + }) + ) + ' "$source" >/dev/null + ;; + *) return 1 ;; + esac +} + +response_uid() { + local source=$1 uid + uid="$("$JQ_BIN" -er '.metadata.uid' "$source")" || return 1 + [[ "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1 + printf '%s\n' "$uid" +} + +local_response_file() { + "$MKTEMP_BIN" "$run_temp/${1}.response.XXXXXXXX" +} + +render_resource() { + case "$1" in networkpolicy) render_network_policy ;; pod) render_pod ;; *) return 1 ;; esac +} +resource_name() { case "$1" in networkpolicy) printf '%s\n' "$policy_name" ;; pod) printf '%s\n' "$pod_name" ;; *) return 1 ;; esac; } + +server_dry_run_resource() { + local kind=$1 name response + name="$(resource_name "$kind")" || return 1 + response="$(local_response_file "${kind}.dry")" || return 1 + "$CHMOD_BIN" 0600 -- "$response" + if ! render_resource "$kind" | "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" create --dry-run=server -f - -o json >"$response"; then + "$RM_BIN" -f -- "$response"; return 1 + fi + validate_object_json "$kind" "$name" "$run_label" "$response" false || { "$RM_BIN" -f -- "$response"; return 1; } + "$RM_BIN" -f -- "$response" +} + +inspect_owned_object() { + local kind=$1 name=$2 destination=$3 response uid rc=0 + response="$(local_response_file "${kind}.inspect")" || return 3 + "$CHMOD_BIN" 0600 -- "$response" + "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" get "$kind" "$name" --ignore-not-found -o json >"$response" || rc=$? + if (( rc != 0 )); then "$RM_BIN" -f -- "$response"; return 3; fi + if [[ ! -s "$response" ]]; then "$RM_BIN" -f -- "$response"; return 1; fi + validate_object_json "$kind" "$name" "$run_label" "$response" true || { "$RM_BIN" -f -- "$response"; return 2; } + uid="$(response_uid "$response")" || { "$RM_BIN" -f -- "$response"; return 2; } + "$RM_BIN" -f -- "$response" + printf -v "$destination" '%s' "$uid" +} + +create_resource() { + local kind=$1 destination=$2 name response uid rc=0 classification + name="$(resource_name "$kind")" || return 1 + response="$(local_response_file "${kind}.create")" || return 1 + "$CHMOD_BIN" 0600 -- "$response" + if render_resource "$kind" | "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" create -f - -o json >"$response"; then + if validate_object_json "$kind" "$name" "$run_label" "$response" true && uid="$(response_uid "$response")"; then + "$RM_BIN" -f -- "$response" + printf -v "$destination" '%s' "$uid" + return 0 + fi + fi + "$RM_BIN" -f -- "$response" + # A syntactically bad success response is still ambiguous: the API may have + # committed the exact object, so classify the live state before cleanup. + if inspect_owned_object "$kind" "$name" "$destination"; then + return 2 + else + classification=$? + [[ "$classification" == 1 ]] && return 1 + return 1 + fi +} + +create_temporary_objects() { + local result + server_dry_run_resource networkpolicy || { fail 'temporary NetworkPolicy server dry-run failed'; return 1; } + if create_resource networkpolicy policy_uid; then :; else + result=$? + [[ "$result" == 2 ]] && fail 'temporary NetworkPolicy create outcome was ambiguous but owned' + [[ "$result" != 2 ]] && fail 'temporary NetworkPolicy create failed' + return 1 + fi + server_dry_run_resource pod || { fail 'temporary Pod server dry-run failed'; return 1; } + if create_resource pod pod_uid; then :; else + result=$? + [[ "$result" == 2 ]] && fail 'temporary Pod create outcome was ambiguous but owned' + [[ "$result" != 2 ]] && fail 'temporary Pod create failed' + return 1 + fi +} + +stop_proxy() { + local pid=$1 attempt + "$KILL_BIN" -TERM "$pid" 2>/dev/null || true + for ((attempt=0; attempt<20; attempt++)); do + if ! "$KILL_BIN" -0 "$pid" 2>/dev/null; then wait "$pid" 2>/dev/null || true; return 0; fi + "$SLEEP_BIN" 0.05 + done + "$KILL_BIN" -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true +} + +api_path_for() { + case "$1" in + pod) printf '/api/v1/namespaces/%s/pods/%s\n' "$NAMESPACE" "$2" ;; + networkpolicy) printf '/apis/networking.k8s.io/v1/namespaces/%s/networkpolicies/%s\n' "$NAMESPACE" "$2" ;; + *) return 1 ;; + esac +} + +object_state() { + local kind=$1 name=$2 expected_uid=$3 response uid rc=0 + response="$(local_response_file "${kind}.state")" || return 4 + "$CHMOD_BIN" 0600 -- "$response" + "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" get "$kind" "$name" --ignore-not-found -o json >"$response" || rc=$? + if (( rc != 0 )); then "$RM_BIN" -f -- "$response"; return 4; fi + if [[ ! -s "$response" ]]; then "$RM_BIN" -f -- "$response"; return 1; fi + uid="$(response_uid "$response")" || { "$RM_BIN" -f -- "$response"; return 4; } + [[ "$uid" == "$expected_uid" ]] || { "$RM_BIN" -f -- "$response"; return 3; } + validate_object_json "$kind" "$name" "$run_label" "$response" true true || { "$RM_BIN" -f -- "$response"; return 4; } + "$RM_BIN" -f -- "$response" + return 0 +} + +wait_for_stable_absence() { + local kind=$1 name=$2 uid=$3 attempt absent=0 state + for ((attempt=0; attempt= 2 )) && return 0 + else + return 1 + fi + fi + "$SLEEP_BIN" 0.05 + done + return 1 +} + +delete_with_uid_precondition() { + local kind=$1 name=$2 uid=$3 api_path socket options response proxy_log proxy_pid='' attempt http_code='' curl_rc=0 + [[ "$name" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ && "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1 + api_path="$(api_path_for "$kind" "$name")" || return 1 + socket="$run_temp/proxy-${kind}.sock" + options="$run_temp/delete-${kind}.json" + response="$run_temp/delete-${kind}.response" + proxy_log="$run_temp/proxy-${kind}.log" + [[ ! -e "$socket" && ! -L "$socket" ]] || return 1 + printf '{"apiVersion":"meta.k8s.io/v1","kind":"DeleteOptions","propagationPolicy":"Background","preconditions":{"uid":"%s"}}\n' "$uid" >"$options" + "$CHMOD_BIN" 0600 -- "$options" + : >"$response"; "$CHMOD_BIN" 0600 -- "$response" + "$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" proxy \ + --unix-socket="$socket" --api-prefix=/ --accept-paths="^${api_path}$" \ + --reject-methods='^(GET|POST|PUT|PATCH)$' >"$proxy_log" 2>&1 & + proxy_pid=$! + for ((attempt=0; attempt/dev/null || { wait "$proxy_pid" 2>/dev/null || true; return 1; } + "$SLEEP_BIN" 0.05 + done + [[ -S "$socket" ]] || { stop_proxy "$proxy_pid"; return 1; } + http_code="$("$CURL_BIN" --silent --show-error --max-time 5 --unix-socket "$socket" --output "$response" --write-out '%{http_code}' \ + --request DELETE --header 'Content-Type: application/json' --data-binary "@$options" "http://localhost${api_path}")" || curl_rc=$? + stop_proxy "$proxy_pid" + # A transport/status ambiguity is not treated as success on its own. Only + # two consecutive exact absence reads reclassify it as a completed delete. + wait_for_stable_absence "$kind" "$name" "$uid" +} + +cleanup() { + local rc=0 + [[ "$cleanup_started" == false ]] || return 0 + cleanup_started=true + if [[ -n "$pod_uid" ]]; then delete_with_uid_precondition pod "$pod_name" "$pod_uid" || rc=1; fi + if [[ -n "$policy_uid" ]]; then delete_with_uid_precondition networkpolicy "$policy_name" "$policy_uid" || rc=1; fi + pod_uid=''; policy_uid='' + (( rc == 0 )) || printf 'ERROR: temporary Blackbox proof object identity is unknown; manual review required\n' >&2 + return "$rc" +} + +on_exit() { + local rc=$? + trap - EXIT HUP INT TERM + if [[ -n "$pod_uid" || -n "$policy_uid" ]]; then + cleanup || { [[ "$rc" == 0 ]] && rc=1; } + fi + cleanup_local_temp || { [[ "$rc" == 0 ]] && rc=1; } + exit "$rc" +} + +run_client_request() { + local host=$1 token=$2 output status rc=0 + output="$("$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" exec "$pod_name" -- sh -ec \ + "wget --no-check-certificate --server-response --spider 'https://$host/?hyeonworks_probe=$token'" 2>&1)" || rc=$? + status="$("$AWK_BIN" '/^ HTTP\// { code=$2 } /^HTTP\// { code=$2 } END { print code }' <<<"$output")" + [[ "$rc" == 1 && "$status" == 403 ]] +} + +collect_log_results() { + local one=$1 two=$2 three=$3 output pair addr status + output="$(root_run "$AWK_BIN" -v one="$one" -v two="$two" -v three="$three" ' + $0 ~ ("hyeonworks_probe=" one "([& ]| HTTP)") { if (++count[1] == 1) value[1]=$1 " " $9 } + $0 ~ ("hyeonworks_probe=" two "([& ]| HTTP)") { if (++count[2] == 1) value[2]=$1 " " $9 } + $0 ~ ("hyeonworks_probe=" three "([& ]| HTTP)") { if (++count[3] == 1) value[3]=$1 " " $9 } + END { if (count[1] != 1 || count[2] != 1 || count[3] != 1) exit 42; print value[1]; print value[2]; print value[3] } + ' /var/log/nginx/access.log)" || return 1 + mapfile -t LOG_RESULTS <<<"$output" + (( ${#LOG_RESULTS[@]} == 3 )) || return 1 + for pair in "${LOG_RESULTS[@]}"; do + read -r addr status <<<"$pair" + [[ "$addr" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ && "$status" == 403 ]] || return 1 + done +} + +write_proof() { + local nginx_sha=$1 tested=$2 local_file root_temporary canonical suffix + local_file="$("$MKTEMP_BIN" "$run_temp/proof.XXXXXXXX")" || return 1 + "$CHMOD_BIN" 0600 -- "$local_file" + printf 'schema=%s\nrollback_id=%s\nnginx_sha256=%s\ntested_at_utc=%s\ngrafana_remote_addr=%s\ngrafana_status=403\nstorage_admin_remote_addr=%s\nstorage_admin_status=403\ndb_admin_remote_addr=%s\ndb_admin_status=403\n' \ + "$PROOF_SCHEMA" "${PLATFORM_OBSERVABILITY_ROLLBACK_ID:?}" "$nginx_sha" "$tested" \ + "${LOG_RESULTS[0]%% *}" "${LOG_RESULTS[1]%% *}" "${LOG_RESULTS[2]%% *}" >"$local_file" + root_temporary="$(root_run "$MKTEMP_BIN" --tmpdir="$rollback_root" '.blackbox-source-proof.env.new.XXXXXXXX')" || { "$RM_BIN" -f -- "$local_file"; return 1; } + suffix=${root_temporary#"$rollback_root/.blackbox-source-proof.env.new."} + [[ "$root_temporary" == "$rollback_root/.blackbox-source-proof.env.new."* && "$suffix" =~ ^[A-Za-z0-9]{8}$ && "$root_temporary" != *$'\n'* ]] || { + "$RM_BIN" -f -- "$local_file"; return 1; + } + canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$root_temporary")" || { "$RM_BIN" -f -- "$local_file"; return 1; } + [[ "$canonical" == "$root_temporary" ]] && root_safe_file "$root_temporary" 600 || { + "$RM_BIN" -f -- "$local_file"; return 1; + } + if ! root_run "$INSTALL_BIN" -o root -g root -m 0600 -- "$local_file" "$root_temporary" || + ! root_safe_file "$root_temporary" 600 || + ! root_run "$CMP_BIN" --silent -- "$local_file" "$root_temporary" || + ! root_run "$LN_BIN" -nT -- "$root_temporary" "$proof_path"; then + root_safe_file "$root_temporary" 600 && root_run "$UNLINK_BIN" -- "$root_temporary" >/dev/null 2>&1 || true + "$RM_BIN" -f -- "$local_file" + return 1 + fi + root_run "$UNLINK_BIN" -- "$root_temporary" || { "$RM_BIN" -f -- "$local_file"; return 1; } + root_safe_file "$proof_path" 600 && root_run "$CMP_BIN" --silent -- "$local_file" "$proof_path" || { + "$RM_BIN" -f -- "$local_file"; return 1; + } + "$RM_BIN" -f -- "$local_file" +} + +execute_proof() { + local nginx_sha stamp random token_one token_two token_three tested + nginx_sha="$(validate_active_guard)" || return 1 + validate_rollback_root || return 1 + require_context_api_auth || return 1 + confirm || return 1 + prepare_local_temp || { fail 'cannot create private validator temporary directory'; return 1; } + stamp="$("$DATE_BIN" -u +%Y%m%dT%H%M%SZ)" || return 1 + random="${RANDOM}${RANDOM}" + run_label="${stamp,,}-${random}" + policy_name="blackbox-edge-source-egress-${run_label}" + pod_name="blackbox-edge-source-${run_label}" + token_one="${stamp}-${RANDOM}${RANDOM}" + token_two="${stamp}-${RANDOM}${RANDOM}" + token_three="${stamp}-${RANDOM}${RANDOM}" + [[ "$token_one" != "$token_two" && "$token_one" != "$token_three" && "$token_two" != "$token_three" ]] || { fail 'could not create unique probe tokens'; return 1; } + # This is the commit gate: it is immediately after confirmation and all + # local preparation, and immediately before the first Kubernetes mutation. + validate_active_guard_expected "$nginx_sha" || return 1 + validate_proof_target_absent || return 1 + create_temporary_objects || return 1 + "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" wait --for=condition=Ready --timeout="$READY_TIMEOUT" "pod/$pod_name" >/dev/null || { + fail 'temporary Pod did not become Ready'; return 1; + } + run_client_request "$GRAFANA_HOST" "$token_one" || { fail 'Grafana client status is not exact BusyBox 403'; return 1; } + run_client_request "$STORAGE_HOST" "$token_two" || { fail 'AIStor client status is not exact BusyBox 403'; return 1; } + run_client_request "$DB_HOST" "$token_three" || { fail 'pgAdmin client status is not exact BusyBox 403'; return 1; } + collect_log_results "$token_one" "$token_two" "$token_three" || { fail 'Nginx log did not contain exactly one 403 result per probe'; return 1; } + tested="$("$DATE_BIN" -u +%Y-%m-%dT%H:%M:%SZ)" || return 1 + write_proof "$nginx_sha" "$tested" || { fail 'cannot atomically write source-proof evidence'; return 1; } + cleanup || { fail 'temporary object cleanup failed'; return 1; } + printf 'BLACKBOX PRIVATE EDGE SOURCE PASS\n' +} + +main() { + parse_args "$@" || return 1 + validate_test_mode || return 1 + require_regular_source "$DENY_GUARD" || return 1 + if [[ "$execute" == false ]]; then + printf 'BLACKBOX_EDGE_SOURCE_DENY_GUARD_SHA256=%s\nBLACKBOX_EDGE_SOURCE_BUSYBOX_IMAGE=%s\nBLACKBOX_PRIVATE_EDGE_SOURCE_DRY_RUN=PASS\n' "$(sha_of "$DENY_GUARD")" "$BUSYBOX_IMAGE" + return 0 + fi + for command_path in "$AWK_BIN" "$CHMOD_BIN" "$CMP_BIN" "$CURL_BIN" "$ID_BIN" "$INSTALL_BIN" "$JQ_BIN" "$KILL_BIN" "$LN_BIN" "$MKTEMP_BIN" "$REALPATH_BIN" "$RM_BIN" "$SHA256SUM_BIN" "$SLEEP_BIN" "$STAT_BIN" "$TEST_BIN" "$TRUE_BIN" "$UNLINK_BIN"; do + [[ -x "$command_path" ]] || { fail "required command is missing: $command_path"; return 1; } + done + [[ -x "$KUBECTL" && -x "$SUDO" && -x "$DATE_BIN" ]] || { fail 'required command boundary is missing'; return 1; } + trap on_exit EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + execute_proof +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then main "$@"; fi diff --git a/services/gitea/README.md b/services/gitea/README.md new file mode 100644 index 0000000..21111a3 --- /dev/null +++ b/services/gitea/README.md @@ -0,0 +1,229 @@ +# Gitea 서비스 + +Gitea는 같은 공식 Helm Chart와 공통 values를 사용하는 두 Kustomize 빌드 루트로 +관리한다. 업스트림 Chart 템플릿은 복제하거나 수정하지 않는다. + +| 용도 | 빌드 루트 | 검증 산출물 | 적용 진입점 | +| --- | --- | --- | --- | +| 신규 설치 baseline | `services/gitea` | `gitea.yaml` | `apply-phase1-gitea.sh` | +| OIDC·브랜딩 목표 상태 | `services/gitea/profiles/oidc` | `gitea-oidc.yaml` | `apply-gitea-oidc.sh` | + +baseline 루트는 OIDC Secret, `id` host alias, Keycloak 전용 egress와 브랜딩을 +포함하지 않는다. OIDC profile만 공통 기반에 이 네 요소를 추가한다. + +상태(2026-07-23): Host Nginx public discovery 전환과 OIDC-enabled Gitea +rollout을 완료했다. 적용 manifest SHA-256은 `d25a757...a157`이며, 활성 +`keycloak` OAuth2 source, 외부 인증 전용 가입 정책, authorization-code redirect와 +아래 선언형 브랜딩의 자동 검증을 모두 통과했다. 실제 realm 사용자의 브라우저 +login/callback/logout, 비상 관리자 실제 로그인과 Pod 재시작 뒤 설정 지속성은 +별도 수용 시험으로 남아 있다. + +## 고정된 업스트림 릴리스 + +| 항목 | 값 | +| --- | --- | +| Chart 저장소 | `https://dl.gitea.com/charts/` | +| Chart | `gitea` | +| Chart 버전 | `12.7.0` | +| Chart 패키지 SHA-256 | `5881ef9c59400bee2d5547e77c4cd0efb925143c2f5d93fb4f38446db76b0167` | +| Gitea 이미지 | `docker.gitea.com/gitea:1.27.0-rootless` | + +Chart는 렌더링할 때 저장소 공통 `.gitignore`에서 제외한 +`profiles/oidc/.helm/charts`에 내려받는다. baseline 루트와 OIDC profile이 이 고정 +SHA 검증 캐시 하나를 각자의 `chartHome`으로 참조한다. values와 Patch는 모두 +`services/gitea` 아래에 있으므로 Kustomize의 기본 `RootOnly` 로드 제한을 유지한다. + +## baseline과 OIDC 로그인 정책 + +두 프로필이 공유하는 `profiles/oidc/values/baseline.yaml`은 다음 정책을 선언한다. + +- 자체 가입을 닫고(`DISABLE_REGISTRATION=true`) 외부 인증 전용 가입도 끈다. +- 가입 버튼을 숨기고 기존 `gitea-admin` 로컬 비상 로그인을 유지한다. +- PostgreSQL, PVC, probe, resource와 restricted securityContext를 공통으로 둔다. + +`profiles/oidc/values/oidc.yaml`은 baseline 위에 다음 항목만 덮어쓴다. + +- `APP_NAME=Hyeonworks Git`으로 브라우저 제목과 애플리케이션 이름을 고정한다. +- `DISABLE_REGISTRATION=false`와 `ALLOW_ONLY_EXTERNAL_REGISTRATION=true`를 함께 사용해 + Keycloak 같은 외부 인증을 통한 신규 사용자 생성만 허용한다. +- `SHOW_REGISTRATION_BUTTON=false`로 Gitea 자체 공개 가입 버튼을 숨긴다. +- `oauth2_client.ENABLE_AUTO_REGISTRATION=true`, `USERNAME=preferred_username`, + `OPENID_CONNECT_SCOPES="profile email"`로 OIDC 최초 로그인 계정을 생성한다. +- baseline의 `ENABLE_PASSWORD_SIGNIN_FORM=true`를 유지해 Keycloak 장애 시에도 로컬 + 비상 관리자가 로그인할 수 있게 한다. + +`ACCOUNT_LINKING=login`은 이메일이 같은 기존 계정을 자동 병합하지 않는다. 사용자가 +기존 Gitea 자격 증명으로 한 번 더 확인해야 연결되므로, 이메일 일치만으로 계정을 +탈취하는 위험을 줄인다. + +## 의존성과 트래픽 + +- `gitea`와 `platform-data` namespace를 먼저 적용한다. +- 이 서비스를 적용하기 전에 `/srv/k3s/ssd/gitea`를 준비하고 `gitea-local-pv` + 기반 구성을 적용한다. `gitea-data` Claim은 `ssd-local-gitea-retain`을 통해 해당 + PV에 바인딩되며 `50Gi`를 요청한다. +- 외부 Database endpoint는 + `platform-postgres-rw.platform-data.svc.cluster.local:5432`다. +- Host Nginx에서 공개 TLS를 종료한다. Traefik은 HTTP를 받아 `gitea-http:3000`으로 + 전달하며, 이 Ingress에는 의도적으로 TLS 섹션이 없다. +- OIDC profile의 Gitea 서버가 discovery와 token endpoint에 접근할 때도 공개 issuer + `https://id.learn.hyeonworks.com/realms/hyeonworks`를 사용한다. 홈 라우터의 NAT + loopback에 의존하지 않도록 Pod의 `hostAliases`는 이 이름만 노드 + `192.168.0.107`로 해석하고, 전용 NetworkPolicy가 해당 `/32`의 TCP 443만 + 허용한다. 연결은 Host Nginx TLS와 Traefik을 그대로 통과한다. +- 첫 배포에서는 Gitea SSH, Actions, Git LFS 및 Package Registry를 비활성화한다. + Chart `12.7.0`은 항상 SSH Service를 렌더링하므로 baseline 루트는 inline patch, + OIDC profile은 `patches/delete-ssh-service.yaml`로 Helm 렌더링 뒤 + `gitea-ssh`를 삭제한다. +- 공통 egress는 클러스터 DNS, 플랫폼 PostgreSQL Pod 및 공개 IPv4 HTTPS 대상으로 + 제한한다. OIDC profile만 Host Nginx 노드 `/32`의 TCP 443을 추가한다. 내부 + HTTPS, SMTP, SSH remote 및 사설 네트워크 webhook을 사용하려면 향후 + NetworkPolicy를 명시적으로 변경해야 한다. + +## 선언형 브랜딩 + +브랜딩은 Gitea 공식 custom path와 Chart의 volume 확장 필드만 사용한다. +Chart 템플릿과 Gitea 전체 페이지 템플릿은 복제하거나 수정하지 않는다. + +```text +branding/ +├── public/assets/ +│ ├── css/hyeonworks.css +│ └── img/ +│ ├── logo.svg +│ └── favicon.svg +└── templates/custom/ + ├── header.tmpl + └── extra_links.tmpl +``` + +`branding/kustomization.yaml`의 `configMapGenerator`가 자산과 템플릿 ConfigMap을 +각각 생성하고 `profiles/oidc/kustomization.yaml`만 이 구성요소를 포함한다. 이름 +suffix hash를 끄지 않았으므로 파일 내용이 바뀌면 이름도 바뀌고, Kustomize가 +Deployment의 ConfigMap 참조를 같은 해시 이름으로 치환한다. + +`profiles/oidc/values/oidc.yaml`은 공식 Chart 확장 지점을 다음과 같이 사용한다. + +- 자산 ConfigMap을 `/data/gitea/public/assets`에 마운트한다. +- 템플릿 ConfigMap을 `/data/gitea/templates/custom`에 마운트한다. +- 두 volume은 rootless Gitea 주 컨테이너에만 `readOnly: true`로 마운트하며, + init container에는 추가하지 않는다. + +`header.tmpl`은 같은 origin의 CSS와 모바일 브라우저 색상만 추가한다. +`extra_links.tmpl`은 실제 Gitea 내부 경로인 `/explore/organizations`만 연결한다. +존재하지 않는 Learning 또는 Identity 서비스 링크는 미리 만들지 않는다. +전체 `home.tmpl`은 덮어쓰지 않아 Gitea 업그레이드 때 template drift 범위를 줄인다. +Gitea는 custom template을 시작할 때 읽으므로 변경 적용에는 Pod 재시작이 필요하다. + +SVG 원본만 저장소에서 관리한다. Gitea가 참조하는 Open Graph, Apple touch icon 및 +구형 브라우저용 PNG는 검증된 변환 절차를 추가하기 전까지 업스트림 기본 자산을 +사용한다. + +- [Gitea 1.27 커스터마이징 공식 문서](https://docs.gitea.com/administration/customizing-gitea) +- [Gitea Chart 12.7.0 공식 values](https://gitea.com/gitea/helm-chart/src/tag/v12.7.0/values.yaml) + +## Secret 계약 + +credential Secret 매니페스트나 credential 값은 이 저장소에 두지 않는다. +`gitea-admin`과 `gitea-db-credentials`는 baseline Deployment 최초 시작 전에, +`gitea-keycloak-oidc`는 OIDC-enabled Deployment rollout 전에 `gitea` +namespace에 있어야 한다. + +### `gitea-admin` + +| 속성 | 필수 값 | +| --- | --- | +| Kubernetes 유형 | `kubernetes.io/basic-auth` | +| `username` | 최초 로컬 비상 관리자 username | +| `password` | 강력한 일회용 관리자 비밀번호 | + +Chart는 `initialOnlyRequireReset`을 사용하므로 최초 로그인 때 초기 비밀번호를 +변경해야 하며 이후에는 조정하지 않는다. + +### `gitea-db-credentials` + +| 속성 | 필수 값 | +| --- | --- | +| Kubernetes 유형 | `kubernetes.io/basic-auth` | +| `username` | `gitea` | +| `password` | PostgreSQL `gitea` Role에 할당한 비밀번호 | + +Kubernetes에서는 namespace를 넘어 Secret을 참조할 수 없다. `platform-data`에 있는 +같은 이름의 Secret은 CloudNativePG/bootstrap 사본이다. `gitea` namespace의 +Secret은 동일한 Database credential을 담아야 하며 Chart가 이 사본을 참조한다. + +### `gitea-keycloak-oidc` + +| 속성 | 필수 값 | +| --- | --- | +| Kubernetes 유형 | `Opaque` | +| `key` | `gitea` | +| `secret` | Keycloak이 생성한 confidential client secret | + +`scripts/bootstrap/configure-keycloak-gitea-oidc.sh`가 Keycloak Admin REST API로 +정확한 callback URI를 가진 client를 생성하거나 갱신하고 이 Secret을 만든다. +공식 Chart의 `gitea.oauth[].existingSecret`이 `key`와 `secret`을 환경 변수로만 +초기화 컨테이너에 전달하고, 인증 소스 이름 `keycloak`을 반복 실행 시 갱신한다. +Secret payload는 values, 렌더 결과 또는 Git에 포함되지 않는다. + +업스트림 Chart는 credential이 아닌 초기화 스크립트와 inline 설정이 담긴 Opaque +Secret도 렌더링한다. 이 생성 리소스들은 고정된 Chart의 일부이며 위 세 credential +계약의 값은 포함하지 않는다. + +## 렌더링과 검사 + +Helm은 `PATH`에 있어야 하지만 릴리스를 직접 설치하는 데 사용하지 않는다. 플랫폼 +저장소 루트에서 고정 Chart SHA를 확인하는 검증기로 렌더링한다. + +```sh +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/validate/render-phase1.sh +``` + +검증기는 `gitea.yaml` baseline과 `gitea-oidc.yaml` 목표 상태를 모두 렌더한다. +두 결과에는 각각 Gitea Deployment 하나, `gitea-http`, `gitea-data`, Traefik +Ingress가 있어야 하고 `gitea-ssh`, NodePort, LoadBalancer와 내부 TLS는 없어야 +한다. 두 Deployment 모두 `gitea-admin`, `gitea-db-credentials`를 참조만 하고 +생성하지 않는다. + +baseline은 `gitea-keycloak-oidc`, ID host alias, Keycloak 전용 NetworkPolicy, +브랜딩 ConfigMap과 외부 인증 전용 가입 설정이 없어야 한다. 반대로 OIDC 산출물은 +`gitea-keycloak-oidc`를 값 노출 없이 참조하고 다음을 모두 만족해야 한다. + +- 두 브랜딩 ConfigMap 이름에 content hash가 있고 Deployment 참조와 일치한다. +- `/data/gitea/public/assets`와 `/data/gitea/templates/custom` 마운트는 read-only다. +- inline 설정에 외부 인증 전용 가입, OIDC 자동 계정 생성, `ACCOUNT_LINKING=login`, + 로컬 비밀번호 로그인 유지가 모두 렌더링된다. +- OAuth 인증 소스 이름은 callback URI의 경로와 같은 `keycloak`이고 discovery + issuer는 `https://id.learn.hyeonworks.com/realms/hyeonworks`다. + +## Keycloak OIDC 안전 적용 + +Host Nginx의 Keycloak discovery가 실제 JSON issuer를 반환하고 위 세 Secret 계약이 +준비된 뒤 다음 전용 진입점으로 적용한다. + +1. Host Nginx의 `id.learn.hyeonworks.com` 정적 hold를 Keycloak proxy 후보로 + 전환한다. +2. public discovery JSON의 `issuer`가 + `https://id.learn.hyeonworks.com/realms/hyeonworks`와 정확히 같은지 확인한다. +3. 다음 전용 스크립트로 OIDC 목표 상태와 브랜딩을 함께 적용한다. + +```sh +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/bootstrap/apply-gitea-oidc.sh --execute +``` + +스크립트는 public discovery와 Secret의 type·key 이름만 먼저 확인하고, 고정 Chart +SHA 검증 결과 중 `gitea-oidc.yaml` 하나만 적용한다. 이후 Deployment·EndpointSlice, +활성 `keycloak` OAuth2 source, 외부 인증 전용 가입 정책, OIDC authorization-code +redirect와 브랜딩 자산 SHA-256을 검증한다. Secret 값, bearer token과 OIDC state가 +포함된 전체 Location은 출력하지 않는다. + +2026-07-23 실제 적용에서는 manifest SHA-256 `d25a757...a157`의 rollout과 위 자동 +검사를 모두 통과했다. 이 redirect 검사는 realm 사용자의 실제 브라우저 +login/callback/logout 시험을 대신하지 않는다. self-registration은 비활성이므로 +named 관리자와 MFA를 준비한 뒤 시험 사용자를 별도로 생성한다. 로컬 비상 관리자 +실제 로그인과 Pod 재시작 뒤 OAuth source·정책·브랜딩 지속성도 별도로 확인한다. + +기존 Gitea에 `apply-phase1-gitea.sh`를 다시 실행해 OIDC 상태를 적용하거나 +복구하지 않는다. diff --git a/services/gitea/base/ingress.yaml b/services/gitea/base/ingress.yaml new file mode 100644 index 0000000..d1e3732 --- /dev/null +++ b/services/gitea/base/ingress.yaml @@ -0,0 +1,25 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: gitea-http + namespace: gitea + labels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + app.kubernetes.io/component: source-control + app.kubernetes.io/part-of: platform + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + ingressClassName: traefik + rules: + - host: git.learn.hyeonworks.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: gitea-http + port: + number: 3000 diff --git a/services/gitea/base/kustomization.yaml b/services/gitea/base/kustomization.yaml new file mode 100644 index 0000000..62cdccc --- /dev/null +++ b/services/gitea/base/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - pvc.yaml + - ingress.yaml + - network-policies.yaml diff --git a/services/gitea/base/network-policies.yaml b/services/gitea/base/network-policies.yaml new file mode 100644 index 0000000..176136c --- /dev/null +++ b/services/gitea/base/network-policies.yaml @@ -0,0 +1,161 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: gitea-default-deny + namespace: gitea + labels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + policyTypes: + - Ingress + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: gitea-allow-traefik-http + namespace: gitea + labels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ports: + - protocol: TCP + port: 3000 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: gitea-allow-dns + namespace: gitea + labels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: gitea-allow-platform-postgres + namespace: gitea + labels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: platform-data + podSelector: + matchLabels: + cnpg.io/cluster: platform-postgres + ports: + - protocol: TCP + port: 5432 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: gitea-allow-public-git-https + namespace: gitea + labels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 169.254.0.0/16 + ports: + - protocol: TCP + port: 443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: gitea-allow-prometheus-metrics + namespace: gitea + labels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: observability-core-kube-pr-prometheus + ports: + - protocol: TCP + port: 3000 diff --git a/services/gitea/base/pvc.yaml b/services/gitea/base/pvc.yaml new file mode 100644 index 0000000..f368981 --- /dev/null +++ b/services/gitea/base/pvc.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: gitea-data + namespace: gitea + labels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + app.kubernetes.io/component: source-control + app.kubernetes.io/part-of: platform +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + storageClassName: ssd-local-gitea-retain + volumeName: gitea-local-pv + resources: + requests: + storage: 50Gi diff --git a/services/gitea/branding/kustomization.yaml b/services/gitea/branding/kustomization.yaml new file mode 100644 index 0000000..ffed365 --- /dev/null +++ b/services/gitea/branding/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: gitea + +configMapGenerator: + - name: gitea-branding-assets + files: + - hyeonworks.css=public/assets/css/hyeonworks.css + - logo.svg=public/assets/img/logo.svg + - favicon.svg=public/assets/img/favicon.svg + - name: gitea-branding-templates + files: + - header.tmpl=templates/custom/header.tmpl + - extra_links.tmpl=templates/custom/extra_links.tmpl diff --git a/services/gitea/branding/public/assets/css/hyeonworks.css b/services/gitea/branding/public/assets/css/hyeonworks.css new file mode 100644 index 0000000..b99e7d7 --- /dev/null +++ b/services/gitea/branding/public/assets/css/hyeonworks.css @@ -0,0 +1,59 @@ +:root { + --hyeonworks-accent: #2563eb; + --hyeonworks-accent-hover: #1d4ed8; + --hyeonworks-focus: rgba(37, 99, 235, 0.35); +} + +#navbar-logo { + gap: 0.5rem; +} + +#navbar-logo img { + width: 2rem; + height: 2rem; +} + +#navbar-logo::after { + content: "Hyeonworks Git"; + font-weight: 700; + letter-spacing: -0.015em; + white-space: nowrap; +} + +#navbar .hw-brand-link { + font-weight: 600; +} + +#navbar .hw-brand-link:hover, +#navbar .hw-brand-link:focus-visible { + color: var(--hyeonworks-accent); +} + +#navbar .hw-brand-link:focus-visible { + border-radius: 0.375rem; + outline: 3px solid var(--hyeonworks-focus); + outline-offset: -3px; +} + +.ui.primary.button, +.ui.primary.buttons .button { + background-color: var(--hyeonworks-accent); +} + +.ui.primary.button:hover, +.ui.primary.buttons .button:hover { + background-color: var(--hyeonworks-accent-hover); +} + +@media (max-width: 767.98px) { + #navbar-logo::after { + display: none; + } +} + +@media (prefers-reduced-motion: reduce) { + #navbar .hw-brand-link, + .ui.button { + transition: none; + } +} diff --git a/services/gitea/branding/public/assets/img/favicon.svg b/services/gitea/branding/public/assets/img/favicon.svg new file mode 100644 index 0000000..c695555 --- /dev/null +++ b/services/gitea/branding/public/assets/img/favicon.svg @@ -0,0 +1,12 @@ + + Hyeonworks Git + + + + + + + + + + diff --git a/services/gitea/branding/public/assets/img/logo.svg b/services/gitea/branding/public/assets/img/logo.svg new file mode 100644 index 0000000..e9c48f8 --- /dev/null +++ b/services/gitea/branding/public/assets/img/logo.svg @@ -0,0 +1,13 @@ + + Hyeonworks Git + 겹쳐진 코드 괄호와 H 문자를 사용한 Hyeonworks Git 로고 + + + + + + + + + + diff --git a/services/gitea/branding/templates/custom/extra_links.tmpl b/services/gitea/branding/templates/custom/extra_links.tmpl new file mode 100644 index 0000000..5fd28db --- /dev/null +++ b/services/gitea/branding/templates/custom/extra_links.tmpl @@ -0,0 +1 @@ +조직 diff --git a/services/gitea/branding/templates/custom/header.tmpl b/services/gitea/branding/templates/custom/header.tmpl new file mode 100644 index 0000000..f6ad4a0 --- /dev/null +++ b/services/gitea/branding/templates/custom/header.tmpl @@ -0,0 +1,2 @@ + + diff --git a/services/gitea/kustomization.yaml b/services/gitea/kustomization.yaml new file mode 100644 index 0000000..eb4f831 --- /dev/null +++ b/services/gitea/kustomization.yaml @@ -0,0 +1,40 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: gitea + +helmGlobals: + chartHome: profiles/oidc/.helm/charts + +helmCharts: + - name: gitea + repo: https://dl.gitea.com/charts/ + version: 12.7.0 + releaseName: gitea + namespace: gitea + valuesFile: profiles/oidc/values/baseline.yaml + includeCRDs: false + skipTests: true + +resources: + - base + +patches: + - path: profiles/oidc/patches/servicemonitor-job-label-patch.yaml + target: + group: monitoring.coreos.com + version: v1 + kind: ServiceMonitor + name: gitea + namespace: gitea + - target: + group: "" + version: v1 + kind: Service + name: gitea-ssh + patch: |- + $patch: delete + apiVersion: v1 + kind: Service + metadata: + name: gitea-ssh diff --git a/services/gitea/profiles/oidc/kustomization.yaml b/services/gitea/profiles/oidc/kustomization.yaml new file mode 100644 index 0000000..c3afa32 --- /dev/null +++ b/services/gitea/profiles/oidc/kustomization.yaml @@ -0,0 +1,39 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: gitea + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: gitea + repo: https://dl.gitea.com/charts/ + version: 12.7.0 + releaseName: gitea + namespace: gitea + valuesFile: values/baseline.yaml + additionalValuesFiles: + - values/oidc.yaml + includeCRDs: false + skipTests: true + +resources: + - ../../base + - ../../branding + - network-policy.yaml + +patches: + - path: patches/servicemonitor-job-label-patch.yaml + target: + group: monitoring.coreos.com + version: v1 + kind: ServiceMonitor + name: gitea + namespace: gitea + - path: patches/delete-ssh-service.yaml + target: + group: "" + version: v1 + kind: Service + name: gitea-ssh diff --git a/services/gitea/profiles/oidc/network-policy.yaml b/services/gitea/profiles/oidc/network-policy.yaml new file mode 100644 index 0000000..7486023 --- /dev/null +++ b/services/gitea/profiles/oidc/network-policy.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: gitea-allow-host-nginx-keycloak + namespace: gitea + labels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + policyTypes: + - Egress + egress: + - to: + - ipBlock: + # CoreDNS split DNS resolves the public issuer name to Host Nginx. + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 443 diff --git a/services/gitea/profiles/oidc/patches/delete-ssh-service.yaml b/services/gitea/profiles/oidc/patches/delete-ssh-service.yaml new file mode 100644 index 0000000..7a56a05 --- /dev/null +++ b/services/gitea/profiles/oidc/patches/delete-ssh-service.yaml @@ -0,0 +1,6 @@ +$patch: delete +apiVersion: v1 +kind: Service +metadata: + name: gitea-ssh + namespace: gitea diff --git a/services/gitea/profiles/oidc/patches/servicemonitor-job-label-patch.yaml b/services/gitea/profiles/oidc/patches/servicemonitor-job-label-patch.yaml new file mode 100644 index 0000000..ecfaf86 --- /dev/null +++ b/services/gitea/profiles/oidc/patches/servicemonitor-job-label-patch.yaml @@ -0,0 +1,7 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: gitea + namespace: gitea +spec: + jobLabel: app.kubernetes.io/name diff --git a/services/gitea/profiles/oidc/values/baseline.yaml b/services/gitea/profiles/oidc/values/baseline.yaml new file mode 100644 index 0000000..1a516ff --- /dev/null +++ b/services/gitea/profiles/oidc/values/baseline.yaml @@ -0,0 +1,189 @@ +namespace: gitea + +replicaCount: 1 + +image: + registry: docker.gitea.com + repository: gitea + tag: "1.27.0" + pullPolicy: IfNotPresent + rootless: true + +# The official chart applies this context to its built-in init containers, +# configure-gitea, and the main container. +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +# A Local RWO volume cannot be attached to an old and a replacement Pod at the +# same time. Terminate the old Pod before creating its replacement. +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + +service: + http: + type: ClusterIP + port: 3000 + clusterIP: "" + nodePort: null + ssh: + type: ClusterIP + port: 22 + clusterIP: "" + nodePort: null + hostPort: null + +# Routing is owned by ../../base/ingress.yaml. Host Nginx terminates TLS before +# forwarding requests to Traefik's HTTP entrypoint. +ingress: + enabled: false + +route: + enabled: false + +gatewayAPI: + enabled: false + +persistence: + enabled: true + create: false + mount: true + claimName: gitea-data + +gitea: + admin: + existingSecret: gitea-admin + email: admin@hyeonworks.com + passwordMode: initialOnlyRequireReset + + config: + server: + PROTOCOL: http + DOMAIN: git.learn.hyeonworks.com + ROOT_URL: https://git.learn.hyeonworks.com/ + HTTP_PORT: 3000 + DISABLE_SSH: true + START_SSH_SERVER: false + LFS_START_SERVER: false + + # Baseline allows only the pre-provisioned local emergency administrator. + # The guarded OIDC profile overrides the first two settings. + service: + DISABLE_REGISTRATION: true + ALLOW_ONLY_EXTERNAL_REGISTRATION: false + SHOW_REGISTRATION_BUTTON: false + ENABLE_PASSWORD_SIGNIN_FORM: true + + database: + DB_TYPE: postgres + HOST: platform-postgres-rw.platform-data.svc.cluster.local:5432 + NAME: gitea + SSL_MODE: disable + + session: + PROVIDER: db + cache: + ADAPTER: memory + queue: + TYPE: level + + actions: + ENABLED: false + packages: + ENABLED: false + repository: + DISABLED_REPO_UNITS: "repo.packages,repo.actions" + + additionalConfigFromEnvs: + - name: GITEA__DATABASE__USER + valueFrom: + secretKeyRef: + name: gitea-db-credentials + key: username + - name: GITEA__DATABASE__PASSWD + valueFrom: + secretKeyRef: + name: gitea-db-credentials + key: password + + metrics: + enabled: true + serviceMonitor: + enabled: true + additionalLabels: + observability.hyeonworks.com/instance: home + interval: 30s + scrapeTimeout: 10s + + livenessProbe: + enabled: true + httpGet: + path: /api/healthz + port: http + initialDelaySeconds: 200 + timeoutSeconds: 5 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + + readinessProbe: + enabled: true + httpGet: + path: /api/healthz + port: http + initialDelaySeconds: 5 + timeoutSeconds: 5 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + + startupProbe: + enabled: true + httpGet: + path: /api/healthz + port: http + initialDelaySeconds: 10 + timeoutSeconds: 5 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 60 + +postgresql: + enabled: false + +postgresql-ha: + enabled: false + +valkey: + enabled: false + +valkey-cluster: + enabled: false + +resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + +initContainers: + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +test: + enabled: false diff --git a/services/gitea/profiles/oidc/values/oidc.yaml b/services/gitea/profiles/oidc/values/oidc.yaml new file mode 100644 index 0000000..e051634 --- /dev/null +++ b/services/gitea/profiles/oidc/values/oidc.yaml @@ -0,0 +1,49 @@ +extraVolumes: + - name: branding-assets + configMap: + name: gitea-branding-assets + defaultMode: 292 + items: + - key: hyeonworks.css + path: css/hyeonworks.css + - key: logo.svg + path: img/logo.svg + - key: favicon.svg + path: img/favicon.svg + - name: branding-templates + configMap: + name: gitea-branding-templates + defaultMode: 292 + items: + - key: header.tmpl + path: header.tmpl + - key: extra_links.tmpl + path: extra_links.tmpl + +extraContainerVolumeMounts: + - name: branding-assets + mountPath: /data/gitea/public/assets + readOnly: true + - name: branding-templates + mountPath: /data/gitea/templates/custom + readOnly: true + +gitea: + oauth: + - name: keycloak + provider: openidConnect + existingSecret: gitea-keycloak-oidc + autoDiscoverUrl: https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration + + config: + APP_NAME: Hyeonworks Git + + service: + DISABLE_REGISTRATION: false + ALLOW_ONLY_EXTERNAL_REGISTRATION: true + + oauth2_client: + ENABLE_AUTO_REGISTRATION: true + USERNAME: preferred_username + ACCOUNT_LINKING: login + OPENID_CONNECT_SCOPES: "profile email" diff --git a/services/keycloak/README.md b/services/keycloak/README.md new file mode 100644 index 0000000..a5fe6ce --- /dev/null +++ b/services/keycloak/README.md @@ -0,0 +1,111 @@ +# Keycloak 서비스 + +이 Kustomization은 `keycloak` namespace에 Keycloak 26.7.0 인스턴스 하나를 +선언한다. 이 인스턴스는 독립적인 OIDC Provider이며 인증 proxy나 Traefik +ForwardAuth middleware가 아니다. + +외부 요청 경로는 다음과 같다. + +```text +Client -> Host Nginx (TLS) -> Traefik web/HTTP -> keycloak-service:8080 +``` + +상태(2026-07-23): Host Nginx의 Keycloak 공개 전환과 Gitea OIDC·브랜딩 rollout을 +완료했다. Nginx 후보 SHA-256은 +`5c5cd74b4992f537fd27c50cf2209573a80a9904e0b19b58c3154717be6ff4a5`, 백업은 +`/etc/nginx/sites-available/learn-services.before-keycloak-20260723160519`이며 +discovery issuer와 Gitea OAuth source·정책·redirect·브랜딩 자동 검증을 통과했다. +실제 realm 사용자의 브라우저 login/callback/logout은 별도 수용 시험으로 남아 있다. + +Keycloak이 생성하는 Ingress는 비활성화한다. 저장소에서 관리하는 Ingress는 +Traefik의 내부 HTTP entrypoint를 통해 `id.learn.hyeonworks.com`만 노출한다. 관리 +포트 `9000`에는 Ingress나 NodePort가 없으며, NetworkPolicy의 출발지는 namespace +범위 Operator로 제한한다. TLS는 Host Nginx에서만 종료하므로 Keycloak은 Traefik이 +선택한 HTTP 경로의 `xforwarded` header만 수락한다. + +## 필수 Secret 계약 + +credential Secret 매니페스트나 값은 이 저장소에 보관하지 않는다. Keycloak +리소스를 생성하기 전에 동일하게 생성한 비밀번호를 사용하는 +`keycloak-db-credentials`를 다음 두 namespace에 생성한다. + +- `platform-data`: CloudNativePG의 `DatabaseRole`에서 사용 +- `keycloak`: Keycloak 서버에서 사용 + +각 Secret의 유형은 `kubernetes.io/basic-auth`, username은 정확히 `keycloak`이어야 +하며 `username`과 `password` 키가 있어야 한다. `platform-data` 사본에는 +`cnpg.io/reload: "true"` label도 있어야 한다. + +인스턴스를 처음 조정할 때 Operator가 `keycloak` namespace에 +`keycloak-initial-admin`을 생성한다. 이 생성된 Secret은 소스 관리하지 않는다. +서비스를 운영 준비 완료 상태로 판단하기 전에 bootstrap credential을 교체하고 +MFA를 활성화한다. + +## 의존성과 렌더링 순서 + +1. `keycloak`, `platform-data`, `cnpg-system` namespace가 존재한다. +2. CloudNativePG와 `platform-postgres`가 Ready 상태다. +3. `keycloak-db-credentials`의 두 사본이 모두 존재한다. +4. Keycloak Operator가 설치되어 Ready 상태다. +5. 이 서비스 Kustomization을 적용한다. +6. Ready 상태와 내부 Traefik routing을 확인한 후에만 Host Nginx에 + `id.learn.hyeonworks.com`을 설정한다. + +Helm 없이 렌더링한다. + +```bash +kubectl kustomize services/keycloak +``` + +배포 후 다음을 검증한다. + +```bash +kubectl -n keycloak wait --for=condition=Ready \ + keycloak/keycloak --timeout=15m +kubectl -n keycloak get keycloak,pod,service,ingress,networkpolicy +curl --fail --silent --show-error \ + https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration \ + | jq --exit-status \ + --arg issuer 'https://id.learn.hyeonworks.com/realms/hyeonworks' \ + '.issuer == $issuer' +``` + +마지막 검사는 HTTP 성공 여부만 보지 않고 JSON의 `issuer`가 공개 issuer와 정확히 +같은지 확인한다. 따라서 전환 전 Host Nginx의 정적 hold 응답이 HTTP 200을 +반환하더라도 성공으로 오인하지 않는다. + +TCP 9000을 대상으로 하는 Ingress, NodePort 또는 LoadBalancer가 없는지도 별도로 +확인한다. + +## 수동 bootstrap 진입점 + +Keycloak-only 적용은 AIStor와 분리한다. 저장소 루트에서 다음 순서를 사용하며 +Secret payload는 명령 인자나 출력에 넣지 않는다. + +```sh +kubectl apply --filename=infrastructure/namespaces/phase2/keycloak.yaml +bash scripts/bootstrap/create-keycloak-secrets.sh --generate --execute +bash scripts/bootstrap/apply-keycloak.sh --execute +bash scripts/bootstrap/configure-keycloak-gitea-oidc.sh --execute +``` + +각 스크립트는 context와 적용 범위를 다시 확인한다. 마지막 스크립트는 +`hyeonworks` realm, confidential Gitea client와 Gitea OIDC Secret을 구성한다. + +public discovery 전환은 클러스터 적용과 분리된 root 작업이며 실제 전환을 +완료했다. + +```sh +sudo bash scripts/bootstrap/apply-host-nginx-keycloak.sh --execute +``` + +상세 명령·출력·실패 경계는 +[중앙 OIDC 실행 원장](../../../docs/platform/runbooks/2026-07-23-keycloak-gitea-oidc-cutover.md)에 +기록한다. + +## 공식 참고 문서 + +- +- +- +- diff --git a/services/keycloak/base/ingress.yaml b/services/keycloak/base/ingress.yaml new file mode 100644 index 0000000..29e4aec --- /dev/null +++ b/services/keycloak/base/ingress.yaml @@ -0,0 +1,23 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: keycloak-http + labels: + app.kubernetes.io/name: keycloak + app.kubernetes.io/instance: keycloak + app.kubernetes.io/component: identity-provider + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + ingressClassName: traefik + rules: + - host: id.learn.hyeonworks.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: keycloak-service + port: + number: 8080 diff --git a/services/keycloak/base/keycloak.yaml b/services/keycloak/base/keycloak.yaml new file mode 100644 index 0000000..c0df799 --- /dev/null +++ b/services/keycloak/base/keycloak.yaml @@ -0,0 +1,94 @@ +apiVersion: k8s.keycloak.org/v2beta1 +kind: Keycloak +metadata: + name: keycloak + labels: + app.kubernetes.io/name: keycloak + app.kubernetes.io/instance: keycloak + app.kubernetes.io/component: identity-provider +spec: + instances: 1 + + db: + vendor: postgres + host: platform-postgres-rw.platform-data.svc.cluster.local + port: 5432 + database: keycloak + schema: public + usernameSecret: + name: keycloak-db-credentials + key: username + passwordSecret: + name: keycloak-db-credentials + key: password + poolInitialSize: 5 + poolMinSize: 5 + poolMaxSize: 20 + + hostname: + hostname: https://id.learn.hyeonworks.com + strict: true + + http: + httpEnabled: true + httpPort: 8080 + serviceHttpPort: 8080 + + proxy: + headers: xforwarded + + ingress: + enabled: false + + additionalOptions: + - name: metrics-enabled + value: "true" + + serviceMonitor: + enabled: true + interval: 30s + scrapeTimeout: 10s + labels: + observability.hyeonworks.com/instance: home + + networkPolicy: + enabled: true + http: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + https: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + management: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: keycloak + podSelector: + matchLabels: + app.kubernetes.io/name: keycloak-operator + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: observability-core-kube-pr-prometheus + + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + + transaction: + xaEnabled: false diff --git a/services/keycloak/base/kustomization.yaml b/services/keycloak/base/kustomization.yaml new file mode 100644 index 0000000..57b1358 --- /dev/null +++ b/services/keycloak/base/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - keycloak.yaml + - ingress.yaml diff --git a/services/keycloak/kustomization.yaml b/services/keycloak/kustomization.yaml new file mode 100644 index 0000000..b5ed130 --- /dev/null +++ b/services/keycloak/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: keycloak + +resources: + - base + +labels: + - pairs: + app.kubernetes.io/part-of: platform + app.kubernetes.io/managed-by: kustomize + includeSelectors: false diff --git a/services/keycloak/overlays/home/README.md b/services/keycloak/overlays/home/README.md new file mode 100644 index 0000000..0e31f6e --- /dev/null +++ b/services/keycloak/overlays/home/README.md @@ -0,0 +1,6 @@ +# 홈 환경 Overlay + +현재 홈 클러스터 구성은 `../base`의 안정적인 리소스로 직접 표현한다. +`services/keycloak`을 유일한 서비스 빌드 루트로 유지한다. 향후 홈 환경 전용 +Patch는 이곳에 추가하고, 독립적으로 렌더링하는 두 번째 애플리케이션을 만들지 말고 +서비스 루트 Kustomization에서 참조한다. diff --git a/services/minio-aistor/README.md b/services/minio-aistor/README.md new file mode 100644 index 0000000..3fed8cc --- /dev/null +++ b/services/minio-aistor/README.md @@ -0,0 +1,81 @@ +# MinIO AIStor ObjectStore + +상태(2026-07-23): + +- baseline ObjectStore와 인증 S3 스모크: 실제 적용 완료 +- admin-oidc profile과 비공개 Console Ingress: 선언형 구현·dry-run 완료, live 적용 대기 + +공식 Chart를 수정하지 않고 Kustomize가 Helm을 렌더한다. + +## 빌드 루트 + +- services/minio-aistor: 현재 live baseline 진입점 +- services/minio-aistor/base/deployment: 공통 Helm baseline과 values +- services/minio-aistor/profiles/admin-oidc: baseline에 OIDC, Console Ingress와 + 추가 NetworkPolicy를 결합한 진입점 + +root와 profile이 같은 형제 baseline을 참조하므로 profile이 조상 root를 다시 +포함하는 Kustomize 순환이 없다. + +## 고정 원본 + +| 항목 | 값 | +|---|---| +| Chart 저장소 | https://helm.min.io/ | +| Chart | aistor-objectstore | +| 버전 | 1.0.16 | +| archive SHA-256 | 50ffa6a4e014cdc48566b237593baab41d039f103cdc50c1b1ac173b8d8bf71e | + +검증기가 만드는 임시 Chart cache는 +base/deployment/.helm/charts/aistor-objectstore-1.0.16 아래에만 존재하고 종료 시 +제거한다. + +## baseline 계약 + +- namespace: object-storage +- ObjectStore: minio-aistor +- pool/server/volume: 각각 1 +- 저장소: aistor-local-xfs-retain, 900Gi RWO +- S3와 Console Service: ClusterIP +- baseline Ingress, NodePort, LoadBalancer: 없음 +- pvcProtection과 Retain Local PV +- 기존 root 계정: 비상용으로 유지 + +Standalone이므로 erasure-code 이중화와 노드 failover는 없다. + +## Secret 계약 + +baseline: + +- object-storage/aistor-root-configuration: config.env + +admin-oidc profile: + +- object-storage/aistor-keycloak-oidc: client-id, client-secret + +평문, Base64, checksum을 Git과 문서에 기록하지 않는다. + +## 렌더와 적용 + +baseline 정적 렌더: + + kubectl kustomize --enable-helm services/minio-aistor + +admin profile 전체 정적 검증: + + PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/validate/render-admin-services.sh + +admin profile 실제 적용은 private DNS, admin 인증서와 Keycloak client/group 준비 +뒤 apply-admin-services.sh가 수행한다. S3 API에는 Ingress를 만들지 않는다. + +현재 baseline은 Initialized/green이고 data0-minio-aistor-pool-0-0 PVC가 +aistor-data-local-pv에 Bound다. S3 회귀 검사는 다음과 같다. + + bash scripts/validate/aistor-s3-smoke.sh --execute + +## 공식 문서 + +- https://docs.min.io/aistor/reference/kubernetes/object-store-helm-chart/ +- https://docs.min.io/aistor/reference/kubernetes/pvc-protection/ +- https://docs.min.io/aistor/administration/iam/identity/keycloak-identity/ diff --git a/services/minio-aistor/base/README.md b/services/minio-aistor/base/README.md new file mode 100644 index 0000000..f34e2b8 --- /dev/null +++ b/services/minio-aistor/base/README.md @@ -0,0 +1,7 @@ +# 기본 확장 지점 + +공통 ObjectStore Helm baseline은 base/deployment에 있다. 저장소, resource, +ClusterIP와 root 비상 계정 경계만 이 계층이 소유한다. + +OIDC 환경 변수와 Console Ingress는 profiles/admin-oidc에서만 추가한다. +Chart 템플릿을 저장소에 복제하거나 직접 수정하지 않는다. diff --git a/services/minio-aistor/base/deployment/README.md b/services/minio-aistor/base/deployment/README.md new file mode 100644 index 0000000..ef5234b --- /dev/null +++ b/services/minio-aistor/base/deployment/README.md @@ -0,0 +1,6 @@ +# AIStor ObjectStore baseline + +AIStor ObjectStore의 공통 Helm baseline입니다. 저장소, root 비상 계정, +리소스 제한과 ClusterIP 경계만 정의합니다. + +Console Ingress와 Keycloak OIDC는 profiles/admin-oidc에서만 추가합니다. diff --git a/services/minio-aistor/base/deployment/kustomization.yaml b/services/minio-aistor/base/deployment/kustomization.yaml new file mode 100644 index 0000000..4f7f920 --- /dev/null +++ b/services/minio-aistor/base/deployment/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: object-storage + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: aistor-objectstore + repo: https://helm.min.io/ + version: 1.0.16 + releaseName: minio-aistor + namespace: object-storage + valuesFile: values/home.yaml + includeCRDs: false + skipTests: true diff --git a/services/minio-aistor/base/deployment/values/home.yaml b/services/minio-aistor/base/deployment/values/home.yaml new file mode 100644 index 0000000..dacd73e --- /dev/null +++ b/services/minio-aistor/base/deployment/values/home.yaml @@ -0,0 +1,67 @@ +secrets: + name: aistor-root-configuration + existingSecret: true + disableRoot: false + +objectStore: + name: minio-aistor + + configuration: + name: aistor-root-configuration + + pools: + - name: pool-0 + servers: 1 + volumesPerServer: 1 + storageNamePrefix: data + size: 900Gi + storageClassName: aistor-local-xfs-retain + nodeSelector: + kubernetes.io/hostname: donghyeon-system-product-name + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: "3" + memory: 8Gi + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + containerSecurityContext: + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + + mountPath: /export + subPath: /data + + certificates: + disableAutoCert: true + + services: + minio: + serviceType: ClusterIP + console: + serviceType: ClusterIP + + pvcProtection: true + prometheusOperator: false + + sideCars: + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/services/minio-aistor/kustomization.yaml b/services/minio-aistor/kustomization.yaml new file mode 100644 index 0000000..9f52c57 --- /dev/null +++ b/services/minio-aistor/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - base/deployment diff --git a/services/minio-aistor/overlays/home/README.md b/services/minio-aistor/overlays/home/README.md new file mode 100644 index 0000000..8df47bd --- /dev/null +++ b/services/minio-aistor/overlays/home/README.md @@ -0,0 +1,8 @@ +# 홈 환경 Overlay 예약 지점 + +현재 홈 환경의 공통 Helm values는 +../../base/deployment/values/home.yaml에 있다. 실제 배포 진입점은 저장소 root와 +profiles/admin-oidc다. + +이 디렉터리는 이후 홈 환경 전용 patch가 필요할 때만 사용하며 현재 활성 +Kustomization 리소스는 없다. diff --git a/services/minio-aistor/profiles/admin-oidc/README.md b/services/minio-aistor/profiles/admin-oidc/README.md new file mode 100644 index 0000000..3831dee --- /dev/null +++ b/services/minio-aistor/profiles/admin-oidc/README.md @@ -0,0 +1,16 @@ +# AIStor Console OIDC 프로필 + +기존 ObjectStore baseline은 그대로 두고 다음 항목만 추가합니다. + +- storage-admin.learn.hyeonworks.com Console Ingress +- Keycloak 표준 OIDC 환경 변수와 policy claim +- Traefik의 Console 접근과 Keycloak HTTPS egress + +S3 API Service에는 Ingress를 만들지 않습니다. aistor-keycloak-oidc +Secret의 client-id, client-secret 값은 적용 스크립트가 만들며 Git에 +저장하지 않습니다. 기존 root 계정은 비상용으로 유지합니다. + +`MINIO_IDENTITY_OPENID_VENDOR=keycloak`은 사용하지 않습니다. 이 옵션은 +AIStor가 Keycloak Admin API로 사용자를 추가 검증할 때 admin URL, realm과 +service-account 권한을 요구합니다. 이 프로필은 Keycloak이 발급한 `policy` +claim을 표준 OIDC로 소비하므로 불필요한 Admin API 권한을 부여하지 않습니다. diff --git a/services/minio-aistor/profiles/admin-oidc/console-ingress.yaml b/services/minio-aistor/profiles/admin-oidc/console-ingress.yaml new file mode 100644 index 0000000..813591c --- /dev/null +++ b/services/minio-aistor/profiles/admin-oidc/console-ingress.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: minio-aistor-console + namespace: object-storage + labels: + app.kubernetes.io/name: minio-aistor + app.kubernetes.io/component: admin-console + app.kubernetes.io/part-of: platform + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + ingressClassName: traefik + rules: + - host: storage-admin.learn.hyeonworks.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: minio-aistor-console + port: + number: 9090 diff --git a/services/minio-aistor/profiles/admin-oidc/kustomization.yaml b/services/minio-aistor/profiles/admin-oidc/kustomization.yaml new file mode 100644 index 0000000..d2aa640 --- /dev/null +++ b/services/minio-aistor/profiles/admin-oidc/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base/deployment + - console-ingress.yaml + - network-policies.yaml + +patches: + - path: objectstore-oidc-patch.yaml + target: + group: aistor.min.io + version: v1 + kind: ObjectStore + name: minio-aistor diff --git a/services/minio-aistor/profiles/admin-oidc/network-policies.yaml b/services/minio-aistor/profiles/admin-oidc/network-policies.yaml new file mode 100644 index 0000000..75e5dff --- /dev/null +++ b/services/minio-aistor/profiles/admin-oidc/network-policies.yaml @@ -0,0 +1,101 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-allow-traefik-console + namespace: object-storage + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: admin-console +spec: + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ports: + - protocol: TCP + port: 9090 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-allow-host-nginx-keycloak + namespace: object-storage + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: oidc-client +spec: + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-allow-blackbox-console-health + namespace: object-storage + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: admin-console +spec: + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + ports: + - protocol: TCP + port: 9090 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-allow-prometheus-metrics + namespace: object-storage + labels: + app.kubernetes.io/part-of: platform + app.kubernetes.io/component: object-storage-metrics +spec: + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: observability-core-kube-pr-prometheus + ports: + - protocol: TCP + port: 9000 diff --git a/services/minio-aistor/profiles/admin-oidc/objectstore-oidc-patch.yaml b/services/minio-aistor/profiles/admin-oidc/objectstore-oidc-patch.yaml new file mode 100644 index 0000000..7008688 --- /dev/null +++ b/services/minio-aistor/profiles/admin-oidc/objectstore-oidc-patch.yaml @@ -0,0 +1,32 @@ +apiVersion: aistor.min.io/v1 +kind: ObjectStore +metadata: + name: minio-aistor +spec: + env: + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: public + - name: MINIO_BROWSER_REDIRECT_URL + value: https://storage-admin.learn.hyeonworks.com + - name: MINIO_IDENTITY_OPENID_CONFIG_URL + value: https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration + - name: MINIO_IDENTITY_OPENID_CLIENT_ID + valueFrom: + secretKeyRef: + name: aistor-keycloak-oidc + key: client-id + - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: aistor-keycloak-oidc + key: client-secret + - name: MINIO_IDENTITY_OPENID_SCOPES + value: openid,profile,email + - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME + value: Hyeonworks ID + - name: MINIO_IDENTITY_OPENID_CLAIM_NAME + value: policy + - name: MINIO_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC + value: "on" + - name: MINIO_BROWSER_SESSION_DURATION + value: 8h diff --git a/services/minio-aistor/values/home.yaml b/services/minio-aistor/values/home.yaml new file mode 100644 index 0000000..dacd73e --- /dev/null +++ b/services/minio-aistor/values/home.yaml @@ -0,0 +1,67 @@ +secrets: + name: aistor-root-configuration + existingSecret: true + disableRoot: false + +objectStore: + name: minio-aistor + + configuration: + name: aistor-root-configuration + + pools: + - name: pool-0 + servers: 1 + volumesPerServer: 1 + storageNamePrefix: data + size: 900Gi + storageClassName: aistor-local-xfs-retain + nodeSelector: + kubernetes.io/hostname: donghyeon-system-product-name + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: "3" + memory: 8Gi + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + containerSecurityContext: + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + + mountPath: /export + subPath: /data + + certificates: + disableAutoCert: true + + services: + minio: + serviceType: ClusterIP + console: + serviceType: ClusterIP + + pvcProtection: true + prometheusOperator: false + + sideCars: + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/services/observability/README.md b/services/observability/README.md new file mode 100644 index 0000000..5d0145b --- /dev/null +++ b/services/observability/README.md @@ -0,0 +1,18 @@ +# Observability core + +This root composes the internal observability workloads deployed after the +namespace, CRD, and Local PV prerequisites: + +- kube-prometheus-stack without Grafana or its bundled node-exporter; +- single-binary Loki and Tempo using isolated AIStor buckets; +- Alloy for Pod logs and internal OTLP/gRPC traces; +- a standalone node-exporter; +- the exact core NetworkPolicy matrix, platform rules, scrape targets, verified + dashboards, and source-only Alertmanager Slack routing. + +It intentionally does not create public ingress, Grafana, OIDC, or credential +Secrets. The Slack routing source references +`alertmanager-slack-webhook/url`; while that Secret is absent, do not apply the +AlertmanagerConfig, its kube-prometheus-stack reference, its public HTTPS +NetworkPolicy, or alert rules. Namespace, CRD, and Local PV roots remain outside +this aggregate because their protected apply order is different. diff --git a/services/observability/alerting/alertmanager-public-https-egress.yaml b/services/observability/alerting/alertmanager-public-https-egress.yaml new file mode 100644 index 0000000..b39303b --- /dev/null +++ b/services/observability/alerting/alertmanager-public-https-egress.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-alertmanager-public-https + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: observability-core-kube-pr-alertmanager + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - protocol: TCP + port: 443 diff --git a/services/observability/alerting/kustomization.yaml b/services/observability/alerting/kustomization.yaml new file mode 100644 index 0000000..7e833cb --- /dev/null +++ b/services/observability/alerting/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: observability + +resources: + - platform-alertmanager.yaml + - alertmanager-public-https-egress.yaml diff --git a/services/observability/alerting/platform-alertmanager.yaml b/services/observability/alerting/platform-alertmanager.yaml new file mode 100644 index 0000000..fe46c64 --- /dev/null +++ b/services/observability/alerting/platform-alertmanager.yaml @@ -0,0 +1,80 @@ +apiVersion: monitoring.coreos.com/v1alpha1 +kind: AlertmanagerConfig +metadata: + name: platform-alertmanager + namespace: observability + labels: + observability.hyeonworks.com/instance: home +spec: + route: + receiver: platform-slack + groupBy: + - cluster + - namespace + - alertname + - severity + groupWait: 30s + groupInterval: 5m + repeatInterval: 4h + routes: + - receiver: platform-null + matchers: + - name: alertname + matchType: "=" + value: InfoInhibitor + inhibitRules: + - sourceMatch: + - name: alertname + matchType: "=" + value: InfoInhibitor + targetMatch: + - name: severity + matchType: "=" + value: info + equal: + - namespace + receivers: + - name: platform-null + - name: platform-slack + slackConfigs: + - apiURL: + name: alertmanager-slack-webhook + key: url + sendResolved: true + linkNames: false + mrkdwnIn: + - text + - fields + fallback: >- + {{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }} + title: >- + {{ if eq .Status "firing" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }} + titleLink: https://grafana.learn.hyeonworks.com/ + color: >- + {{ if eq .Status "resolved" }}good{{ else if or (eq .CommonLabels.severity "emergency") (eq .CommonLabels.severity "critical") }}danger{{ else if eq .CommonLabels.severity "warning" }}warning{{ else }}#439FE0{{ end }} + fields: + - title: Status + value: '{{ .Status | toUpper }}' + short: true + - title: Severity + value: '{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}' + short: true + - title: Location + value: '{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}' + short: true + - title: Alert count + value: '{{ len .Alerts }}' + short: true + text: |- + {{ range .Alerts }} + *Alert status:* {{ .Status | toUpper }} + *Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }} + *Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }} + *Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }} + *Started:* {{ .StartsAt.Format "2006-01-02T15:04:05Z07:00" }} + {{ if eq .Status "resolved" }}*Ended:* {{ .EndsAt.Format "2006-01-02T15:04:05Z07:00" }}{{ end }} + {{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }} + + {{ end }} + · + footer: hyeonworks observability · Alertmanager diff --git a/services/observability/alloy/README.md b/services/observability/alloy/README.md new file mode 100644 index 0000000..e9f12c2 --- /dev/null +++ b/services/observability/alloy/README.md @@ -0,0 +1,12 @@ +# Alloy + +Grafana Alloy runs as a single-node DaemonSet in `observability-agent`. +It tails only `/var/log/pods`, writes logs to the in-cluster Loki service, and +accepts OTLP/gRPC traces on port `4317` for the in-cluster Tempo service. + +The deployment deliberately disables host namespaces, automatic service +account token mounts, CRD creation, the config reloader, public exposure, and +persistent OpenTelemetry queues. The 2 GiB PVC stores only Alloy file +positions and local state. Alloy keeps primary UID/GID `473:473`; supplemental +group `0` permits read-only traversal of k3s' `root:root 0750` Pod log tree +without changing host permissions or running the container as root. diff --git a/services/observability/alloy/config.alloy b/services/observability/alloy/config.alloy new file mode 100644 index 0000000..a6e256e --- /dev/null +++ b/services/observability/alloy/config.alloy @@ -0,0 +1,214 @@ +discovery.kubernetes "pods" { + role = "pod" + + selectors { + role = "pod" + field = "spec.nodeName=" + sys.env("HOSTNAME") + } +} + +discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pods.targets + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + regex = "(.+)" + target_label = "app" + } + rule { + source_labels = [ + "__meta_kubernetes_pod_label_app_kubernetes_io_name", + "__meta_kubernetes_pod_label_app", + ] + separator = ";" + regex = "^;(.+)$" + replacement = "$1" + target_label = "app" + } + rule { + source_labels = [ + "__meta_kubernetes_pod_uid", + "__meta_kubernetes_pod_container_name", + ] + separator = "/" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = `^(\S+):\/\/.+$` + replacement = "$1" + target_label = "container_runtime" + } +} + +local.file_match "pod_logs" { + path_targets = discovery.relabel.pod_logs.output +} + +loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] +} + +loki.process "pod_logs" { + stage.match { + selector = `{container_runtime=~"containerd|cri-o"}` + stage.cri {} + } + stage.match { + selector = `{container_runtime="docker"}` + stage.docker {} + } + + stage.replace { + expression = `(?i)"(?:authorization|bearer|jwt|password|passwd|client[_-]?secret)"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"` + replace = "[REDACTED]" + } + stage.replace { + expression = `(?i)(?:authorization\s*[:=]\s*)(?:bearer\s+)?([A-Za-z0-9._~+/=-]+)` + replace = "[REDACTED]" + } + stage.replace { + expression = `(?i)\bbearer\s+([A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+){0,2})` + replace = "[REDACTED]" + } + stage.replace { + expression = `(?i)(?:password|passwd|client[_-]?secret)\s*[:=]\s*([^\s"'{},]+)` + replace = "[REDACTED]" + } + + stage.json { + expressions = { + level = "level", + } + } + stage.labels { + values = { + level = "", + } + } + stage.match { + selector = `{level!~"(?i)^(trace|debug|info|warn|warning|error|fatal|panic)$"}` + stage.label_drop { + values = ["level"] + } + } + stage.static_labels { + values = { + cluster = "home", + } + } + stage.label_keep { + values = ["cluster", "namespace", "app", "pod", "container", "level"] + } + + forward_to = [loki.write.local.receiver] +} + +loki.write "local" { + endpoint { + url = "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push" + batch_size = "256KiB" + batch_wait = "1s" + remote_timeout = "10s" + min_backoff_period = "500ms" + max_backoff_period = "30s" + max_backoff_retries = 10 + } +} + +otelcol.receiver.otlp "traces" { + grpc { + endpoint = "0.0.0.0:4317" + } + + output { + traces = [otelcol.processor.k8sattributes.traces.input] + } +} + +otelcol.processor.k8sattributes "traces" { + auth_type = "serviceAccount" + + extract { + deployment_name_from_replicaset = false + metadata = [ + "k8s.namespace.name", + "k8s.deployment.name", + "k8s.node.name", + "k8s.pod.name", + "k8s.pod.uid", + "k8s.pod.start_time", + "k8s.container.name", + ] + } + + pod_association { + source { + from = "resource_attribute" + name = "k8s.pod.uid" + } + } + pod_association { + source { + from = "resource_attribute" + name = "k8s.pod.ip" + } + } + pod_association { + source { + from = "connection" + } + } + + output { + traces = [otelcol.processor.batch.traces.input] + } +} + +otelcol.processor.batch "traces" { + timeout = "1s" + send_batch_size = 512 + send_batch_max_size = 1024 + + output { + traces = [otelcol.exporter.otlp.tempo.input] + } +} + +otelcol.exporter.otlp "tempo" { + client { + endpoint = "tempo.observability.svc.cluster.local:4317" + tls { + insecure = true + } + } + + retry_on_failure { + enabled = true + initial_interval = "1s" + max_interval = "15s" + max_elapsed_time = "5m" + } + + sending_queue { + enabled = true + queue_size = 256 + num_consumers = 2 + sizer = "requests" + block_on_overflow = true + } +} diff --git a/services/observability/alloy/kustomization.yaml b/services/observability/alloy/kustomization.yaml new file mode 100644 index 0000000..5beac13 --- /dev/null +++ b/services/observability/alloy/kustomization.yaml @@ -0,0 +1,51 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: observability-agent + +resources: + - pvc.yaml + +configMapGenerator: + - name: alloy-config + files: + - config.alloy=config.alloy + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: alloy + repo: https://grafana.github.io/helm-charts + version: 1.11.0 + releaseName: alloy + namespace: observability-agent + valuesFile: values/home.yaml + includeCRDs: false + skipTests: true + +images: + - name: docker.io/grafana/alloy + newName: docker.io/grafana/alloy + digest: sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17 + +patches: + - target: + group: apps + version: v1 + kind: DaemonSet + name: alloy + patch: |- + - op: add + path: /spec/template/spec/automountServiceAccountToken + value: false + - op: add + path: /spec/template/spec/hostNetwork + value: false + - op: add + path: /spec/template/spec/hostPID + value: false + - op: add + path: /spec/template/spec/hostIPC + value: false + diff --git a/services/observability/alloy/pvc.yaml b/services/observability/alloy/pvc.yaml new file mode 100644 index 0000000..e63cfa2 --- /dev/null +++ b/services/observability/alloy/pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: alloy-storage + namespace: observability-agent + labels: + app.kubernetes.io/name: alloy + observability.hyeonworks.com/instance: home +spec: + accessModes: + - ReadWriteOnce + storageClassName: ssd-local-observability-alloy-retain + volumeName: observability-alloy-local-pv + resources: + requests: + storage: 2Gi + diff --git a/services/observability/alloy/values/home.yaml b/services/observability/alloy/values/home.yaml new file mode 100644 index 0000000..37a316a --- /dev/null +++ b/services/observability/alloy/values/home.yaml @@ -0,0 +1,155 @@ +crds: + create: false + +global: + podSecurityContext: + runAsNonRoot: true + runAsUser: 473 + runAsGroup: 473 + fsGroup: 473 + fsGroupChangePolicy: OnRootMismatch + # k3s keeps /var/log/pods root:root 0750. Retain Alloy's non-root primary + # identity while granting read-only traversal of that single hostPath. + supplementalGroups: + - 0 + seccompProfile: + type: RuntimeDefault + +alloy: + configMap: + create: false + name: alloy-config + key: config.alloy + clustering: + enabled: false + stabilityLevel: generally-available + storagePath: /var/lib/alloy + enableHttpServerPort: true + listenAddr: 0.0.0.0 + listenPort: 12345 + enableReporting: false + extraPorts: + - name: otlp-grpc + port: 4317 + targetPort: 4317 + protocol: TCP + mounts: + varlog: false + dockercontainers: false + extra: + - name: pod-logs + mountPath: /var/log/pods + readOnly: true + - name: alloy-storage + mountPath: /var/lib/alloy + - name: tmp + mountPath: /tmp + - name: kubernetes-api-access + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + securityContext: + runAsNonRoot: true + runAsUser: 473 + runAsGroup: 473 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 200m + memory: 384Mi + limits: + cpu: "1" + memory: 768Mi + +image: + registry: docker.io + repository: grafana/alloy + tag: v1.18.0 + digest: sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17 + pullPolicy: IfNotPresent + +rbac: + create: true + namespaces: [] + rules: + - apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch + clusterRules: + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - list + - watch + +serviceAccount: + create: true + name: alloy + automountServiceAccountToken: false + +configReloader: + enabled: false + +controller: + type: daemonset + hostNetwork: false + hostPID: false + volumes: + extra: + - name: pod-logs + hostPath: + path: /var/log/pods + type: Directory + - name: alloy-storage + persistentVolumeClaim: + claimName: alloy-storage + - name: tmp + emptyDir: {} + - name: kubernetes-api-access + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + expirationSeconds: 3600 + path: token + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt + - downwardAPI: + items: + - path: namespace + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + +networkPolicy: + enabled: false + +service: + enabled: true + type: ClusterIP + +serviceMonitor: + enabled: true + interval: 30s + additionalLabels: + observability.hyeonworks.com/instance: home + +ingress: + enabled: false diff --git a/services/observability/blackbox-exporter/README.md b/services/observability/blackbox-exporter/README.md new file mode 100644 index 0000000..5d9577d --- /dev/null +++ b/services/observability/blackbox-exporter/README.md @@ -0,0 +1,22 @@ +# Blackbox exporter + +This component deploys the pinned Prometheus blackbox exporter as an internal +`ClusterIP` service in the `observability` namespace. The `/probe` handler is +reachable only from the Prometheus Pod; it has no Ingress, NodePort, host port, +or service-account token. + +Only three fixed `Probe` groups are declared: + +- `platform-public-edge`: Gitea health and Keycloak discovery through Host Nginx +- `platform-private-edge`: Grafana, pgAdmin, and AIStor Console must return 403 +- `platform-private-internal`: the same private services must return 200 through + their ClusterIP Services + +The internal Grafana and pgAdmin targets use the blackbox exporter's `hostname` +parameter to set their required HTTP Host header. Target URLs and modules are +declarative; arbitrary callers are not granted access to port 9115. + +The private-edge source boundary was proven separately and recorded as a +root-only `platform-blackbox-source-v1` evidence file. Applying this component +must remain gated on that evidence matching the active deny-only Nginx config. + diff --git a/services/observability/blackbox-exporter/kustomization.yaml b/services/observability/blackbox-exporter/kustomization.yaml new file mode 100644 index 0000000..dee31a4 --- /dev/null +++ b/services/observability/blackbox-exporter/kustomization.yaml @@ -0,0 +1,45 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: observability + +resources: + - probes-public.yaml + - probes-private-edge.yaml + - probes-private-internal.yaml + - network-policy.yaml + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: prometheus-blackbox-exporter + repo: https://prometheus-community.github.io/helm-charts + version: 11.16.0 + releaseName: blackbox-exporter + namespace: observability + valuesFile: values/home.yaml + includeCRDs: false + skipTests: true + +patches: + - target: + version: v1 + kind: ServiceAccount + name: blackbox-exporter + patch: |- + - op: add + path: /automountServiceAccountToken + value: false + - target: + group: apps + version: v1 + kind: Deployment + name: blackbox-exporter + patch: |- + - op: add + path: /spec/template/spec/hostPID + value: false + - op: add + path: /spec/template/spec/hostIPC + value: false diff --git a/services/observability/blackbox-exporter/network-policy.yaml b/services/observability/blackbox-exporter/network-policy.yaml new file mode 100644 index 0000000..60fbcf0 --- /dev/null +++ b/services/observability/blackbox-exporter/network-policy.yaml @@ -0,0 +1,88 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-prometheus-to-blackbox + namespace: observability + labels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/part-of: platform-observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: observability-core-kube-pr-prometheus + ports: + - protocol: TCP + port: 9115 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-blackbox-egress + namespace: observability + labels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/part-of: platform-observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - ipBlock: + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 443 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + ports: + - protocol: TCP + port: 3000 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: platform-admin + podSelector: + matchLabels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/instance: pgadmin + ports: + - protocol: TCP + port: 5050 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: object-storage + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + ports: + - protocol: TCP + port: 9090 diff --git a/services/observability/blackbox-exporter/probes-private-edge.yaml b/services/observability/blackbox-exporter/probes-private-edge.yaml new file mode 100644 index 0000000..2f6eef4 --- /dev/null +++ b/services/observability/blackbox-exporter/probes-private-edge.yaml @@ -0,0 +1,28 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: platform-private-edge + namespace: observability + labels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + app.kubernetes.io/part-of: platform-observability + observability.hyeonworks.com/instance: home +spec: + jobName: blackbox-private-edge + interval: 30s + scrapeTimeout: 15s + module: http_private_edge_403 + prober: + url: blackbox-exporter.observability.svc.cluster.local:9115 + scheme: http + path: /probe + proxyFromEnvironment: false + targets: + staticConfig: + labels: + observability.hyeonworks.com/probe-group: private-edge + static: + - https://grafana.learn.hyeonworks.com/ + - https://storage-admin.learn.hyeonworks.com/ + - https://db-admin.learn.hyeonworks.com/ diff --git a/services/observability/blackbox-exporter/probes-private-internal.yaml b/services/observability/blackbox-exporter/probes-private-internal.yaml new file mode 100644 index 0000000..a62e6fa --- /dev/null +++ b/services/observability/blackbox-exporter/probes-private-internal.yaml @@ -0,0 +1,44 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: platform-private-internal + namespace: observability + labels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + app.kubernetes.io/part-of: platform-observability + observability.hyeonworks.com/instance: home +spec: + jobName: blackbox-private-internal + interval: 30s + scrapeTimeout: 15s + module: http_private_internal_200 + prober: + url: blackbox-exporter.observability.svc.cluster.local:9115 + scheme: http + path: /probe + proxyFromEnvironment: false + targets: + staticConfig: + labels: + observability.hyeonworks.com/probe-group: private-internal + static: + - http://grafana.observability.svc.cluster.local/api/health + - http://pgadmin.platform-admin.svc.cluster.local/misc/ping + - http://minio-aistor-console.object-storage.svc.cluster.local:9090/ + relabelingConfigs: + - sourceLabels: + - __param_target + regex: http://grafana[.]observability[.]svc[.]cluster[.]local/api/health + targetLabel: __param_hostname + replacement: grafana.learn.hyeonworks.com + - sourceLabels: + - __param_target + regex: http://pgadmin[.]platform-admin[.]svc[.]cluster[.]local/misc/ping + targetLabel: __param_hostname + replacement: db-admin.learn.hyeonworks.com + - sourceLabels: + - __param_target + regex: http://minio-aistor-console[.]object-storage[.]svc[.]cluster[.]local:9090/ + targetLabel: __param_hostname + replacement: storage-admin.learn.hyeonworks.com diff --git a/services/observability/blackbox-exporter/probes-public.yaml b/services/observability/blackbox-exporter/probes-public.yaml new file mode 100644 index 0000000..cd072e1 --- /dev/null +++ b/services/observability/blackbox-exporter/probes-public.yaml @@ -0,0 +1,27 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: platform-public-edge + namespace: observability + labels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + app.kubernetes.io/part-of: platform-observability + observability.hyeonworks.com/instance: home +spec: + jobName: blackbox-public-edge + interval: 30s + scrapeTimeout: 15s + module: http_2xx + prober: + url: blackbox-exporter.observability.svc.cluster.local:9115 + scheme: http + path: /probe + proxyFromEnvironment: false + targets: + staticConfig: + labels: + observability.hyeonworks.com/probe-group: public-edge + static: + - https://git.learn.hyeonworks.com/api/healthz + - https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration diff --git a/services/observability/blackbox-exporter/values/home.yaml b/services/observability/blackbox-exporter/values/home.yaml new file mode 100644 index 0000000..a6426b8 --- /dev/null +++ b/services/observability/blackbox-exporter/values/home.yaml @@ -0,0 +1,128 @@ +fullnameOverride: blackbox-exporter + +commonLabels: + observability.hyeonworks.com/instance: home + +kind: Deployment +replicas: 1 +revisionHistoryLimit: 3 + +strategy: + type: Recreate + rollingUpdate: null + +automountServiceAccountToken: false +hostNetwork: false +hostPort: 0 + +image: + registry: quay.io + repository: prometheus/blackbox-exporter + tag: v0.28.0 + digest: sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68 + pullPolicy: IfNotPresent + +podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + +securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + +serviceAccount: + create: true + name: blackbox-exporter + +service: + type: ClusterIP + port: 9115 + +ingress: + enabled: false + +route: + main: + enabled: false + +networkPolicy: + enabled: false + +configReloader: + enabled: false + +serviceMonitor: + enabled: false + selfMonitor: + enabled: true + port: http + interval: 30s + scrapeTimeout: 10s + +config: + modules: + http_2xx: + prober: http + timeout: 10s + http: + method: GET + valid_status_codes: + - 200 + valid_http_versions: + - HTTP/1.1 + - HTTP/2.0 + follow_redirects: true + enable_http2: true + preferred_ip_protocol: ip4 + ip_protocol_fallback: false + tls_config: + insecure_skip_verify: false + http_private_edge_403: + prober: http + timeout: 10s + http: + method: GET + valid_status_codes: + - 403 + valid_http_versions: + - HTTP/1.1 + - HTTP/2.0 + follow_redirects: true + enable_http2: true + preferred_ip_protocol: ip4 + ip_protocol_fallback: false + tls_config: + insecure_skip_verify: false + http_private_internal_200: + prober: http + timeout: 10s + http: + method: GET + valid_status_codes: + - 200 + valid_http_versions: + - HTTP/1.1 + - HTTP/2.0 + follow_redirects: true + enable_http2: true + preferred_ip_protocol: ip4 + ip_protocol_fallback: false diff --git a/services/observability/dashboards/platform/https-endpoints.json b/services/observability/dashboards/platform/https-endpoints.json new file mode 100644 index 0000000..efdd53e --- /dev/null +++ b/services/observability/dashboards/platform/https-endpoints.json @@ -0,0 +1,208 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min by (job, instance) (probe_success{})", + "legendFormat": "{{job}} / {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTPS Probe Success", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (job, instance) (probe_duration_seconds{})", + "legendFormat": "{{job}} / {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTPS Probe Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (instance) (probe_http_status_code{job=\"blackbox-private-edge\"})", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Observed HTTP Status (403 Is Expected at the Private Edge)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(probe_ssl_earliest_cert_expiry{} - time()) / 86400", + "legendFormat": "{{job}} / {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "TLS Certificate Days Remaining", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "platform", + "observability", + "https" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Platform / HTTPS Endpoints", + "uid": "platform-https-endpoints", + "version": 1 +} diff --git a/services/observability/dashboards/platform/kubernetes-node.json b/services/observability/dashboards/platform/kubernetes-node.json new file mode 100644 index 0000000..50a2a24 --- /dev/null +++ b/services/observability/dashboards/platform/kubernetes-node.json @@ -0,0 +1,266 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * (1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])))", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Node CPU Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * (1 - node_memory_MemAvailable_bytes{} / node_memory_MemTotal_bytes{})", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Node Memory Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})", + "legendFormat": "{{instance}} {{device}}", + "range": true, + "refId": "A" + } + ], + "title": "Root Filesystem Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * (1 - node_filesystem_files_free{mountpoint=\"/\"} / node_filesystem_files{mountpoint=\"/\"})", + "legendFormat": "{{instance}} {{device}}", + "range": true, + "refId": "A" + } + ], + "title": "Root Filesystem Inodes Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!=\"lo\"}[5m]))", + "legendFormat": "{{instance}} receive", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]))", + "legendFormat": "{{instance}} transmit", + "range": true, + "refId": "B" + } + ], + "title": "Node Network Throughput", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "platform", + "observability", + "node" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Platform / Kubernetes Node", + "uid": "platform-kubernetes-node", + "version": 1 +} diff --git a/services/observability/dashboards/platform/kustomization.yaml b/services/observability/dashboards/platform/kustomization.yaml new file mode 100644 index 0000000..a0ac9d9 --- /dev/null +++ b/services/observability/dashboards/platform/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: observability + +generatorOptions: + disableNameSuffixHash: true + labels: + grafana_dashboard: "1" + observability.hyeonworks.com/instance: home + observability.hyeonworks.com/owner: platform-observability + +configMapGenerator: + - name: grafana-dashboard-kubernetes-node + files: + - kubernetes-node.json + - name: grafana-dashboard-workload-health + files: + - workload-health.json + - name: grafana-dashboard-platform-services + files: + - platform-services.json + - name: grafana-dashboard-observability-backends + files: + - observability-backends.json + - name: grafana-dashboard-https-endpoints + files: + - https-endpoints.json diff --git a/services/observability/dashboards/platform/observability-backends.json b/services/observability/dashboards/platform/observability-backends.json new file mode 100644 index 0000000..44d84f6 --- /dev/null +++ b/services/observability/dashboards/platform/observability-backends.json @@ -0,0 +1,384 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(prometheus_tsdb_head_series{})", + "legendFormat": "head series", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(prometheus_engine_queries{})", + "legendFormat": "active queries", + "range": true, + "refId": "B" + } + ], + "title": "Prometheus Head Series and Active Queries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(prometheus_notifications_errors_total{}[5m]))", + "legendFormat": "errors / second", + "range": true, + "refId": "A" + } + ], + "title": "Prometheus Notification Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(loki_ingester_memory_chunks{})", + "legendFormat": "memory chunks", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(loki_ingester_flush_queue_length{})", + "legendFormat": "flush queue", + "range": true, + "refId": "B" + } + ], + "title": "Loki Ingester State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - max(loki_objstore_bucket_last_successful_upload_time{})", + "legendFormat": "seconds since upload", + "range": true, + "refId": "A" + } + ], + "title": "Loki Object Store Upload Age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(tempo_ingester_live_traces{})", + "legendFormat": "live traces", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(tempo_ingester_flush_queue_length{})", + "legendFormat": "flush queue", + "range": true, + "refId": "B" + } + ], + "title": "Tempo Ingester State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(alloy_config_last_load_successful{})", + "legendFormat": "last load successful", + "range": true, + "refId": "A" + } + ], + "title": "Alloy Configuration State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(loki_write_batch_retries_total{}[5m]))", + "legendFormat": "retry / second", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(loki_write_dropped_entries_total{}[5m]))", + "legendFormat": "dropped / second", + "range": true, + "refId": "B" + } + ], + "title": "Alloy Delivery Retry and Drop Rate", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "platform", + "observability", + "backends" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Platform / Observability Backends", + "uid": "platform-observability-backends", + "version": 1 +} diff --git a/services/observability/dashboards/platform/platform-services.json b/services/observability/dashboards/platform/platform-services.json new file mode 100644 index 0000000..5222978 --- /dev/null +++ b/services/observability/dashboards/platform/platform-services.json @@ -0,0 +1,419 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(gitea_repositories{})", + "legendFormat": "repositories", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(gitea_users{})", + "legendFormat": "users", + "range": true, + "refId": "B" + } + ], + "title": "Gitea Inventory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(agroal_active_count{})", + "legendFormat": "active", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(agroal_available_count{})", + "legendFormat": "available", + "range": true, + "refId": "B" + } + ], + "title": "Keycloak Database Pool", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(cnpg_collector_up{})", + "legendFormat": "collector up", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(cnpg_backends_total{})", + "legendFormat": "backends", + "range": true, + "refId": "B" + } + ], + "title": "CloudNativePG Collector and Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (datname) (cnpg_pg_database_size_bytes{})", + "legendFormat": "{{datname}}", + "range": true, + "refId": "A" + } + ], + "title": "CloudNativePG Database Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * max by (bucket) (minio_cluster_usage_buckets_total_bytes{bucket=~\"loki|tempo\"}) / max by (bucket) (minio_cluster_usage_buckets_quota_total_bytes{bucket=~\"loki|tempo\"} > 0)", + "legendFormat": "{{bucket}}", + "range": true, + "refId": "A" + } + ], + "title": "AIStor Bucket Quota Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (entrypoint) (rate(traefik_entrypoint_requests_total{}[5m]))", + "legendFormat": "{{entrypoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Traefik Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * sum(rate(traefik_entrypoint_requests_total{code=~\"5..\"}[5m])) / clamp_min(sum(rate(traefik_entrypoint_requests_total{}[5m])), 1e-9)", + "legendFormat": "5xx ratio", + "range": true, + "refId": "A" + } + ], + "title": "Traefik 5xx Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min by (job) (probe_success{})", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Platform HTTPS Boundary Health", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "platform", + "observability", + "services" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Platform / Services", + "uid": "platform-services", + "version": 1 +} diff --git a/services/observability/dashboards/platform/workload-health.json b/services/observability/dashboards/platform/workload-health.json new file mode 100644 index 0000000..b0091c7 --- /dev/null +++ b/services/observability/dashboards/platform/workload-health.json @@ -0,0 +1,218 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespace, deployment) (kube_deployment_status_replicas_unavailable{})", + "legendFormat": "{{namespace}} / {{deployment}}", + "range": true, + "refId": "A" + } + ], + "title": "Deployment Unavailable Replicas", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "clamp_min(sum by (namespace, statefulset) (kube_statefulset_replicas{}) - sum by (namespace, statefulset) (kube_statefulset_status_replicas_ready{}), 0)", + "legendFormat": "{{namespace}} / {{statefulset}}", + "range": true, + "refId": "A" + } + ], + "title": "StatefulSet Replicas Not Ready", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{}[15m]))", + "legendFormat": "{{namespace}} / {{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Container Restarts in 15 Minutes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_persistentvolumeclaim_status_phase{phase=\"Pending\"})", + "legendFormat": "Pending", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_persistentvolumeclaim_status_phase{phase=\"Lost\"})", + "legendFormat": "Lost", + "range": true, + "refId": "B" + } + ], + "title": "PersistentVolumeClaim Health", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "platform", + "observability", + "workloads" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Platform / Workload Health", + "uid": "platform-workload-health", + "version": 1 +} diff --git a/services/observability/grafana/README.md b/services/observability/grafana/README.md new file mode 100644 index 0000000..3153fb7 --- /dev/null +++ b/services/observability/grafana/README.md @@ -0,0 +1,24 @@ +# Grafana + +Private Grafana for the home observability stack. The pinned Grafana chart is +rendered as the independent `grafana` release in `observability`; the +kube-prometheus-stack embedded Grafana remains disabled. + +The workload consumes, but never renders, these existing Secret contracts: + +- `grafana-admin`: `admin-user`, `admin-password` +- `grafana-keycloak-oidc`: `client-id`, `client-secret` + +The `grafana` PVC is fixed to `observability-grafana-local-pv` (2Gi, Retain). +Host Nginx terminates the exact-SAN certificate and forwards plain HTTP through +Traefik; the Kubernetes Ingress therefore contains no TLS material. + +The Pod does not automount a service-account token. A 3600-second projected +token, CA, and namespace are mounted only into `grafana-sc-dashboard`, whose +namespaced Role can read ConfigMaps but not Secrets. Dashboard ConfigMaps must +carry `grafana_dashboard: "1"` in `observability`. + +Provisioned datasource UIDs are `prometheus`, `loki`, and `tempo`. Tempo service +map configuration is present for future use, but the metrics-generator and +Prometheus remote-write path are intentionally not enabled, so service-graph +availability is not an initial acceptance condition. diff --git a/services/observability/grafana/dashboard-sidecar-rbac.yaml b/services/observability/grafana/dashboard-sidecar-rbac.yaml new file mode 100644 index 0000000..d4db54e --- /dev/null +++ b/services/observability/grafana/dashboard-sidecar-rbac.yaml @@ -0,0 +1,27 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: grafana-dashboard-sidecar + namespace: observability +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: grafana-dashboard-sidecar + namespace: observability +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: grafana-dashboard-sidecar +subjects: + - kind: ServiceAccount + name: grafana + namespace: observability diff --git a/services/observability/grafana/dashboard-sidecar-token-patch.yaml b/services/observability/grafana/dashboard-sidecar-token-patch.yaml new file mode 100644 index 0000000..03c06cb --- /dev/null +++ b/services/observability/grafana/dashboard-sidecar-token-patch.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: observability +spec: + template: + spec: + automountServiceAccountToken: false + hostNetwork: false + hostPID: false + hostIPC: false + containers: + - name: grafana-sc-dashboard + volumeMounts: + - name: dashboard-sidecar-api-access + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + volumes: + - name: dashboard-sidecar-api-access + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + expirationSeconds: 3600 + path: token + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt + - downwardAPI: + items: + - path: namespace + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace diff --git a/services/observability/grafana/ingress-class-patch.yaml b/services/observability/grafana/ingress-class-patch.yaml new file mode 100644 index 0000000..ba11dc8 --- /dev/null +++ b/services/observability/grafana/ingress-class-patch.yaml @@ -0,0 +1,7 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana + namespace: observability +spec: + ingressClassName: traefik diff --git a/services/observability/grafana/kustomization.yaml b/services/observability/grafana/kustomization.yaml new file mode 100644 index 0000000..483cd80 --- /dev/null +++ b/services/observability/grafana/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: observability + +resources: + - dashboard-sidecar-rbac.yaml + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: grafana + repo: https://grafana-community.github.io/helm-charts + version: 12.10.1 + releaseName: grafana + namespace: observability + valuesFile: values/home.yaml + includeCRDs: false + skipTests: true + +patches: + - path: ingress-class-patch.yaml + - path: dashboard-sidecar-token-patch.yaml diff --git a/services/observability/grafana/values/home.yaml b/services/observability/grafana/values/home.yaml new file mode 100644 index 0000000..06400ed --- /dev/null +++ b/services/observability/grafana/values/home.yaml @@ -0,0 +1,262 @@ +replicas: 1 + +extraLabels: + observability.hyeonworks.com/instance: home + +deploymentStrategy: + type: Recreate + +revisionHistoryLimit: 3 + +automountServiceAccountToken: false +enableServiceLinks: false + +rbac: + create: false + +serviceAccount: + create: true + name: grafana + automountServiceAccountToken: false + +image: + registry: docker.io + repository: grafana/grafana + tag: 13.1.1 + sha: f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b + pullPolicy: IfNotPresent + +testFramework: + enabled: false + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + +containerSecurityContext: + allowPrivilegeEscalation: false + privileged: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +initChownData: + enabled: false + +persistence: + type: pvc + enabled: true + storageClassName: ssd-local-observability-grafana-retain + volumeName: observability-grafana-local-pv + lookupVolumeName: false + accessModes: + - ReadWriteOnce + size: 2Gi + finalizers: + - kubernetes.io/pvc-protection + +admin: + existingSecret: grafana-admin + userKey: admin-user + passwordKey: admin-password + +envValueFrom: + GF_AUTH_GENERIC_OAUTH_CLIENT_ID: + secretKeyRef: + name: grafana-keycloak-oidc + key: client-id + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: + secretKeyRef: + name: grafana-keycloak-oidc + key: client-secret + +service: + enabled: true + type: ClusterIP + port: 80 + targetPort: 3000 + portName: service + +serviceMonitor: + enabled: true + interval: 30s + path: /metrics + +ingress: + enabled: true + annotations: {} + path: / + pathType: Prefix + hosts: + - grafana.learn.hyeonworks.com + tls: [] + +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: false + check_for_plugin_updates: false + reporting_enabled: false + plugins: + preinstall_disabled: true + server: + domain: grafana.learn.hyeonworks.com + root_url: https://grafana.learn.hyeonworks.com/ + enforce_domain: true + security: + cookie_secure: true + cookie_samesite: lax + auth: + disable_login_form: false + oauth_auto_login: false + login_maximum_lifetime_duration: 8h + login_maximum_inactive_lifetime_duration: 30m + auth.anonymous: + enabled: false + auth.basic: + enabled: true + auth.generic_oauth: + enabled: true + name: Keycloak + allow_sign_up: true + use_pkce: true + scopes: openid profile email + groups_attribute_path: groups + allowed_groups: /platform-observability-admins /platform-observability-viewers + role_attribute_strict: true + allow_assign_grafana_admin: false + skip_org_role_sync: false + validate_id_token: true + use_refresh_token: true + role_attribute_path: "contains(groups[*], '/platform-observability-admins') && 'Admin' || contains(groups[*], '/platform-observability-viewers') && 'Viewer' || null" + auth_url: https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/auth + token_url: https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/token + api_url: https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/userinfo + signout_redirect_url: https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/logout + +datasources: + datasources.yaml: + apiVersion: 1 + deleteDatasources: + - name: Prometheus + orgId: 1 + - name: Loki + orgId: 1 + - name: Tempo + orgId: 1 + datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://observability-core-kube-pr-prometheus.observability.svc.cluster.local:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + timeInterval: 30s + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki.observability.svc.cluster.local:3100 + isDefault: false + editable: false + jsonData: + derivedFields: + - name: trace_id + matcherRegex: '"trace_id"[[:space:]]*:[[:space:]]*"([0-9a-f]{32})"' + datasourceUid: tempo + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo.observability.svc.cluster.local:3200 + isDefault: false + editable: false + jsonData: + httpMethod: GET + nodeGraph: + enabled: true + serviceMap: + datasourceUid: prometheus + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -1m + spanEndTimeShift: 1m + tags: + - key: k8s.namespace.name + value: namespace + - key: k8s.pod.name + value: pod + filterByTraceID: true + filterBySpanID: false + +sidecar: + image: + registry: quay.io + repository: kiwigrid/k8s-sidecar + tag: 2.10.0 + sha: 129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + dashboards: + enabled: true + label: grafana_dashboard + labelValue: "1" + searchNamespace: + - observability + resource: configmap + watchMethod: WATCH + skipReload: true + provider: + name: sidecarProvider + orgid: 1 + folder: "" + folderUid: "" + type: file + disableDelete: false + allowUiUpdates: false + foldersFromFilesStructure: false + +imageRenderer: + enabled: false + +networkPolicy: + enabled: false + +assertNoLeakedSecrets: true diff --git a/services/observability/kube-prometheus-stack/README.md b/services/observability/kube-prometheus-stack/README.md new file mode 100644 index 0000000..c6fbd92 --- /dev/null +++ b/services/observability/kube-prometheus-stack/README.md @@ -0,0 +1,15 @@ +# kube-prometheus-stack + +고정 chart `88.0.1`을 `observability` namespace에 렌더한다. Grafana와 +node-exporter, control-plane에 존재하지 않는 scheduler/controller-manager/etcd/ +kube-proxy monitor는 끈다. Prometheus와 Alertmanager 데이터는 각각의 `Retain` +Local PV에 보존하며 CRD는 별도 root에서 먼저 적용한다. + +Prometheus discovery는 namespace label +`observability.hyeonworks.com/enabled=true`와 resource label +`observability.hyeonworks.com/instance=home`의 교집합만 선택한다. + +Alertmanager source는 selector discovery 대신 global +`alertmanagerConfiguration.name=platform-alertmanager`를 참조한다. 참조 대상의 +Slack Secret이 준비되기 전에는 이 변경을 live에 적용하지 않고 기존 null 구성을 +유지한다. diff --git a/services/observability/kube-prometheus-stack/kustomization.yaml b/services/observability/kube-prometheus-stack/kustomization.yaml new file mode 100644 index 0000000..560efaf --- /dev/null +++ b/services/observability/kube-prometheus-stack/kustomization.yaml @@ -0,0 +1,37 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: observability + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: kube-prometheus-stack + repo: https://prometheus-community.github.io/helm-charts + version: 88.0.1 + releaseName: observability-core + namespace: observability + valuesFile: values/home.yaml + includeCRDs: false + skipTests: true + +images: + - name: quay.io/prometheus-operator/prometheus-operator + newName: quay.io/prometheus-operator/prometheus-operator + digest: sha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2 + - name: registry.k8s.io/kube-state-metrics/kube-state-metrics + newName: registry.k8s.io/kube-state-metrics/kube-state-metrics + digest: sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4 + +patches: + - target: + group: monitoring.coreos.com + version: v1 + kind: Alertmanager + name: observability-core-kube-pr-alertmanager + patch: |- + - op: remove + path: /spec/alertmanagerConfigSelector + - op: remove + path: /spec/alertmanagerConfigNamespaceSelector diff --git a/services/observability/kube-prometheus-stack/values/home.yaml b/services/observability/kube-prometheus-stack/values/home.yaml new file mode 100644 index 0000000..6994a76 --- /dev/null +++ b/services/observability/kube-prometheus-stack/values/home.yaml @@ -0,0 +1,282 @@ +commonLabels: + observability.hyeonworks.com/instance: home + +crds: + enabled: false + +grafana: + enabled: false + +nodeExporter: + enabled: false + +kubeApiServer: + enabled: true + +kubelet: + enabled: true + +coreDns: + enabled: true + +kubeControllerManager: + enabled: false + +kubeScheduler: + enabled: false + +kubeEtcd: + enabled: false + +kubeProxy: + enabled: false + +defaultRules: + disabled: + Watchdog: true + rules: + etcd: false + kubeControllerManager: false + kubeProxy: false + kubeSchedulerAlerting: false + kubeSchedulerRecording: false + windows: false + +prometheusOperator: + admissionWebhooks: + enabled: false + patch: + enabled: false + tls: + enabled: false + image: + registry: quay.io + repository: prometheus-operator/prometheus-operator + tag: v0.93.0 + sha: 64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + serviceAccount: + automountServiceAccountToken: true + prometheusConfigReloader: + image: + registry: quay.io + repository: prometheus-operator/prometheus-config-reloader + tag: v0.93.0 + sha: 65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + +kubeStateMetrics: + enabled: true + +kube-state-metrics: + image: + registry: registry.k8s.io + repository: kube-state-metrics/kube-state-metrics + tag: v2.19.1 + sha: sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + enabled: true + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + automountServiceAccountToken: true + collectors: + - certificatesigningrequests + - configmaps + - cronjobs + - daemonsets + - deployments + - endpointslices + - horizontalpodautoscalers + - ingresses + - jobs + - leases + - limitranges + - mutatingwebhookconfigurations + - namespaces + - networkpolicies + - nodes + - persistentvolumeclaims + - persistentvolumes + - poddisruptionbudgets + - pods + - replicasets + - replicationcontrollers + - resourcequotas + - services + - statefulsets + - storageclasses + - validatingwebhookconfigurations + - volumeattachments + prometheus: + monitor: + additionalLabels: + observability.hyeonworks.com/instance: home + +alertmanager: + enabled: true + config: + global: + resolve_timeout: 5m + route: + group_by: + - namespace + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: "null" + routes: [] + receivers: + - name: "null" + serviceAccount: + automountServiceAccountToken: false + alertmanagerSpec: + replicas: 1 + retention: 120h + automountServiceAccountToken: false + alertmanagerConfiguration: + name: platform-alertmanager + image: + registry: quay.io + repository: prometheus/alertmanager + tag: v0.33.1 + sha: a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 2000 + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + storage: + volumeClaimTemplate: + spec: + storageClassName: ssd-local-observability-alertmanager-retain + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + +prometheus: + enabled: true + prometheusSpec: + replicas: 1 + retention: 15d + retentionSize: 16GB + scrapeInterval: 30s + evaluationInterval: 30s + walCompression: true + automountServiceAccountToken: true + image: + registry: quay.io + repository: prometheus/prometheus + tag: v3.13.2-distroless + sha: ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b + resources: + requests: + cpu: 750m + memory: 2Gi + limits: + cpu: "2" + memory: 3Gi + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 2000 + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: ssd-local-observability-prometheus-retain + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + probeSelectorNilUsesHelmValues: false + ruleSelectorNilUsesHelmValues: false + serviceMonitorNamespaceSelector: + matchLabels: + observability.hyeonworks.com/enabled: "true" + podMonitorNamespaceSelector: + matchLabels: + observability.hyeonworks.com/enabled: "true" + probeNamespaceSelector: + matchLabels: + observability.hyeonworks.com/enabled: "true" + ruleNamespaceSelector: + matchLabels: + observability.hyeonworks.com/enabled: "true" + serviceMonitorSelector: + matchLabels: + observability.hyeonworks.com/instance: home + podMonitorSelector: + matchLabels: + observability.hyeonworks.com/instance: home + probeSelector: + matchLabels: + observability.hyeonworks.com/instance: home + ruleSelector: + matchLabels: + observability.hyeonworks.com/instance: home diff --git a/services/observability/kustomization.yaml b/services/observability/kustomization.yaml new file mode 100644 index 0000000..aa2a157 --- /dev/null +++ b/services/observability/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - kube-prometheus-stack + - loki + - tempo + - alloy + - node-exporter + - grafana + - blackbox-exporter + - network-policies + - targets + - rules/platform + - alerting + - dashboards/platform diff --git a/services/observability/loki/README.md b/services/observability/loki/README.md new file mode 100644 index 0000000..df71e1c --- /dev/null +++ b/services/observability/loki/README.md @@ -0,0 +1,16 @@ +# Loki + +This root renders `grafana-community/loki` chart `18.7.1` as one Loki +`3.7.4` monolithic StatefulSet in `observability`. + +- The image is pinned to the reviewed amd64 digest. +- Logs use the `loki` AIStor bucket through `loki-object-storage`; no root + credential is referenced. +- Retention is 30 days. The WAL and working data use the dedicated 5Gi + Retain Local PV. +- Gateway, canary, caches, embedded MinIO, tests, and rule sidecars are + disabled. +- The only client-facing service is ClusterIP port 3100. + +Do not delete the PVC or bucket during rollback. Scale the StatefulSet down, +preserve its Retain volume, and investigate S3/WAL state before reapplying. diff --git a/services/observability/loki/kustomization.yaml b/services/observability/loki/kustomization.yaml new file mode 100644 index 0000000..f0542a9 --- /dev/null +++ b/services/observability/loki/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: observability + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: loki + repo: https://grafana-community.github.io/helm-charts + version: 18.7.1 + releaseName: loki + namespace: observability + valuesFile: values/home.yaml + includeCRDs: false + skipTests: true diff --git a/services/observability/loki/values/home.yaml b/services/observability/loki/values/home.yaml new file mode 100644 index 0000000..9266338 --- /dev/null +++ b/services/observability/loki/values/home.yaml @@ -0,0 +1,146 @@ +deploymentMode: Monolithic + +commonLabels: + observability.hyeonworks.com/instance: home + +defaults: + automountServiceAccountToken: false + +serviceAccount: + automountServiceAccountToken: false + +loki: + auth_enabled: false + image: + registry: docker.io + repository: grafana/loki + tag: "3.7.4" + digest: sha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e + podSecurityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + commonConfig: + path_prefix: /var/loki + replication_factor: 1 + schemaConfig: + configs: + - from: "2026-07-01" + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: index_ + period: 24h + storage: + type: s3 + bucketNames: + chunks: loki + ruler: loki + admin: loki + s3: + endpoint: http://minio.object-storage.svc.cluster.local + s3ForcePathStyle: true + insecure: true + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} + useTestSchema: false + limits_config: + retention_period: 720h + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_line_size: 256KB + discover_log_levels: false + discover_service_name: [] + ingester: + chunk_idle_period: 30m + max_chunk_age: 1h + wal: + checkpoint_duration: 5m + replay_memory_ceiling: 512MB + disk_full_threshold: 0.90 + flush_on_shutdown: true + compactor: + retention_enabled: true + delete_request_store: s3 + retention_delete_delay: 2h + retention_delete_worker_count: 10 + max_compaction_parallelism: 1 + +singleBinary: + replicas: 1 + sidecar: false + podLabels: + platform.hyeonworks.com/aistor-client: "true" + service: + type: ClusterIP + resources: + requests: + cpu: 350m + memory: 768Mi + limits: + cpu: 1500m + memory: 1536Mi + extraEnv: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: loki-object-storage + key: access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: loki-object-storage + key: secret-access-key + persistence: + enabled: true + size: 5Gi + storageClass: ssd-local-observability-loki-retain + enableStatefulSetAutoDeletePVC: true + whenScaled: Retain + whenDeleted: Retain + +read: + replicas: 0 +write: + replicas: 0 +backend: + replicas: 0 + +ruler: + sidecar: false + +gateway: + enabled: false +lokiCanary: + enabled: false +chunksCache: + enabled: false +resultsCache: + enabled: false +memcached: + enabled: false +minio: + enabled: false +test: + enabled: false +sidecar: + rules: + enabled: false + +monitoring: + serviceMonitor: + enabled: true + interval: 30s diff --git a/services/observability/network-policies/kustomization.yaml b/services/observability/network-policies/kustomization.yaml new file mode 100644 index 0000000..3f42dce --- /dev/null +++ b/services/observability/network-policies/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - observability.yaml + - observability-agent.yaml + - object-storage-counterpart.yaml + diff --git a/services/observability/network-policies/object-storage-counterpart.yaml b/services/observability/network-policies/object-storage-counterpart.yaml new file mode 100644 index 0000000..65853c1 --- /dev/null +++ b/services/observability/network-policies/object-storage-counterpart.yaml @@ -0,0 +1,31 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-allow-observability-s3 + namespace: object-storage + labels: + app.kubernetes.io/component: observability-object-storage + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + platform.hyeonworks.com/aistor-client: "true" + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - loki + - tempo + ports: + - protocol: TCP + port: 9000 diff --git a/services/observability/network-policies/observability-agent.yaml b/services/observability/network-policies/observability-agent.yaml new file mode 100644 index 0000000..a15f5ad --- /dev/null +++ b/services/observability/network-policies/observability-agent.yaml @@ -0,0 +1,140 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-agent-allow-alloy-dns + namespace: observability-agent +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: alloy + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-agent-allow-alloy-api + namespace: observability-agent +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: alloy + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - protocol: TCP + port: 443 + - to: + - ipBlock: + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 6443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-agent-allow-alloy-backends + namespace: observability-agent +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: alloy + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: loki + ports: + - protocol: TCP + port: 3100 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: tempo + ports: + - protocol: TCP + port: 4317 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-agent-allow-prometheus-to-alloy + namespace: observability-agent +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: alloy + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 12345 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-agent-allow-prometheus-to-node-exporter + namespace: observability-agent +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-node-exporter + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 9100 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-agent-default-deny + namespace: observability-agent +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + diff --git a/services/observability/network-policies/observability.yaml b/services/observability/network-policies/observability.yaml new file mode 100644 index 0000000..e092e02 --- /dev/null +++ b/services/observability/network-policies/observability.yaml @@ -0,0 +1,548 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-dns + namespace: observability +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-kubernetes-api + namespace: observability +spec: + podSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - kube-prometheus-stack-prometheus-operator + - kube-state-metrics + - prometheus + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - protocol: TCP + port: 443 + - to: + - ipBlock: + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 6443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-prometheus-egress + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + policyTypes: + - Egress + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: kube-prometheus-stack-prometheus-operator + ports: + - protocol: TCP + port: 8080 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + ports: + - protocol: TCP + port: 8080 + - protocol: TCP + port: 8081 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 9090 + - protocol: TCP + port: 8080 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: alertmanager + ports: + - protocol: TCP + port: 9093 + - protocol: TCP + port: 8080 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: loki + ports: + - protocol: TCP + port: 3100 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: tempo + ports: + - protocol: TCP + port: 3200 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + ports: + - protocol: TCP + port: 3000 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + ports: + - protocol: TCP + port: 9115 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability-agent + podSelector: + matchLabels: + app.kubernetes.io/name: alloy + ports: + - protocol: TCP + port: 12345 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability-agent + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-node-exporter + ports: + - protocol: TCP + port: 9100 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: TCP + port: 9153 + - to: + - ipBlock: + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 10250 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-prometheus-to-operator + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: kube-prometheus-stack-prometheus-operator + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 8080 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-prometheus-to-ksm + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 8080 + - protocol: TCP + port: 8081 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-prometheus-self + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 9090 + - protocol: TCP + port: 8080 + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + ports: + - protocol: TCP + port: 9090 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-alertmanager + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: alertmanager + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 9093 + - protocol: TCP + port: 8080 + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: alertmanager + ports: + - protocol: TCP + port: 9094 + - protocol: UDP + port: 9094 + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: alertmanager + ports: + - protocol: TCP + port: 9094 + - protocol: UDP + port: 9094 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-loki + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: loki + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 3100 + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + ports: + - protocol: TCP + port: 3100 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability-agent + podSelector: + matchLabels: + app.kubernetes.io/name: alloy + ports: + - protocol: TCP + port: 3100 + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: loki + ports: + - protocol: TCP + port: 7946 + - protocol: UDP + port: 7946 + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: loki + ports: + - protocol: TCP + port: 7946 + - protocol: UDP + port: 7946 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-tempo + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: tempo + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 3200 + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + ports: + - protocol: TCP + port: 3200 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability-agent + podSelector: + matchLabels: + app.kubernetes.io/name: alloy + ports: + - protocol: TCP + port: 4317 + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: tempo + ports: + - protocol: TCP + port: 7946 + - protocol: UDP + port: 7946 + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: tempo + ports: + - protocol: TCP + port: 7946 + - protocol: UDP + port: 7946 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-aistor-egress + namespace: observability +spec: + podSelector: + matchLabels: + platform.hyeonworks.com/aistor-client: "true" + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - loki + - tempo + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 10.43.124.248/32 + ports: + - protocol: TCP + port: 80 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: object-storage + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + ports: + - protocol: TCP + port: 9000 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-grafana-ingress + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + app.kubernetes.io/instance: traefik-kube-system + ports: + - protocol: TCP + port: 3000 + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 3000 + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + ports: + - protocol: TCP + port: 3000 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-grafana-datasources + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + policyTypes: + - Egress + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 9090 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: loki + ports: + - protocol: TCP + port: 3100 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: tempo + ports: + - protocol: TCP + port: 3200 + - to: + - ipBlock: + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-grafana-dashboard-api + namespace: observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - protocol: TCP + port: 443 + - to: + - ipBlock: + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 6443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-default-deny + namespace: observability +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress diff --git a/services/observability/node-exporter/README.md b/services/observability/node-exporter/README.md new file mode 100644 index 0000000..e8816b7 --- /dev/null +++ b/services/observability/node-exporter/README.md @@ -0,0 +1,11 @@ +# node-exporter + +The standalone node-exporter runs in `observability-agent` with the embedded +kube-prometheus-stack copy and kube-rbac-proxy disabled. It mounts only +`/proc`, `/sys`, and `/` from the node, all read-only, and exposes an internal +ClusterIP metrics Service on port `9100`. + +Host network, PID, and IPC namespaces remain disabled by the platform security +contract. Consequently, network-namespace metrics are treated as a known +single-node visibility limitation rather than authoritative host-network data. + diff --git a/services/observability/node-exporter/kustomization.yaml b/services/observability/node-exporter/kustomization.yaml new file mode 100644 index 0000000..f1408b4 --- /dev/null +++ b/services/observability/node-exporter/kustomization.yaml @@ -0,0 +1,23 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: observability-agent + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: prometheus-node-exporter + repo: https://prometheus-community.github.io/helm-charts + version: 4.56.1 + releaseName: node-exporter + namespace: observability-agent + valuesFile: values/home.yaml + includeCRDs: false + skipTests: true + +images: + - name: quay.io/prometheus/node-exporter + newName: quay.io/prometheus/node-exporter + digest: sha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798 + diff --git a/services/observability/node-exporter/values/home.yaml b/services/observability/node-exporter/values/home.yaml new file mode 100644 index 0000000..d994b14 --- /dev/null +++ b/services/observability/node-exporter/values/home.yaml @@ -0,0 +1,78 @@ +fullnameOverride: node-exporter +namespaceOverride: observability-agent + +image: + registry: quay.io + repository: prometheus/node-exporter + tag: v1.12.1 + pullPolicy: IfNotPresent + +kubeRBACProxy: + enabled: false + +service: + enabled: true + type: ClusterIP + port: 9100 + targetPort: 9100 + portName: metrics + listenOnAllInterfaces: true + +prometheus: + monitor: + enabled: true + additionalLabels: + observability.hyeonworks.com/instance: home + interval: 30s + podMonitor: + enabled: false + +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + +serviceAccount: + create: true + automountServiceAccountToken: false + +rbac: + create: true + +securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + +hostNetwork: false +hostPID: false +hostIPC: false + +hostRootFsMount: + enabled: true + mountPropagation: None +hostProcFsMount: + mountPropagation: None +hostSysFsMount: + mountPropagation: None + +nodeSelector: + kubernetes.io/os: linux + +tolerations: + - effect: NoSchedule + operator: Exists diff --git a/services/observability/object-storage/README.md b/services/observability/object-storage/README.md new file mode 100644 index 0000000..7155dae --- /dev/null +++ b/services/observability/object-storage/README.md @@ -0,0 +1,10 @@ +# Observability object storage + +Loki와 Tempo는 AIStor에서 서로 다른 bucket, policy, user와 Kubernetes Secret을 +사용한다. Loki quota는 200Gi, Tempo quota는 100Gi다. 정책은 자기 bucket의 위치·목록과 +자기 object의 읽기·쓰기·삭제·multipart 처리만 허용한다. + +운영 적용 전에는 K3s Secret 암호화와 격리 복구 증거, AIStor root credential의 암호화된 +외부 복구 가능성을 각각 다시 확인한다. 기존 bucket·object·policy·user·quota는 자동 +삭제하거나 credential을 자동 회전하지 않는다. 한쪽 Secret만 존재하는 partial 상태에서는 +추가 mutation을 중단한다. diff --git a/services/observability/object-storage/policies/loki.json b/services/observability/object-storage/policies/loki.json new file mode 100644 index 0000000..e534c52 --- /dev/null +++ b/services/observability/object-storage/policies/loki.json @@ -0,0 +1,29 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:ListBucketMultipartUploads" + ], + "Resource": [ + "arn:aws:s3:::loki" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::loki/*" + ] + } + ] +} diff --git a/services/observability/object-storage/policies/tempo.json b/services/observability/object-storage/policies/tempo.json new file mode 100644 index 0000000..215466b --- /dev/null +++ b/services/observability/object-storage/policies/tempo.json @@ -0,0 +1,29 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:ListBucketMultipartUploads" + ], + "Resource": [ + "arn:aws:s3:::tempo" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::tempo/*" + ] + } + ] +} diff --git a/services/observability/rules/platform/certificate-rules.yaml b/services/observability/rules/platform/certificate-rules.yaml new file mode 100644 index 0000000..91016d8 --- /dev/null +++ b/services/observability/rules/platform/certificate-rules.yaml @@ -0,0 +1,87 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: platform-certificate-probes + namespace: observability + labels: + app.kubernetes.io/name: platform-observability-rules + app.kubernetes.io/part-of: observability + observability.hyeonworks.com/instance: home +spec: + groups: + - name: platform.blackbox + rules: + - alert: PlatformPublicEdgeProbeFailed + expr: >- + probe_success{ + namespace="observability", + "observability.hyeonworks.com/probe-group"="public-edge" + } == 0 + for: 5m + labels: + severity: critical + annotations: + summary: 외부 엔드포인트 프로브 실패 + description: '{{ $labels.instance }} 외부 엔드포인트 프로브가 5분간 실패했습니다.' + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformPrivateEdgeProbeFailed + expr: >- + probe_success{ + namespace="observability", + "observability.hyeonworks.com/probe-group"="private-edge" + } == 0 + for: 5m + labels: + severity: critical + annotations: + summary: 비공개 경계 프로브 실패 + description: '{{ $labels.instance }} 비공개 경계의 예상 응답이 5분간 실패했습니다. HTTP 403이면서 probe_success=1이면 정상입니다.' + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformInternalHealthProbeFailed + expr: >- + probe_success{ + namespace="observability", + "observability.hyeonworks.com/probe-group"="private-internal" + } == 0 + for: 5m + labels: + severity: critical + annotations: + summary: 내부 상태 확인 프로브 실패 + description: '{{ $labels.instance }} 내부 상태 확인 프로브가 5분간 실패했습니다.' + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - name: platform.certificates + rules: + - alert: PlatformCertificateExpiry + expr: >- + ( + probe_ssl_earliest_cert_expiry{ + namespace="observability", + "observability.hyeonworks.com/probe-group"=~"public-edge|private-edge" + } - time() + ) >= 14 * 24 * 60 * 60 + and + ( + probe_ssl_earliest_cert_expiry{ + namespace="observability", + "observability.hyeonworks.com/probe-group"=~"public-edge|private-edge" + } - time() + ) < 30 * 24 * 60 * 60 + labels: + severity: warning + annotations: + summary: TLS 인증서가 30일 이내 만료됨 + description: '{{ $labels.instance }} 인증서 중 가장 이른 만료 시점이 14일 이상 30일 미만 남았습니다.' + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformCertificateExpiry + expr: >- + probe_ssl_earliest_cert_expiry{ + namespace="observability", + "observability.hyeonworks.com/probe-group"=~"public-edge|private-edge" + } - time() < 14 * 24 * 60 * 60 + labels: + severity: critical + annotations: + summary: TLS 인증서가 14일 이내 만료됨 + description: '{{ $labels.instance }} 인증서 중 가장 이른 만료 시점이 14일 미만 남았습니다.' + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md diff --git a/services/observability/rules/platform/core-rules.yaml b/services/observability/rules/platform/core-rules.yaml new file mode 100644 index 0000000..846b484 --- /dev/null +++ b/services/observability/rules/platform/core-rules.yaml @@ -0,0 +1,117 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: platform-observability-core + namespace: observability + labels: + app.kubernetes.io/name: platform-observability-rules + app.kubernetes.io/part-of: observability + observability.hyeonworks.com/instance: home +spec: + groups: + - name: platform.storage + rules: + - alert: PlatformPersistentVolumeClaimNotBound + expr: kube_persistentvolumeclaim_status_phase{phase=~"Pending|Lost"} == 1 + for: 10m + labels: + severity: warning + annotations: + summary: PersistentVolumeClaim이 바인딩되지 않음 + description: 플랫폼 PersistentVolumeClaim이 Pending 또는 Lost 상태로 10분간 유지됐습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformRootFilesystemUsage + expr: >- + 100 * (1 - + node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} + / + node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} + ) > 75 + for: 30m + labels: + severity: info + threshold: "75" + annotations: + summary: 루트 파일시스템 사용률이 75%를 초과함 + description: 노드 루트 파일시스템 사용률이 정보 임계치인 75%를 30분간 초과했습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformRootFilesystemUsage + expr: >- + 100 * (1 - + node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} + / + node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} + ) > 85 + for: 15m + labels: + severity: warning + threshold: "85" + annotations: + summary: 루트 파일시스템 사용률이 85%를 초과함 + description: 노드 루트 파일시스템 사용률이 경고 임계치인 85%를 15분간 초과했습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformRootFilesystemUsage + expr: >- + 100 * (1 - + node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} + / + node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} + ) > 90 + for: 5m + labels: + severity: critical + threshold: "90" + annotations: + summary: 루트 파일시스템 사용률이 90%를 초과함 + description: 노드 루트 파일시스템 사용률이 심각 임계치인 90%를 5분간 초과했습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformRootFilesystemInodes + expr: >- + 100 * (1 - + node_filesystem_files_free{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} + / + node_filesystem_files{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} + ) > 80 + for: 30m + labels: + severity: warning + threshold: "80" + annotations: + summary: 루트 파일시스템 inode 사용률이 80%를 초과함 + description: 노드 루트 파일시스템 inode 사용률이 경고 임계치인 80%를 30분간 초과했습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformRootFilesystemInodes + expr: >- + 100 * (1 - + node_filesystem_files_free{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} + / + node_filesystem_files{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} + ) > 90 + for: 15m + labels: + severity: critical + threshold: "90" + annotations: + summary: 루트 파일시스템 inode 사용률이 90%를 초과함 + description: 노드 루트 파일시스템 inode 사용률이 심각 임계치인 90%를 15분간 초과했습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - name: platform.workloads + rules: + - alert: PlatformPodRestartBurst + expr: increase(kube_pod_container_status_restarts_total[15m]) > 3 + for: 5m + labels: + severity: warning + annotations: + summary: 컨테이너 재시작 급증 감지 + description: 한 컨테이너가 최근 15분 동안 3회를 초과해 재시작했습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformPodOOMKilled + expr: kube_pod_container_status_last_terminated_reason{reason="OOMKilled"} == 1 + for: 5m + labels: + severity: critical + annotations: + summary: 컨테이너가 OOM Killer에 의해 종료됨 + description: 컨테이너의 최근 종료 사유가 OOMKilled인 상태로 5분간 유지됐습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md diff --git a/services/observability/rules/platform/kustomization.yaml b/services/observability/rules/platform/kustomization.yaml new file mode 100644 index 0000000..cbae294 --- /dev/null +++ b/services/observability/rules/platform/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: observability + +resources: + - core-rules.yaml + - storage-quota-rules.yaml + - certificate-rules.yaml + - verified-service-rules.yaml diff --git a/services/observability/rules/platform/storage-quota-rules.yaml b/services/observability/rules/platform/storage-quota-rules.yaml new file mode 100644 index 0000000..f975147 --- /dev/null +++ b/services/observability/rules/platform/storage-quota-rules.yaml @@ -0,0 +1,55 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: platform-aistor-storage-quota + namespace: observability + labels: + app.kubernetes.io/name: platform-observability-rules + app.kubernetes.io/part-of: observability + observability.hyeonworks.com/instance: home +spec: + groups: + - name: platform.aistor-storage-quota + rules: + - record: platform:aistor_bucket_quota_usage_percent + expr: >- + 100 + * max by (bucket) ( + minio_cluster_usage_buckets_total_bytes{bucket=~"loki|tempo"} + ) + / max by (bucket) ( + minio_cluster_usage_buckets_quota_total_bytes{bucket=~"loki|tempo"} > 0 + ) + - alert: PlatformAIStorBucketQuotaUsage + expr: >- + platform:aistor_bucket_quota_usage_percent >= 70 + and + platform:aistor_bucket_quota_usage_percent < 85 + for: 15m + labels: + severity: warning + annotations: + summary: AIStor 버킷 할당량 사용률 경고 + description: '{{ $labels.bucket }} 버킷 사용률이 70% 이상 85% 미만으로 15분간 유지됐습니다.' + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformAIStorBucketQuotaUsage + expr: >- + platform:aistor_bucket_quota_usage_percent >= 85 + and + platform:aistor_bucket_quota_usage_percent < 95 + for: 10m + labels: + severity: critical + annotations: + summary: AIStor 버킷 할당량 사용률 심각 + description: '{{ $labels.bucket }} 버킷 사용률이 85% 이상 95% 미만으로 10분간 유지됐습니다.' + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformAIStorBucketQuotaUsage + expr: platform:aistor_bucket_quota_usage_percent >= 95 + for: 5m + labels: + severity: emergency + annotations: + summary: AIStor 버킷 할당량 사용률 긴급 + description: '대응 절차에 따라 {{ $labels.bucket }} 버킷의 수집 경로를 중지하고 객체는 자동으로 삭제하지 마세요.' + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md diff --git a/services/observability/rules/platform/verified-service-rules.yaml b/services/observability/rules/platform/verified-service-rules.yaml new file mode 100644 index 0000000..9ebfd90 --- /dev/null +++ b/services/observability/rules/platform/verified-service-rules.yaml @@ -0,0 +1,88 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: platform-verified-services + namespace: observability + labels: + app.kubernetes.io/name: platform-observability-rules + app.kubernetes.io/part-of: observability + observability.hyeonworks.com/instance: home +spec: + groups: + - name: platform.verified-services + rules: + - alert: PlatformCNPGCollectorDown + expr: >- + min by (cluster, namespace) ( + cnpg_collector_up{ + namespace="platform-data", + cluster="platform-postgres" + } + ) == 0 + for: 5m + labels: + severity: critical + annotations: + summary: CloudNativePG 수집기가 PostgreSQL 중단을 보고함 + description: '{{ $labels.cluster }} CloudNativePG 수집기가 PostgreSQL 중단 상태를 5분간 보고했습니다.' + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformAlloyConfigLoadFailed + expr: >- + min by (namespace) ( + alloy_config_last_load_successful{namespace="observability-agent"} + ) == 0 + for: 5m + labels: + severity: critical + annotations: + summary: Alloy 설정 로드 실패 + description: Alloy의 마지막 설정 로드 실패 상태가 5분간 유지됐습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformAlloyLogDeliveryRetries + expr: >- + sum by (namespace) ( + increase(loki_write_batch_retries_total{namespace="observability-agent"}[15m]) + ) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: Alloy 로그 배치 전송 재시도 발생 + description: Alloy가 최근 15분 동안 Loki 쓰기 배치를 한 번 이상 재시도했습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformAlloyLogEntriesDropped + expr: >- + sum by (namespace) ( + increase(loki_write_dropped_entries_total{namespace="observability-agent"}[15m]) + ) > 0 + for: 5m + labels: + severity: critical + annotations: + summary: Alloy 로그 항목 유실 발생 + description: Alloy가 최근 15분 동안 재시도를 모두 소진한 뒤 로그 항목을 유실했습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformLokiRuntimeConfigReloadFailed + expr: >- + min by (namespace) ( + loki_runtime_config_last_reload_successful{namespace="observability"} + ) == 0 + for: 5m + labels: + severity: critical + annotations: + summary: Loki 런타임 설정 재로드 실패 + description: Loki의 마지막 런타임 설정 재로드 실패 상태가 5분간 유지됐습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - alert: PlatformLokiWALDiskUsageHigh + expr: >- + max by (namespace) ( + loki_ingester_wal_disk_usage_percent{namespace="observability"} + ) >= 0.8 + for: 15m + labels: + severity: warning + annotations: + summary: Loki WAL 디스크 사용률 높음 + description: Loki WAL 디스크 사용률이 80% 이상으로 15분간 유지됐습니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md diff --git a/services/observability/targets/README.md b/services/observability/targets/README.md new file mode 100644 index 0000000..83797a2 --- /dev/null +++ b/services/observability/targets/README.md @@ -0,0 +1,24 @@ +# Platform metric targets + +This component declares the manually owned monitor resources and counterpart +NetworkPolicy needed to onboard existing platform services into the home +Prometheus instance. + +The owners remain authoritative for their endpoint activation: + +- Gitea chart: `/metrics` and its generated ServiceMonitor +- Keycloak Operator: management metrics and its generated ServiceMonitor +- CloudNativePG: the declared `metrics` Pod port through a manual PodMonitor +- AIStor: the v3 per-bucket usage endpoint through a manual ServiceMonitor +- packaged Traefik chart: its dedicated metrics Service and ServiceMonitor + +The AIStor endpoint uses `MINIO_PROMETHEUS_AUTH_TYPE=public`, but no public +route is created. Port 9000 remains protected by the namespace default-deny and +the exact Prometheus/client allow policies. Because S3 and metrics share that +L4 port, an already-authorized S3 client can also reach the credential-less +metrics path; HTTP-path isolation is not expressible with NetworkPolicy. + +The first inventory is captured before blackbox is deployed. Grafana already +exists and must be recorded explicitly as a preexisting, verify-only substrate +target; it is not owned or rolled back by this component. + diff --git a/services/observability/targets/aistor/service-monitor.yaml b/services/observability/targets/aistor/service-monitor.yaml new file mode 100644 index 0000000..6897101 --- /dev/null +++ b/services/observability/targets/aistor/service-monitor.yaml @@ -0,0 +1,28 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: aistor-bucket-usage + namespace: object-storage + labels: + app.kubernetes.io/name: minio-aistor + app.kubernetes.io/component: bucket-usage-metrics + app.kubernetes.io/part-of: platform + observability.hyeonworks.com/instance: home +spec: + jobLabel: aistor.min.io/objectStore + selector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + namespaceSelector: + matchNames: + - object-storage + endpoints: + - port: http-minio + path: /minio/metrics/v3/cluster/usage/buckets + interval: 60s + scrapeTimeout: 10s + relabelings: + - action: keep + sourceLabels: + - __meta_kubernetes_service_name + regex: minio diff --git a/services/observability/targets/cloudnative-pg/pod-monitor.yaml b/services/observability/targets/cloudnative-pg/pod-monitor.yaml new file mode 100644 index 0000000..9b577b7 --- /dev/null +++ b/services/observability/targets/cloudnative-pg/pod-monitor.yaml @@ -0,0 +1,24 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: platform-postgres + namespace: platform-data + labels: + app.kubernetes.io/name: platform-postgres + app.kubernetes.io/component: database-metrics + app.kubernetes.io/part-of: platform + observability.hyeonworks.com/instance: home +spec: + jobLabel: cnpg.io/cluster + selector: + matchLabels: + cnpg.io/cluster: platform-postgres + namespaceSelector: + matchNames: + - platform-data + podMetricsEndpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s + diff --git a/services/observability/targets/counterpart-network-policies.yaml b/services/observability/targets/counterpart-network-policies.yaml new file mode 100644 index 0000000..decdc5e --- /dev/null +++ b/services/observability/targets/counterpart-network-policies.yaml @@ -0,0 +1,108 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: traefik-preserve-ingress-and-allow-prometheus-metrics + namespace: kube-system + labels: + app.kubernetes.io/name: traefik + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + app.kubernetes.io/instance: traefik-kube-system + policyTypes: + - Ingress + ingress: + # Preserve the pre-policy HTTP, HTTPS, and health reachability. The new + # policy isolates the selected Pod, so omitting these ports would break the + # existing NodePort and controller health paths. + - ports: + - protocol: TCP + port: 8000 + - protocol: TCP + port: 8443 + - protocol: TCP + port: 8080 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: observability-core-kube-pr-prometheus + ports: + - protocol: TCP + port: 9100 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-prometheus-platform-targets + namespace: observability + labels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: platform-observability +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: observability-core-kube-pr-prometheus + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: gitea + podSelector: + matchLabels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: gitea + ports: + - protocol: TCP + port: 3000 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: keycloak + podSelector: + matchLabels: + app: keycloak + app.kubernetes.io/instance: keycloak + app.kubernetes.io/managed-by: keycloak-operator + ports: + - protocol: TCP + port: 9000 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: platform-data + podSelector: + matchLabels: + cnpg.io/cluster: platform-postgres + ports: + - protocol: TCP + port: 9187 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: object-storage + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + ports: + - protocol: TCP + port: 9000 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + app.kubernetes.io/instance: traefik-kube-system + ports: + - protocol: TCP + port: 9100 diff --git a/services/observability/targets/kustomization.yaml b/services/observability/targets/kustomization.yaml new file mode 100644 index 0000000..652196b --- /dev/null +++ b/services/observability/targets/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cloudnative-pg/pod-monitor.yaml + - aistor/service-monitor.yaml + - counterpart-network-policies.yaml + diff --git a/services/observability/tempo/README.md b/services/observability/tempo/README.md new file mode 100644 index 0000000..3b1d1be --- /dev/null +++ b/services/observability/tempo/README.md @@ -0,0 +1,15 @@ +# Tempo + +This root renders `grafana-community/tempo` chart `2.2.3` as one Tempo +`2.10.7` StatefulSet in `observability`. + +- The image is rewritten to the reviewed amd64 digest. +- Traces use the `tempo` AIStor bucket through `tempo-object-storage`; no root + credential is referenced. +- Retention is seven days. WAL and local working data use the dedicated 5Gi + Retain Local PV. +- Only the query/metrics port 3200 and OTLP/gRPC port 4317 are exposed. +- OTLP/HTTP, Jaeger, Zipkin, metrics-generator, and tempo-query are disabled. + +Do not delete the PVC or bucket during rollback. Stop the StatefulSet and +preserve its Retain volume while S3/WAL state is investigated. diff --git a/services/observability/tempo/kustomization.yaml b/services/observability/tempo/kustomization.yaml new file mode 100644 index 0000000..114f52f --- /dev/null +++ b/services/observability/tempo/kustomization.yaml @@ -0,0 +1,56 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: observability + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: tempo + repo: https://grafana-community.github.io/helm-charts + version: 2.2.3 + releaseName: tempo + namespace: observability + valuesFile: values/home.yaml + includeCRDs: false + skipTests: true + +images: + - name: docker.io/grafana/tempo + newName: docker.io/grafana/tempo + digest: sha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c + +patches: + - target: + group: "" + version: v1 + kind: Service + name: tempo + patch: |- + - op: replace + path: /spec/ports + value: + - name: tempo-prom-metrics + port: 3200 + protocol: TCP + targetPort: 3200 + - name: grpc-tempo-otlp + port: 4317 + protocol: TCP + targetPort: 4317 + - target: + group: apps + version: v1 + kind: StatefulSet + name: tempo + patch: |- + - op: replace + path: /spec/template/spec/containers/0/ports + value: + - containerPort: 3200 + name: prom-metrics + protocol: TCP + - containerPort: 4317 + name: otlp-grpc + protocol: TCP diff --git a/services/observability/tempo/values/home.yaml b/services/observability/tempo/values/home.yaml new file mode 100644 index 0000000..d342353 --- /dev/null +++ b/services/observability/tempo/values/home.yaml @@ -0,0 +1,125 @@ +global: + commonLabels: + observability.hyeonworks.com/instance: home + +replicas: 1 + +config: | + stream_over_http_enabled: false + memberlist: + cluster_label: "{{ .Release.Name }}.{{ .Release.Namespace }}" + multitenancy_enabled: false + usage_report: + reporting_enabled: false + compactor: + compaction: + block_retention: 168h + distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + ingester: + trace_idle_period: 10s + max_block_duration: 5m + max_block_bytes: 100000000 + complete_block_timeout: 15m + flush_all_on_shutdown: true + server: + http_listen_port: 3200 + storage: + trace: + backend: s3 + s3: + bucket: tempo + endpoint: minio.object-storage.svc.cluster.local:80 + insecure: true + forcepathstyle: true + access_key: ${AWS_ACCESS_KEY_ID} + secret_key: ${AWS_SECRET_ACCESS_KEY} + wal: + path: /var/tempo/wal + querier: {} + query_frontend: {} + overrides: + defaults: + ingestion: + rate_limit_bytes: 5000000 + burst_size_bytes: 10000000 + max_traces_per_user: 5000 + global: + max_bytes_per_trace: 5000000 + +tempo: + registry: docker.io + repository: grafana/tempo + tag: 2.10.7 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + memBallastSizeMbs: 0 + reportingEnabled: false + metricsGenerator: + enabled: false + extraArgs: + config.expand-env: true + extraEnv: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: tempo-object-storage + key: access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: tempo-object-storage + key: secret-access-key + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +tempoQuery: + enabled: false + +securityContext: + runAsUser: 10001 + runAsGroup: 10001 + runAsNonRoot: true + fsGroup: 10001 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + +serviceAccount: + automountServiceAccountToken: false + +service: + type: ClusterIP + +serviceMonitor: + enabled: true + interval: 30s + additionalLabels: + observability.hyeonworks.com/instance: home + +persistence: + enabled: true + enableStatefulSetAutoDeletePVC: false + storageClassName: ssd-local-observability-tempo-retain + accessModes: + - ReadWriteOnce + size: 5Gi + +podLabels: + platform.hyeonworks.com/aistor-client: "true" diff --git a/services/pgadmin/README.md b/services/pgadmin/README.md new file mode 100644 index 0000000..f332e4c --- /dev/null +++ b/services/pgadmin/README.md @@ -0,0 +1,19 @@ +# pgAdmin + +PostgreSQL 관리 화면을 platform-admin 네임스페이스에 배포합니다. + +- 주소: db-admin.learn.hyeonworks.com +- 실제 접근 범위: 호스트 Nginx에서 LAN·Tailscale만 허용 +- 차트: dpage/pgadmin4-helm:9.16.0 +- 이미지: dpage/pgadmin4:9.16의 amd64 digest 고정 +- 저장소: 2Gi SSD Local PV, Retain +- 배포 전략: 단일 replica, Recreate + +Keycloak 그룹 /platform-db-admins가 있는 사용자만 OIDC 로그인이 +가능합니다. 내부 관리자는 비상용으로 유지합니다. 공유 서버 정의에는 +비밀번호가 없으며, DB 비밀번호 저장도 비활성화합니다. + +필요한 Secret은 다음 두 개뿐이며 Git에 저장하지 않습니다. + +- pgadmin-keycloak-oidc: client-id, client-secret +- pgadmin-bootstrap: password diff --git a/services/pgadmin/config.yaml b/services/pgadmin/config.yaml new file mode 100644 index 0000000..13320a6 --- /dev/null +++ b/services/pgadmin/config.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pgadmin-config + namespace: platform-admin + labels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/instance: pgadmin + app.kubernetes.io/part-of: platform +data: + config_local.py: | + import os + + AUTHENTICATION_SOURCES = ['oauth2', 'internal'] + OAUTH2_AUTO_CREATE_USER = True + OAUTH2_CONFIG = [{ + 'OAUTH2_NAME': 'keycloak', + 'OAUTH2_DISPLAY_NAME': 'Hyeonworks ID', + 'OAUTH2_CLIENT_ID': os.environ['PGADMIN_OIDC_CLIENT_ID'], + 'OAUTH2_CLIENT_SECRET': os.environ['PGADMIN_OIDC_CLIENT_SECRET'], + 'OAUTH2_SERVER_METADATA_URL': ( + 'https://id.learn.hyeonworks.com/realms/hyeonworks/' + '.well-known/openid-configuration' + ), + 'OAUTH2_SCOPE': 'openid email profile', + 'OAUTH2_USERNAME_CLAIM': 'preferred_username', + 'OAUTH2_ADDITIONAL_CLAIMS': { + 'groups': ['/platform-db-admins'], + }, + }] + + ALLOW_SAVE_PASSWORD = False + ENHANCED_COOKIE_PROTECTION = False + SESSION_COOKIE_SECURE = True + SESSION_COOKIE_HTTPONLY = True + SESSION_COOKIE_SAMESITE = 'Lax' + ALLOWED_HOSTS = ['db-admin.learn.hyeonworks.com'] + UPGRADE_CHECK_ENABLED = False + SHOW_GRAVATAR_IMAGE = False + LLM_ENABLED = False + ENABLE_PSQL = False + ENABLE_SERVER_PASS_EXEC_CMD = False diff --git a/services/pgadmin/ingress-class-patch.yaml b/services/pgadmin/ingress-class-patch.yaml new file mode 100644 index 0000000..ffaf79f --- /dev/null +++ b/services/pgadmin/ingress-class-patch.yaml @@ -0,0 +1,5 @@ +- op: remove + path: /metadata/annotations/kubernetes.io~1ingress.class +- op: add + path: /spec/ingressClassName + value: traefik diff --git a/services/pgadmin/kustomization.yaml b/services/pgadmin/kustomization.yaml new file mode 100644 index 0000000..02fb1e0 --- /dev/null +++ b/services/pgadmin/kustomization.yaml @@ -0,0 +1,38 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +helmGlobals: + chartHome: .helm/charts + +helmCharts: + - name: pgadmin4-helm + repo: oci://docker.io/dpage + version: 9.16.0 + releaseName: pgadmin + namespace: platform-admin + valuesFile: values/home.yaml + includeCRDs: false + skipTests: true + +resources: + - config.yaml + - network-policies + +patches: + - patch: |- + - op: add + path: /metadata/namespace + value: platform-admin + target: + labelSelector: app.kubernetes.io/instance=pgadmin + - path: ingress-class-patch.yaml + target: + group: networking.k8s.io + version: v1 + kind: Ingress + name: pgadmin + +images: + - name: docker.io/dpage/pgadmin4 + newName: docker.io/dpage/pgadmin4 + digest: sha256:66a300a7ecdcc1f325af0c430315329bca46cd4a7067227d6899802238167c6e diff --git a/services/pgadmin/network-policies/kustomization.yaml b/services/pgadmin/network-policies/kustomization.yaml new file mode 100644 index 0000000..7e74d38 --- /dev/null +++ b/services/pgadmin/network-policies/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - pgadmin.yaml + - platform-postgres.yaml diff --git a/services/pgadmin/network-policies/pgadmin.yaml b/services/pgadmin/network-policies/pgadmin.yaml new file mode 100644 index 0000000..7c1989a --- /dev/null +++ b/services/pgadmin/network-policies/pgadmin.yaml @@ -0,0 +1,150 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: pgadmin-default-deny + namespace: platform-admin + labels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/instance: pgadmin + policyTypes: + - Ingress + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: pgadmin-allow-traefik + namespace: platform-admin + labels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/instance: pgadmin + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ports: + - protocol: TCP + port: 5050 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: pgadmin-allow-dns + namespace: platform-admin + labels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/instance: pgadmin + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: pgadmin-allow-keycloak + namespace: platform-admin + labels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/instance: pgadmin + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 192.168.0.107/32 + ports: + - protocol: TCP + port: 443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: pgadmin-allow-platform-postgres + namespace: platform-admin + labels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/instance: pgadmin + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: platform-data + podSelector: + matchLabels: + cnpg.io/cluster: platform-postgres + ports: + - protocol: TCP + port: 5432 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: pgadmin-allow-blackbox-health + namespace: platform-admin + labels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/instance: pgadmin + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-blackbox-exporter + app.kubernetes.io/instance: blackbox-exporter + ports: + - protocol: TCP + port: 5050 diff --git a/services/pgadmin/network-policies/platform-postgres.yaml b/services/pgadmin/network-policies/platform-postgres.yaml new file mode 100644 index 0000000..8755465 --- /dev/null +++ b/services/pgadmin/network-policies/platform-postgres.yaml @@ -0,0 +1,27 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: platform-postgres-allow-pgadmin + namespace: platform-data + labels: + app.kubernetes.io/name: platform-postgres + app.kubernetes.io/component: database-network-policy + app.kubernetes.io/part-of: platform +spec: + podSelector: + matchLabels: + cnpg.io/cluster: platform-postgres + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: platform-admin + podSelector: + matchLabels: + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/instance: pgadmin + ports: + - protocol: TCP + port: 5432 diff --git a/services/pgadmin/values/home.yaml b/services/pgadmin/values/home.yaml new file mode 100644 index 0000000..ad2937f --- /dev/null +++ b/services/pgadmin/values/home.yaml @@ -0,0 +1,175 @@ +fullname: pgadmin + +image: + registry: docker.io + repository: dpage/pgadmin4 + tag: "9.16" + pullPolicy: IfNotPresent + +commonLabels: + app: pgadmin4 + app.kubernetes.io/name: pgadmin4 + app.kubernetes.io/instance: pgadmin + app.kubernetes.io/component: database-admin + app.kubernetes.io/part-of: platform + +replicas: 1 +strategy: + type: Recreate + +containerPort: 5050 +disablePostfix: true +enableServiceLinks: false + +auth: + email: platform-admin@hyeonworks.com + existingSecret: pgadmin-bootstrap + passwordKey: password + +extraEnvVars: + - name: PGADMIN_OIDC_CLIENT_ID + valueFrom: + secretKeyRef: + name: pgadmin-keycloak-oidc + key: client-id + - name: PGADMIN_OIDC_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: pgadmin-keycloak-oidc + key: client-secret + - name: PGADMIN_REPLACE_SERVERS_ON_STARTUP + value: "True" + +config_local: + enabled: false + +extraVolumes: + - name: config-local + configMap: + name: pgadmin-config + defaultMode: 292 + items: + - key: config_local.py + path: config_local.py + +extraVolumeMounts: + - name: config-local + mountPath: /pgadmin4/config_local.py + subPath: config_local.py + readOnly: true + +serverDefinitions: + enabled: true + data: + "1": + Name: Hyeonworks Gitea + Group: Platform + Host: platform-postgres-rw.platform-data.svc.cluster.local + Port: 5432 + MaintenanceDB: gitea + Username: gitea + Shared: true + SharedUsername: gitea + ConnectionParameters: + sslmode: require + connect_timeout: 10 + "2": + Name: Hyeonworks Keycloak + Group: Platform + Host: platform-postgres-rw.platform-data.svc.cluster.local + Port: 5432 + MaintenanceDB: keycloak + Username: keycloak + Shared: true + SharedUsername: keycloak + ConnectionParameters: + sslmode: require + connect_timeout: 10 + +resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + +persistence: + enabled: true + size: 2Gi + storageClass: ssd-local-pgadmin-retain + accessModes: + - ReadWriteOnce + +service: + type: ClusterIP + port: 80 + +serviceAccount: + create: true + automountServiceAccountToken: false + +ingress: + enabled: true + hostname: db-admin.learn.hyeonworks.com + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: web + tlsSecret: "" + +startupProbe: + enabled: true + httpGet: + path: /misc/ping + port: http + httpHeaders: + - name: Host + value: db-admin.learn.hyeonworks.com + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 30 + +readinessProbe: + enabled: true + httpGet: + path: /misc/ping + port: http + httpHeaders: + - name: Host + value: db-admin.learn.hyeonworks.com + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + +livenessProbe: + enabled: true + httpGet: + path: /misc/ping + port: http + httpHeaders: + - name: Host + value: db-admin.learn.hyeonworks.com + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 6 + +podSecurityContext: + enabled: true + fsGroupChangePolicy: OnRootMismatch + fsGroup: 1001 + +containerSecurityContext: + enabled: true + runAsUser: 1001 + runAsGroup: 1001 + runAsNonRoot: true + privileged: false + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + appArmorProfile: + type: RuntimeDefault diff --git a/services/platform-postgres-keycloak/README.md b/services/platform-postgres-keycloak/README.md new file mode 100644 index 0000000..c774836 --- /dev/null +++ b/services/platform-postgres-keycloak/README.md @@ -0,0 +1,42 @@ +# 플랫폼 PostgreSQL — Keycloak Phase 2 확장 + +상태: 2026-07-23 실제 적용, `DatabaseRole`·`Database` 모두 `applied=true` +additive NetworkPolicy도 적용됐으며 기존 PostgreSQL Cluster와 Gitea DB 리소스는 +교체하지 않았다. + +이 디렉터리는 이미 생성된 `platform-postgres` CloudNativePG Cluster에 추가하는 +Kustomize 빌드 루트다. 의도적으로 `Cluster` 리소스를 포함하지 않으며 Phase 1 +NetworkPolicy를 Patch하거나 교체하지 않는다. + +렌더링 결과에는 다음 항목만 있다. + +- `DatabaseRole/platform-postgres-keycloak` +- `Database/platform-postgres-keycloak` +- 기존 PostgreSQL operand Pod를 선택하고 `keycloak` namespace에서 TCP 5432로 + 접근하는 것을 허용하는 `NetworkPolicy/platform-postgres-keycloak-ingress` + +NetworkPolicy ingress 규칙은 누적되므로 이 확장은 Phase 1의 CloudNativePG, 동일 +Cluster 및 Gitea 허용 규칙을 보존한다. 기존 공통 Cluster에는 Keycloak이 자기 +Database에 SCRAM으로 접근하도록 허용하는 정확한 규칙과, 바로 다음에 같은 Role의 +다른 모든 Database 접근을 거부하는 HBA 규칙이 이미 있다. 이 확장은 Cluster를 +Patch하지 않는다. + +이 루트를 적용하기 전에 `platform-data/keycloak-db-credentials`를 생성한다. 유형은 +`kubernetes.io/basic-auth`, label은 `cnpg.io/reload: "true"`, username은 +`keycloak`이어야 하며 비밀번호는 Git 외부에서 제공한다. 이에 대응하는 애플리케이션 +측 사본은 `keycloak` namespace에 둔다. + +기본 RootOnly 로드 제한으로 다음과 같이 렌더링한다. + +```sh +kubectl kustomize services/platform-postgres-keycloak +``` + +Phase 1 Cluster가 Ready 상태이고 Keycloak Secret 계약이 존재할 때만 이 확장을 +적용한다. 렌더링 결과에 `Cluster` 리소스가 없어야 한다. + +## 공식 참고 문서 + +- +- +- diff --git a/services/platform-postgres-keycloak/base/keycloak-database-role.yaml b/services/platform-postgres-keycloak/base/keycloak-database-role.yaml new file mode 100644 index 0000000..3cba323 --- /dev/null +++ b/services/platform-postgres-keycloak/base/keycloak-database-role.yaml @@ -0,0 +1,22 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: DatabaseRole +metadata: + name: platform-postgres-keycloak + labels: + app.kubernetes.io/name: keycloak + app.kubernetes.io/component: database-role +spec: + cluster: + name: platform-postgres + name: keycloak + comment: Keycloak identity provider database owner + login: true + superuser: false + createdb: false + createrole: false + replication: false + bypassrls: false + connectionLimit: 50 + passwordSecret: + name: keycloak-db-credentials + databaseRoleReclaimPolicy: retain diff --git a/services/platform-postgres-keycloak/base/keycloak-database.yaml b/services/platform-postgres-keycloak/base/keycloak-database.yaml new file mode 100644 index 0000000..96cbd0c --- /dev/null +++ b/services/platform-postgres-keycloak/base/keycloak-database.yaml @@ -0,0 +1,14 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: platform-postgres-keycloak + labels: + app.kubernetes.io/name: keycloak + app.kubernetes.io/component: database +spec: + cluster: + name: platform-postgres + name: keycloak + owner: keycloak + ensure: present + databaseReclaimPolicy: retain diff --git a/services/platform-postgres-keycloak/base/kustomization.yaml b/services/platform-postgres-keycloak/base/kustomization.yaml new file mode 100644 index 0000000..bd1ff06 --- /dev/null +++ b/services/platform-postgres-keycloak/base/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: platform-data + +resources: + - keycloak-database-role.yaml + - keycloak-database.yaml + - platform-postgres-keycloak-network-policy.yaml + +labels: + - pairs: + app.kubernetes.io/part-of: platform + app.kubernetes.io/managed-by: kustomize + includeSelectors: false diff --git a/services/platform-postgres-keycloak/base/platform-postgres-keycloak-network-policy.yaml b/services/platform-postgres-keycloak/base/platform-postgres-keycloak-network-policy.yaml new file mode 100644 index 0000000..abd736e --- /dev/null +++ b/services/platform-postgres-keycloak/base/platform-postgres-keycloak-network-policy.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: platform-postgres-keycloak-ingress + labels: + app.kubernetes.io/name: platform-postgres + app.kubernetes.io/component: database-network-policy +spec: + podSelector: + matchLabels: + cnpg.io/cluster: platform-postgres + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: keycloak + ports: + - protocol: TCP + port: 5432 diff --git a/services/platform-postgres-keycloak/kustomization.yaml b/services/platform-postgres-keycloak/kustomization.yaml new file mode 100644 index 0000000..271aa1a --- /dev/null +++ b/services/platform-postgres-keycloak/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - base diff --git a/services/platform-postgres/README.md b/services/platform-postgres/README.md new file mode 100644 index 0000000..ebb0f2e --- /dev/null +++ b/services/platform-postgres/README.md @@ -0,0 +1,73 @@ +# 플랫폼 PostgreSQL — Phase 1 빌드 루트 + +이 Kustomization은 `platform-data` namespace에 Phase 1 CloudNativePG 기반 구성을 +선언한다. + +- `platform-postgres`라는 `Cluster` 하나 +- Gitea용 `DatabaseRole`과 `Database` +- CloudNativePG, 동일 Cluster 트래픽 및 Gitea만 허용하는 기본 ingress + NetworkPolicy + +PostgreSQL 17.9 인스턴스 하나로 시작하며 20 GiB Claim을 홈 서버의 정적 +`ssd-local-postgres-retain` StorageClass에 바인딩한다. CloudNativePG에서는 필수 +read/write Service만 유지한다. + +```text +platform-postgres-rw.platform-data.svc.cluster.local:5432 +``` + +이미지 `ghcr.io/cloudnative-pg/postgresql:17.9-standard-trixie`는 공식 PostgreSQL +17 멀티 아키텍처 이미지로 고정되어 있다. 백업 및 복원 시험 후 의도적으로 태그를 +업그레이드한다. + +Cluster의 사용자 정의 `pg_hba` 규칙은 `gitea`가 `gitea` Database에만, +`keycloak`이 `keycloak` Database에만 인증하도록 허용한 뒤 각 Role의 다른 모든 +Database 접근을 거부한다. 이 규칙들은 CloudNativePG의 기본 fallback 인증 규칙보다 +앞에 배치된다. + +## Phase 1 Secret 계약 + +Cluster를 생성하기 전에 `platform-data`에 `gitea-db-credentials`를 생성한다. + +- 유형: `kubernetes.io/basic-auth` +- label: `cnpg.io/reload: "true"` +- `username`: 정확히 `gitea` +- `password`: Git 외부에서 제공하는 생성된 비밀번호 + +동일한 credential 값을 `gitea` namespace에 복사한다. `initdb` bootstrap은 최초 +소유자와 Database를 만들 때 이 Secret을 사용하고 data checksum을 활성화한다. +Gitea Role과 Database는 모두 `retain` 조정 정책을 사용한다. + +Keycloak `DatabaseRole`, `Database`, Secret 참조 및 namespace 전용 ingress 허용은 +이 Phase 1 빌드 루트에 의도적으로 포함하지 않았다. 공통 Cluster의 Keycloak HBA +허용·거부 규칙 쌍은 이곳에 유지하고, 독립적인 형제 빌드 루트 +`services/platform-postgres-keycloak`이 Keycloak login Role, Database 및 +namespace 전용 ingress를 추가한다. 2026-07-23 기준 이 Keycloak 확장은 실제 +클러스터에 적용됐다. 두 빌드 루트가 분리돼 있으므로 +`platform-data/keycloak-db-credentials`가 없어도 신규 Phase 1 baseline만은 +배포할 수 있다. + +## 의존성과 렌더링 + +1. Phase 1 namespace를 생성한다. +2. PostgreSQL SSD Local PV 기반 구성을 준비하고 적용한다. +3. CloudNativePG Operator를 설치하고 준비될 때까지 기다린다. +4. Gitea credential Secret 계약을 생성한다. +5. 이 디렉터리를 렌더링하고 적용한다. + +```sh +kubectl kustomize services/platform-postgres +``` + +렌더링 결과에는 Cluster 하나와 Gitea의 DatabaseRole 및 Database만 있어야 한다. +Keycloak DatabaseRole, Database, Secret 참조 또는 Keycloak namespace ingress +허용은 없어야 하며, Keycloak용 HBA 정책만 존재해야 한다. + +## 공식 참고 문서 + +- +- +- +- +- +- diff --git a/services/platform-postgres/base/cluster.yaml b/services/platform-postgres/base/cluster.yaml new file mode 100644 index 0000000..070617d --- /dev/null +++ b/services/platform-postgres/base/cluster.yaml @@ -0,0 +1,46 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: platform-postgres + labels: + app.kubernetes.io/name: platform-postgres + app.kubernetes.io/component: database +spec: + description: Shared platform PostgreSQL instance initialized for Gitea + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:17.9-standard-trixie + + bootstrap: + initdb: + dataChecksums: true + database: gitea + owner: gitea + secret: + name: gitea-db-credentials + + postgresql: + pg_hba: + - host gitea gitea all scram-sha-256 + - host all gitea all reject + - host keycloak keycloak all scram-sha-256 + - host all keycloak all reject + + storage: + size: 20Gi + + managed: + services: + disabledDefaultServices: + - ro + - r + + affinity: + enablePodAntiAffinity: false + + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 4Gi diff --git a/services/platform-postgres/base/gitea-database-role.yaml b/services/platform-postgres/base/gitea-database-role.yaml new file mode 100644 index 0000000..6f46424 --- /dev/null +++ b/services/platform-postgres/base/gitea-database-role.yaml @@ -0,0 +1,22 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: DatabaseRole +metadata: + name: platform-postgres-gitea + labels: + app.kubernetes.io/name: gitea + app.kubernetes.io/component: database-role +spec: + cluster: + name: platform-postgres + name: gitea + comment: Gitea application database owner + login: true + superuser: false + createdb: false + createrole: false + replication: false + bypassrls: false + connectionLimit: 50 + passwordSecret: + name: gitea-db-credentials + databaseRoleReclaimPolicy: retain diff --git a/services/platform-postgres/base/gitea-database.yaml b/services/platform-postgres/base/gitea-database.yaml new file mode 100644 index 0000000..921c547 --- /dev/null +++ b/services/platform-postgres/base/gitea-database.yaml @@ -0,0 +1,14 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: platform-postgres-gitea + labels: + app.kubernetes.io/name: gitea + app.kubernetes.io/component: database +spec: + cluster: + name: platform-postgres + name: gitea + owner: gitea + ensure: present + databaseReclaimPolicy: retain diff --git a/services/platform-postgres/base/kustomization.yaml b/services/platform-postgres/base/kustomization.yaml new file mode 100644 index 0000000..6ce28e4 --- /dev/null +++ b/services/platform-postgres/base/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: platform-data + +resources: + - cluster.yaml + - gitea-database-role.yaml + - gitea-database.yaml + - platform-postgres-network-policy.yaml + +labels: + - pairs: + app.kubernetes.io/part-of: platform + app.kubernetes.io/managed-by: kustomize + includeSelectors: false diff --git a/services/platform-postgres/base/platform-postgres-network-policy.yaml b/services/platform-postgres/base/platform-postgres-network-policy.yaml new file mode 100644 index 0000000..18b4dc1 --- /dev/null +++ b/services/platform-postgres/base/platform-postgres-network-policy.yaml @@ -0,0 +1,53 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: platform-postgres-ingress + labels: + app.kubernetes.io/name: platform-postgres + app.kubernetes.io/component: database-network-policy +spec: + podSelector: + matchLabels: + cnpg.io/cluster: platform-postgres + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: cnpg-system + podSelector: + matchLabels: + app.kubernetes.io/name: cloudnative-pg + ports: + - protocol: TCP + port: 8000 + - protocol: TCP + port: 5432 + - from: + - podSelector: + matchLabels: + cnpg.io/cluster: platform-postgres + ports: + - protocol: TCP + port: 8000 + - protocol: TCP + port: 5432 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: gitea + ports: + - protocol: TCP + port: 5432 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: observability-core-kube-pr-prometheus + ports: + - protocol: TCP + port: 9187 diff --git a/services/platform-postgres/kustomization.yaml b/services/platform-postgres/kustomization.yaml new file mode 100644 index 0000000..a629b31 --- /dev/null +++ b/services/platform-postgres/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - overlays/home diff --git a/services/platform-postgres/overlays/home/kustomization.yaml b/services/platform-postgres/overlays/home/kustomization.yaml new file mode 100644 index 0000000..b8fd861 --- /dev/null +++ b/services/platform-postgres/overlays/home/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +patches: + - path: platform-postgres-storage-patch.yaml diff --git a/services/platform-postgres/overlays/home/platform-postgres-storage-patch.yaml b/services/platform-postgres/overlays/home/platform-postgres-storage-patch.yaml new file mode 100644 index 0000000..ae3c6fa --- /dev/null +++ b/services/platform-postgres/overlays/home/platform-postgres-storage-patch.yaml @@ -0,0 +1,8 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: platform-postgres +spec: + storage: + storageClass: ssd-local-postgres-retain + size: 20Gi