From 9751db0e978eac2bb1d1a75061f0f50bbfc43179 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 01:25:24 +0900 Subject: [PATCH 1/2] feat(keycloak): pin the operator source and both images by digest The Keycloak Operator remote resource is pinned to the tag 26.7.0, and that tag's content changed once (live build-timestamp 2026-07-09, later renders 2026-07-23). Both images use the 26.7.0 tag with imagePullPolicy Always, so every restart depends on quay.io and could pull different bits. Pin the remote resource to the tag's commit, the operator image and RELATED_IMAGE_KEYCLOAK to the digests that were running on 2026-11-12 (pod imageID) and set the operator to IfNotPresent. The env value is set with a strategic merge patch keyed by name; a JSON patch by list index hit KUBERNETES_NAMESPACE instead. render-phase2.sh and apply-keycloak.sh check the commit and the digest. Risk: applying restarts the Keycloak operator once (restart window). Keycloak itself keeps running. Verification (desktop): render-phase2.sh passes; the rendered Deployment has the pinned image, IfNotPresent and the digest in RELATED_IMAGE_KEYCLOAK with the other env entries unchanged. --- .../controllers/keycloak-operator/README.md | 5 +++ .../keycloak-operator/kustomization.yaml | 32 ++++++++++++++++++- scripts/bootstrap/apply-keycloak.sh | 4 +-- scripts/validate/render-phase2.sh | 3 +- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/infrastructure/controllers/keycloak-operator/README.md b/infrastructure/controllers/keycloak-operator/README.md index 1ef2c2f..c08c4ad 100644 --- a/infrastructure/controllers/keycloak-operator/README.md +++ b/infrastructure/controllers/keycloak-operator/README.md @@ -14,6 +14,11 @@ Kubernetes 리소스를 직접 참조하며 업스트림 매니페스트를 복 - 저장소: - 리소스 경로: `kubernetes` - Keycloak 및 Operator 버전: `26.7.0` +- 원격 ref: 태그가 아니라 그 태그의 커밋 `38fd7a0c84e15d4b66b37d1c2ea728ca2f5416a4`(2026-11-12). 같은 태그의 내용이 + 한 번 바뀌었다(build-timestamp 07-09 → 07-23, 성숙도 점검 A5). +- 이미지: Operator·Keycloak 모두 2026-11-12에 돌던 digest로 고정, Operator `imagePullPolicy: IfNotPresent`. + Keycloak CR의 `spec.image`를 정하면 Operator가 최적화 빌드된 이미지로 보므로 `startOptimized: false`를 같이 둔다. + 올릴 때는 태그·커밋·두 digest와 `scripts/validate/render-phase2.sh`·`scripts/bootstrap/apply-keycloak.sh`의 검사 문자열을 함께 바꾼다. - 설치 모드: `keycloak` namespace 범위 업스트림 namespace 범위 Deployment는 각 컨트롤러의 감시 namespace를 diff --git a/infrastructure/controllers/keycloak-operator/kustomization.yaml b/infrastructure/controllers/keycloak-operator/kustomization.yaml index d8c89fc..4c8fce9 100644 --- a/infrastructure/controllers/keycloak-operator/kustomization.yaml +++ b/infrastructure/controllers/keycloak-operator/kustomization.yaml @@ -3,8 +3,38 @@ kind: Kustomization namespace: keycloak +# keycloak-k8s-resources tag 26.7.0 pointed at this commit on 2026-11-12. The tag +# content changed once (live build-timestamp 2026-07-09, later renders 2026-07-23), +# so the commit is pinned instead of the tag. resources: - - github.com/keycloak/keycloak-k8s-resources/kubernetes?ref=26.7.0 + - github.com/keycloak/keycloak-k8s-resources/kubernetes?ref=38fd7a0c84e15d4b66b37d1c2ea728ca2f5416a4 + +# The 26.7.0 images that were running on 2026-11-12 (pod imageID), by digest. +images: + - name: quay.io/keycloak/keycloak-operator + digest: sha256:257ee324da2c701b7fc1ed1b2ce0e03979d1bf149952383bdcf6be904ae5b4f3 + +patches: + # Merged by container and env name (strategic merge), not by list index. + - target: + group: apps + version: v1 + kind: Deployment + name: keycloak-operator + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: keycloak-operator + spec: + template: + spec: + containers: + - name: keycloak-operator + imagePullPolicy: IfNotPresent + env: + - name: RELATED_IMAGE_KEYCLOAK + value: quay.io/keycloak/keycloak@sha256:0f198be292568439d700cdbfb893e69a6009bb43a94a06a945b1d3d506c76b13 labels: - pairs: diff --git a/scripts/bootstrap/apply-keycloak.sh b/scripts/bootstrap/apply-keycloak.sh index aa17f67..ca2558e 100755 --- a/scripts/bootstrap/apply-keycloak.sh +++ b/scripts/bootstrap/apply-keycloak.sh @@ -189,8 +189,8 @@ assert_regex_count \ assert_regex_count \ "${REPOSITORY_ROOT}/infrastructure/controllers/keycloak-operator/kustomization.yaml" \ - 'github\.com/keycloak/keycloak-k8s-resources/kubernetes\?ref=26\.7\.0' 1 \ - "Keycloak Operator 26.7.0 source pin" + 'github\.com/keycloak/keycloak-k8s-resources/kubernetes\?ref=38fd7a0c84e15d4b66b37d1c2ea728ca2f5416a4' 1 \ + "Keycloak Operator 26.7.0 commit pin" assert_regex_count \ "${render_temp_dir}/keycloak-operator.yaml" \ '^kind:[[:space:]]CustomResourceDefinition$' 4 \ diff --git a/scripts/validate/render-phase2.sh b/scripts/validate/render-phase2.sh index a214de2..2190de8 100755 --- a/scripts/validate/render-phase2.sh +++ b/scripts/validate/render-phase2.sh @@ -266,7 +266,8 @@ aistor_objectstore_kustomization="${REPOSITORY_ROOT}/services/minio-aistor/base/ aistor_operator_values="${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/values/home.yaml" aistor_objectstore_values="${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/values/home.yaml" -assert_regex_count "$keycloak_operator_kustomization" '^[[:space:]]*-[[:space:]]github\.com/keycloak/keycloak-k8s-resources/kubernetes\?ref=26\.7\.0[[:space:]]*$' 1 "Keycloak Operator remote ref must be 26.7.0" +assert_regex_count "$keycloak_operator_kustomization" '^[[:space:]]*-[[:space:]]github\.com/keycloak/keycloak-k8s-resources/kubernetes\?ref=38fd7a0c84e15d4b66b37d1c2ea728ca2f5416a4[[:space:]]*$' 1 "Keycloak Operator remote ref must be the 26.7.0 commit" +assert_regex_count "$keycloak_operator_kustomization" '^[[:space:]]*digest:[[:space:]]sha256:257ee324da2c701b7fc1ed1b2ce0e03979d1bf149952383bdcf6be904ae5b4f3[[:space:]]*$' 1 "Keycloak Operator image must be pinned by digest" assert_regex_count "$keycloak_operator_kustomization" 'github\.com/keycloak/keycloak-k8s-resources/kubernetes' 1 "exactly one Keycloak Operator remote resource is allowed" assert_regex_count "$keycloak_operator_kustomization" '\?ref=' 1 "exactly one pinned remote ref is allowed in the Keycloak Operator root" -- 2.54.0 From de600a0bb168cc6f93b5a07facd76d128756ae0a Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 01:25:24 +0900 Subject: [PATCH 2/2] feat(priority): attach PriorityClasses to Keycloak, Gitea, Loki, Tempo The PriorityClasses exist in the repository but no pod uses them, so every platform pod still evicts at priority 0. Attach platform-critical to Keycloak (CR spec.scheduling) and Gitea (chart priorityClassName), and platform-low to Loki (singleBinary) and Tempo. The Keycloak CR also gets spec.image at the running 26.7.0 digest with startOptimized: false: with spec.image the operator otherwise assumes a pre-built optimized image, and this is the stock one. Grafana and pgAdmin have no top-level priorityClassName in their charts and are left for later; platform-postgres is W1's file and tech-log belongs to its deploy/ directory. Risk: applying restarts Keycloak, Gitea, Loki and Tempo once, so it belongs in the k3s restart window, after the PriorityClasses exist (a pod that names a missing class is rejected). The accepted-drift list holds these objects until 2026-11-30. Verification (desktop): render-phase1/2 and render-observability-core pass with the four priorityClassName values; kubectl diff (server dry-run) against live shows only these fields. --- scripts/validate/live-drift-accepted.txt | 6 +++++- scripts/validate/render-phase2.sh | 3 +++ services/gitea/profiles/oidc/values/baseline.yaml | 3 +++ services/keycloak/base/keycloak.yaml | 9 +++++++++ services/observability/loki/values/home.yaml | 2 ++ services/observability/tempo/values/home.yaml | 3 +++ services/priority-classes/README.md | 7 ++++--- 7 files changed, 29 insertions(+), 4 deletions(-) diff --git a/scripts/validate/live-drift-accepted.txt b/scripts/validate/live-drift-accepted.txt index 762302b..cd416e9 100644 --- a/scripts/validate/live-drift-accepted.txt +++ b/scripts/validate/live-drift-accepted.txt @@ -3,8 +3,12 @@ Cluster platform-data platform-postgres 2026-11-30 pg_hba의 techlog 두 줄이 repo에 없다. w1이 CNPG 백업과 함께 넣는다(1c4ef9a). DatabaseRole platform-data platform-postgres-gitea 2026-11-30 렌더가 false 기본값 필드를 명시할 뿐 값은 같다. 다음 Phase 1 적용 때 사라진다. DatabaseRole platform-data platform-postgres-keycloak 2026-11-30 렌더가 false 기본값 필드를 명시할 뿐 값은 같다. 다음 Phase 2 적용 때 사라진다. -Deployment keycloak keycloak-operator 2026-11-30 원격 ref 26.7.0 태그 내용이 바뀌어 build-timestamp 라벨만 다르다. 재시작 창에 커밋 SHA 고정과 함께 적용한다. +Deployment keycloak keycloak-operator 2026-11-30 커밋 SHA·이미지 digest·IfNotPresent 고정(재시작). 재시작 창에 apply-keycloak.sh로 적용한다. Service keycloak keycloak-operator 2026-11-30 위와 같은 build-timestamp 라벨. ServiceAccount keycloak keycloak-operator 2026-11-30 위와 같은 build-timestamp 라벨. Deployment observability observability-core-kube-pr-operator 2026-11-30 이미지 문자열이 tag@digest와 digest로 다르고 digest는 같다. 적용하면 재시작되므로 재시작 창에. Deployment observability observability-core-kube-state-metrics 2026-11-30 위와 같은 이미지 문자열 차이. +Keycloak keycloak keycloak 2026-11-30 이미지 digest·startOptimized false·platform-critical(Keycloak 재시작). 재시작 창에 PriorityClass를 먼저 만든 뒤 적용한다. +Deployment gitea gitea 2026-11-30 priorityClassName platform-critical(Gitea 재시작). 재시작 창에 apply-gitea-oidc.sh로 적용한다. +StatefulSet observability loki 2026-11-30 priorityClassName platform-low(Loki 재시작). 재시작 창에 관측 core 적용으로. +StatefulSet observability tempo 2026-11-30 priorityClassName platform-low(Tempo 재시작). 재시작 창에 관측 core 적용으로. diff --git a/scripts/validate/render-phase2.sh b/scripts/validate/render-phase2.sh index 2190de8..547d642 100755 --- a/scripts/validate/render-phase2.sh +++ b/scripts/validate/render-phase2.sh @@ -343,6 +343,9 @@ assert_regex_count "${render_dir}/keycloak.yaml" '^kind: Ingress$' 1 "Keycloak I assert_regex_count "${render_dir}/minio-aistor.yaml" '^kind: ObjectStore$' 1 "AIStor ObjectStore count" assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*instances:[[:space:]]1[[:space:]]*$' 1 "Keycloak instance count field" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*image:[[:space:]]quay\.io/keycloak/keycloak@sha256:[0-9a-f]{64}[[:space:]]*$' 1 "Keycloak server image must be pinned by digest" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*startOptimized:[[:space:]]false[[:space:]]*$' 1 "the stock Keycloak image must not start optimized" +assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*priorityClassName:[[:space:]]platform-critical[[:space:]]*$' 1 "Keycloak priority class" assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*hostname:[[:space:]]https://id\.learn\.hyeonworks\.com[[:space:]]*$' 1 "Keycloak external hostname" assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*strict:[[:space:]]true[[:space:]]*$' 1 "Keycloak strict hostname mode" assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*httpEnabled:[[:space:]]true[[:space:]]*$' 1 "Keycloak internal HTTP mode" diff --git a/services/gitea/profiles/oidc/values/baseline.yaml b/services/gitea/profiles/oidc/values/baseline.yaml index da779e3..d570fcd 100644 --- a/services/gitea/profiles/oidc/values/baseline.yaml +++ b/services/gitea/profiles/oidc/values/baseline.yaml @@ -2,6 +2,9 @@ namespace: gitea replicaCount: 1 +# services/priority-classes: evicted after everything that is not critical. +priorityClassName: platform-critical + image: registry: docker.gitea.com repository: gitea diff --git a/services/keycloak/base/keycloak.yaml b/services/keycloak/base/keycloak.yaml index c0df799..95fa949 100644 --- a/services/keycloak/base/keycloak.yaml +++ b/services/keycloak/base/keycloak.yaml @@ -9,6 +9,15 @@ metadata: spec: instances: 1 + # The 26.7.0 server image running on 2026-11-12, by digest. With spec.image set + # the operator assumes a pre-built (optimized) image; this is the stock image, + # so startOptimized stays false and Keycloak builds at start as before. + image: quay.io/keycloak/keycloak@sha256:0f198be292568439d700cdbfb893e69a6009bb43a94a06a945b1d3d506c76b13 + startOptimized: false + + scheduling: + priorityClassName: platform-critical + db: vendor: postgres host: platform-postgres-rw.platform-data.svc.cluster.local diff --git a/services/observability/loki/values/home.yaml b/services/observability/loki/values/home.yaml index 9266338..082aca7 100644 --- a/services/observability/loki/values/home.yaml +++ b/services/observability/loki/values/home.yaml @@ -80,6 +80,8 @@ loki: max_compaction_parallelism: 1 singleBinary: + # services/priority-classes: logs can be lost first under node pressure. + priorityClassName: platform-low replicas: 1 sidecar: false podLabels: diff --git a/services/observability/tempo/values/home.yaml b/services/observability/tempo/values/home.yaml index d342353..673d1de 100644 --- a/services/observability/tempo/values/home.yaml +++ b/services/observability/tempo/values/home.yaml @@ -4,6 +4,9 @@ global: replicas: 1 +# services/priority-classes: traces can be lost first under node pressure. +priorityClassName: platform-low + config: | stream_over_http_enabled: false memberlist: diff --git a/services/priority-classes/README.md b/services/priority-classes/README.md index 26eb2dc..92ac343 100644 --- a/services/priority-classes/README.md +++ b/services/priority-classes/README.md @@ -6,13 +6,14 @@ | 이름 | 값 | 선점 | 붙일 곳 | | --- | --- | --- | --- | -| `platform-critical` | 1000000 | 낮은 것을 선점 | platform-postgres, Gitea, Keycloak, tech-log | -| `platform-low` | -1000 | 선점 안 함 | Grafana, pgAdmin, Loki, Tempo | +| `platform-critical` | 1000000 | 낮은 것을 선점 | Gitea(`services/gitea` values), Keycloak(CR `spec.scheduling`). platform-postgres는 W1, tech-log는 tech-log `deploy/` | +| `platform-low` | -1000 | 선점 안 함 | Loki(`singleBinary`), Tempo. Grafana·pgAdmin은 chart에 최상위 키가 없어 아직 안 붙였다 | - 아무것도 안 붙인 파드는 기본값 0이다. 시스템 클래스(`system-*`, 20억)는 그대로 가장 높다. - `globalDefault`는 둘 다 끈다. 새 파드가 모르게 우선순위를 얻지 않게 하려는 것이다. - 클래스를 만드는 것은 아무 파드도 바꾸지 않는다. **붙이는 것은 파드 재시작을 부르므로** - k3s 재시작 창에 한다(`infrastructure/host/k3s/README.md`). + k3s 재시작 창에 한다(`docs/runbooks/k3s-restart-window.md`). +- **클래스를 먼저 만든다.** 없는 클래스를 가리키는 파드는 생성이 거절돼 Deployment가 새 파드를 못 만든다. ```sh kubectl apply -k services/priority-classes -- 2.54.0