Add platform infrastructure configuration

2026-08-28 17:35:41 +09:00
parent fa76531e5b
commit 16c337bcc9
302 changed files with 83259 additions and 1 deletions
@@ -0,0 +1,87 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: platform-certificate-probes
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: platform-observability-rules
+    app.kubernetes.io/part-of: observability
+    observability.hyeonworks.com/instance: home
+spec:
+  groups:
+    - name: platform.blackbox
+      rules:
+        - alert: PlatformPublicEdgeProbeFailed
+          expr: >-
+            probe_success{
+              namespace="observability",
+              "observability.hyeonworks.com/probe-group"="public-edge"
+            } == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: 외부 엔드포인트 프로브 실패
+            description: '{{ $labels.instance }} 외부 엔드포인트 프로브가 5분간 실패했습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformPrivateEdgeProbeFailed
+          expr: >-
+            probe_success{
+              namespace="observability",
+              "observability.hyeonworks.com/probe-group"="private-edge"
+            } == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: 비공개 경계 프로브 실패
+            description: '{{ $labels.instance }} 비공개 경계의 예상 응답이 5분간 실패했습니다. HTTP 403이면서 probe_success=1이면 정상입니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformInternalHealthProbeFailed
+          expr: >-
+            probe_success{
+              namespace="observability",
+              "observability.hyeonworks.com/probe-group"="private-internal"
+            } == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: 내부 상태 확인 프로브 실패
+            description: '{{ $labels.instance }} 내부 상태 확인 프로브가 5분간 실패했습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+    - name: platform.certificates
+      rules:
+        - alert: PlatformCertificateExpiry
+          expr: >-
+            (
+              probe_ssl_earliest_cert_expiry{
+                namespace="observability",
+                "observability.hyeonworks.com/probe-group"=~"public-edge|private-edge"
+              } - time()
+            ) >= 14 * 24 * 60 * 60
+            and
+            (
+              probe_ssl_earliest_cert_expiry{
+                namespace="observability",
+                "observability.hyeonworks.com/probe-group"=~"public-edge|private-edge"
+              } - time()
+            ) < 30 * 24 * 60 * 60
+          labels:
+            severity: warning
+          annotations:
+            summary: TLS 인증서가 30일 이내 만료됨
+            description: '{{ $labels.instance }} 인증서 중 가장 이른 만료 시점이 14일 이상 30일 미만 남았습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformCertificateExpiry
+          expr: >-
+            probe_ssl_earliest_cert_expiry{
+              namespace="observability",
+              "observability.hyeonworks.com/probe-group"=~"public-edge|private-edge"
+            } - time() < 14 * 24 * 60 * 60
+          labels:
+            severity: critical
+          annotations:
+            summary: TLS 인증서가 14일 이내 만료됨
+            description: '{{ $labels.instance }} 인증서 중 가장 이른 만료 시점이 14일 미만 남았습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
@@ -0,0 +1,117 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: platform-observability-core
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: platform-observability-rules
+    app.kubernetes.io/part-of: observability
+    observability.hyeonworks.com/instance: home
+spec:
+  groups:
+    - name: platform.storage
+      rules:
+        - alert: PlatformPersistentVolumeClaimNotBound
+          expr: kube_persistentvolumeclaim_status_phase{phase=~"Pending|Lost"} == 1
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: PersistentVolumeClaim이 바인딩되지 않음
+            description: 플랫폼 PersistentVolumeClaim이 Pending 또는 Lost 상태로 10분간 유지됐습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformRootFilesystemUsage
+          expr: >-
+            100 * (1 -
+              node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+              /
+              node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+            ) > 75
+          for: 30m
+          labels:
+            severity: info
+            threshold: "75"
+          annotations:
+            summary: 루트 파일시스템 사용률이 75%를 초과함
+            description: 노드 루트 파일시스템 사용률이 정보 임계치인 75%를 30분간 초과했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformRootFilesystemUsage
+          expr: >-
+            100 * (1 -
+              node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+              /
+              node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+            ) > 85
+          for: 15m
+          labels:
+            severity: warning
+            threshold: "85"
+          annotations:
+            summary: 루트 파일시스템 사용률이 85%를 초과함
+            description: 노드 루트 파일시스템 사용률이 경고 임계치인 85%를 15분간 초과했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformRootFilesystemUsage
+          expr: >-
+            100 * (1 -
+              node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+              /
+              node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+            ) > 90
+          for: 5m
+          labels:
+            severity: critical
+            threshold: "90"
+          annotations:
+            summary: 루트 파일시스템 사용률이 90%를 초과함
+            description: 노드 루트 파일시스템 사용률이 심각 임계치인 90%를 5분간 초과했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformRootFilesystemInodes
+          expr: >-
+            100 * (1 -
+              node_filesystem_files_free{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+              /
+              node_filesystem_files{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+            ) > 80
+          for: 30m
+          labels:
+            severity: warning
+            threshold: "80"
+          annotations:
+            summary: 루트 파일시스템 inode 사용률이 80%를 초과함
+            description: 노드 루트 파일시스템 inode 사용률이 경고 임계치인 80%를 30분간 초과했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformRootFilesystemInodes
+          expr: >-
+            100 * (1 -
+              node_filesystem_files_free{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+              /
+              node_filesystem_files{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+            ) > 90
+          for: 15m
+          labels:
+            severity: critical
+            threshold: "90"
+          annotations:
+            summary: 루트 파일시스템 inode 사용률이 90%를 초과함
+            description: 노드 루트 파일시스템 inode 사용률이 심각 임계치인 90%를 15분간 초과했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+    - name: platform.workloads
+      rules:
+        - alert: PlatformPodRestartBurst
+          expr: increase(kube_pod_container_status_restarts_total[15m]) > 3
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: 컨테이너 재시작 급증 감지
+            description: 한 컨테이너가 최근 15분 동안 3회를 초과해 재시작했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformPodOOMKilled
+          expr: kube_pod_container_status_last_terminated_reason{reason="OOMKilled"} == 1
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: 컨테이너가 OOM Killer에 의해 종료됨
+            description: 컨테이너의 최근 종료 사유가 OOMKilled인 상태로 5분간 유지됐습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability
+
+resources:
+  - core-rules.yaml
+  - storage-quota-rules.yaml
+  - certificate-rules.yaml
+  - verified-service-rules.yaml
@@ -0,0 +1,55 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: platform-aistor-storage-quota
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: platform-observability-rules
+    app.kubernetes.io/part-of: observability
+    observability.hyeonworks.com/instance: home
+spec:
+  groups:
+    - name: platform.aistor-storage-quota
+      rules:
+        - record: platform:aistor_bucket_quota_usage_percent
+          expr: >-
+            100
+            * max by (bucket) (
+              minio_cluster_usage_buckets_total_bytes{bucket=~"loki|tempo"}
+            )
+            / max by (bucket) (
+              minio_cluster_usage_buckets_quota_total_bytes{bucket=~"loki|tempo"} > 0
+            )
+        - alert: PlatformAIStorBucketQuotaUsage
+          expr: >-
+            platform:aistor_bucket_quota_usage_percent >= 70
+            and
+            platform:aistor_bucket_quota_usage_percent < 85
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: AIStor 버킷 할당량 사용률 경고
+            description: '{{ $labels.bucket }} 버킷 사용률이 70% 이상 85% 미만으로 15분간 유지됐습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformAIStorBucketQuotaUsage
+          expr: >-
+            platform:aistor_bucket_quota_usage_percent >= 85
+            and
+            platform:aistor_bucket_quota_usage_percent < 95
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: AIStor 버킷 할당량 사용률 심각
+            description: '{{ $labels.bucket }} 버킷 사용률이 85% 이상 95% 미만으로 10분간 유지됐습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformAIStorBucketQuotaUsage
+          expr: platform:aistor_bucket_quota_usage_percent >= 95
+          for: 5m
+          labels:
+            severity: emergency
+          annotations:
+            summary: AIStor 버킷 할당량 사용률 긴급
+            description: '대응 절차에 따라 {{ $labels.bucket }} 버킷의 수집 경로를 중지하고 객체는 자동으로 삭제하지 마세요.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
@@ -0,0 +1,88 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: platform-verified-services
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: platform-observability-rules
+    app.kubernetes.io/part-of: observability
+    observability.hyeonworks.com/instance: home
+spec:
+  groups:
+    - name: platform.verified-services
+      rules:
+        - alert: PlatformCNPGCollectorDown
+          expr: >-
+            min by (cluster, namespace) (
+              cnpg_collector_up{
+                namespace="platform-data",
+                cluster="platform-postgres"
+              }
+            ) == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: CloudNativePG 수집기가 PostgreSQL 중단을 보고함
+            description: '{{ $labels.cluster }} CloudNativePG 수집기가 PostgreSQL 중단 상태를 5분간 보고했습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformAlloyConfigLoadFailed
+          expr: >-
+            min by (namespace) (
+              alloy_config_last_load_successful{namespace="observability-agent"}
+            ) == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: Alloy 설정 로드 실패
+            description: Alloy의 마지막 설정 로드 실패 상태가 5분간 유지됐습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformAlloyLogDeliveryRetries
+          expr: >-
+            sum by (namespace) (
+              increase(loki_write_batch_retries_total{namespace="observability-agent"}[15m])
+            ) > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: Alloy 로그 배치 전송 재시도 발생
+            description: Alloy가 최근 15분 동안 Loki 쓰기 배치를 한 번 이상 재시도했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformAlloyLogEntriesDropped
+          expr: >-
+            sum by (namespace) (
+              increase(loki_write_dropped_entries_total{namespace="observability-agent"}[15m])
+            ) > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: Alloy 로그 항목 유실 발생
+            description: Alloy가 최근 15분 동안 재시도를 모두 소진한 뒤 로그 항목을 유실했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformLokiRuntimeConfigReloadFailed
+          expr: >-
+            min by (namespace) (
+              loki_runtime_config_last_reload_successful{namespace="observability"}
+            ) == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: Loki 런타임 설정 재로드 실패
+            description: Loki의 마지막 런타임 설정 재로드 실패 상태가 5분간 유지됐습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformLokiWALDiskUsageHigh
+          expr: >-
+            max by (namespace) (
+              loki_ingester_wal_disk_usage_percent{namespace="observability"}
+            ) >= 0.8
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: Loki WAL 디스크 사용률 높음
+            description: Loki WAL 디스크 사용률이 80% 이상으로 15분간 유지됐습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md