From 0bb0e0ac49817ce937adf1b03da4ea40570d01a6 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 4 Sep 2026 09:13:41 +0900 Subject: [PATCH] feat: add Prometheus, node-exporter and Grafana for fault-injection observability Co-Authored-By: Claude Opus 5 --- .../console-2026-09-03T07-41-09-675Z.log | 1 - .../console-2026-09-03T07-41-27-913Z.log | 1 - .../console-2026-09-03T07-41-40-247Z.log | 1 - .../console-2026-09-03T07-41-50-414Z.log | 1 - .../console-2026-09-03T07-42-00-948Z.log | 1 - .../page-2026-09-03T07-41-09-984Z.yml | 25 -- .../page-2026-09-03T07-41-27-980Z.yml | 0 .../page-2026-09-03T07-41-40-313Z.yml | 0 .../page-2026-09-03T07-41-50-480Z.yml | 0 .../page-2026-09-03T07-42-01-011Z.yml | 0 deploy/lab/k8s/observability.yaml | 368 ++++++++++++++++++ 11 files changed, 368 insertions(+), 30 deletions(-) delete mode 100644 .playwright-mcp/console-2026-09-03T07-41-09-675Z.log delete mode 100644 .playwright-mcp/console-2026-09-03T07-41-27-913Z.log delete mode 100644 .playwright-mcp/console-2026-09-03T07-41-40-247Z.log delete mode 100644 .playwright-mcp/console-2026-09-03T07-41-50-414Z.log delete mode 100644 .playwright-mcp/console-2026-09-03T07-42-00-948Z.log delete mode 100644 .playwright-mcp/page-2026-09-03T07-41-09-984Z.yml delete mode 100644 .playwright-mcp/page-2026-09-03T07-41-27-980Z.yml delete mode 100644 .playwright-mcp/page-2026-09-03T07-41-40-313Z.yml delete mode 100644 .playwright-mcp/page-2026-09-03T07-41-50-480Z.yml delete mode 100644 .playwright-mcp/page-2026-09-03T07-42-01-011Z.yml create mode 100644 deploy/lab/k8s/observability.yaml diff --git a/.playwright-mcp/console-2026-09-03T07-41-09-675Z.log b/.playwright-mcp/console-2026-09-03T07-41-09-675Z.log deleted file mode 100644 index e06e4d0..0000000 --- a/.playwright-mcp/console-2026-09-03T07-41-09-675Z.log +++ /dev/null @@ -1 +0,0 @@ -[ 295ms] [ERROR] Failed to load resource: the server responded with a status of 401 (Unauthorized) @ https://hyeonworks.com/api/v1/studio/session:0 diff --git a/.playwright-mcp/console-2026-09-03T07-41-27-913Z.log b/.playwright-mcp/console-2026-09-03T07-41-27-913Z.log deleted file mode 100644 index 8fcfb02..0000000 --- a/.playwright-mcp/console-2026-09-03T07-41-27-913Z.log +++ /dev/null @@ -1 +0,0 @@ -[ 125ms] [ERROR] Failed to load resource: the server responded with a status of 401 (Unauthorized) @ https://hyeonworks.com/api/v1/studio/session:0 diff --git a/.playwright-mcp/console-2026-09-03T07-41-40-247Z.log b/.playwright-mcp/console-2026-09-03T07-41-40-247Z.log deleted file mode 100644 index 96bf008..0000000 --- a/.playwright-mcp/console-2026-09-03T07-41-40-247Z.log +++ /dev/null @@ -1 +0,0 @@ -[ 95ms] [ERROR] Failed to load resource: the server responded with a status of 401 (Unauthorized) @ https://hyeonworks.com/api/v1/studio/session:0 diff --git a/.playwright-mcp/console-2026-09-03T07-41-50-414Z.log b/.playwright-mcp/console-2026-09-03T07-41-50-414Z.log deleted file mode 100644 index d9ccac2..0000000 --- a/.playwright-mcp/console-2026-09-03T07-41-50-414Z.log +++ /dev/null @@ -1 +0,0 @@ -[ 91ms] [ERROR] Failed to load resource: the server responded with a status of 401 (Unauthorized) @ https://hyeonworks.com/api/v1/studio/session:0 diff --git a/.playwright-mcp/console-2026-09-03T07-42-00-948Z.log b/.playwright-mcp/console-2026-09-03T07-42-00-948Z.log deleted file mode 100644 index 490cc1e..0000000 --- a/.playwright-mcp/console-2026-09-03T07-42-00-948Z.log +++ /dev/null @@ -1 +0,0 @@ -[ 88ms] [ERROR] Failed to load resource: the server responded with a status of 401 (Unauthorized) @ https://hyeonworks.com/api/v1/studio/session:0 diff --git a/.playwright-mcp/page-2026-09-03T07-41-09-984Z.yml b/.playwright-mcp/page-2026-09-03T07-41-09-984Z.yml deleted file mode 100644 index 997b2a2..0000000 --- a/.playwright-mcp/page-2026-09-03T07-41-09-984Z.yml +++ /dev/null @@ -1,25 +0,0 @@ -- generic [ref=f7e3]: - - link "본문으로 건너뛰기" [ref=f7e4] [cursor=pointer]: - - /url: "#main-content" - - banner [ref=f7e5]: - - generic [ref=f7e6]: - - link "TechLog 홈" [ref=f7e8] [cursor=pointer]: - - /url: / - - text: TechLog - - generic [ref=f7e9]: - - button "TechLog 검색 열기" [ref=f7e11] [cursor=pointer]: 검색 - - group [ref=f7e12]: - - generic "메뉴" [ref=f7e13] [cursor=pointer] - - generic [ref=f7e14]: - - paragraph [ref=f7e15]: 화면을 준비하고 있습니다. - - generic [ref=f7e16]: TechLog 로딩 중 - - contentinfo [ref=f7e17]: - - generic [ref=f7e18]: - - generic [ref=f7e19]: - - paragraph [ref=f7e20]: 동현 - - paragraph [ref=f7e21]: 문제를 재현하고 검증해 실제 운영에 적용할 수 있는 형태로 정리합니다. - - generic [ref=f7e22]: - - link "프로필" [ref=f7e23] [cursor=pointer]: - - /url: /profile - - link "변경 기록" [ref=f7e24] [cursor=pointer]: - - /url: /releases \ No newline at end of file diff --git a/.playwright-mcp/page-2026-09-03T07-41-27-980Z.yml b/.playwright-mcp/page-2026-09-03T07-41-27-980Z.yml deleted file mode 100644 index e69de29..0000000 diff --git a/.playwright-mcp/page-2026-09-03T07-41-40-313Z.yml b/.playwright-mcp/page-2026-09-03T07-41-40-313Z.yml deleted file mode 100644 index e69de29..0000000 diff --git a/.playwright-mcp/page-2026-09-03T07-41-50-480Z.yml b/.playwright-mcp/page-2026-09-03T07-41-50-480Z.yml deleted file mode 100644 index e69de29..0000000 diff --git a/.playwright-mcp/page-2026-09-03T07-42-01-011Z.yml b/.playwright-mcp/page-2026-09-03T07-42-01-011Z.yml deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/lab/k8s/observability.yaml b/deploy/lab/k8s/observability.yaml new file mode 100644 index 0000000..499d2e2 --- /dev/null +++ b/deploy/lab/k8s/observability.yaml @@ -0,0 +1,368 @@ +# Prometheus + node-exporter + Grafana. +# +# Purpose: during a fault-injection experiment, know *which signal moved first*. +# Without a metrics store the only record is whatever scrolled past in a terminal, +# and "the cluster recovered in about a minute" is not a measurement. +# +# kubectl apply -f deploy/lab/k8s/observability.yaml +# kubectl -n observability rollout status deployment/prometheus --timeout=300s +# +# Placement decision — Prometheus and Grafana are pinned to the control-plane +# node (kc-lab-1). An observability stack must not share a failure domain with +# the thing it observes. With only two nodes that cannot be fully avoided, so the +# rule here is: the node that gets killed in experiments is the *agent* +# (kc-lab-2, holding keycloak-0 and postgres), and everything needed to watch +# that happen lives on the server node. +apiVersion: v1 +kind: Namespace +metadata: + name: observability +--- +# Prometheus discovers scrape targets by querying the Kubernetes API, so it +# needs read access to nodes, services, endpoints and pods. Without this the +# kubernetes_sd_configs below silently return no targets. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: observability +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: + - apiGroups: [""] + resources: [nodes, nodes/metrics, services, endpoints, pods] + verbs: [get, list, watch] + - nonResourceURLs: ["/metrics"] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: observability +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: observability +data: + prometheus.yml: | + global: + # 15s is short for production but right here: a node loss should show up + # within a couple of samples, not a minute later. + scrape_interval: 15s + evaluation_interval: 15s + + scrape_configs: + # Prometheus scraping itself. Useful as a control: if this target is down, + # the problem is Prometheus, not the thing being measured. + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] + + # Keycloak. Metrics live on the management port 9000, not 8080 — the same + # split that the health probes use. KC_METRICS_ENABLED=true is already set + # on the StatefulSet. + # + # Discovery is by endpoints rather than a static list because pod IPs + # change on every restart; that was observed directly when the lab was + # power-cycled and every pod came back with a new address. + - job_name: keycloak + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: [keycloak-lab] + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: keycloak-headless;management + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + + # node-exporter, one per node via DaemonSet. This is what answers + # "did the machine die or did the process die". + - job_name: node-exporter + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: [observability] + relabel_configs: + - source_labels: [__meta_kubernetes_service_name] + action: keep + regex: node-exporter + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + + # The kubelet's own metrics, reached through the API server proxy so no + # extra port needs opening. + - job_name: kubelet + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: prometheus-data + namespace: observability +spec: + accessModes: [ReadWriteOnce] + storageClassName: local-path + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: observability +spec: + replicas: 1 + strategy: + type: Recreate # RWO volume; two pods cannot mount it at once + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + serviceAccountName: prometheus + # See the placement note at the top of this file. + nodeSelector: + node-role.kubernetes.io/control-plane: "true" + securityContext: + fsGroup: 65534 # the image runs as nobody and must own the volume + containers: + - name: prometheus + image: prom/prometheus:v3.1.0 + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + # 7 days is far more than an experiment needs and keeps the volume + # small enough that it never becomes the reason a node fills up. + - --storage.tsdb.retention.time=7d + - --web.enable-lifecycle + ports: + - containerPort: 9090 + name: http + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: data + mountPath: /prometheus + readinessProbe: + httpGet: { path: /-/ready, port: http } + initialDelaySeconds: 10 + livenessProbe: + httpGet: { path: /-/healthy, port: http } + initialDelaySeconds: 30 + resources: + requests: { memory: 256Mi, cpu: 50m } + limits: { memory: 640Mi } + volumes: + - name: config + configMap: + name: prometheus-config + - name: data + persistentVolumeClaim: + claimName: prometheus-data +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: observability +spec: + selector: + app: prometheus + ports: + - port: 9090 + targetPort: http +--- +# node-exporter. A DaemonSet so every node reports, including one that is about +# to be killed — the last samples before it goes silent are the interesting part. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-exporter + namespace: observability +spec: + selector: + matchLabels: + app: node-exporter + template: + metadata: + labels: + app: node-exporter + spec: + # Host namespaces: the point is to measure the machine, not the container. + hostNetwork: true + hostPID: true + tolerations: + - operator: Exists # must also run on tainted nodes + containers: + - name: node-exporter + image: prom/node-exporter:v1.8.2 + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + - --path.rootfs=/host/root + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + ports: + - containerPort: 9100 + name: metrics + hostPort: 9100 + volumeMounts: + - { name: proc, mountPath: /host/proc, readOnly: true } + - { name: sys, mountPath: /host/sys, readOnly: true } + - { name: rootfs, mountPath: /host/root, readOnly: true, mountPropagation: HostToContainer } + resources: + requests: { memory: 32Mi, cpu: 20m } + limits: { memory: 96Mi } + volumes: + - { name: proc, hostPath: { path: /proc } } + - { name: sys, hostPath: { path: /sys } } + - { name: rootfs, hostPath: { path: / } } +--- +apiVersion: v1 +kind: Service +metadata: + name: node-exporter + namespace: observability +spec: + clusterIP: None # headless: Prometheus wants each pod, not a VIP + selector: + app: node-exporter + ports: + - port: 9100 + targetPort: metrics + name: metrics +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: observability +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "true" + containers: + - name: grafana + image: grafana/grafana:11.4.0 + ports: + - containerPort: 3000 + name: http + env: + - name: GF_SECURITY_ADMIN_USER + value: admin + - name: GF_SECURITY_ADMIN_PASSWORD + value: lab-grafana-change-me + # Grafana builds absolute URLs for redirects and asset paths. Behind + # the nginx -> Traefik chain it must be told the external address, + # for exactly the reason Keycloak needs KC_HOSTNAME. Without it, + # login redirects come back as http://:3000. + - name: GF_SERVER_ROOT_URL + value: https://app2.hyeonworks.com + volumeMounts: + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + readinessProbe: + httpGet: { path: /api/health, port: http } + initialDelaySeconds: 15 + resources: + requests: { memory: 128Mi, cpu: 50m } + limits: { memory: 320Mi } + volumes: + - name: datasources + configMap: + name: grafana-datasources +--- +# Provisioning the datasource as a file means Grafana comes up already wired to +# Prometheus. Clicking through the UI would leave the configuration only in +# Grafana's own database, which is emptyDir here and disappears on restart. +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + namespace: observability +data: + prometheus.yaml: | + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus.observability.svc:9090 + isDefault: true +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: observability +spec: + selector: + app: grafana + ports: + - port: 3000 + targetPort: http +--- +# Grafana is published on app2.hyeonworks.com because that name is already in +# the wildcard-free certificate (auth / app1 / app2) and is otherwise unused. +# It moves when app2 is needed for the SSO experiment. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana + namespace: observability +spec: + ingressClassName: traefik + rules: + - host: app2.hyeonworks.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana + port: + number: 3000