document-haness/docs/keycloak-session-store/source/deploy/lab/k8s/observability.yaml

# Prometheus + node-exporter + Grafana.
#
# Purpose: during a fault-injection experiment, know *which signal moved first*.
# Without a metrics store the only record is whatever scrolled past in a terminal,
# and "the cluster recovered in about a minute" is not a measurement.
#
#   kubectl apply -f deploy/lab/k8s/observability.yaml
#   kubectl -n observability rollout status deployment/prometheus --timeout=300s
#
# Placement decision — Prometheus and Grafana are pinned to the control-plane
# node (kc-lab-1). An observability stack must not share a failure domain with
# the thing it observes. With only two nodes that cannot be fully avoided, so the
# rule here is: the node that gets killed in experiments is the *agent*
# (kc-lab-2, holding keycloak-0 and postgres), and everything needed to watch
# that happen lives on the server node.
apiVersion: v1
kind: Namespace
metadata:
  name: observability
---
# Prometheus discovers scrape targets by querying the Kubernetes API, so it
# needs read access to nodes, services, endpoints and pods. Without this the
# kubernetes_sd_configs below silently return no targets.
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: observability
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
  - apiGroups: [""]
    # nodes/proxy is required in addition to nodes/metrics: the kubelet job
    # reaches each node through the API server's proxy subresource
    # (/api/v1/nodes/<name>/proxy/metrics). Without it every kubelet target
    # fails with 403 Forbidden while the other jobs stay green — a partial
    # failure that is easy to miss unless the target list is checked.
    resources: [nodes, nodes/metrics, nodes/proxy, services, endpoints, pods]
    verbs: [get, list, watch]
  - nonResourceURLs: ["/metrics"]
    verbs: [get]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
  - kind: ServiceAccount
    name: prometheus
    namespace: observability
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: observability
data:
  prometheus.yml: |
    global:
      # 15s is short for production but right here: a node loss should show up
      # within a couple of samples, not a minute later.
      scrape_interval: 15s
      evaluation_interval: 15s

    scrape_configs:
      # Prometheus scraping itself. Useful as a control: if this target is down,
      # the problem is Prometheus, not the thing being measured.
      - job_name: prometheus
        static_configs:
          - targets: ['localhost:9090']

      # Keycloak. Metrics live on the management port 9000, not 8080 — the same
      # split that the health probes use. KC_METRICS_ENABLED=true is already set
      # on the StatefulSet.
      #
      # Discovery is by endpoints rather than a static list because pod IPs
      # change on every restart; that was observed directly when the lab was
      # power-cycled and every pod came back with a new address.
      - job_name: keycloak
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names: [keycloak-lab]
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
            action: keep
            regex: keycloak-headless;management
          - source_labels: [__meta_kubernetes_pod_name]
            target_label: pod
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node

      # node-exporter, one per node via DaemonSet. This is what answers
      # "did the machine die or did the process die".
      - job_name: node-exporter
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names: [observability]
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_name]
            action: keep
            regex: node-exporter
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node

      # The kubelet's own metrics, reached through the API server proxy so no
      # extra port needs opening.
      - job_name: kubelet
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
          - role: node
        relabel_configs:
          - action: labelmap
            regex: __meta_kubernetes_node_label_(.+)
          - target_label: __address__
            replacement: kubernetes.default.svc:443
          - source_labels: [__meta_kubernetes_node_name]
            regex: (.+)
            target_label: __metrics_path__
            replacement: /api/v1/nodes/${1}/proxy/metrics
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: prometheus-data
  namespace: observability
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: local-path
  resources:
    requests:
      storage: 5Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: observability
spec:
  replicas: 1
  strategy:
    type: Recreate            # RWO volume; two pods cannot mount it at once
  selector:
    matchLabels:
      app: prometheus
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      serviceAccountName: prometheus
      # See the placement note at the top of this file.
      nodeSelector:
        node-role.kubernetes.io/control-plane: "true"
      securityContext:
        fsGroup: 65534        # the image runs as nobody and must own the volume
      containers:
        - name: prometheus
          image: prom/prometheus:v3.1.0
          args:
            - --config.file=/etc/prometheus/prometheus.yml
            - --storage.tsdb.path=/prometheus
            # 7 days is far more than an experiment needs and keeps the volume
            # small enough that it never becomes the reason a node fills up.
            - --storage.tsdb.retention.time=7d
            - --web.enable-lifecycle
          ports:
            - containerPort: 9090
              name: http
          volumeMounts:
            - name: config
              mountPath: /etc/prometheus
            - name: data
              mountPath: /prometheus
          readinessProbe:
            httpGet: { path: /-/ready, port: http }
            initialDelaySeconds: 10
          livenessProbe:
            httpGet: { path: /-/healthy, port: http }
            initialDelaySeconds: 30
          resources:
            requests: { memory: 256Mi, cpu: 50m }
            limits:   { memory: 640Mi }
      volumes:
        - name: config
          configMap:
            name: prometheus-config
        - name: data
          persistentVolumeClaim:
            claimName: prometheus-data
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: observability
spec:
  selector:
    app: prometheus
  ports:
    - port: 9090
      targetPort: http
---
# node-exporter. A DaemonSet so every node reports, including one that is about
# to be killed — the last samples before it goes silent are the interesting part.
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-exporter
  namespace: observability
spec:
  selector:
    matchLabels:
      app: node-exporter
  template:
    metadata:
      labels:
        app: node-exporter
    spec:
      # Host namespaces: the point is to measure the machine, not the container.
      hostNetwork: true
      hostPID: true
      tolerations:
        - operator: Exists      # must also run on tainted nodes
      containers:
        - name: node-exporter
          image: prom/node-exporter:v1.8.2
          args:
            - --path.procfs=/host/proc
            - --path.sysfs=/host/sys
            - --path.rootfs=/host/root
            - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/)
          ports:
            - containerPort: 9100
              name: metrics
              hostPort: 9100
          volumeMounts:
            - { name: proc,   mountPath: /host/proc, readOnly: true }
            - { name: sys,    mountPath: /host/sys,  readOnly: true }
            - { name: rootfs, mountPath: /host/root, readOnly: true, mountPropagation: HostToContainer }
          resources:
            requests: { memory: 32Mi, cpu: 20m }
            limits:   { memory: 96Mi }
      volumes:
        - { name: proc,   hostPath: { path: /proc } }
        - { name: sys,    hostPath: { path: /sys } }
        - { name: rootfs, hostPath: { path: / } }
---
apiVersion: v1
kind: Service
metadata:
  name: node-exporter
  namespace: observability
spec:
  clusterIP: None             # headless: Prometheus wants each pod, not a VIP
  selector:
    app: node-exporter
  ports:
    - port: 9100
      targetPort: metrics
      name: metrics
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: grafana
  namespace: observability
spec:
  replicas: 1
  selector:
    matchLabels:
      app: grafana
  template:
    metadata:
      labels:
        app: grafana
    spec:
      nodeSelector:
        node-role.kubernetes.io/control-plane: "true"
      containers:
        - name: grafana
          image: grafana/grafana:11.4.0
          ports:
            - containerPort: 3000
              name: http
          env:
            - name: GF_SECURITY_ADMIN_USER
              value: admin
            - name: GF_SECURITY_ADMIN_PASSWORD
              value: lab-grafana-change-me
            # Grafana builds absolute URLs for redirects and asset paths. Behind
            # the nginx -> Traefik chain it must be told the external address,
            # for exactly the reason Keycloak needs KC_HOSTNAME. Without it,
            # login redirects come back as http://<pod-ip>:3000.
            - name: GF_SERVER_ROOT_URL
              value: https://app2.hyeonworks.com
          volumeMounts:
            - name: datasources
              mountPath: /etc/grafana/provisioning/datasources
          readinessProbe:
            httpGet: { path: /api/health, port: http }
            initialDelaySeconds: 15
          resources:
            requests: { memory: 128Mi, cpu: 50m }
            limits:   { memory: 320Mi }
      volumes:
        - name: datasources
          configMap:
            name: grafana-datasources
---
# Provisioning the datasource as a file means Grafana comes up already wired to
# Prometheus. Clicking through the UI would leave the configuration only in
# Grafana's own database, which is emptyDir here and disappears on restart.
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-datasources
  namespace: observability
data:
  prometheus.yaml: |
    apiVersion: 1
    datasources:
      - name: Prometheus
        type: prometheus
        access: proxy
        url: http://prometheus.observability.svc:9090
        isDefault: true
---
apiVersion: v1
kind: Service
metadata:
  name: grafana
  namespace: observability
spec:
  selector:
    app: grafana
  ports:
    - port: 3000
      targetPort: http
---
# Grafana is published on app2.hyeonworks.com because that name is already in
# the wildcard-free certificate (auth / app1 / app2) and is otherwise unused.
# It moves when app2 is needed for the SSO experiment.
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: grafana
  namespace: observability
spec:
  ingressClassName: traefik
  rules:
    - host: app2.hyeonworks.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: grafana
                port:
                  number: 3000