Add platform infrastructure configuration

2026-08-28 17:35:41 +09:00
parent fa76531e5b
commit 16c337bcc9
302 changed files with 83259 additions and 1 deletions
@@ -0,0 +1,18 @@
+# Observability core
+
+This root composes the internal observability workloads deployed after the
+namespace, CRD, and Local PV prerequisites:
+
+- kube-prometheus-stack without Grafana or its bundled node-exporter;
+- single-binary Loki and Tempo using isolated AIStor buckets;
+- Alloy for Pod logs and internal OTLP/gRPC traces;
+- a standalone node-exporter;
+- the exact core NetworkPolicy matrix, platform rules, scrape targets, verified
+  dashboards, and source-only Alertmanager Slack routing.
+
+It intentionally does not create public ingress, Grafana, OIDC, or credential
+Secrets. The Slack routing source references
+`alertmanager-slack-webhook/url`; while that Secret is absent, do not apply the
+AlertmanagerConfig, its kube-prometheus-stack reference, its public HTTPS
+NetworkPolicy, or alert rules. Namespace, CRD, and Local PV roots remain outside
+this aggregate because their protected apply order is different.
@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-alertmanager-public-https
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: alertmanager
+      app.kubernetes.io/instance: observability-core-kube-pr-alertmanager
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except:
+              - 10.0.0.0/8
+              - 100.64.0.0/10
+              - 172.16.0.0/12
+              - 192.168.0.0/16
+      ports:
+        - protocol: TCP
+          port: 443
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability
+
+resources:
+  - platform-alertmanager.yaml
+  - alertmanager-public-https-egress.yaml
@@ -0,0 +1,80 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: platform-alertmanager
+  namespace: observability
+  labels:
+    observability.hyeonworks.com/instance: home
+spec:
+  route:
+    receiver: platform-slack
+    groupBy:
+      - cluster
+      - namespace
+      - alertname
+      - severity
+    groupWait: 30s
+    groupInterval: 5m
+    repeatInterval: 4h
+    routes:
+      - receiver: platform-null
+        matchers:
+          - name: alertname
+            matchType: "="
+            value: InfoInhibitor
+  inhibitRules:
+    - sourceMatch:
+        - name: alertname
+          matchType: "="
+          value: InfoInhibitor
+      targetMatch:
+        - name: severity
+          matchType: "="
+          value: info
+      equal:
+        - namespace
+  receivers:
+    - name: platform-null
+    - name: platform-slack
+      slackConfigs:
+        - apiURL:
+            name: alertmanager-slack-webhook
+            key: url
+          sendResolved: true
+          linkNames: false
+          mrkdwnIn:
+            - text
+            - fields
+          fallback: >-
+            {{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}
+          title: >-
+            {{ if eq .Status "firing" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}
+          titleLink: https://grafana.learn.hyeonworks.com/
+          color: >-
+            {{ if eq .Status "resolved" }}good{{ else if or (eq .CommonLabels.severity "emergency") (eq .CommonLabels.severity "critical") }}danger{{ else if eq .CommonLabels.severity "warning" }}warning{{ else }}#439FE0{{ end }}
+          fields:
+            - title: Status
+              value: '{{ .Status | toUpper }}'
+              short: true
+            - title: Severity
+              value: '{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}'
+              short: true
+            - title: Location
+              value: '{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}'
+              short: true
+            - title: Alert count
+              value: '{{ len .Alerts }}'
+              short: true
+          text: |-
+            {{ range .Alerts }}
+            *Alert status:* {{ .Status | toUpper }}
+            *Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}
+            *Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}
+            *Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}
+            *Started:* {{ .StartsAt.Format "2006-01-02T15:04:05Z07:00" }}
+            {{ if eq .Status "resolved" }}*Ended:* {{ .EndsAt.Format "2006-01-02T15:04:05Z07:00" }}{{ end }}
+            {{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}
+
+            {{ end }}
+            <https://grafana.learn.hyeonworks.com/|Grafana> · <https://grafana.learn.hyeonworks.com/explore|Explore>
+          footer: hyeonworks observability · Alertmanager
@@ -0,0 +1,12 @@
+# Alloy
+
+Grafana Alloy runs as a single-node DaemonSet in `observability-agent`.
+It tails only `/var/log/pods`, writes logs to the in-cluster Loki service, and
+accepts OTLP/gRPC traces on port `4317` for the in-cluster Tempo service.
+
+The deployment deliberately disables host namespaces, automatic service
+account token mounts, CRD creation, the config reloader, public exposure, and
+persistent OpenTelemetry queues. The 2 GiB PVC stores only Alloy file
+positions and local state. Alloy keeps primary UID/GID `473:473`; supplemental
+group `0` permits read-only traversal of k3s' `root:root 0750` Pod log tree
+without changing host permissions or running the container as root.
@@ -0,0 +1,214 @@
+discovery.kubernetes "pods" {
+  role = "pod"
+
+  selectors {
+    role  = "pod"
+    field = "spec.nodeName=" + sys.env("HOSTNAME")
+  }
+}
+
+discovery.relabel "pod_logs" {
+  targets = discovery.kubernetes.pods.targets
+
+  rule {
+    source_labels = ["__meta_kubernetes_namespace"]
+    target_label  = "namespace"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_name"]
+    target_label  = "pod"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_container_name"]
+    target_label  = "container"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+    regex         = "(.+)"
+    target_label  = "app"
+  }
+  rule {
+    source_labels = [
+      "__meta_kubernetes_pod_label_app_kubernetes_io_name",
+      "__meta_kubernetes_pod_label_app",
+    ]
+    separator   = ";"
+    regex       = "^;(.+)$"
+    replacement = "$1"
+    target_label = "app"
+  }
+  rule {
+    source_labels = [
+      "__meta_kubernetes_pod_uid",
+      "__meta_kubernetes_pod_container_name",
+    ]
+    separator   = "/"
+    replacement = "/var/log/pods/*$1/*.log"
+    target_label = "__path__"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_container_id"]
+    regex         = `^(\S+):\/\/.+$`
+    replacement   = "$1"
+    target_label  = "container_runtime"
+  }
+}
+
+local.file_match "pod_logs" {
+  path_targets = discovery.relabel.pod_logs.output
+}
+
+loki.source.file "pod_logs" {
+  targets    = local.file_match.pod_logs.targets
+  forward_to = [loki.process.pod_logs.receiver]
+}
+
+loki.process "pod_logs" {
+  stage.match {
+    selector = `{container_runtime=~"containerd|cri-o"}`
+    stage.cri {}
+  }
+  stage.match {
+    selector = `{container_runtime="docker"}`
+    stage.docker {}
+  }
+
+  stage.replace {
+    expression = `(?i)"(?:authorization|bearer|jwt|password|passwd|client[_-]?secret)"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"`
+    replace    = "[REDACTED]"
+  }
+  stage.replace {
+    expression = `(?i)(?:authorization\s*[:=]\s*)(?:bearer\s+)?([A-Za-z0-9._~+/=-]+)`
+    replace    = "[REDACTED]"
+  }
+  stage.replace {
+    expression = `(?i)\bbearer\s+([A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+){0,2})`
+    replace    = "[REDACTED]"
+  }
+  stage.replace {
+    expression = `(?i)(?:password|passwd|client[_-]?secret)\s*[:=]\s*([^\s"'{},]+)`
+    replace    = "[REDACTED]"
+  }
+
+  stage.json {
+    expressions = {
+      level = "level",
+    }
+  }
+  stage.labels {
+    values = {
+      level = "",
+    }
+  }
+  stage.match {
+    selector = `{level!~"(?i)^(trace|debug|info|warn|warning|error|fatal|panic)$"}`
+    stage.label_drop {
+      values = ["level"]
+    }
+  }
+  stage.static_labels {
+    values = {
+      cluster = "home",
+    }
+  }
+  stage.label_keep {
+    values = ["cluster", "namespace", "app", "pod", "container", "level"]
+  }
+
+  forward_to = [loki.write.local.receiver]
+}
+
+loki.write "local" {
+  endpoint {
+    url                 = "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push"
+    batch_size          = "256KiB"
+    batch_wait          = "1s"
+    remote_timeout      = "10s"
+    min_backoff_period  = "500ms"
+    max_backoff_period  = "30s"
+    max_backoff_retries = 10
+  }
+}
+
+otelcol.receiver.otlp "traces" {
+  grpc {
+    endpoint = "0.0.0.0:4317"
+  }
+
+  output {
+    traces = [otelcol.processor.k8sattributes.traces.input]
+  }
+}
+
+otelcol.processor.k8sattributes "traces" {
+  auth_type = "serviceAccount"
+
+  extract {
+    deployment_name_from_replicaset = false
+    metadata = [
+      "k8s.namespace.name",
+      "k8s.deployment.name",
+      "k8s.node.name",
+      "k8s.pod.name",
+      "k8s.pod.uid",
+      "k8s.pod.start_time",
+      "k8s.container.name",
+    ]
+  }
+
+  pod_association {
+    source {
+      from = "resource_attribute"
+      name = "k8s.pod.uid"
+    }
+  }
+  pod_association {
+    source {
+      from = "resource_attribute"
+      name = "k8s.pod.ip"
+    }
+  }
+  pod_association {
+    source {
+      from = "connection"
+    }
+  }
+
+  output {
+    traces = [otelcol.processor.batch.traces.input]
+  }
+}
+
+otelcol.processor.batch "traces" {
+  timeout             = "1s"
+  send_batch_size     = 512
+  send_batch_max_size = 1024
+
+  output {
+    traces = [otelcol.exporter.otlp.tempo.input]
+  }
+}
+
+otelcol.exporter.otlp "tempo" {
+  client {
+    endpoint = "tempo.observability.svc.cluster.local:4317"
+    tls {
+      insecure = true
+    }
+  }
+
+  retry_on_failure {
+    enabled          = true
+    initial_interval = "1s"
+    max_interval     = "15s"
+    max_elapsed_time = "5m"
+  }
+
+  sending_queue {
+    enabled           = true
+    queue_size        = 256
+    num_consumers     = 2
+    sizer             = "requests"
+    block_on_overflow = true
+  }
+}
@@ -0,0 +1,51 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability-agent
+
+resources:
+  - pvc.yaml
+
+configMapGenerator:
+  - name: alloy-config
+    files:
+      - config.alloy=config.alloy
+
+helmGlobals:
+  chartHome: .helm/charts
+
+helmCharts:
+  - name: alloy
+    repo: https://grafana.github.io/helm-charts
+    version: 1.11.0
+    releaseName: alloy
+    namespace: observability-agent
+    valuesFile: values/home.yaml
+    includeCRDs: false
+    skipTests: true
+
+images:
+  - name: docker.io/grafana/alloy
+    newName: docker.io/grafana/alloy
+    digest: sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17
+
+patches:
+  - target:
+      group: apps
+      version: v1
+      kind: DaemonSet
+      name: alloy
+    patch: |-
+      - op: add
+        path: /spec/template/spec/automountServiceAccountToken
+        value: false
+      - op: add
+        path: /spec/template/spec/hostNetwork
+        value: false
+      - op: add
+        path: /spec/template/spec/hostPID
+        value: false
+      - op: add
+        path: /spec/template/spec/hostIPC
+        value: false
+
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: alloy-storage
+  namespace: observability-agent
+  labels:
+    app.kubernetes.io/name: alloy
+    observability.hyeonworks.com/instance: home
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: ssd-local-observability-alloy-retain
+  volumeName: observability-alloy-local-pv
+  resources:
+    requests:
+      storage: 2Gi
+
@@ -0,0 +1,155 @@
+crds:
+  create: false
+
+global:
+  podSecurityContext:
+    runAsNonRoot: true
+    runAsUser: 473
+    runAsGroup: 473
+    fsGroup: 473
+    fsGroupChangePolicy: OnRootMismatch
+    # k3s keeps /var/log/pods root:root 0750. Retain Alloy's non-root primary
+    # identity while granting read-only traversal of that single hostPath.
+    supplementalGroups:
+      - 0
+    seccompProfile:
+      type: RuntimeDefault
+
+alloy:
+  configMap:
+    create: false
+    name: alloy-config
+    key: config.alloy
+  clustering:
+    enabled: false
+  stabilityLevel: generally-available
+  storagePath: /var/lib/alloy
+  enableHttpServerPort: true
+  listenAddr: 0.0.0.0
+  listenPort: 12345
+  enableReporting: false
+  extraPorts:
+    - name: otlp-grpc
+      port: 4317
+      targetPort: 4317
+      protocol: TCP
+  mounts:
+    varlog: false
+    dockercontainers: false
+    extra:
+      - name: pod-logs
+        mountPath: /var/log/pods
+        readOnly: true
+      - name: alloy-storage
+        mountPath: /var/lib/alloy
+      - name: tmp
+        mountPath: /tmp
+      - name: kubernetes-api-access
+        mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+        readOnly: true
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 473
+    runAsGroup: 473
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop:
+        - ALL
+    seccompProfile:
+      type: RuntimeDefault
+  resources:
+    requests:
+      cpu: 200m
+      memory: 384Mi
+    limits:
+      cpu: "1"
+      memory: 768Mi
+
+image:
+  registry: docker.io
+  repository: grafana/alloy
+  tag: v1.18.0
+  digest: sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17
+  pullPolicy: IfNotPresent
+
+rbac:
+  create: true
+  namespaces: []
+  rules:
+    - apiGroups:
+        - ""
+      resources:
+        - pods
+        - namespaces
+      verbs:
+        - get
+        - list
+        - watch
+  clusterRules:
+    - apiGroups:
+        - apps
+      resources:
+        - replicasets
+      verbs:
+        - get
+        - list
+        - watch
+
+serviceAccount:
+  create: true
+  name: alloy
+  automountServiceAccountToken: false
+
+configReloader:
+  enabled: false
+
+controller:
+  type: daemonset
+  hostNetwork: false
+  hostPID: false
+  volumes:
+    extra:
+      - name: pod-logs
+        hostPath:
+          path: /var/log/pods
+          type: Directory
+      - name: alloy-storage
+        persistentVolumeClaim:
+          claimName: alloy-storage
+      - name: tmp
+        emptyDir: {}
+      - name: kubernetes-api-access
+        projected:
+          defaultMode: 420
+          sources:
+            - serviceAccountToken:
+                expirationSeconds: 3600
+                path: token
+            - configMap:
+                name: kube-root-ca.crt
+                items:
+                  - key: ca.crt
+                    path: ca.crt
+            - downwardAPI:
+                items:
+                  - path: namespace
+                    fieldRef:
+                      apiVersion: v1
+                      fieldPath: metadata.namespace
+
+networkPolicy:
+  enabled: false
+
+service:
+  enabled: true
+  type: ClusterIP
+
+serviceMonitor:
+  enabled: true
+  interval: 30s
+  additionalLabels:
+    observability.hyeonworks.com/instance: home
+
+ingress:
+  enabled: false
@@ -0,0 +1,22 @@
+# Blackbox exporter
+
+This component deploys the pinned Prometheus blackbox exporter as an internal
+`ClusterIP` service in the `observability` namespace. The `/probe` handler is
+reachable only from the Prometheus Pod; it has no Ingress, NodePort, host port,
+or service-account token.
+
+Only three fixed `Probe` groups are declared:
+
+- `platform-public-edge`: Gitea health and Keycloak discovery through Host Nginx
+- `platform-private-edge`: Grafana, pgAdmin, and AIStor Console must return 403
+- `platform-private-internal`: the same private services must return 200 through
+  their ClusterIP Services
+
+The internal Grafana and pgAdmin targets use the blackbox exporter's `hostname`
+parameter to set their required HTTP Host header. Target URLs and modules are
+declarative; arbitrary callers are not granted access to port 9115.
+
+The private-edge source boundary was proven separately and recorded as a
+root-only `platform-blackbox-source-v1` evidence file. Applying this component
+must remain gated on that evidence matching the active deny-only Nginx config.
+
@@ -0,0 +1,45 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability
+
+resources:
+  - probes-public.yaml
+  - probes-private-edge.yaml
+  - probes-private-internal.yaml
+  - network-policy.yaml
+
+helmGlobals:
+  chartHome: .helm/charts
+
+helmCharts:
+  - name: prometheus-blackbox-exporter
+    repo: https://prometheus-community.github.io/helm-charts
+    version: 11.16.0
+    releaseName: blackbox-exporter
+    namespace: observability
+    valuesFile: values/home.yaml
+    includeCRDs: false
+    skipTests: true
+
+patches:
+  - target:
+      version: v1
+      kind: ServiceAccount
+      name: blackbox-exporter
+    patch: |-
+      - op: add
+        path: /automountServiceAccountToken
+        value: false
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      name: blackbox-exporter
+    patch: |-
+      - op: add
+        path: /spec/template/spec/hostPID
+        value: false
+      - op: add
+        path: /spec/template/spec/hostIPC
+        value: false
@@ -0,0 +1,88 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-prometheus-to-blackbox
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: prometheus-blackbox-exporter
+    app.kubernetes.io/part-of: platform-observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: prometheus-blackbox-exporter
+      app.kubernetes.io/instance: blackbox-exporter
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+              app.kubernetes.io/instance: observability-core-kube-pr-prometheus
+      ports:
+        - protocol: TCP
+          port: 9115
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-blackbox-egress
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: prometheus-blackbox-exporter
+    app.kubernetes.io/part-of: platform-observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: prometheus-blackbox-exporter
+      app.kubernetes.io/instance: blackbox-exporter
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+    - to:
+        - ipBlock:
+            cidr: 192.168.0.107/32
+      ports:
+        - protocol: TCP
+          port: 443
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: grafana
+              app.kubernetes.io/instance: grafana
+      ports:
+        - protocol: TCP
+          port: 3000
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: platform-admin
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: pgadmin4
+              app.kubernetes.io/instance: pgadmin
+      ports:
+        - protocol: TCP
+          port: 5050
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: object-storage
+          podSelector:
+            matchLabels:
+              aistor.min.io/objectStore: minio-aistor
+      ports:
+        - protocol: TCP
+          port: 9090
@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Probe
+metadata:
+  name: platform-private-edge
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: prometheus-blackbox-exporter
+    app.kubernetes.io/instance: blackbox-exporter
+    app.kubernetes.io/part-of: platform-observability
+    observability.hyeonworks.com/instance: home
+spec:
+  jobName: blackbox-private-edge
+  interval: 30s
+  scrapeTimeout: 15s
+  module: http_private_edge_403
+  prober:
+    url: blackbox-exporter.observability.svc.cluster.local:9115
+    scheme: http
+    path: /probe
+    proxyFromEnvironment: false
+  targets:
+    staticConfig:
+      labels:
+        observability.hyeonworks.com/probe-group: private-edge
+      static:
+        - https://grafana.learn.hyeonworks.com/
+        - https://storage-admin.learn.hyeonworks.com/
+        - https://db-admin.learn.hyeonworks.com/
@@ -0,0 +1,44 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Probe
+metadata:
+  name: platform-private-internal
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: prometheus-blackbox-exporter
+    app.kubernetes.io/instance: blackbox-exporter
+    app.kubernetes.io/part-of: platform-observability
+    observability.hyeonworks.com/instance: home
+spec:
+  jobName: blackbox-private-internal
+  interval: 30s
+  scrapeTimeout: 15s
+  module: http_private_internal_200
+  prober:
+    url: blackbox-exporter.observability.svc.cluster.local:9115
+    scheme: http
+    path: /probe
+    proxyFromEnvironment: false
+  targets:
+    staticConfig:
+      labels:
+        observability.hyeonworks.com/probe-group: private-internal
+      static:
+        - http://grafana.observability.svc.cluster.local/api/health
+        - http://pgadmin.platform-admin.svc.cluster.local/misc/ping
+        - http://minio-aistor-console.object-storage.svc.cluster.local:9090/
+      relabelingConfigs:
+        - sourceLabels:
+            - __param_target
+          regex: http://grafana[.]observability[.]svc[.]cluster[.]local/api/health
+          targetLabel: __param_hostname
+          replacement: grafana.learn.hyeonworks.com
+        - sourceLabels:
+            - __param_target
+          regex: http://pgadmin[.]platform-admin[.]svc[.]cluster[.]local/misc/ping
+          targetLabel: __param_hostname
+          replacement: db-admin.learn.hyeonworks.com
+        - sourceLabels:
+            - __param_target
+          regex: http://minio-aistor-console[.]object-storage[.]svc[.]cluster[.]local:9090/
+          targetLabel: __param_hostname
+          replacement: storage-admin.learn.hyeonworks.com
@@ -0,0 +1,27 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Probe
+metadata:
+  name: platform-public-edge
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: prometheus-blackbox-exporter
+    app.kubernetes.io/instance: blackbox-exporter
+    app.kubernetes.io/part-of: platform-observability
+    observability.hyeonworks.com/instance: home
+spec:
+  jobName: blackbox-public-edge
+  interval: 30s
+  scrapeTimeout: 15s
+  module: http_2xx
+  prober:
+    url: blackbox-exporter.observability.svc.cluster.local:9115
+    scheme: http
+    path: /probe
+    proxyFromEnvironment: false
+  targets:
+    staticConfig:
+      labels:
+        observability.hyeonworks.com/probe-group: public-edge
+      static:
+        - https://git.learn.hyeonworks.com/api/healthz
+        - https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration
@@ -0,0 +1,128 @@
+fullnameOverride: blackbox-exporter
+
+commonLabels:
+  observability.hyeonworks.com/instance: home
+
+kind: Deployment
+replicas: 1
+revisionHistoryLimit: 3
+
+strategy:
+  type: Recreate
+  rollingUpdate: null
+
+automountServiceAccountToken: false
+hostNetwork: false
+hostPort: 0
+
+image:
+  registry: quay.io
+  repository: prometheus/blackbox-exporter
+  tag: v0.28.0
+  digest: sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68
+  pullPolicy: IfNotPresent
+
+podSecurityContext:
+  runAsNonRoot: true
+  runAsUser: 1000
+  runAsGroup: 1000
+  seccompProfile:
+    type: RuntimeDefault
+
+securityContext:
+  runAsNonRoot: true
+  runAsUser: 1000
+  runAsGroup: 1000
+  readOnlyRootFilesystem: true
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+      - ALL
+  seccompProfile:
+    type: RuntimeDefault
+
+resources:
+  requests:
+    cpu: 25m
+    memory: 32Mi
+  limits:
+    cpu: 200m
+    memory: 128Mi
+
+serviceAccount:
+  create: true
+  name: blackbox-exporter
+
+service:
+  type: ClusterIP
+  port: 9115
+
+ingress:
+  enabled: false
+
+route:
+  main:
+    enabled: false
+
+networkPolicy:
+  enabled: false
+
+configReloader:
+  enabled: false
+
+serviceMonitor:
+  enabled: false
+  selfMonitor:
+    enabled: true
+    port: http
+    interval: 30s
+    scrapeTimeout: 10s
+
+config:
+  modules:
+    http_2xx:
+      prober: http
+      timeout: 10s
+      http:
+        method: GET
+        valid_status_codes:
+          - 200
+        valid_http_versions:
+          - HTTP/1.1
+          - HTTP/2.0
+        follow_redirects: true
+        enable_http2: true
+        preferred_ip_protocol: ip4
+        ip_protocol_fallback: false
+        tls_config:
+          insecure_skip_verify: false
+    http_private_edge_403:
+      prober: http
+      timeout: 10s
+      http:
+        method: GET
+        valid_status_codes:
+          - 403
+        valid_http_versions:
+          - HTTP/1.1
+          - HTTP/2.0
+        follow_redirects: true
+        enable_http2: true
+        preferred_ip_protocol: ip4
+        ip_protocol_fallback: false
+        tls_config:
+          insecure_skip_verify: false
+    http_private_internal_200:
+      prober: http
+      timeout: 10s
+      http:
+        method: GET
+        valid_status_codes:
+          - 200
+        valid_http_versions:
+          - HTTP/1.1
+          - HTTP/2.0
+        follow_redirects: true
+        enable_http2: true
+        preferred_ip_protocol: ip4
+        ip_protocol_fallback: false
@@ -0,0 +1,208 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "max": 1,
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "min by (job, instance) (probe_success{})",
+          "legendFormat": "{{job}} / {{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "HTTPS Probe Success",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "max by (job, instance) (probe_duration_seconds{})",
+          "legendFormat": "{{job}} / {{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "HTTPS Probe Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "max by (instance) (probe_http_status_code{job=\"blackbox-private-edge\"})",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Observed HTTP Status (403 Is Expected at the Private Edge)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "d"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "(probe_ssl_earliest_cert_expiry{} - time()) / 86400",
+          "legendFormat": "{{job}} / {{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "TLS Certificate Days Remaining",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 42,
+  "tags": [
+    "platform",
+    "observability",
+    "https"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Platform / HTTPS Endpoints",
+  "uid": "platform-https-endpoints",
+  "version": 1
+}
@@ -0,0 +1,266 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "max": 100,
+          "min": 0,
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])))",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Node CPU Used",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "max": 100,
+          "min": 0,
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - node_memory_MemAvailable_bytes{} / node_memory_MemTotal_bytes{})",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Node Memory Used",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "max": 100,
+          "min": 0,
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})",
+          "legendFormat": "{{instance}} {{device}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Root Filesystem Used",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "max": 100,
+          "min": 0,
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - node_filesystem_files_free{mountpoint=\"/\"} / node_filesystem_files{mountpoint=\"/\"})",
+          "legendFormat": "{{instance}} {{device}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Root Filesystem Inodes Used",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!=\"lo\"}[5m]))",
+          "legendFormat": "{{instance}} receive",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]))",
+          "legendFormat": "{{instance}} transmit",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Node Network Throughput",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 42,
+  "tags": [
+    "platform",
+    "observability",
+    "node"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Platform / Kubernetes Node",
+  "uid": "platform-kubernetes-node",
+  "version": 1
+}
@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability
+
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    grafana_dashboard: "1"
+    observability.hyeonworks.com/instance: home
+    observability.hyeonworks.com/owner: platform-observability
+
+configMapGenerator:
+  - name: grafana-dashboard-kubernetes-node
+    files:
+      - kubernetes-node.json
+  - name: grafana-dashboard-workload-health
+    files:
+      - workload-health.json
+  - name: grafana-dashboard-platform-services
+    files:
+      - platform-services.json
+  - name: grafana-dashboard-observability-backends
+    files:
+      - observability-backends.json
+  - name: grafana-dashboard-https-endpoints
+    files:
+      - https-endpoints.json
@@ -0,0 +1,384 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "max(prometheus_tsdb_head_series{})",
+          "legendFormat": "head series",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(prometheus_engine_queries{})",
+          "legendFormat": "active queries",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Prometheus Head Series and Active Queries",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(prometheus_notifications_errors_total{}[5m]))",
+          "legendFormat": "errors / second",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Prometheus Notification Error Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(loki_ingester_memory_chunks{})",
+          "legendFormat": "memory chunks",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(loki_ingester_flush_queue_length{})",
+          "legendFormat": "flush queue",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Loki Ingester State",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "time() - max(loki_objstore_bucket_last_successful_upload_time{})",
+          "legendFormat": "seconds since upload",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Loki Object Store Upload Age",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(tempo_ingester_live_traces{})",
+          "legendFormat": "live traces",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(tempo_ingester_flush_queue_length{})",
+          "legendFormat": "flush queue",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Tempo Ingester State",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "max": 1,
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "min(alloy_config_last_load_successful{})",
+          "legendFormat": "last load successful",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Alloy Configuration State",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(loki_write_batch_retries_total{}[5m]))",
+          "legendFormat": "retry / second",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(loki_write_dropped_entries_total{}[5m]))",
+          "legendFormat": "dropped / second",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Alloy Delivery Retry and Drop Rate",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 42,
+  "tags": [
+    "platform",
+    "observability",
+    "backends"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Platform / Observability Backends",
+  "uid": "platform-observability-backends",
+  "version": 1
+}
@@ -0,0 +1,419 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "max(gitea_repositories{})",
+          "legendFormat": "repositories",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(gitea_users{})",
+          "legendFormat": "users",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Gitea Inventory",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(agroal_active_count{})",
+          "legendFormat": "active",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(agroal_available_count{})",
+          "legendFormat": "available",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Keycloak Database Pool",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "min(cnpg_collector_up{})",
+          "legendFormat": "collector up",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(cnpg_backends_total{})",
+          "legendFormat": "backends",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "CloudNativePG Collector and Connections",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum by (datname) (cnpg_pg_database_size_bytes{})",
+          "legendFormat": "{{datname}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "CloudNativePG Database Size",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "max": 100,
+          "min": 0,
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "100 * max by (bucket) (minio_cluster_usage_buckets_total_bytes{bucket=~\"loki|tempo\"}) / max by (bucket) (minio_cluster_usage_buckets_quota_total_bytes{bucket=~\"loki|tempo\"} > 0)",
+          "legendFormat": "{{bucket}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "AIStor Bucket Quota Used",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum by (entrypoint) (rate(traefik_entrypoint_requests_total{}[5m]))",
+          "legendFormat": "{{entrypoint}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Traefik Request Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "max": 100,
+          "min": 0,
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "100 * sum(rate(traefik_entrypoint_requests_total{code=~\"5..\"}[5m])) / clamp_min(sum(rate(traefik_entrypoint_requests_total{}[5m])), 1e-9)",
+          "legendFormat": "5xx ratio",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Traefik 5xx Ratio",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "max": 1,
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "min by (job) (probe_success{})",
+          "legendFormat": "{{job}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Platform HTTPS Boundary Health",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 42,
+  "tags": [
+    "platform",
+    "observability",
+    "services"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Platform / Services",
+  "uid": "platform-services",
+  "version": 1
+}
@@ -0,0 +1,218 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum by (namespace, deployment) (kube_deployment_status_replicas_unavailable{})",
+          "legendFormat": "{{namespace}} / {{deployment}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Deployment Unavailable Replicas",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "clamp_min(sum by (namespace, statefulset) (kube_statefulset_replicas{}) - sum by (namespace, statefulset) (kube_statefulset_status_replicas_ready{}), 0)",
+          "legendFormat": "{{namespace}} / {{statefulset}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "StatefulSet Replicas Not Ready",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{}[15m]))",
+          "legendFormat": "{{namespace}} / {{pod}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Container Restarts in 15 Minutes",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(kube_persistentvolumeclaim_status_phase{phase=\"Pending\"})",
+          "legendFormat": "Pending",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(kube_persistentvolumeclaim_status_phase{phase=\"Lost\"})",
+          "legendFormat": "Lost",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "PersistentVolumeClaim Health",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 42,
+  "tags": [
+    "platform",
+    "observability",
+    "workloads"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Platform / Workload Health",
+  "uid": "platform-workload-health",
+  "version": 1
+}
@@ -0,0 +1,24 @@
+# Grafana
+
+Private Grafana for the home observability stack. The pinned Grafana chart is
+rendered as the independent `grafana` release in `observability`; the
+kube-prometheus-stack embedded Grafana remains disabled.
+
+The workload consumes, but never renders, these existing Secret contracts:
+
+- `grafana-admin`: `admin-user`, `admin-password`
+- `grafana-keycloak-oidc`: `client-id`, `client-secret`
+
+The `grafana` PVC is fixed to `observability-grafana-local-pv` (2Gi, Retain).
+Host Nginx terminates the exact-SAN certificate and forwards plain HTTP through
+Traefik; the Kubernetes Ingress therefore contains no TLS material.
+
+The Pod does not automount a service-account token. A 3600-second projected
+token, CA, and namespace are mounted only into `grafana-sc-dashboard`, whose
+namespaced Role can read ConfigMaps but not Secrets. Dashboard ConfigMaps must
+carry `grafana_dashboard: "1"` in `observability`.
+
+Provisioned datasource UIDs are `prometheus`, `loki`, and `tempo`. Tempo service
+map configuration is present for future use, but the metrics-generator and
+Prometheus remote-write path are intentionally not enabled, so service-graph
+availability is not an initial acceptance condition.
@@ -0,0 +1,27 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: grafana-dashboard-sidecar
+  namespace: observability
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: grafana-dashboard-sidecar
+  namespace: observability
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: grafana-dashboard-sidecar
+subjects:
+  - kind: ServiceAccount
+    name: grafana
+    namespace: observability
@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana
+  namespace: observability
+spec:
+  template:
+    spec:
+      automountServiceAccountToken: false
+      hostNetwork: false
+      hostPID: false
+      hostIPC: false
+      containers:
+        - name: grafana-sc-dashboard
+          volumeMounts:
+            - name: dashboard-sidecar-api-access
+              mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+              readOnly: true
+      volumes:
+        - name: dashboard-sidecar-api-access
+          projected:
+            defaultMode: 420
+            sources:
+              - serviceAccountToken:
+                  expirationSeconds: 3600
+                  path: token
+              - configMap:
+                  name: kube-root-ca.crt
+                  items:
+                    - key: ca.crt
+                      path: ca.crt
+              - downwardAPI:
+                  items:
+                    - path: namespace
+                      fieldRef:
+                        apiVersion: v1
+                        fieldPath: metadata.namespace
@@ -0,0 +1,7 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: grafana
+  namespace: observability
+spec:
+  ingressClassName: traefik
@@ -0,0 +1,24 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability
+
+resources:
+  - dashboard-sidecar-rbac.yaml
+
+helmGlobals:
+  chartHome: .helm/charts
+
+helmCharts:
+  - name: grafana
+    repo: https://grafana-community.github.io/helm-charts
+    version: 12.10.1
+    releaseName: grafana
+    namespace: observability
+    valuesFile: values/home.yaml
+    includeCRDs: false
+    skipTests: true
+
+patches:
+  - path: ingress-class-patch.yaml
+  - path: dashboard-sidecar-token-patch.yaml
@@ -0,0 +1,262 @@
+replicas: 1
+
+extraLabels:
+  observability.hyeonworks.com/instance: home
+
+deploymentStrategy:
+  type: Recreate
+
+revisionHistoryLimit: 3
+
+automountServiceAccountToken: false
+enableServiceLinks: false
+
+rbac:
+  create: false
+
+serviceAccount:
+  create: true
+  name: grafana
+  automountServiceAccountToken: false
+
+image:
+  registry: docker.io
+  repository: grafana/grafana
+  tag: 13.1.1
+  sha: f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b
+  pullPolicy: IfNotPresent
+
+testFramework:
+  enabled: false
+
+securityContext:
+  runAsNonRoot: true
+  runAsUser: 472
+  runAsGroup: 472
+  fsGroup: 472
+  fsGroupChangePolicy: OnRootMismatch
+  seccompProfile:
+    type: RuntimeDefault
+
+containerSecurityContext:
+  allowPrivilegeEscalation: false
+  privileged: false
+  capabilities:
+    drop:
+      - ALL
+  seccompProfile:
+    type: RuntimeDefault
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 256Mi
+  limits:
+    cpu: 500m
+    memory: 512Mi
+
+initChownData:
+  enabled: false
+
+persistence:
+  type: pvc
+  enabled: true
+  storageClassName: ssd-local-observability-grafana-retain
+  volumeName: observability-grafana-local-pv
+  lookupVolumeName: false
+  accessModes:
+    - ReadWriteOnce
+  size: 2Gi
+  finalizers:
+    - kubernetes.io/pvc-protection
+
+admin:
+  existingSecret: grafana-admin
+  userKey: admin-user
+  passwordKey: admin-password
+
+envValueFrom:
+  GF_AUTH_GENERIC_OAUTH_CLIENT_ID:
+    secretKeyRef:
+      name: grafana-keycloak-oidc
+      key: client-id
+  GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:
+    secretKeyRef:
+      name: grafana-keycloak-oidc
+      key: client-secret
+
+service:
+  enabled: true
+  type: ClusterIP
+  port: 80
+  targetPort: 3000
+  portName: service
+
+serviceMonitor:
+  enabled: true
+  interval: 30s
+  path: /metrics
+
+ingress:
+  enabled: true
+  annotations: {}
+  path: /
+  pathType: Prefix
+  hosts:
+    - grafana.learn.hyeonworks.com
+  tls: []
+
+grafana.ini:
+  paths:
+    data: /var/lib/grafana/
+    logs: /var/log/grafana
+    plugins: /var/lib/grafana/plugins
+    provisioning: /etc/grafana/provisioning
+  analytics:
+    check_for_updates: false
+    check_for_plugin_updates: false
+    reporting_enabled: false
+  plugins:
+    preinstall_disabled: true
+  server:
+    domain: grafana.learn.hyeonworks.com
+    root_url: https://grafana.learn.hyeonworks.com/
+    enforce_domain: true
+  security:
+    cookie_secure: true
+    cookie_samesite: lax
+  auth:
+    disable_login_form: false
+    oauth_auto_login: false
+    login_maximum_lifetime_duration: 8h
+    login_maximum_inactive_lifetime_duration: 30m
+  auth.anonymous:
+    enabled: false
+  auth.basic:
+    enabled: true
+  auth.generic_oauth:
+    enabled: true
+    name: Keycloak
+    allow_sign_up: true
+    use_pkce: true
+    scopes: openid profile email
+    groups_attribute_path: groups
+    allowed_groups: /platform-observability-admins /platform-observability-viewers
+    role_attribute_strict: true
+    allow_assign_grafana_admin: false
+    skip_org_role_sync: false
+    validate_id_token: true
+    use_refresh_token: true
+    role_attribute_path: "contains(groups[*], '/platform-observability-admins') && 'Admin' || contains(groups[*], '/platform-observability-viewers') && 'Viewer' || null"
+    auth_url: https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/auth
+    token_url: https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/token
+    api_url: https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/userinfo
+    signout_redirect_url: https://id.learn.hyeonworks.com/realms/hyeonworks/protocol/openid-connect/logout
+
+datasources:
+  datasources.yaml:
+    apiVersion: 1
+    deleteDatasources:
+      - name: Prometheus
+        orgId: 1
+      - name: Loki
+        orgId: 1
+      - name: Tempo
+        orgId: 1
+    datasources:
+      - name: Prometheus
+        uid: prometheus
+        type: prometheus
+        access: proxy
+        url: http://observability-core-kube-pr-prometheus.observability.svc.cluster.local:9090
+        isDefault: true
+        editable: false
+        jsonData:
+          httpMethod: POST
+          timeInterval: 30s
+      - name: Loki
+        uid: loki
+        type: loki
+        access: proxy
+        url: http://loki.observability.svc.cluster.local:3100
+        isDefault: false
+        editable: false
+        jsonData:
+          derivedFields:
+            - name: trace_id
+              matcherRegex: '"trace_id"[[:space:]]*:[[:space:]]*"([0-9a-f]{32})"'
+              datasourceUid: tempo
+              url: '$${__value.raw}'
+      - name: Tempo
+        uid: tempo
+        type: tempo
+        access: proxy
+        url: http://tempo.observability.svc.cluster.local:3200
+        isDefault: false
+        editable: false
+        jsonData:
+          httpMethod: GET
+          nodeGraph:
+            enabled: true
+          serviceMap:
+            datasourceUid: prometheus
+          tracesToLogsV2:
+            datasourceUid: loki
+            spanStartTimeShift: -1m
+            spanEndTimeShift: 1m
+            tags:
+              - key: k8s.namespace.name
+                value: namespace
+              - key: k8s.pod.name
+                value: pod
+            filterByTraceID: true
+            filterBySpanID: false
+
+sidecar:
+  image:
+    registry: quay.io
+    repository: kiwigrid/k8s-sidecar
+    tag: 2.10.0
+    sha: 129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3
+  imagePullPolicy: IfNotPresent
+  resources:
+    requests:
+      cpu: 25m
+      memory: 64Mi
+    limits:
+      cpu: 100m
+      memory: 128Mi
+  securityContext:
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop:
+        - ALL
+    seccompProfile:
+      type: RuntimeDefault
+  dashboards:
+    enabled: true
+    label: grafana_dashboard
+    labelValue: "1"
+    searchNamespace:
+      - observability
+    resource: configmap
+    watchMethod: WATCH
+    skipReload: true
+    provider:
+      name: sidecarProvider
+      orgid: 1
+      folder: ""
+      folderUid: ""
+      type: file
+      disableDelete: false
+      allowUiUpdates: false
+      foldersFromFilesStructure: false
+
+imageRenderer:
+  enabled: false
+
+networkPolicy:
+  enabled: false
+
+assertNoLeakedSecrets: true
@@ -0,0 +1,15 @@
+# kube-prometheus-stack
+
+고정 chart `88.0.1`을 `observability` namespace에 렌더한다. Grafana와
+node-exporter, control-plane에 존재하지 않는 scheduler/controller-manager/etcd/
+kube-proxy monitor는 끈다. Prometheus와 Alertmanager 데이터는 각각의 `Retain`
+Local PV에 보존하며 CRD는 별도 root에서 먼저 적용한다.
+
+Prometheus discovery는 namespace label
+`observability.hyeonworks.com/enabled=true`와 resource label
+`observability.hyeonworks.com/instance=home`의 교집합만 선택한다.
+
+Alertmanager source는 selector discovery 대신 global
+`alertmanagerConfiguration.name=platform-alertmanager`를 참조한다. 참조 대상의
+Slack Secret이 준비되기 전에는 이 변경을 live에 적용하지 않고 기존 null 구성을
+유지한다.
@@ -0,0 +1,37 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability
+
+helmGlobals:
+  chartHome: .helm/charts
+
+helmCharts:
+  - name: kube-prometheus-stack
+    repo: https://prometheus-community.github.io/helm-charts
+    version: 88.0.1
+    releaseName: observability-core
+    namespace: observability
+    valuesFile: values/home.yaml
+    includeCRDs: false
+    skipTests: true
+
+images:
+  - name: quay.io/prometheus-operator/prometheus-operator
+    newName: quay.io/prometheus-operator/prometheus-operator
+    digest: sha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2
+  - name: registry.k8s.io/kube-state-metrics/kube-state-metrics
+    newName: registry.k8s.io/kube-state-metrics/kube-state-metrics
+    digest: sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4
+
+patches:
+  - target:
+      group: monitoring.coreos.com
+      version: v1
+      kind: Alertmanager
+      name: observability-core-kube-pr-alertmanager
+    patch: |-
+      - op: remove
+        path: /spec/alertmanagerConfigSelector
+      - op: remove
+        path: /spec/alertmanagerConfigNamespaceSelector
@@ -0,0 +1,282 @@
+commonLabels:
+  observability.hyeonworks.com/instance: home
+
+crds:
+  enabled: false
+
+grafana:
+  enabled: false
+
+nodeExporter:
+  enabled: false
+
+kubeApiServer:
+  enabled: true
+
+kubelet:
+  enabled: true
+
+coreDns:
+  enabled: true
+
+kubeControllerManager:
+  enabled: false
+
+kubeScheduler:
+  enabled: false
+
+kubeEtcd:
+  enabled: false
+
+kubeProxy:
+  enabled: false
+
+defaultRules:
+  disabled:
+    Watchdog: true
+  rules:
+    etcd: false
+    kubeControllerManager: false
+    kubeProxy: false
+    kubeSchedulerAlerting: false
+    kubeSchedulerRecording: false
+    windows: false
+
+prometheusOperator:
+  admissionWebhooks:
+    enabled: false
+    patch:
+      enabled: false
+  tls:
+    enabled: false
+  image:
+    registry: quay.io
+    repository: prometheus-operator/prometheus-operator
+    tag: v0.93.0
+    sha: 64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 256Mi
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
+    runAsGroup: 65534
+    fsGroup: 65534
+    seccompProfile:
+      type: RuntimeDefault
+  containerSecurityContext:
+    allowPrivilegeEscalation: false
+    capabilities:
+      drop:
+        - ALL
+    readOnlyRootFilesystem: true
+  serviceAccount:
+    automountServiceAccountToken: true
+  prometheusConfigReloader:
+    image:
+      registry: quay.io
+      repository: prometheus-operator/prometheus-config-reloader
+      tag: v0.93.0
+      sha: 65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905
+    resources:
+      requests:
+        cpu: 25m
+        memory: 32Mi
+      limits:
+        cpu: 100m
+        memory: 64Mi
+
+kubeStateMetrics:
+  enabled: true
+
+kube-state-metrics:
+  image:
+    registry: registry.k8s.io
+    repository: kube-state-metrics/kube-state-metrics
+    tag: v2.19.1
+    sha: sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 256Mi
+  securityContext:
+    enabled: true
+    runAsNonRoot: true
+    runAsUser: 65534
+    runAsGroup: 65534
+    fsGroup: 65534
+    seccompProfile:
+      type: RuntimeDefault
+  containerSecurityContext:
+    allowPrivilegeEscalation: false
+    capabilities:
+      drop:
+        - ALL
+    readOnlyRootFilesystem: true
+  automountServiceAccountToken: true
+  collectors:
+    - certificatesigningrequests
+    - configmaps
+    - cronjobs
+    - daemonsets
+    - deployments
+    - endpointslices
+    - horizontalpodautoscalers
+    - ingresses
+    - jobs
+    - leases
+    - limitranges
+    - mutatingwebhookconfigurations
+    - namespaces
+    - networkpolicies
+    - nodes
+    - persistentvolumeclaims
+    - persistentvolumes
+    - poddisruptionbudgets
+    - pods
+    - replicasets
+    - replicationcontrollers
+    - resourcequotas
+    - services
+    - statefulsets
+    - storageclasses
+    - validatingwebhookconfigurations
+    - volumeattachments
+  prometheus:
+    monitor:
+      additionalLabels:
+        observability.hyeonworks.com/instance: home
+
+alertmanager:
+  enabled: true
+  config:
+    global:
+      resolve_timeout: 5m
+    route:
+      group_by:
+        - namespace
+      group_wait: 30s
+      group_interval: 5m
+      repeat_interval: 12h
+      receiver: "null"
+      routes: []
+    receivers:
+      - name: "null"
+  serviceAccount:
+    automountServiceAccountToken: false
+  alertmanagerSpec:
+    replicas: 1
+    retention: 120h
+    automountServiceAccountToken: false
+    alertmanagerConfiguration:
+      name: platform-alertmanager
+    image:
+      registry: quay.io
+      repository: prometheus/alertmanager
+      tag: v0.33.1
+      sha: a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b
+    resources:
+      requests:
+        cpu: 50m
+        memory: 128Mi
+      limits:
+        cpu: 250m
+        memory: 256Mi
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 1000
+      runAsGroup: 2000
+      fsGroup: 2000
+      fsGroupChangePolicy: OnRootMismatch
+      seccompProfile:
+        type: RuntimeDefault
+    persistentVolumeClaimRetentionPolicy:
+      whenDeleted: Retain
+      whenScaled: Retain
+    storage:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: ssd-local-observability-alertmanager-retain
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 1Gi
+
+prometheus:
+  enabled: true
+  prometheusSpec:
+    replicas: 1
+    retention: 15d
+    retentionSize: 16GB
+    scrapeInterval: 30s
+    evaluationInterval: 30s
+    walCompression: true
+    automountServiceAccountToken: true
+    image:
+      registry: quay.io
+      repository: prometheus/prometheus
+      tag: v3.13.2-distroless
+      sha: ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
+    resources:
+      requests:
+        cpu: 750m
+        memory: 2Gi
+      limits:
+        cpu: "2"
+        memory: 3Gi
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 1000
+      runAsGroup: 2000
+      fsGroup: 2000
+      fsGroupChangePolicy: OnRootMismatch
+      seccompProfile:
+        type: RuntimeDefault
+    persistentVolumeClaimRetentionPolicy:
+      whenDeleted: Retain
+      whenScaled: Retain
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: ssd-local-observability-prometheus-retain
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 20Gi
+    serviceMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelectorNilUsesHelmValues: false
+    probeSelectorNilUsesHelmValues: false
+    ruleSelectorNilUsesHelmValues: false
+    serviceMonitorNamespaceSelector:
+      matchLabels:
+        observability.hyeonworks.com/enabled: "true"
+    podMonitorNamespaceSelector:
+      matchLabels:
+        observability.hyeonworks.com/enabled: "true"
+    probeNamespaceSelector:
+      matchLabels:
+        observability.hyeonworks.com/enabled: "true"
+    ruleNamespaceSelector:
+      matchLabels:
+        observability.hyeonworks.com/enabled: "true"
+    serviceMonitorSelector:
+      matchLabels:
+        observability.hyeonworks.com/instance: home
+    podMonitorSelector:
+      matchLabels:
+        observability.hyeonworks.com/instance: home
+    probeSelector:
+      matchLabels:
+        observability.hyeonworks.com/instance: home
+    ruleSelector:
+      matchLabels:
+        observability.hyeonworks.com/instance: home
@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - kube-prometheus-stack
+  - loki
+  - tempo
+  - alloy
+  - node-exporter
+  - grafana
+  - blackbox-exporter
+  - network-policies
+  - targets
+  - rules/platform
+  - alerting
+  - dashboards/platform
@@ -0,0 +1,16 @@
+# Loki
+
+This root renders `grafana-community/loki` chart `18.7.1` as one Loki
+`3.7.4` monolithic StatefulSet in `observability`.
+
+- The image is pinned to the reviewed amd64 digest.
+- Logs use the `loki` AIStor bucket through `loki-object-storage`; no root
+  credential is referenced.
+- Retention is 30 days. The WAL and working data use the dedicated 5Gi
+  Retain Local PV.
+- Gateway, canary, caches, embedded MinIO, tests, and rule sidecars are
+  disabled.
+- The only client-facing service is ClusterIP port 3100.
+
+Do not delete the PVC or bucket during rollback. Scale the StatefulSet down,
+preserve its Retain volume, and investigate S3/WAL state before reapplying.
@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability
+
+helmGlobals:
+  chartHome: .helm/charts
+
+helmCharts:
+  - name: loki
+    repo: https://grafana-community.github.io/helm-charts
+    version: 18.7.1
+    releaseName: loki
+    namespace: observability
+    valuesFile: values/home.yaml
+    includeCRDs: false
+    skipTests: true
@@ -0,0 +1,146 @@
+deploymentMode: Monolithic
+
+commonLabels:
+  observability.hyeonworks.com/instance: home
+
+defaults:
+  automountServiceAccountToken: false
+
+serviceAccount:
+  automountServiceAccountToken: false
+
+loki:
+  auth_enabled: false
+  image:
+    registry: docker.io
+    repository: grafana/loki
+    tag: "3.7.4"
+    digest: sha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e
+  podSecurityContext:
+    runAsNonRoot: true
+    runAsUser: 10001
+    runAsGroup: 10001
+    fsGroup: 10001
+    fsGroupChangePolicy: OnRootMismatch
+    seccompProfile:
+      type: RuntimeDefault
+  containerSecurityContext:
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop:
+        - ALL
+    seccompProfile:
+      type: RuntimeDefault
+  commonConfig:
+    path_prefix: /var/loki
+    replication_factor: 1
+  schemaConfig:
+    configs:
+      - from: "2026-07-01"
+        store: tsdb
+        object_store: s3
+        schema: v13
+        index:
+          prefix: index_
+          period: 24h
+  storage:
+    type: s3
+    bucketNames:
+      chunks: loki
+      ruler: loki
+      admin: loki
+    s3:
+      endpoint: http://minio.object-storage.svc.cluster.local
+      s3ForcePathStyle: true
+      insecure: true
+      accessKeyId: ${AWS_ACCESS_KEY_ID}
+      secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
+  useTestSchema: false
+  limits_config:
+    retention_period: 720h
+    ingestion_rate_mb: 4
+    ingestion_burst_size_mb: 6
+    max_line_size: 256KB
+    discover_log_levels: false
+    discover_service_name: []
+  ingester:
+    chunk_idle_period: 30m
+    max_chunk_age: 1h
+    wal:
+      checkpoint_duration: 5m
+      replay_memory_ceiling: 512MB
+      disk_full_threshold: 0.90
+      flush_on_shutdown: true
+  compactor:
+    retention_enabled: true
+    delete_request_store: s3
+    retention_delete_delay: 2h
+    retention_delete_worker_count: 10
+    max_compaction_parallelism: 1
+
+singleBinary:
+  replicas: 1
+  sidecar: false
+  podLabels:
+    platform.hyeonworks.com/aistor-client: "true"
+  service:
+    type: ClusterIP
+  resources:
+    requests:
+      cpu: 350m
+      memory: 768Mi
+    limits:
+      cpu: 1500m
+      memory: 1536Mi
+  extraEnv:
+    - name: AWS_ACCESS_KEY_ID
+      valueFrom:
+        secretKeyRef:
+          name: loki-object-storage
+          key: access-key-id
+    - name: AWS_SECRET_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: loki-object-storage
+          key: secret-access-key
+  persistence:
+    enabled: true
+    size: 5Gi
+    storageClass: ssd-local-observability-loki-retain
+    enableStatefulSetAutoDeletePVC: true
+    whenScaled: Retain
+    whenDeleted: Retain
+
+read:
+  replicas: 0
+write:
+  replicas: 0
+backend:
+  replicas: 0
+
+ruler:
+  sidecar: false
+
+gateway:
+  enabled: false
+lokiCanary:
+  enabled: false
+chunksCache:
+  enabled: false
+resultsCache:
+  enabled: false
+memcached:
+  enabled: false
+minio:
+  enabled: false
+test:
+  enabled: false
+sidecar:
+  rules:
+    enabled: false
+
+monitoring:
+  serviceMonitor:
+    enabled: true
+    interval: 30s
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - observability.yaml
+  - observability-agent.yaml
+  - object-storage-counterpart.yaml
+
@@ -0,0 +1,31 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: object-storage-allow-observability-s3
+  namespace: object-storage
+  labels:
+    app.kubernetes.io/component: observability-object-storage
+    app.kubernetes.io/part-of: platform
+spec:
+  podSelector:
+    matchLabels:
+      aistor.min.io/objectStore: minio-aistor
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: observability
+          podSelector:
+            matchLabels:
+              platform.hyeonworks.com/aistor-client: "true"
+            matchExpressions:
+              - key: app.kubernetes.io/name
+                operator: In
+                values:
+                  - loki
+                  - tempo
+      ports:
+        - protocol: TCP
+          port: 9000
@@ -0,0 +1,140 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-agent-allow-alloy-dns
+  namespace: observability-agent
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: alloy
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-agent-allow-alloy-api
+  namespace: observability-agent
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: alloy
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - ipBlock:
+            cidr: 10.43.0.1/32
+      ports:
+        - protocol: TCP
+          port: 443
+    - to:
+        - ipBlock:
+            cidr: 192.168.0.107/32
+      ports:
+        - protocol: TCP
+          port: 6443
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-agent-allow-alloy-backends
+  namespace: observability-agent
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: alloy
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: observability
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: loki
+      ports:
+        - protocol: TCP
+          port: 3100
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: observability
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: tempo
+      ports:
+        - protocol: TCP
+          port: 4317
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-agent-allow-prometheus-to-alloy
+  namespace: observability-agent
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: alloy
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: observability
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 12345
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-agent-allow-prometheus-to-node-exporter
+  namespace: observability-agent
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: prometheus-node-exporter
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: observability
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 9100
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-agent-default-deny
+  namespace: observability-agent
+spec:
+  podSelector: {}
+  policyTypes:
+    - Ingress
+    - Egress
+
@@ -0,0 +1,548 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-dns
+  namespace: observability
+spec:
+  podSelector: {}
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-kubernetes-api
+  namespace: observability
+spec:
+  podSelector:
+    matchExpressions:
+      - key: app.kubernetes.io/name
+        operator: In
+        values:
+          - kube-prometheus-stack-prometheus-operator
+          - kube-state-metrics
+          - prometheus
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - ipBlock:
+            cidr: 10.43.0.1/32
+      ports:
+        - protocol: TCP
+          port: 443
+    - to:
+        - ipBlock:
+            cidr: 192.168.0.107/32
+      ports:
+        - protocol: TCP
+          port: 6443
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-prometheus-egress
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: prometheus
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: kube-prometheus-stack-prometheus-operator
+      ports:
+        - protocol: TCP
+          port: 8080
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: kube-state-metrics
+      ports:
+        - protocol: TCP
+          port: 8080
+        - protocol: TCP
+          port: 8081
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 9090
+        - protocol: TCP
+          port: 8080
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: alertmanager
+      ports:
+        - protocol: TCP
+          port: 9093
+        - protocol: TCP
+          port: 8080
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: loki
+      ports:
+        - protocol: TCP
+          port: 3100
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: tempo
+      ports:
+        - protocol: TCP
+          port: 3200
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: grafana
+              app.kubernetes.io/instance: grafana
+      ports:
+        - protocol: TCP
+          port: 3000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus-blackbox-exporter
+              app.kubernetes.io/instance: blackbox-exporter
+      ports:
+        - protocol: TCP
+          port: 9115
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: observability-agent
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: alloy
+      ports:
+        - protocol: TCP
+          port: 12345
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: observability-agent
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus-node-exporter
+      ports:
+        - protocol: TCP
+          port: 9100
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: TCP
+          port: 9153
+    - to:
+        - ipBlock:
+            cidr: 192.168.0.107/32
+      ports:
+        - protocol: TCP
+          port: 10250
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-prometheus-to-operator
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: kube-prometheus-stack-prometheus-operator
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 8080
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-prometheus-to-ksm
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: kube-state-metrics
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 8080
+        - protocol: TCP
+          port: 8081
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-prometheus-self
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: prometheus
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 9090
+        - protocol: TCP
+          port: 8080
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: grafana
+              app.kubernetes.io/instance: grafana
+      ports:
+        - protocol: TCP
+          port: 9090
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-alertmanager
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: alertmanager
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 9093
+        - protocol: TCP
+          port: 8080
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: alertmanager
+      ports:
+        - protocol: TCP
+          port: 9094
+        - protocol: UDP
+          port: 9094
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: alertmanager
+      ports:
+        - protocol: TCP
+          port: 9094
+        - protocol: UDP
+          port: 9094
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-loki
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: loki
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 3100
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: grafana
+              app.kubernetes.io/instance: grafana
+      ports:
+        - protocol: TCP
+          port: 3100
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: observability-agent
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: alloy
+      ports:
+        - protocol: TCP
+          port: 3100
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: loki
+      ports:
+        - protocol: TCP
+          port: 7946
+        - protocol: UDP
+          port: 7946
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: loki
+      ports:
+        - protocol: TCP
+          port: 7946
+        - protocol: UDP
+          port: 7946
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-tempo
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: tempo
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 3200
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: grafana
+              app.kubernetes.io/instance: grafana
+      ports:
+        - protocol: TCP
+          port: 3200
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: observability-agent
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: alloy
+      ports:
+        - protocol: TCP
+          port: 4317
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: tempo
+      ports:
+        - protocol: TCP
+          port: 7946
+        - protocol: UDP
+          port: 7946
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: tempo
+      ports:
+        - protocol: TCP
+          port: 7946
+        - protocol: UDP
+          port: 7946
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-aistor-egress
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      platform.hyeonworks.com/aistor-client: "true"
+    matchExpressions:
+      - key: app.kubernetes.io/name
+        operator: In
+        values:
+          - loki
+          - tempo
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - ipBlock:
+            cidr: 10.43.124.248/32
+      ports:
+        - protocol: TCP
+          port: 80
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: object-storage
+          podSelector:
+            matchLabels:
+              aistor.min.io/objectStore: minio-aistor
+      ports:
+        - protocol: TCP
+          port: 9000
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-grafana-ingress
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: grafana
+      app.kubernetes.io/instance: grafana
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: traefik
+              app.kubernetes.io/instance: traefik-kube-system
+      ports:
+        - protocol: TCP
+          port: 3000
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 3000
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus-blackbox-exporter
+              app.kubernetes.io/instance: blackbox-exporter
+      ports:
+        - protocol: TCP
+          port: 3000
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-grafana-datasources
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: grafana
+      app.kubernetes.io/instance: grafana
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 9090
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: loki
+      ports:
+        - protocol: TCP
+          port: 3100
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: tempo
+      ports:
+        - protocol: TCP
+          port: 3200
+    - to:
+        - ipBlock:
+            cidr: 192.168.0.107/32
+      ports:
+        - protocol: TCP
+          port: 443
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-grafana-dashboard-api
+  namespace: observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: grafana
+      app.kubernetes.io/instance: grafana
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - ipBlock:
+            cidr: 10.43.0.1/32
+      ports:
+        - protocol: TCP
+          port: 443
+    - to:
+        - ipBlock:
+            cidr: 192.168.0.107/32
+      ports:
+        - protocol: TCP
+          port: 6443
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-default-deny
+  namespace: observability
+spec:
+  podSelector: {}
+  policyTypes:
+    - Ingress
+    - Egress
@@ -0,0 +1,11 @@
+# node-exporter
+
+The standalone node-exporter runs in `observability-agent` with the embedded
+kube-prometheus-stack copy and kube-rbac-proxy disabled. It mounts only
+`/proc`, `/sys`, and `/` from the node, all read-only, and exposes an internal
+ClusterIP metrics Service on port `9100`.
+
+Host network, PID, and IPC namespaces remain disabled by the platform security
+contract. Consequently, network-namespace metrics are treated as a known
+single-node visibility limitation rather than authoritative host-network data.
+
@@ -0,0 +1,23 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability-agent
+
+helmGlobals:
+  chartHome: .helm/charts
+
+helmCharts:
+  - name: prometheus-node-exporter
+    repo: https://prometheus-community.github.io/helm-charts
+    version: 4.56.1
+    releaseName: node-exporter
+    namespace: observability-agent
+    valuesFile: values/home.yaml
+    includeCRDs: false
+    skipTests: true
+
+images:
+  - name: quay.io/prometheus/node-exporter
+    newName: quay.io/prometheus/node-exporter
+    digest: sha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798
+
@@ -0,0 +1,78 @@
+fullnameOverride: node-exporter
+namespaceOverride: observability-agent
+
+image:
+  registry: quay.io
+  repository: prometheus/node-exporter
+  tag: v1.12.1
+  pullPolicy: IfNotPresent
+
+kubeRBACProxy:
+  enabled: false
+
+service:
+  enabled: true
+  type: ClusterIP
+  port: 9100
+  targetPort: 9100
+  portName: metrics
+  listenOnAllInterfaces: true
+
+prometheus:
+  monitor:
+    enabled: true
+    additionalLabels:
+      observability.hyeonworks.com/instance: home
+    interval: 30s
+  podMonitor:
+    enabled: false
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: 250m
+    memory: 256Mi
+
+serviceAccount:
+  create: true
+  automountServiceAccountToken: false
+
+rbac:
+  create: true
+
+securityContext:
+  fsGroup: 65534
+  runAsGroup: 65534
+  runAsNonRoot: true
+  runAsUser: 65534
+  seccompProfile:
+    type: RuntimeDefault
+
+containerSecurityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+      - ALL
+  privileged: false
+  readOnlyRootFilesystem: true
+
+hostNetwork: false
+hostPID: false
+hostIPC: false
+
+hostRootFsMount:
+  enabled: true
+  mountPropagation: None
+hostProcFsMount:
+  mountPropagation: None
+hostSysFsMount:
+  mountPropagation: None
+
+nodeSelector:
+  kubernetes.io/os: linux
+
+tolerations:
+  - effect: NoSchedule
+    operator: Exists
@@ -0,0 +1,10 @@
+# Observability object storage
+
+Loki와 Tempo는 AIStor에서 서로 다른 bucket, policy, user와 Kubernetes Secret을
+사용한다. Loki quota는 200Gi, Tempo quota는 100Gi다. 정책은 자기 bucket의 위치·목록과
+자기 object의 읽기·쓰기·삭제·multipart 처리만 허용한다.
+
+운영 적용 전에는 K3s Secret 암호화와 격리 복구 증거, AIStor root credential의 암호화된
+외부 복구 가능성을 각각 다시 확인한다. 기존 bucket·object·policy·user·quota는 자동
+삭제하거나 credential을 자동 회전하지 않는다. 한쪽 Secret만 존재하는 partial 상태에서는
+추가 mutation을 중단한다.
@@ -0,0 +1,29 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetBucketLocation",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads"
+      ],
+      "Resource": [
+        "arn:aws:s3:::loki"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetObject",
+        "s3:PutObject",
+        "s3:DeleteObject",
+        "s3:AbortMultipartUpload",
+        "s3:ListMultipartUploadParts"
+      ],
+      "Resource": [
+        "arn:aws:s3:::loki/*"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,29 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetBucketLocation",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads"
+      ],
+      "Resource": [
+        "arn:aws:s3:::tempo"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetObject",
+        "s3:PutObject",
+        "s3:DeleteObject",
+        "s3:AbortMultipartUpload",
+        "s3:ListMultipartUploadParts"
+      ],
+      "Resource": [
+        "arn:aws:s3:::tempo/*"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,87 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: platform-certificate-probes
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: platform-observability-rules
+    app.kubernetes.io/part-of: observability
+    observability.hyeonworks.com/instance: home
+spec:
+  groups:
+    - name: platform.blackbox
+      rules:
+        - alert: PlatformPublicEdgeProbeFailed
+          expr: >-
+            probe_success{
+              namespace="observability",
+              "observability.hyeonworks.com/probe-group"="public-edge"
+            } == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: 외부 엔드포인트 프로브 실패
+            description: '{{ $labels.instance }} 외부 엔드포인트 프로브가 5분간 실패했습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformPrivateEdgeProbeFailed
+          expr: >-
+            probe_success{
+              namespace="observability",
+              "observability.hyeonworks.com/probe-group"="private-edge"
+            } == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: 비공개 경계 프로브 실패
+            description: '{{ $labels.instance }} 비공개 경계의 예상 응답이 5분간 실패했습니다. HTTP 403이면서 probe_success=1이면 정상입니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformInternalHealthProbeFailed
+          expr: >-
+            probe_success{
+              namespace="observability",
+              "observability.hyeonworks.com/probe-group"="private-internal"
+            } == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: 내부 상태 확인 프로브 실패
+            description: '{{ $labels.instance }} 내부 상태 확인 프로브가 5분간 실패했습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+    - name: platform.certificates
+      rules:
+        - alert: PlatformCertificateExpiry
+          expr: >-
+            (
+              probe_ssl_earliest_cert_expiry{
+                namespace="observability",
+                "observability.hyeonworks.com/probe-group"=~"public-edge|private-edge"
+              } - time()
+            ) >= 14 * 24 * 60 * 60
+            and
+            (
+              probe_ssl_earliest_cert_expiry{
+                namespace="observability",
+                "observability.hyeonworks.com/probe-group"=~"public-edge|private-edge"
+              } - time()
+            ) < 30 * 24 * 60 * 60
+          labels:
+            severity: warning
+          annotations:
+            summary: TLS 인증서가 30일 이내 만료됨
+            description: '{{ $labels.instance }} 인증서 중 가장 이른 만료 시점이 14일 이상 30일 미만 남았습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformCertificateExpiry
+          expr: >-
+            probe_ssl_earliest_cert_expiry{
+              namespace="observability",
+              "observability.hyeonworks.com/probe-group"=~"public-edge|private-edge"
+            } - time() < 14 * 24 * 60 * 60
+          labels:
+            severity: critical
+          annotations:
+            summary: TLS 인증서가 14일 이내 만료됨
+            description: '{{ $labels.instance }} 인증서 중 가장 이른 만료 시점이 14일 미만 남았습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
@@ -0,0 +1,117 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: platform-observability-core
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: platform-observability-rules
+    app.kubernetes.io/part-of: observability
+    observability.hyeonworks.com/instance: home
+spec:
+  groups:
+    - name: platform.storage
+      rules:
+        - alert: PlatformPersistentVolumeClaimNotBound
+          expr: kube_persistentvolumeclaim_status_phase{phase=~"Pending|Lost"} == 1
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: PersistentVolumeClaim이 바인딩되지 않음
+            description: 플랫폼 PersistentVolumeClaim이 Pending 또는 Lost 상태로 10분간 유지됐습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformRootFilesystemUsage
+          expr: >-
+            100 * (1 -
+              node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+              /
+              node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+            ) > 75
+          for: 30m
+          labels:
+            severity: info
+            threshold: "75"
+          annotations:
+            summary: 루트 파일시스템 사용률이 75%를 초과함
+            description: 노드 루트 파일시스템 사용률이 정보 임계치인 75%를 30분간 초과했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformRootFilesystemUsage
+          expr: >-
+            100 * (1 -
+              node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+              /
+              node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+            ) > 85
+          for: 15m
+          labels:
+            severity: warning
+            threshold: "85"
+          annotations:
+            summary: 루트 파일시스템 사용률이 85%를 초과함
+            description: 노드 루트 파일시스템 사용률이 경고 임계치인 85%를 15분간 초과했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformRootFilesystemUsage
+          expr: >-
+            100 * (1 -
+              node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+              /
+              node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+            ) > 90
+          for: 5m
+          labels:
+            severity: critical
+            threshold: "90"
+          annotations:
+            summary: 루트 파일시스템 사용률이 90%를 초과함
+            description: 노드 루트 파일시스템 사용률이 심각 임계치인 90%를 5분간 초과했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformRootFilesystemInodes
+          expr: >-
+            100 * (1 -
+              node_filesystem_files_free{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+              /
+              node_filesystem_files{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+            ) > 80
+          for: 30m
+          labels:
+            severity: warning
+            threshold: "80"
+          annotations:
+            summary: 루트 파일시스템 inode 사용률이 80%를 초과함
+            description: 노드 루트 파일시스템 inode 사용률이 경고 임계치인 80%를 30분간 초과했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformRootFilesystemInodes
+          expr: >-
+            100 * (1 -
+              node_filesystem_files_free{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+              /
+              node_filesystem_files{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
+            ) > 90
+          for: 15m
+          labels:
+            severity: critical
+            threshold: "90"
+          annotations:
+            summary: 루트 파일시스템 inode 사용률이 90%를 초과함
+            description: 노드 루트 파일시스템 inode 사용률이 심각 임계치인 90%를 15분간 초과했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+    - name: platform.workloads
+      rules:
+        - alert: PlatformPodRestartBurst
+          expr: increase(kube_pod_container_status_restarts_total[15m]) > 3
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: 컨테이너 재시작 급증 감지
+            description: 한 컨테이너가 최근 15분 동안 3회를 초과해 재시작했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformPodOOMKilled
+          expr: kube_pod_container_status_last_terminated_reason{reason="OOMKilled"} == 1
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: 컨테이너가 OOM Killer에 의해 종료됨
+            description: 컨테이너의 최근 종료 사유가 OOMKilled인 상태로 5분간 유지됐습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability
+
+resources:
+  - core-rules.yaml
+  - storage-quota-rules.yaml
+  - certificate-rules.yaml
+  - verified-service-rules.yaml
@@ -0,0 +1,55 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: platform-aistor-storage-quota
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: platform-observability-rules
+    app.kubernetes.io/part-of: observability
+    observability.hyeonworks.com/instance: home
+spec:
+  groups:
+    - name: platform.aistor-storage-quota
+      rules:
+        - record: platform:aistor_bucket_quota_usage_percent
+          expr: >-
+            100
+            * max by (bucket) (
+              minio_cluster_usage_buckets_total_bytes{bucket=~"loki|tempo"}
+            )
+            / max by (bucket) (
+              minio_cluster_usage_buckets_quota_total_bytes{bucket=~"loki|tempo"} > 0
+            )
+        - alert: PlatformAIStorBucketQuotaUsage
+          expr: >-
+            platform:aistor_bucket_quota_usage_percent >= 70
+            and
+            platform:aistor_bucket_quota_usage_percent < 85
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: AIStor 버킷 할당량 사용률 경고
+            description: '{{ $labels.bucket }} 버킷 사용률이 70% 이상 85% 미만으로 15분간 유지됐습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformAIStorBucketQuotaUsage
+          expr: >-
+            platform:aistor_bucket_quota_usage_percent >= 85
+            and
+            platform:aistor_bucket_quota_usage_percent < 95
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: AIStor 버킷 할당량 사용률 심각
+            description: '{{ $labels.bucket }} 버킷 사용률이 85% 이상 95% 미만으로 10분간 유지됐습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformAIStorBucketQuotaUsage
+          expr: platform:aistor_bucket_quota_usage_percent >= 95
+          for: 5m
+          labels:
+            severity: emergency
+          annotations:
+            summary: AIStor 버킷 할당량 사용률 긴급
+            description: '대응 절차에 따라 {{ $labels.bucket }} 버킷의 수집 경로를 중지하고 객체는 자동으로 삭제하지 마세요.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
@@ -0,0 +1,88 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: platform-verified-services
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: platform-observability-rules
+    app.kubernetes.io/part-of: observability
+    observability.hyeonworks.com/instance: home
+spec:
+  groups:
+    - name: platform.verified-services
+      rules:
+        - alert: PlatformCNPGCollectorDown
+          expr: >-
+            min by (cluster, namespace) (
+              cnpg_collector_up{
+                namespace="platform-data",
+                cluster="platform-postgres"
+              }
+            ) == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: CloudNativePG 수집기가 PostgreSQL 중단을 보고함
+            description: '{{ $labels.cluster }} CloudNativePG 수집기가 PostgreSQL 중단 상태를 5분간 보고했습니다.'
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformAlloyConfigLoadFailed
+          expr: >-
+            min by (namespace) (
+              alloy_config_last_load_successful{namespace="observability-agent"}
+            ) == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: Alloy 설정 로드 실패
+            description: Alloy의 마지막 설정 로드 실패 상태가 5분간 유지됐습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformAlloyLogDeliveryRetries
+          expr: >-
+            sum by (namespace) (
+              increase(loki_write_batch_retries_total{namespace="observability-agent"}[15m])
+            ) > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: Alloy 로그 배치 전송 재시도 발생
+            description: Alloy가 최근 15분 동안 Loki 쓰기 배치를 한 번 이상 재시도했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformAlloyLogEntriesDropped
+          expr: >-
+            sum by (namespace) (
+              increase(loki_write_dropped_entries_total{namespace="observability-agent"}[15m])
+            ) > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: Alloy 로그 항목 유실 발생
+            description: Alloy가 최근 15분 동안 재시도를 모두 소진한 뒤 로그 항목을 유실했습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformLokiRuntimeConfigReloadFailed
+          expr: >-
+            min by (namespace) (
+              loki_runtime_config_last_reload_successful{namespace="observability"}
+            ) == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: Loki 런타임 설정 재로드 실패
+            description: Loki의 마지막 런타임 설정 재로드 실패 상태가 5분간 유지됐습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
+        - alert: PlatformLokiWALDiskUsageHigh
+          expr: >-
+            max by (namespace) (
+              loki_ingester_wal_disk_usage_percent{namespace="observability"}
+            ) >= 0.8
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: Loki WAL 디스크 사용률 높음
+            description: Loki WAL 디스크 사용률이 80% 이상으로 15분간 유지됐습니다.
+            runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
@@ -0,0 +1,24 @@
+# Platform metric targets
+
+This component declares the manually owned monitor resources and counterpart
+NetworkPolicy needed to onboard existing platform services into the home
+Prometheus instance.
+
+The owners remain authoritative for their endpoint activation:
+
+- Gitea chart: `/metrics` and its generated ServiceMonitor
+- Keycloak Operator: management metrics and its generated ServiceMonitor
+- CloudNativePG: the declared `metrics` Pod port through a manual PodMonitor
+- AIStor: the v3 per-bucket usage endpoint through a manual ServiceMonitor
+- packaged Traefik chart: its dedicated metrics Service and ServiceMonitor
+
+The AIStor endpoint uses `MINIO_PROMETHEUS_AUTH_TYPE=public`, but no public
+route is created. Port 9000 remains protected by the namespace default-deny and
+the exact Prometheus/client allow policies. Because S3 and metrics share that
+L4 port, an already-authorized S3 client can also reach the credential-less
+metrics path; HTTP-path isolation is not expressible with NetworkPolicy.
+
+The first inventory is captured before blackbox is deployed. Grafana already
+exists and must be recorded explicitly as a preexisting, verify-only substrate
+target; it is not owned or rolled back by this component.
+
@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: aistor-bucket-usage
+  namespace: object-storage
+  labels:
+    app.kubernetes.io/name: minio-aistor
+    app.kubernetes.io/component: bucket-usage-metrics
+    app.kubernetes.io/part-of: platform
+    observability.hyeonworks.com/instance: home
+spec:
+  jobLabel: aistor.min.io/objectStore
+  selector:
+    matchLabels:
+      aistor.min.io/objectStore: minio-aistor
+  namespaceSelector:
+    matchNames:
+      - object-storage
+  endpoints:
+    - port: http-minio
+      path: /minio/metrics/v3/cluster/usage/buckets
+      interval: 60s
+      scrapeTimeout: 10s
+      relabelings:
+        - action: keep
+          sourceLabels:
+            - __meta_kubernetes_service_name
+          regex: minio
@@ -0,0 +1,24 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: platform-postgres
+  namespace: platform-data
+  labels:
+    app.kubernetes.io/name: platform-postgres
+    app.kubernetes.io/component: database-metrics
+    app.kubernetes.io/part-of: platform
+    observability.hyeonworks.com/instance: home
+spec:
+  jobLabel: cnpg.io/cluster
+  selector:
+    matchLabels:
+      cnpg.io/cluster: platform-postgres
+  namespaceSelector:
+    matchNames:
+      - platform-data
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s
+
@@ -0,0 +1,108 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: traefik-preserve-ingress-and-allow-prometheus-metrics
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: traefik
+    app.kubernetes.io/part-of: platform
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: traefik
+      app.kubernetes.io/instance: traefik-kube-system
+  policyTypes:
+    - Ingress
+  ingress:
+    # Preserve the pre-policy HTTP, HTTPS, and health reachability. The new
+    # policy isolates the selected Pod, so omitting these ports would break the
+    # existing NodePort and controller health paths.
+    - ports:
+        - protocol: TCP
+          port: 8000
+        - protocol: TCP
+          port: 8443
+        - protocol: TCP
+          port: 8080
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: observability
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+              app.kubernetes.io/instance: observability-core-kube-pr-prometheus
+      ports:
+        - protocol: TCP
+          port: 9100
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: observability-allow-prometheus-platform-targets
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/part-of: platform-observability
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: prometheus
+      app.kubernetes.io/instance: observability-core-kube-pr-prometheus
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: gitea
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: gitea
+              app.kubernetes.io/instance: gitea
+      ports:
+        - protocol: TCP
+          port: 3000
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: keycloak
+          podSelector:
+            matchLabels:
+              app: keycloak
+              app.kubernetes.io/instance: keycloak
+              app.kubernetes.io/managed-by: keycloak-operator
+      ports:
+        - protocol: TCP
+          port: 9000
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: platform-data
+          podSelector:
+            matchLabels:
+              cnpg.io/cluster: platform-postgres
+      ports:
+        - protocol: TCP
+          port: 9187
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: object-storage
+          podSelector:
+            matchLabels:
+              aistor.min.io/objectStore: minio-aistor
+      ports:
+        - protocol: TCP
+          port: 9000
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: traefik
+              app.kubernetes.io/instance: traefik-kube-system
+      ports:
+        - protocol: TCP
+          port: 9100
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - cloudnative-pg/pod-monitor.yaml
+  - aistor/service-monitor.yaml
+  - counterpart-network-policies.yaml
+
@@ -0,0 +1,15 @@
+# Tempo
+
+This root renders `grafana-community/tempo` chart `2.2.3` as one Tempo
+`2.10.7` StatefulSet in `observability`.
+
+- The image is rewritten to the reviewed amd64 digest.
+- Traces use the `tempo` AIStor bucket through `tempo-object-storage`; no root
+  credential is referenced.
+- Retention is seven days. WAL and local working data use the dedicated 5Gi
+  Retain Local PV.
+- Only the query/metrics port 3200 and OTLP/gRPC port 4317 are exposed.
+- OTLP/HTTP, Jaeger, Zipkin, metrics-generator, and tempo-query are disabled.
+
+Do not delete the PVC or bucket during rollback. Stop the StatefulSet and
+preserve its Retain volume while S3/WAL state is investigated.
@@ -0,0 +1,56 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability
+
+helmGlobals:
+  chartHome: .helm/charts
+
+helmCharts:
+  - name: tempo
+    repo: https://grafana-community.github.io/helm-charts
+    version: 2.2.3
+    releaseName: tempo
+    namespace: observability
+    valuesFile: values/home.yaml
+    includeCRDs: false
+    skipTests: true
+
+images:
+  - name: docker.io/grafana/tempo
+    newName: docker.io/grafana/tempo
+    digest: sha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c
+
+patches:
+  - target:
+      group: ""
+      version: v1
+      kind: Service
+      name: tempo
+    patch: |-
+      - op: replace
+        path: /spec/ports
+        value:
+          - name: tempo-prom-metrics
+            port: 3200
+            protocol: TCP
+            targetPort: 3200
+          - name: grpc-tempo-otlp
+            port: 4317
+            protocol: TCP
+            targetPort: 4317
+  - target:
+      group: apps
+      version: v1
+      kind: StatefulSet
+      name: tempo
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/containers/0/ports
+        value:
+          - containerPort: 3200
+            name: prom-metrics
+            protocol: TCP
+          - containerPort: 4317
+            name: otlp-grpc
+            protocol: TCP
@@ -0,0 +1,125 @@
+global:
+  commonLabels:
+    observability.hyeonworks.com/instance: home
+
+replicas: 1
+
+config: |
+  stream_over_http_enabled: false
+  memberlist:
+    cluster_label: "{{ .Release.Name }}.{{ .Release.Namespace }}"
+  multitenancy_enabled: false
+  usage_report:
+    reporting_enabled: false
+  compactor:
+    compaction:
+      block_retention: 168h
+  distributor:
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+  ingester:
+    trace_idle_period: 10s
+    max_block_duration: 5m
+    max_block_bytes: 100000000
+    complete_block_timeout: 15m
+    flush_all_on_shutdown: true
+  server:
+    http_listen_port: 3200
+  storage:
+    trace:
+      backend: s3
+      s3:
+        bucket: tempo
+        endpoint: minio.object-storage.svc.cluster.local:80
+        insecure: true
+        forcepathstyle: true
+        access_key: ${AWS_ACCESS_KEY_ID}
+        secret_key: ${AWS_SECRET_ACCESS_KEY}
+      wal:
+        path: /var/tempo/wal
+  querier: {}
+  query_frontend: {}
+  overrides:
+    defaults:
+      ingestion:
+        rate_limit_bytes: 5000000
+        burst_size_bytes: 10000000
+        max_traces_per_user: 5000
+      global:
+        max_bytes_per_trace: 5000000
+
+tempo:
+  registry: docker.io
+  repository: grafana/tempo
+  tag: 2.10.7
+  resources:
+    requests:
+      cpu: 250m
+      memory: 512Mi
+    limits:
+      cpu: "1"
+      memory: 1Gi
+  memBallastSizeMbs: 0
+  reportingEnabled: false
+  metricsGenerator:
+    enabled: false
+  extraArgs:
+    config.expand-env: true
+  extraEnv:
+    - name: AWS_ACCESS_KEY_ID
+      valueFrom:
+        secretKeyRef:
+          name: tempo-object-storage
+          key: access-key-id
+    - name: AWS_SECRET_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: tempo-object-storage
+          key: secret-access-key
+  securityContext:
+    runAsNonRoot: true
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop:
+        - ALL
+    seccompProfile:
+      type: RuntimeDefault
+
+tempoQuery:
+  enabled: false
+
+securityContext:
+  runAsUser: 10001
+  runAsGroup: 10001
+  runAsNonRoot: true
+  fsGroup: 10001
+  fsGroupChangePolicy: OnRootMismatch
+  seccompProfile:
+    type: RuntimeDefault
+
+serviceAccount:
+  automountServiceAccountToken: false
+
+service:
+  type: ClusterIP
+
+serviceMonitor:
+  enabled: true
+  interval: 30s
+  additionalLabels:
+    observability.hyeonworks.com/instance: home
+
+persistence:
+  enabled: true
+  enableStatefulSetAutoDeletePVC: false
+  storageClassName: ssd-local-observability-tempo-retain
+  accessModes:
+    - ReadWriteOnce
+  size: 5Gi
+
+podLabels:
+  platform.hyeonworks.com/aistor-client: "true"