From ebdf33f07282ad4109dae643719b829b2ca63e0a Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 00:29:18 +0900 Subject: [PATCH] feat(tenants): declare the tech-log objects inside platform namespaces The tech-log tenant needs objects inside platform namespaces: its database and role, ingress to Postgres and AIStor, blackbox probes, alert rules, a dashboard and two observability NetworkPolicies. They were only ever applied with kubectl and no repository holds them, so a deleted object cannot be recreated and a platform re-apply can drop tech-log access without anyone noticing. Declare them in services/tenants/techlog, copied from each object's last-applied configuration. The dashboard follows the repository's dashboard pattern (JSON file + configMapGenerator, same name, same labels). The hyeonworks-prod Namespace moves here as the tenant boundary and gets PSA enforce=baseline, warn/audit=restricted. Rejected: enforce=restricted now. A server dry-run shows all three tech-log pods would violate it; baseline has zero violations. Risk: the render matches the live objects exactly except the added PSA labels. The pg_hba rules and the Secret stay out of this root (Cluster owner and Git policy). Verification (desktop): - render-tenants-and-policies.sh passes. - kubectl diff of the render shows only the Namespace PSA labels. - kubectl label --dry-run=server enforce=baseline: no warnings. --- .../validate/render-tenants-and-policies.sh | 126 ++++++++++++++++++ services/tenants/techlog/README.md | 44 ++++++ .../dashboards/techlog-operations.json | 1 + services/tenants/techlog/kustomization.yaml | 23 ++++ services/tenants/techlog/namespace.yaml | 12 ++ services/tenants/techlog/object-storage.yaml | 22 +++ .../observability-network-policies.yaml | 45 +++++++ .../tenants/techlog/observability-probes.yaml | 46 +++++++ .../tenants/techlog/observability-rules.yaml | 67 ++++++++++ services/tenants/techlog/platform-data.yaml | 55 ++++++++ 10 files changed, 441 insertions(+) create mode 100644 scripts/validate/render-tenants-and-policies.sh create mode 100644 services/tenants/techlog/README.md create mode 100644 services/tenants/techlog/dashboards/techlog-operations.json create mode 100644 services/tenants/techlog/kustomization.yaml create mode 100644 services/tenants/techlog/namespace.yaml create mode 100644 services/tenants/techlog/object-storage.yaml create mode 100644 services/tenants/techlog/observability-network-policies.yaml create mode 100644 services/tenants/techlog/observability-probes.yaml create mode 100644 services/tenants/techlog/observability-rules.yaml create mode 100644 services/tenants/techlog/platform-data.yaml diff --git a/scripts/validate/render-tenants-and-policies.sh b/scripts/validate/render-tenants-and-policies.sh new file mode 100644 index 0000000..49bd860 --- /dev/null +++ b/scripts/validate/render-tenants-and-policies.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Render the tech-log tenant root, the platform PriorityClasses and the keycloak +# namespace NetworkPolicies, and check their invariants. Static: no cluster access. +set -Eeuo pipefail + +readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1" +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +kustomize_version="$(kubectl version --client --output=yaml | sed -n 's/^kustomizeVersion: //p')" +[[ "$kustomize_version" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \ + fail "expected Kustomize ${EXPECTED_KUSTOMIZE_VERSION}, found ${kustomize_version:-unknown}" + +work="$(mktemp -d "${TMPDIR:-/tmp}/platform-tenants-render.XXXXXX")" +trap 'rm -rf -- "$work"' EXIT + +kubectl kustomize "$REPOSITORY_ROOT/services/tenants/techlog" >"$work/techlog.yaml" +kubectl kustomize "$REPOSITORY_ROOT/services/priority-classes" >"$work/priority-classes.yaml" +kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" + +python3 -I - "$work" <<'PY' +import sys, pathlib, yaml + +work = pathlib.Path(sys.argv[1]) +errors = [] + +def load(name): + return [d for d in yaml.safe_load_all((work / name).read_text()) if d] + +def keyset(docs): + return sorted((d["kind"], d["metadata"].get("namespace", ""), d["metadata"]["name"]) for d in docs) + +for name in ("techlog.yaml", "priority-classes.yaml", "keycloak-network-policies.yaml"): + for d in load(name): + if d["kind"] == "Secret": + errors.append(f"{name}: Secret objects must not be rendered") + +techlog = load("techlog.yaml") +expected = sorted([ + ("Namespace", "", "hyeonworks-prod"), + ("Database", "platform-data", "platform-postgres-techlog"), + ("DatabaseRole", "platform-data", "platform-postgres-techlog"), + ("NetworkPolicy", "platform-data", "allow-techlog-backend"), + ("NetworkPolicy", "object-storage", "allow-techlog-backend"), + ("Probe", "observability", "techlog-edge"), + ("Probe", "observability", "techlog-readiness"), + ("PrometheusRule", "observability", "techlog"), + ("ConfigMap", "observability", "grafana-dashboard-techlog"), + ("NetworkPolicy", "observability", "blackbox-allow-techlog-health"), + ("NetworkPolicy", "observability", "observability-allow-techlog-scrape"), +]) +if keyset(techlog) != expected: + errors.append(f"tech-log tenant objects differ: {keyset(techlog)}") +for d in techlog: + kind = d["kind"] + if kind == "Namespace": + labels = d["metadata"].get("labels", {}) + want = { + "pod-security.kubernetes.io/enforce": "baseline", + "pod-security.kubernetes.io/warn": "restricted", + "pod-security.kubernetes.io/audit": "restricted", + "observability.hyeonworks.com/enabled": "true", + } + for k, v in want.items(): + if labels.get(k) != v: + errors.append(f"hyeonworks-prod label {k} must be {v}") + if kind == "DatabaseRole": + spec = d["spec"] + if spec.get("passwordSecret", {}).get("name") != "techlog-db-credentials": + errors.append("techlog DatabaseRole must reference techlog-db-credentials") + for flag in ("superuser", "createdb", "createrole", "replication", "bypassrls"): + if spec.get(flag) is not False: + errors.append(f"techlog DatabaseRole {flag} must be false") + if kind in ("Database", "DatabaseRole"): + policy = d["spec"].get("databaseReclaimPolicy") or d["spec"].get("databaseRoleReclaimPolicy") + if policy != "retain": + errors.append(f"{kind} must keep the retain reclaim policy") + if kind == "ConfigMap": + labels = d["metadata"].get("labels", {}) + if labels.get("grafana_dashboard") != "1": + errors.append("tech-log dashboard ConfigMap needs grafana_dashboard=1") + if d["metadata"]["name"] != "grafana-dashboard-techlog": + errors.append("tech-log dashboard ConfigMap must keep its name (no hash suffix)") + +classes = {d["metadata"]["name"]: d for d in load("priority-classes.yaml")} +if sorted(classes) != ["platform-critical", "platform-low"]: + errors.append(f"PriorityClasses differ: {sorted(classes)}") +else: + crit, low = classes["platform-critical"], classes["platform-low"] + if not (0 < crit["value"] < 1_000_000_000): + errors.append("platform-critical must be above 0 and below the system classes") + if not low["value"] < 0: + errors.append("platform-low must be below the default priority 0") + if low.get("preemptionPolicy") != "Never": + errors.append("platform-low must never preempt") + for d in classes.values(): + if d.get("globalDefault") is not False: + errors.append(f"{d['metadata']['name']} must not be the global default") + +policies = {d["metadata"]["name"]: d for d in load("keycloak-network-policies.yaml")} +deny = policies.get("keycloak-default-deny") +if not deny or deny["spec"].get("podSelector") != {} or sorted(deny["spec"].get("policyTypes", [])) != ["Egress", "Ingress"]: + errors.append("keycloak needs a namespace-wide default deny for ingress and egress") +for d in policies.values(): + if d["metadata"].get("namespace") != "keycloak": + errors.append(f"{d['metadata']['name']} must live in the keycloak namespace") + for rule in d["spec"].get("egress", []) or []: + for peer in rule.get("to", []): + cidr = peer.get("ipBlock", {}).get("cidr") + if cidr and cidr not in ("10.43.0.1/32", "192.168.0.107/32"): + errors.append(f"{d['metadata']['name']}: unexpected egress CIDR {cidr}") + if not rule.get("ports"): + errors.append(f"{d['metadata']['name']}: every egress rule must name its ports") + +if errors: + for e in errors: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) +print(f"tech-log tenant: {len(techlog)} objects; PriorityClasses: {len(classes)}; keycloak NetworkPolicies: {len(policies)}") +PY + +echo "TENANTS AND POLICIES STATIC RENDER PASS" diff --git a/services/tenants/techlog/README.md b/services/tenants/techlog/README.md new file mode 100644 index 0000000..3eca649 --- /dev/null +++ b/services/tenants/techlog/README.md @@ -0,0 +1,44 @@ +# tech-log 테넌트 — 플랫폼 쪽 객체 + +tech-log(hyeonworks.com 운영)가 플랫폼을 쓰려면 플랫폼 namespace 안에 허용 객체가 있어야 한다. +이 빌드 루트가 그 객체를 소유한다. 앱 자체(Deployment·Service·Ingress·ServiceMonitor·앱 +namespace 안의 NetworkPolicy)는 tech-log 저장소의 `deploy/`가 소유한다. + +2026-11-11 점검(`docs/reviews/2026-11-11-k3s-maturity-review.md` 4.3절) 전까지 이 객체들은 +`kubectl apply`로만 들어가 있었고 어느 저장소에도 없었다. 처음 내용은 라이브 객체의 +`last-applied-configuration`을 그대로 옮겼다. 라이브와 다른 것은 `hyeonworks-prod`의 PSA 라벨뿐이다. + +| 파일 | 객체 | +| --- | --- | +| `namespace.yaml` | Namespace `hyeonworks-prod`: 관측 대상 라벨, PSA `enforce=baseline`·`warn=restricted`·`audit=restricted` | +| `platform-data.yaml` | `techlog` Database와 DatabaseRole(로그인만, 연결 30), Postgres로 들어오는 ingress | +| `object-storage.yaml` | AIStor 9000으로 들어오는 ingress | +| `observability-probes.yaml` | blackbox Probe 둘(공개 HTTPS, 내부 readiness) | +| `observability-rules.yaml` | PrometheusRule `techlog` 일곱 개 | +| `dashboards/techlog-operations.json` | Grafana 대시보드(`grafana-dashboard-techlog` ConfigMap으로 생성) | +| `observability-network-policies.yaml` | blackbox → readiness, Prometheus → management scrape egress | + +## 여기에 없는 것 + +- `platform-data/platform-postgres`의 `pg_hba` techlog 두 줄: Cluster 객체는 + `services/platform-postgres`가 소유한다. +- Secret: `platform-data/techlog-db-credentials`(`kubernetes.io/basic-auth`, `username`은 정확히 + `techlog`)는 DatabaseRole이 참조한다. 앱 쪽 `hyeonworks-prod/hyeonworks-secrets`는 tech-log가 + 소유한다. 둘 다 Git에 두지 않는다. + +## PSA + +`enforce=baseline`은 2026-11-12에 서버 dry-run(`kubectl label --dry-run=server`)으로 위반 0을 +확인하고 붙였다. `restricted`로 강제하면 지금 파드 셋(redis, backend, frontend)이 모두 걸린다 +(`allowPrivilegeEscalation`, capabilities, `runAsNonRoot`, seccomp). tech-log 쪽 securityContext가 +고쳐진 뒤 같은 dry-run이 경고 없이 통과하면 `enforce`를 `restricted`로 올린다. + +## 렌더·적용 + +```sh +bash scripts/validate/render-tenants-and-policies.sh +kubectl kustomize services/tenants/techlog | kubectl diff -f - +kubectl apply -k services/tenants/techlog +``` + +새로 설치할 때는 이 루트를 tech-log `deploy/`보다 먼저 적용한다(namespace를 여기서 만든다). diff --git a/services/tenants/techlog/dashboards/techlog-operations.json b/services/tenants/techlog/dashboards/techlog-operations.json new file mode 100644 index 0000000..8dd5929 --- /dev/null +++ b/services/tenants/techlog/dashboards/techlog-operations.json @@ -0,0 +1 @@ +{"uid": "techlog-operations", "title": "TechLog · 운영 상태", "tags": ["techlog", "home", "application"], "timezone": "browser", "schemaVersion": 39, "version": 1, "refresh": "30s", "time": {"from": "now-1h", "to": "now"}, "panels": [{"id": 1, "title": "API 메트릭 수집", "type": "stat", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 0, "y": 0, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "up{namespace=\"hyeonworks-prod\",service=\"techlog-management\"}", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 2, "title": "의존성 readiness", "type": "stat", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 12, "y": 0, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "probe_success{job=\"techlog-readiness\"}", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 3, "title": "웹·API HTTPS 상태", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 0, "y": 7, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "probe_success{job=\"techlog-edge\"}", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 4, "title": "인증서 남은 기간", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 12, "y": 7, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "(probe_ssl_earliest_cert_expiry{job=\"techlog-edge\"}-time())/86400", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "d"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 5, "title": "HTTP 요청률", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 0, "y": 14, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "sum by(uri,status)(rate(http_server_requests_seconds_count{namespace=\"hyeonworks-prod\",uri!~\"/actuator.*\"}[$__rate_interval]))", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 6, "title": "HTTP p95·p99 지연", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 12, "y": 14, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "histogram_quantile(0.99,sum by(le)(rate(http_server_requests_seconds_bucket{namespace=\"hyeonworks-prod\",uri!~\"/actuator.*\"}[$__rate_interval])))", "legendFormat": "p99"}, {"refId": "B", "expr": "histogram_quantile(0.95,sum by(le)(rate(http_server_requests_seconds_bucket{namespace=\"hyeonworks-prod\",uri!~\"/actuator.*\"}[$__rate_interval])))", "legendFormat": "p95"}], "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 7, "title": "HTTP 상태별 응답", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 0, "y": 21, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "sum by(status)(rate(http_server_requests_seconds_count{namespace=\"hyeonworks-prod\"}[$__rate_interval]))", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 8, "title": "JVM 메모리", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 12, "y": 21, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "sum by(area)(jvm_memory_used_bytes{namespace=\"hyeonworks-prod\"})", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 9, "title": "JVM GC 시간", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 0, "y": 28, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "sum by(action)(rate(jvm_gc_pause_seconds_sum{namespace=\"hyeonworks-prod\"}[$__rate_interval]))", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 10, "title": "프로세스 CPU", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 12, "y": 28, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "process_cpu_usage{namespace=\"hyeonworks-prod\"}", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 11, "title": "DB 연결 풀 사용·대기", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 0, "y": 35, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "hikaricp_connections_active{namespace=\"hyeonworks-prod\"}", "legendFormat": "active"}, {"refId": "B", "expr": "hikaricp_connections_pending{namespace=\"hyeonworks-prod\"}", "legendFormat": "pending"}], "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 12, "title": "DB 연결 획득 지연", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 12, "y": 35, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "rate(hikaricp_connections_acquire_seconds_sum{namespace=\"hyeonworks-prod\"}[$__rate_interval])/clamp_min(rate(hikaricp_connections_acquire_seconds_count{namespace=\"hyeonworks-prod\"}[$__rate_interval]),0.001)", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 13, "title": "Pod 재시작", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 0, "y": 42, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "kube_pod_container_status_restarts_total{namespace=\"hyeonworks-prod\"}", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 14, "title": "분산 추적 샘플링 비율", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "prometheus"}, "gridPos": {"x": 12, "y": 42, "w": 12, "h": 7}, "targets": [{"refId": "A", "expr": "tracing_sampling_rate{namespace=\"hyeonworks-prod\"}", "legendFormat": "{{uri}} {{status}} {{pool}} {{id}}"}], "fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, {"id": 15, "title": "TechLog 구조화 로그 · 요청 ID / 추적 ID", "type": "logs", "datasource": {"type": "loki", "uid": "loki"}, "gridPos": {"x": 0, "y": 49, "w": 24, "h": 12}, "targets": [{"refId": "A", "expr": "{namespace=\"hyeonworks-prod\",app=\"techlog-backend\"}", "queryType": "range"}], "options": {"showTime": true, "showLabels": false, "showCommonLabels": false, "wrapLogMessage": true, "prettifyLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true}}, {"id": 16, "type": "text", "title": "추적 수집 상태", "gridPos": {"x": 0, "y": 61, "w": 24, "h": 4}, "options": {"mode": "markdown", "content": "요청 ID와 추적 ID는 JSON 로그에서 조회할 수 있습니다. **Tempo span 저장은 현재 이미지에서 검증되지 않았습니다.** 메트릭·로그·의존성 health 수집은 동작합니다."}}], "links": []} \ No newline at end of file diff --git a/services/tenants/techlog/kustomization.yaml b/services/tenants/techlog/kustomization.yaml new file mode 100644 index 0000000..6ede4d1 --- /dev/null +++ b/services/tenants/techlog/kustomization.yaml @@ -0,0 +1,23 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# tech-log(hyeonworks.com) 테넌트의 플랫폼 쪽 객체. 앱 쪽 매니페스트는 tech-log repo가 소유한다. +resources: + - namespace.yaml + - platform-data.yaml + - object-storage.yaml + - observability-probes.yaml + - observability-rules.yaml + - observability-network-policies.yaml + +generatorOptions: + disableNameSuffixHash: true + labels: + app.kubernetes.io/part-of: techlog + grafana_dashboard: "1" + +configMapGenerator: + - name: grafana-dashboard-techlog + namespace: observability + files: + - dashboards/techlog-operations.json diff --git a/services/tenants/techlog/namespace.yaml b/services/tenants/techlog/namespace.yaml new file mode 100644 index 0000000..7374c1b --- /dev/null +++ b/services/tenants/techlog/namespace.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: hyeonworks-prod + labels: + observability.hyeonworks.com/enabled: 'true' + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest diff --git a/services/tenants/techlog/object-storage.yaml b/services/tenants/techlog/object-storage.yaml new file mode 100644 index 0000000..8774f13 --- /dev/null +++ b/services/tenants/techlog/object-storage.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-techlog-backend + namespace: object-storage +spec: + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hyeonworks-prod + podSelector: + matchLabels: + app: techlog-backend + ports: + - port: 9000 + protocol: TCP + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + policyTypes: + - Ingress diff --git a/services/tenants/techlog/observability-network-policies.yaml b/services/tenants/techlog/observability-network-policies.yaml new file mode 100644 index 0000000..bc4e88f --- /dev/null +++ b/services/tenants/techlog/observability-network-policies.yaml @@ -0,0 +1,45 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: blackbox-allow-techlog-health + namespace: observability +spec: + egress: + - ports: + - port: 9001 + protocol: TCP + to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hyeonworks-prod + podSelector: + matchLabels: + app: techlog-backend + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-blackbox-exporter + policyTypes: + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: observability-allow-techlog-scrape + namespace: observability +spec: + egress: + - ports: + - port: 9001 + protocol: TCP + to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hyeonworks-prod + podSelector: + matchLabels: + app: techlog-backend + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + policyTypes: + - Egress diff --git a/services/tenants/techlog/observability-probes.yaml b/services/tenants/techlog/observability-probes.yaml new file mode 100644 index 0000000..71e0bcd --- /dev/null +++ b/services/tenants/techlog/observability-probes.yaml @@ -0,0 +1,46 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + labels: + observability.hyeonworks.com/instance: home + name: techlog-edge + namespace: observability +spec: + interval: 30s + jobName: techlog-edge + module: http_2xx + prober: + path: /probe + scheme: http + url: blackbox-exporter.observability.svc.cluster.local:9115 + scrapeTimeout: 15s + targets: + staticConfig: + labels: + application: techlog + static: + - https://hyeonworks.com/ + - https://hyeonworks.com/api/healthcheck +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + labels: + observability.hyeonworks.com/instance: home + name: techlog-readiness + namespace: observability +spec: + interval: 30s + jobName: techlog-readiness + module: http_2xx + prober: + path: /probe + scheme: http + url: blackbox-exporter.observability.svc.cluster.local:9115 + scrapeTimeout: 15s + targets: + staticConfig: + labels: + application: techlog + static: + - http://techlog-management.hyeonworks-prod.svc.cluster.local:9001/actuator/health/readiness diff --git a/services/tenants/techlog/observability-rules.yaml b/services/tenants/techlog/observability-rules.yaml new file mode 100644 index 0000000..89eec74 --- /dev/null +++ b/services/tenants/techlog/observability-rules.yaml @@ -0,0 +1,67 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + observability.hyeonworks.com/instance: home + name: techlog + namespace: observability +spec: + groups: + - name: techlog + rules: + - alert: TechLogMetricsDown + annotations: + summary: TechLog metrics target is unreachable + expr: up{namespace="hyeonworks-prod",service="techlog-management"} == 0 + for: 5m + labels: + application: techlog + severity: warning + - alert: TechLogReadinessDown + annotations: + summary: TechLog dependency readiness is DOWN + expr: probe_success{job="techlog-readiness"} == 0 + for: 3m + labels: + application: techlog + severity: critical + - alert: TechLogHTTPSDown + annotations: + summary: TechLog HTTPS or API health is failing + expr: probe_success{job="techlog-edge"} == 0 + for: 3m + labels: + application: techlog + severity: critical + - alert: TechLogHTTP5xx + annotations: + summary: TechLog HTTP 5xx exceeds 5% + expr: (sum(rate(http_server_requests_seconds_count{namespace="hyeonworks-prod",status=~"5.."}[5m])) / clamp_min(sum(rate(http_server_requests_seconds_count{namespace="hyeonworks-prod"}[5m])),0.001)) > 0.05 and sum(increase(http_server_requests_seconds_count{namespace="hyeonworks-prod"}[5m])) > 20 + for: 5m + labels: + application: techlog + severity: warning + - alert: TechLogLatencyHigh + annotations: + summary: TechLog p99 latency exceeds 2 seconds + expr: histogram_quantile(0.99,sum by(le)(rate(http_server_requests_seconds_bucket{namespace="hyeonworks-prod",uri!~"/actuator.*"}[5m]))) > 2 + for: 10m + labels: + application: techlog + severity: warning + - alert: TechLogHeapPressure + annotations: + summary: TechLog heap usage exceeds 85% + expr: sum(jvm_memory_used_bytes{namespace="hyeonworks-prod",area="heap"}) / sum(jvm_memory_max_bytes{namespace="hyeonworks-prod",area="heap"}) > 0.85 + for: 10m + labels: + application: techlog + severity: warning + - alert: TechLogDatabasePoolWaiting + annotations: + summary: TechLog database connections are queued + expr: hikaricp_connections_pending{namespace="hyeonworks-prod"} > 0 + for: 5m + labels: + application: techlog + severity: warning diff --git a/services/tenants/techlog/platform-data.yaml b/services/tenants/techlog/platform-data.yaml new file mode 100644 index 0000000..4b1bc54 --- /dev/null +++ b/services/tenants/techlog/platform-data.yaml @@ -0,0 +1,55 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: platform-postgres-techlog + namespace: platform-data +spec: + cluster: + name: platform-postgres + databaseReclaimPolicy: retain + ensure: present + name: techlog + owner: techlog +--- +apiVersion: postgresql.cnpg.io/v1 +kind: DatabaseRole +metadata: + name: platform-postgres-techlog + namespace: platform-data +spec: + bypassrls: false + cluster: + name: platform-postgres + connectionLimit: 30 + createdb: false + createrole: false + databaseRoleReclaimPolicy: retain + login: true + name: techlog + passwordSecret: + name: techlog-db-credentials + replication: false + superuser: false +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-techlog-backend + namespace: platform-data +spec: + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hyeonworks-prod + podSelector: + matchLabels: + app: techlog-backend + ports: + - port: 5432 + protocol: TCP + podSelector: + matchLabels: + cnpg.io/cluster: platform-postgres + policyTypes: + - Ingress