Add platform infrastructure configuration
This commit is contained in:
donghyeon-ka
@@ -0,0 +1,12 @@
|
||||
# Alloy
|
||||
|
||||
Grafana Alloy runs as a single-node DaemonSet in `observability-agent`.
|
||||
It tails only `/var/log/pods`, writes logs to the in-cluster Loki service, and
|
||||
accepts OTLP/gRPC traces on port `4317` for the in-cluster Tempo service.
|
||||
|
||||
The deployment deliberately disables host namespaces, automatic service
|
||||
account token mounts, CRD creation, the config reloader, public exposure, and
|
||||
persistent OpenTelemetry queues. The 2 GiB PVC stores only Alloy file
|
||||
positions and local state. Alloy keeps primary UID/GID `473:473`; supplemental
|
||||
group `0` permits read-only traversal of k3s' `root:root 0750` Pod log tree
|
||||
without changing host permissions or running the container as root.
|
||||
@@ -0,0 +1,214 @@
|
||||
discovery.kubernetes "pods" {
|
||||
role = "pod"
|
||||
|
||||
selectors {
|
||||
role = "pod"
|
||||
field = "spec.nodeName=" + sys.env("HOSTNAME")
|
||||
}
|
||||
}
|
||||
|
||||
discovery.relabel "pod_logs" {
|
||||
targets = discovery.kubernetes.pods.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
target_label = "namespace"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_name"]
|
||||
target_label = "pod"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
|
||||
regex = "(.+)"
|
||||
target_label = "app"
|
||||
}
|
||||
rule {
|
||||
source_labels = [
|
||||
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
|
||||
"__meta_kubernetes_pod_label_app",
|
||||
]
|
||||
separator = ";"
|
||||
regex = "^;(.+)$"
|
||||
replacement = "$1"
|
||||
target_label = "app"
|
||||
}
|
||||
rule {
|
||||
source_labels = [
|
||||
"__meta_kubernetes_pod_uid",
|
||||
"__meta_kubernetes_pod_container_name",
|
||||
]
|
||||
separator = "/"
|
||||
replacement = "/var/log/pods/*$1/*.log"
|
||||
target_label = "__path__"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_container_id"]
|
||||
regex = `^(\S+):\/\/.+$`
|
||||
replacement = "$1"
|
||||
target_label = "container_runtime"
|
||||
}
|
||||
}
|
||||
|
||||
local.file_match "pod_logs" {
|
||||
path_targets = discovery.relabel.pod_logs.output
|
||||
}
|
||||
|
||||
loki.source.file "pod_logs" {
|
||||
targets = local.file_match.pod_logs.targets
|
||||
forward_to = [loki.process.pod_logs.receiver]
|
||||
}
|
||||
|
||||
loki.process "pod_logs" {
|
||||
stage.match {
|
||||
selector = `{container_runtime=~"containerd|cri-o"}`
|
||||
stage.cri {}
|
||||
}
|
||||
stage.match {
|
||||
selector = `{container_runtime="docker"}`
|
||||
stage.docker {}
|
||||
}
|
||||
|
||||
stage.replace {
|
||||
expression = `(?i)"(?:authorization|bearer|jwt|password|passwd|client[_-]?secret)"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"`
|
||||
replace = "[REDACTED]"
|
||||
}
|
||||
stage.replace {
|
||||
expression = `(?i)(?:authorization\s*[:=]\s*)(?:bearer\s+)?([A-Za-z0-9._~+/=-]+)`
|
||||
replace = "[REDACTED]"
|
||||
}
|
||||
stage.replace {
|
||||
expression = `(?i)\bbearer\s+([A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+){0,2})`
|
||||
replace = "[REDACTED]"
|
||||
}
|
||||
stage.replace {
|
||||
expression = `(?i)(?:password|passwd|client[_-]?secret)\s*[:=]\s*([^\s"'{},]+)`
|
||||
replace = "[REDACTED]"
|
||||
}
|
||||
|
||||
stage.json {
|
||||
expressions = {
|
||||
level = "level",
|
||||
}
|
||||
}
|
||||
stage.labels {
|
||||
values = {
|
||||
level = "",
|
||||
}
|
||||
}
|
||||
stage.match {
|
||||
selector = `{level!~"(?i)^(trace|debug|info|warn|warning|error|fatal|panic)$"}`
|
||||
stage.label_drop {
|
||||
values = ["level"]
|
||||
}
|
||||
}
|
||||
stage.static_labels {
|
||||
values = {
|
||||
cluster = "home",
|
||||
}
|
||||
}
|
||||
stage.label_keep {
|
||||
values = ["cluster", "namespace", "app", "pod", "container", "level"]
|
||||
}
|
||||
|
||||
forward_to = [loki.write.local.receiver]
|
||||
}
|
||||
|
||||
loki.write "local" {
|
||||
endpoint {
|
||||
url = "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push"
|
||||
batch_size = "256KiB"
|
||||
batch_wait = "1s"
|
||||
remote_timeout = "10s"
|
||||
min_backoff_period = "500ms"
|
||||
max_backoff_period = "30s"
|
||||
max_backoff_retries = 10
|
||||
}
|
||||
}
|
||||
|
||||
otelcol.receiver.otlp "traces" {
|
||||
grpc {
|
||||
endpoint = "0.0.0.0:4317"
|
||||
}
|
||||
|
||||
output {
|
||||
traces = [otelcol.processor.k8sattributes.traces.input]
|
||||
}
|
||||
}
|
||||
|
||||
otelcol.processor.k8sattributes "traces" {
|
||||
auth_type = "serviceAccount"
|
||||
|
||||
extract {
|
||||
deployment_name_from_replicaset = false
|
||||
metadata = [
|
||||
"k8s.namespace.name",
|
||||
"k8s.deployment.name",
|
||||
"k8s.node.name",
|
||||
"k8s.pod.name",
|
||||
"k8s.pod.uid",
|
||||
"k8s.pod.start_time",
|
||||
"k8s.container.name",
|
||||
]
|
||||
}
|
||||
|
||||
pod_association {
|
||||
source {
|
||||
from = "resource_attribute"
|
||||
name = "k8s.pod.uid"
|
||||
}
|
||||
}
|
||||
pod_association {
|
||||
source {
|
||||
from = "resource_attribute"
|
||||
name = "k8s.pod.ip"
|
||||
}
|
||||
}
|
||||
pod_association {
|
||||
source {
|
||||
from = "connection"
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
traces = [otelcol.processor.batch.traces.input]
|
||||
}
|
||||
}
|
||||
|
||||
otelcol.processor.batch "traces" {
|
||||
timeout = "1s"
|
||||
send_batch_size = 512
|
||||
send_batch_max_size = 1024
|
||||
|
||||
output {
|
||||
traces = [otelcol.exporter.otlp.tempo.input]
|
||||
}
|
||||
}
|
||||
|
||||
otelcol.exporter.otlp "tempo" {
|
||||
client {
|
||||
endpoint = "tempo.observability.svc.cluster.local:4317"
|
||||
tls {
|
||||
insecure = true
|
||||
}
|
||||
}
|
||||
|
||||
retry_on_failure {
|
||||
enabled = true
|
||||
initial_interval = "1s"
|
||||
max_interval = "15s"
|
||||
max_elapsed_time = "5m"
|
||||
}
|
||||
|
||||
sending_queue {
|
||||
enabled = true
|
||||
queue_size = 256
|
||||
num_consumers = 2
|
||||
sizer = "requests"
|
||||
block_on_overflow = true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: observability-agent
|
||||
|
||||
resources:
|
||||
- pvc.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: alloy-config
|
||||
files:
|
||||
- config.alloy=config.alloy
|
||||
|
||||
helmGlobals:
|
||||
chartHome: .helm/charts
|
||||
|
||||
helmCharts:
|
||||
- name: alloy
|
||||
repo: https://grafana.github.io/helm-charts
|
||||
version: 1.11.0
|
||||
releaseName: alloy
|
||||
namespace: observability-agent
|
||||
valuesFile: values/home.yaml
|
||||
includeCRDs: false
|
||||
skipTests: true
|
||||
|
||||
images:
|
||||
- name: docker.io/grafana/alloy
|
||||
newName: docker.io/grafana/alloy
|
||||
digest: sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17
|
||||
|
||||
patches:
|
||||
- target:
|
||||
group: apps
|
||||
version: v1
|
||||
kind: DaemonSet
|
||||
name: alloy
|
||||
patch: |-
|
||||
- op: add
|
||||
path: /spec/template/spec/automountServiceAccountToken
|
||||
value: false
|
||||
- op: add
|
||||
path: /spec/template/spec/hostNetwork
|
||||
value: false
|
||||
- op: add
|
||||
path: /spec/template/spec/hostPID
|
||||
value: false
|
||||
- op: add
|
||||
path: /spec/template/spec/hostIPC
|
||||
value: false
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: alloy-storage
|
||||
namespace: observability-agent
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
observability.hyeonworks.com/instance: home
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: ssd-local-observability-alloy-retain
|
||||
volumeName: observability-alloy-local-pv
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
crds:
|
||||
create: false
|
||||
|
||||
global:
|
||||
podSecurityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 473
|
||||
runAsGroup: 473
|
||||
fsGroup: 473
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
# k3s keeps /var/log/pods root:root 0750. Retain Alloy's non-root primary
|
||||
# identity while granting read-only traversal of that single hostPath.
|
||||
supplementalGroups:
|
||||
- 0
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
|
||||
alloy:
|
||||
configMap:
|
||||
create: false
|
||||
name: alloy-config
|
||||
key: config.alloy
|
||||
clustering:
|
||||
enabled: false
|
||||
stabilityLevel: generally-available
|
||||
storagePath: /var/lib/alloy
|
||||
enableHttpServerPort: true
|
||||
listenAddr: 0.0.0.0
|
||||
listenPort: 12345
|
||||
enableReporting: false
|
||||
extraPorts:
|
||||
- name: otlp-grpc
|
||||
port: 4317
|
||||
targetPort: 4317
|
||||
protocol: TCP
|
||||
mounts:
|
||||
varlog: false
|
||||
dockercontainers: false
|
||||
extra:
|
||||
- name: pod-logs
|
||||
mountPath: /var/log/pods
|
||||
readOnly: true
|
||||
- name: alloy-storage
|
||||
mountPath: /var/lib/alloy
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
- name: kubernetes-api-access
|
||||
mountPath: /var/run/secrets/kubernetes.io/serviceaccount
|
||||
readOnly: true
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 473
|
||||
runAsGroup: 473
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 384Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 768Mi
|
||||
|
||||
image:
|
||||
registry: docker.io
|
||||
repository: grafana/alloy
|
||||
tag: v1.18.0
|
||||
digest: sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
rbac:
|
||||
create: true
|
||||
namespaces: []
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- namespaces
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
clusterRules:
|
||||
- apiGroups:
|
||||
- apps
|
||||
resources:
|
||||
- replicasets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
|
||||
serviceAccount:
|
||||
create: true
|
||||
name: alloy
|
||||
automountServiceAccountToken: false
|
||||
|
||||
configReloader:
|
||||
enabled: false
|
||||
|
||||
controller:
|
||||
type: daemonset
|
||||
hostNetwork: false
|
||||
hostPID: false
|
||||
volumes:
|
||||
extra:
|
||||
- name: pod-logs
|
||||
hostPath:
|
||||
path: /var/log/pods
|
||||
type: Directory
|
||||
- name: alloy-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: alloy-storage
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
- name: kubernetes-api-access
|
||||
projected:
|
||||
defaultMode: 420
|
||||
sources:
|
||||
- serviceAccountToken:
|
||||
expirationSeconds: 3600
|
||||
path: token
|
||||
- configMap:
|
||||
name: kube-root-ca.crt
|
||||
items:
|
||||
- key: ca.crt
|
||||
path: ca.crt
|
||||
- downwardAPI:
|
||||
items:
|
||||
- path: namespace
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: metadata.namespace
|
||||
|
||||
networkPolicy:
|
||||
enabled: false
|
||||
|
||||
service:
|
||||
enabled: true
|
||||
type: ClusterIP
|
||||
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
interval: 30s
|
||||
additionalLabels:
|
||||
observability.hyeonworks.com/instance: home
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
Reference in New Issue
Block a user