56 lines
3.0 KiB
Bash
56 lines
3.0 KiB
Bash
#!/usr/bin/env bash
|
|
echo "### df"
|
|
df -hT / /srv/k3s/aistor /mnt/hdd-archive
|
|
echo
|
|
echo "### 파드 안에서 잰 PVC 실사용량 (du -sh)"
|
|
kubectl -n gitea exec gitea-77f645475b-bcpzp -c gitea -- du -sh /data
|
|
kubectl -n platform-data exec platform-postgres-1 -c postgres -- du -sh /var/lib/postgresql/data
|
|
kubectl -n platform-admin exec pgadmin-b7c5b4df8-gqmhf -c pgadmin4 -- du -sh /var/lib/pgadmin
|
|
kubectl -n observability exec grafana-6bbf699897-9gwzz -c grafana -- du -sh /var/lib/grafana
|
|
kubectl -n observability-agent exec alloy-69nq2 -c alloy -- du -sh /var/lib/alloy
|
|
kubectl -n observability exec alertmanager-observability-core-kube-pr-alertmanager-0 -c alertmanager -- du -sh /alertmanager
|
|
echo "(loki · tempo · prometheus 는 distroless 라 du 도 ls 도 없다 — 자체 metric 으로 대신한다)"
|
|
echo
|
|
echo "### distroless 세 편의 자체 metric (bytes)"
|
|
kubectl -n observability port-forward svc/observability-core-kube-pr-prometheus 19090:9090 >/dev/null 2>&1 &
|
|
PF=$!
|
|
sleep 4
|
|
for Q in prometheus_tsdb_storage_blocks_bytes prometheus_tsdb_wal_storage_size_bytes \
|
|
prometheus_tsdb_head_chunks_storage_size_bytes loki_ingester_wal_disk_usage_percent; do
|
|
echo "-- $Q"
|
|
curl -s --get --data-urlencode "query=$Q" http://127.0.0.1:19090/api/v1/query \
|
|
| python3 -c 'import sys,json
|
|
for r in json.load(sys.stdin)["data"]["result"]:
|
|
m=r["metric"]; print(" %-24s %-46s %s" % (m.get("job","?"), m.get("pod","?"), r["value"][1]))'
|
|
done
|
|
echo "-- kubelet_volume_stats_used_bytes / capacity (local PV 는 파일시스템 전체를 되풀이한다)"
|
|
for Q in kubelet_volume_stats_used_bytes kubelet_volume_stats_capacity_bytes; do
|
|
echo " [$Q]"
|
|
curl -s --get --data-urlencode "query=$Q" http://127.0.0.1:19090/api/v1/query \
|
|
| python3 -c 'import sys,json
|
|
seen=set()
|
|
for r in json.load(sys.stdin)["data"]["result"]:
|
|
k=r["metric"].get("persistentvolumeclaim","?")
|
|
if k in seen: continue
|
|
seen.add(k); print(" %-70s %s" % (k[:70], r["value"][1]))'
|
|
done
|
|
kill $PF 2>/dev/null
|
|
wait $PF 2>/dev/null
|
|
echo
|
|
echo "### 루트 파일시스템 최상위 (node-exporter, du -shx, uid 65534 read-only)"
|
|
for d in var usr snap opt srv boot etc tmp home mnt; do
|
|
printf "%-8s " "/$d"
|
|
kubectl -n observability-agent exec node-exporter-xfwqx -- du -shx /host/root/$d 2>/dev/null | tail -1
|
|
done
|
|
echo
|
|
echo "### 사용자 홈 (본인 권한)"
|
|
du -shx /home/donghyeon
|
|
du -shx /home/donghyeon/* 2>/dev/null | sort -h | tail -6
|
|
echo
|
|
echo "### /var 안쪽 (읽히는 것만)"
|
|
for d in lib log cache; do printf "%-12s " "/var/$d"; kubectl -n observability-agent exec node-exporter-xfwqx -- du -shx /host/root/var/$d 2>/dev/null | tail -1; done
|
|
kubectl -n observability-agent exec node-exporter-xfwqx -- du -shx /host/root/var/lib/snapd /host/root/var/lib/rancher /host/root/var/lib/apt 2>/dev/null | tail -3
|
|
echo
|
|
echo "### 읽지 못한 디렉터리 (여기에 미귀속분이 있다)"
|
|
kubectl -n observability-agent exec node-exporter-xfwqx -- du -shx /host/root/var/lib/docker /host/root/var/lib/containerd /host/root/root 2>&1 | grep -i "permission denied"
|