init: 폴더구조 설계 및 인프라 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,659 @@
|
||||
# resources / probes / availability 예시
|
||||
|
||||
아래 예시는 1000+ 서비스를 운영하는 기준선이다. 모든 YAML은 그대로 `kubectl apply -f` 가능한 형태이며, 라벨 / probe / PDB / HPA / topologySpread / ServiceMonitor / NetworkPolicy가 한 세트로 맞물린다.
|
||||
|
||||
---
|
||||
|
||||
## 좋은 예시 1: auth-server 완전 매니페스트 세트 (Burstable + HPA)
|
||||
|
||||
### Deployment
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: auth-server
|
||||
namespace: auth-prod
|
||||
labels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
app.kubernetes.io/version: "1.24.0"
|
||||
app.kubernetes.io/component: api
|
||||
app.kubernetes.io/part-of: identity-platform
|
||||
app.kubernetes.io/managed-by: argocd
|
||||
spec:
|
||||
replicas: 3
|
||||
revisionHistoryLimit: 10
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 25%
|
||||
maxUnavailable: 0
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
app.kubernetes.io/version: "1.24.0"
|
||||
app.kubernetes.io/component: api
|
||||
app.kubernetes.io/part-of: identity-platform
|
||||
spec:
|
||||
serviceAccountName: auth-server
|
||||
terminationGracePeriodSeconds: 45
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
fsGroup: 10001
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: topology.kubernetes.io/zone
|
||||
whenUnsatisfiable: DoNotSchedule
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: ScheduleAnyway
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
containers:
|
||||
- name: auth-server
|
||||
image: registry.example.com/identity/auth-server:1.24.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
- name: metrics
|
||||
containerPort: 8081
|
||||
env:
|
||||
- name: JAVA_OPTS
|
||||
value: "-XX:MaxRAMPercentage=75.0 -XX:+UseG1GC"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "500m"
|
||||
memory: "1Gi"
|
||||
limits:
|
||||
memory: "1536Mi"
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/started
|
||||
port: http
|
||||
periodSeconds: 5
|
||||
failureThreshold: 24
|
||||
timeoutSeconds: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: http
|
||||
periodSeconds: 5
|
||||
failureThreshold: 3
|
||||
timeoutSeconds: 2
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: http
|
||||
periodSeconds: 15
|
||||
failureThreshold: 3
|
||||
timeoutSeconds: 3
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command: ["/bin/sh", "-c", "sleep 15"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
```
|
||||
|
||||
**왜 좋은가:**
|
||||
|
||||
- QoS는 의도적으로 Burstable (CPU limit 생략으로 throttling 회피, memory는 1.5× headroom).
|
||||
- startup probe가 최대 120초 (24×5) cold start를 덮으며 그 전까지 readiness/liveness는 실행되지 않는다.
|
||||
- topologySpreadConstraints로 zone 장애 격리 + host 분산.
|
||||
- app.kubernetes.io/* 표준 라벨 full set.
|
||||
- preStop sleep 15초로 endpoint 제거 전파 시간을 확보한다.
|
||||
- rolling update `maxUnavailable: 0`으로 항상 N replica 이상 유지.
|
||||
|
||||
### Service + PDB
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: auth-server
|
||||
namespace: auth-prod
|
||||
labels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: http
|
||||
- name: metrics
|
||||
port: 8081
|
||||
targetPort: metrics
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: auth-server
|
||||
namespace: auth-prod
|
||||
labels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
spec:
|
||||
maxUnavailable: 1
|
||||
unhealthyPodEvictionPolicy: AlwaysAllow
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
```
|
||||
|
||||
### HorizontalPodAutoscaler v2 with behavior block
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: auth-server
|
||||
namespace: auth-prod
|
||||
labels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: auth-server
|
||||
minReplicas: 3
|
||||
maxReplicas: 20
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
- type: Pods
|
||||
pods:
|
||||
metric:
|
||||
name: http_requests_in_flight
|
||||
target:
|
||||
type: AverageValue
|
||||
averageValue: "50"
|
||||
behavior:
|
||||
scaleUp:
|
||||
stabilizationWindowSeconds: 0
|
||||
policies:
|
||||
- type: Percent
|
||||
value: 100
|
||||
periodSeconds: 30
|
||||
- type: Pods
|
||||
value: 4
|
||||
periodSeconds: 30
|
||||
selectPolicy: Max
|
||||
scaleDown:
|
||||
stabilizationWindowSeconds: 300
|
||||
policies:
|
||||
- type: Percent
|
||||
value: 25
|
||||
periodSeconds: 60
|
||||
selectPolicy: Max
|
||||
```
|
||||
|
||||
**왜 좋은가:**
|
||||
|
||||
- Resource metric과 custom Pods metric을 동시에 평가.
|
||||
- scaleUp stabilization 0s → 스파이크에 즉시 반응.
|
||||
- scaleDown 300s stabilization + 25%/min rate → flapping 방지.
|
||||
- Pods metric은 pod당 in-flight request 수 (label cardinality 안전).
|
||||
|
||||
### ServiceMonitor (kube-prometheus-stack)
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: auth-server
|
||||
namespace: auth-prod
|
||||
labels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- auth-prod
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
endpoints:
|
||||
- port: metrics
|
||||
path: /actuator/prometheus
|
||||
scheme: http
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
honorLabels: false
|
||||
relabelings:
|
||||
- sourceLabels: [__meta_kubernetes_pod_name]
|
||||
targetLabel: pod
|
||||
- sourceLabels: [__meta_kubernetes_namespace]
|
||||
targetLabel: namespace
|
||||
- action: labeldrop
|
||||
regex: "pod_template_hash"
|
||||
```
|
||||
|
||||
### NetworkPolicy (scrape만 prometheus namespace에서 허용)
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: auth-server-metrics-from-prom
|
||||
namespace: auth-prod
|
||||
labels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: auth-server
|
||||
app.kubernetes.io/instance: auth-server-prod
|
||||
policyTypes: ["Ingress"]
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: monitoring
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
ports:
|
||||
- port: metrics
|
||||
protocol: TCP
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: ingress-nginx
|
||||
ports:
|
||||
- port: http
|
||||
protocol: TCP
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 좋은 예시 2: Keycloak — Guaranteed QoS + slow startup
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: identity
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
app.kubernetes.io/instance: keycloak-prod
|
||||
app.kubernetes.io/version: "24.0.4"
|
||||
app.kubernetes.io/component: identity-provider
|
||||
app.kubernetes.io/part-of: identity-platform
|
||||
spec:
|
||||
serviceName: keycloak-headless
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
app.kubernetes.io/instance: keycloak-prod
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
app.kubernetes.io/instance: keycloak-prod
|
||||
app.kubernetes.io/version: "24.0.4"
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 60
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: topology.kubernetes.io/zone
|
||||
whenUnsatisfiable: DoNotSchedule
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
app.kubernetes.io/instance: keycloak-prod
|
||||
containers:
|
||||
- name: keycloak
|
||||
image: quay.io/keycloak/keycloak:24.0.4
|
||||
args: ["start"]
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
- name: mgmt
|
||||
containerPort: 9000
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: "2Gi"
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: "2Gi"
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health/started
|
||||
port: mgmt
|
||||
periodSeconds: 10
|
||||
failureThreshold: 30
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health/ready
|
||||
port: mgmt
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health/live
|
||||
port: mgmt
|
||||
periodSeconds: 30
|
||||
failureThreshold: 3
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumeMounts:
|
||||
- { name: tmp, mountPath: /tmp }
|
||||
- { name: data, mountPath: /opt/keycloak/data }
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: identity
|
||||
spec:
|
||||
maxUnavailable: 1
|
||||
unhealthyPodEvictionPolicy: AlwaysAllow
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
app.kubernetes.io/instance: keycloak-prod
|
||||
```
|
||||
|
||||
**왜 좋은가:**
|
||||
|
||||
- Guaranteed QoS (request == limit 모든 리소스) → eviction 우선순위 최고.
|
||||
- startup budget = 10s × 30 = 300s, Keycloak cold boot p99 덮음.
|
||||
- management port 9000에만 health, HTTP 8080은 traffic 전용.
|
||||
- PDB maxUnavailable: 1로 3-node infinispan cluster 중 최소 2개 생존 보장.
|
||||
|
||||
---
|
||||
|
||||
## 좋은 예시 3: 1.29+ native sidecar (log forwarder)
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: report-worker
|
||||
namespace: reporting
|
||||
labels:
|
||||
app.kubernetes.io/name: report-worker
|
||||
app.kubernetes.io/instance: report-worker-prod
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: report-worker
|
||||
app.kubernetes.io/instance: report-worker-prod
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: report-worker
|
||||
app.kubernetes.io/instance: report-worker-prod
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
- name: schema-check
|
||||
image: registry.example.com/tools/schema-check:1.2.0
|
||||
command: ["/bin/schema-check", "--fail-fast"]
|
||||
resources:
|
||||
requests: { cpu: 50m, memory: 64Mi }
|
||||
limits: { memory: 128Mi }
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
- name: log-forwarder
|
||||
image: grafana/alloy:v1.2.0
|
||||
restartPolicy: Always # <- native sidecar (1.29+)
|
||||
args: ["run", "/etc/alloy/config.alloy"]
|
||||
resources:
|
||||
requests: { cpu: 50m, memory: 128Mi }
|
||||
limits: { memory: 256Mi }
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumeMounts:
|
||||
- name: alloy-config
|
||||
mountPath: /etc/alloy
|
||||
- name: shared-logs
|
||||
mountPath: /var/log/app
|
||||
containers:
|
||||
- name: worker
|
||||
image: registry.example.com/reporting/worker:2.3.1
|
||||
resources:
|
||||
requests:
|
||||
cpu: "200m"
|
||||
memory: "512Mi"
|
||||
limits:
|
||||
memory: "768Mi"
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumeMounts:
|
||||
- name: shared-logs
|
||||
mountPath: /var/log/app
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: alloy-config
|
||||
configMap:
|
||||
name: alloy-config
|
||||
- name: shared-logs
|
||||
emptyDir: {}
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
```
|
||||
|
||||
**왜 좋은가:**
|
||||
|
||||
- `restartPolicy: Always` on init container = native sidecar 패턴 (1.29+).
|
||||
- init container 순서: schema-check 완료 → log-forwarder sidecar 시작 → main container.
|
||||
- sidecar는 main 종료 후 SIGTERM 받음 (log flush 가능).
|
||||
|
||||
---
|
||||
|
||||
## 나쁜 예시 1: CPU limit 기계적 설정 (throttling 유발)
|
||||
|
||||
```yaml
|
||||
resources:
|
||||
requests:
|
||||
cpu: "500m"
|
||||
memory: "1Gi"
|
||||
limits:
|
||||
cpu: "500m" # BAD
|
||||
memory: "1Gi"
|
||||
```
|
||||
|
||||
**문제:**
|
||||
|
||||
- Linux CFS quota가 100ms period 내 burst만으로도 throttle을 발생시킨다.
|
||||
- p99 latency가 간헐적으로 튀어도 원인이 숨는다 (metric은 평균 usage 기준).
|
||||
- Google SRE / Tim Hockin 공식 가이드: "대부분의 워크로드에서 CPU limit를 제거하라".
|
||||
|
||||
**Fix:** CPU는 request만, memory만 limit로.
|
||||
|
||||
---
|
||||
|
||||
## 나쁜 예시 2: liveness로 readiness 대신함
|
||||
|
||||
```yaml
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health # BAD - deep check
|
||||
port: 8080
|
||||
periodSeconds: 5
|
||||
failureThreshold: 2
|
||||
# readiness 없음
|
||||
```
|
||||
|
||||
**문제:**
|
||||
|
||||
- deep `/actuator/health`는 DB/외부 의존성 포함. DB blip → 모든 Pod 재시작 → cascading failure.
|
||||
- 트래픽 수용 준비 상태를 표현할 수단이 없다.
|
||||
|
||||
**Fix:** startup / readiness / liveness 세 축 분리. liveness는 `/health/live` 같은 shallow check.
|
||||
|
||||
---
|
||||
|
||||
## 나쁜 예시 3: podAntiAffinity로 spread 시도 (legacy)
|
||||
|
||||
```yaml
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values: ["auth-server"]
|
||||
topologyKey: kubernetes.io/hostname
|
||||
```
|
||||
|
||||
**문제:**
|
||||
|
||||
- replica 수가 노드 수보다 많으면 스케줄 불가.
|
||||
- zone 분산이 회계되지 않는다 (skew 개념 없음).
|
||||
- maxSkew 튜닝 불가.
|
||||
|
||||
**Fix:** topologySpreadConstraints 사용 (좋은 예시 1 참조).
|
||||
|
||||
---
|
||||
|
||||
## 나쁜 예시 4: replica 1 서비스에 PDB
|
||||
|
||||
```yaml
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
spec:
|
||||
minAvailable: 1 # BAD - with replica=1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: singleton-app
|
||||
```
|
||||
|
||||
**문제:**
|
||||
|
||||
- node drain이 영구 블록된다 (`PDB violation`).
|
||||
- Kubernetes 업그레이드가 불가능해진다.
|
||||
|
||||
**Fix:** replica 1은 PDB 제거. 필요 시 replica 2+로 늘리고 PDB 적용.
|
||||
|
||||
---
|
||||
|
||||
## 나쁜 예시 5: HPA v1 스타일 (behavior 없음)
|
||||
|
||||
```yaml
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: auth-server
|
||||
minReplicas: 1
|
||||
maxReplicas: 10
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
# behavior block 없음
|
||||
```
|
||||
|
||||
**문제:**
|
||||
|
||||
- 기본 scale-down stabilization 300s지만 scale-up도 쓸데없이 보수적.
|
||||
- 트래픽 burst에 대응 지연.
|
||||
- 트래픽 drop 뒤 flapping 발생 가능 (policy 정의 없음).
|
||||
|
||||
**Fix:** `behavior` block 필수 (좋은 예시 1 참조).
|
||||
|
||||
---
|
||||
|
||||
## 나쁜 예시 6: limit만 있고 request 없음
|
||||
|
||||
```yaml
|
||||
resources:
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: "1Gi"
|
||||
```
|
||||
|
||||
**문제:**
|
||||
|
||||
- Kubernetes가 request = limit로 복사 → 암묵적 Guaranteed.
|
||||
- 스케줄러 회계가 과대 평가되어 cluster density 저하.
|
||||
- 의도한 QoS class와 다름.
|
||||
|
||||
**Fix:** requests 명시 필수.
|
||||
Reference in New Issue
Block a user