chore: initialize from backend template 0a6dd0e
This commit is contained in:
@@ -0,0 +1,127 @@
|
||||
# Storage certification job.
|
||||
#
|
||||
# A PersistentVolumeClaim is not a filesystem contract. Whether an atomic rename, a same-file-store
|
||||
# guarantee, or symlink refusal actually holds depends on the CSI driver, the StorageClass, the
|
||||
# access mode, the backend, and the mount options — so this job records all five alongside the probe
|
||||
# result. A certification without that tuple is not transferable to another cluster.
|
||||
#
|
||||
# The job writes a machine-readable result to the claim itself so the evidence lives with the volume
|
||||
# it describes.
|
||||
#
|
||||
# kubectl apply -f infra/fileserver/kubernetes/pvc-certification-job.yaml
|
||||
# kubectl logs job/fileserver-pvc-certification
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: fileserver-certification
|
||||
labels:
|
||||
app.kubernetes.io/name: fileserver
|
||||
app.kubernetes.io/component: certification
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
# Left unset on purpose: the certification is only meaningful for the class it actually ran on,
|
||||
# so the operator names it explicitly rather than inheriting a cluster default.
|
||||
storageClassName: ""
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: fileserver-pvc-certification
|
||||
labels:
|
||||
app.kubernetes.io/name: fileserver
|
||||
app.kubernetes.io/component: certification
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: fileserver
|
||||
app.kubernetes.io/component: certification
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
fsGroup: 10001
|
||||
containers:
|
||||
- name: certify
|
||||
image: eclipse-temurin:21-jdk
|
||||
env:
|
||||
- name: FILESERVER_STORAGE_ROOT
|
||||
value: /var/lib/backend/files
|
||||
- name: KUBERNETES_VERSION
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.annotations['certification.fileserver/kubernetes-version']
|
||||
- name: CSI_DRIVER
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.annotations['certification.fileserver/csi-driver']
|
||||
- name: STORAGE_CLASS
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.annotations['certification.fileserver/storage-class']
|
||||
- name: ACCESS_MODE
|
||||
value: ReadWriteOnce
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
ROOT="${FILESERVER_STORAGE_ROOT}"
|
||||
mkdir -p "${ROOT}/staging" "${ROOT}/content"
|
||||
|
||||
# Atomic rename within one file store is the property the publish path depends on.
|
||||
echo probe > "${ROOT}/staging/probe"
|
||||
if mv "${ROOT}/staging/probe" "${ROOT}/content/probe" 2>/dev/null; then
|
||||
ATOMIC_MOVE=true
|
||||
else
|
||||
ATOMIC_MOVE=false
|
||||
fi
|
||||
|
||||
# Same device means a rename is a metadata operation rather than a copy.
|
||||
STAGING_DEV=$(stat -c %d "${ROOT}/staging")
|
||||
CONTENT_DEV=$(stat -c %d "${ROOT}/content")
|
||||
[ "${STAGING_DEV}" = "${CONTENT_DEV}" ] && SAME_STORE=true || SAME_STORE=false
|
||||
|
||||
# O_EXCL create is what makes a publish create-only rather than an overwrite.
|
||||
if (set -o noclobber; echo x > "${ROOT}/content/excl") 2>/dev/null; then
|
||||
ATOMIC_CREATE=true
|
||||
else
|
||||
ATOMIC_CREATE=false
|
||||
fi
|
||||
|
||||
cat > "${ROOT}/certification-result.json" <<RESULT
|
||||
{
|
||||
"kubernetesVersion": "${KUBERNETES_VERSION:-unknown}",
|
||||
"csiDriver": "${CSI_DRIVER:-unknown}",
|
||||
"storageClass": "${STORAGE_CLASS:-unknown}",
|
||||
"accessMode": "${ACCESS_MODE}",
|
||||
"backend": "$(stat -f -c %T "${ROOT}")",
|
||||
"mountOptions": "$(findmnt -no OPTIONS --target "${ROOT}" || echo unknown)",
|
||||
"atomicMove": ${ATOMIC_MOVE},
|
||||
"sameFileStore": ${SAME_STORE},
|
||||
"atomicCreate": ${ATOMIC_CREATE}
|
||||
}
|
||||
RESULT
|
||||
cat "${ROOT}/certification-result.json"
|
||||
|
||||
# Fail closed: a volume that cannot publish atomically must not be certified silently.
|
||||
[ "${SAME_STORE}" = "true" ] || { echo "staging and content are on different stores"; exit 1; }
|
||||
volumeMounts:
|
||||
- name: storage
|
||||
mountPath: /var/lib/backend/files
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumes:
|
||||
- name: storage
|
||||
persistentVolumeClaim:
|
||||
claimName: fileserver-certification
|
||||
@@ -0,0 +1,61 @@
|
||||
# Network-filesystem certification environment.
|
||||
#
|
||||
# A local filesystem cannot reproduce the failures this environment exists to test: a rename whose
|
||||
# acknowledgement is lost, a stale file handle after the server restarts, and a client that keeps
|
||||
# writing across a network cut. Those are precisely the cases where "the write failed, retry it" is
|
||||
# the wrong conclusion, so they are certified against a real NFS server rather than a mock.
|
||||
#
|
||||
# Opt in with FILESERVER_NFS_TESTS=true; the default test run does not start this.
|
||||
#
|
||||
# docker compose -f infra/fileserver/nfs/compose.yml up -d
|
||||
# FILESERVER_NFS_TESTS=true ./gradlew :adapter:outbound:fileserver:test
|
||||
#
|
||||
# To exercise the ambiguity paths:
|
||||
# docker compose -f infra/fileserver/nfs/compose.yml restart nfs-server # stale handles
|
||||
# docker network disconnect fileserver-nfs <client> # lost responses
|
||||
|
||||
services:
|
||||
nfs-server:
|
||||
image: erichough/nfs-server:2.2.1
|
||||
container_name: fileserver-nfs-server
|
||||
privileged: true
|
||||
environment:
|
||||
NFS_EXPORT_0: "/exports *(rw,sync,no_subtree_check,no_root_squash,fsid=0)"
|
||||
NFS_VERSION: "4.2"
|
||||
NFS_LOG_LEVEL: DEBUG
|
||||
volumes:
|
||||
- nfs-exports:/exports
|
||||
ports:
|
||||
- "2049:2049"
|
||||
networks:
|
||||
- fileserver-nfs
|
||||
healthcheck:
|
||||
test: ["CMD", "rpcinfo", "-t", "localhost", "nfs", "4"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
nfs-client:
|
||||
image: eclipse-temurin:21-jdk
|
||||
container_name: fileserver-nfs-client
|
||||
privileged: true
|
||||
depends_on:
|
||||
nfs-server:
|
||||
condition: service_healthy
|
||||
# hard,intr is the correct production mount: a soft mount turns a slow server into a silent
|
||||
# short write, which is exactly the corruption the design refuses to accept.
|
||||
command: >
|
||||
bash -c "mkdir -p /mnt/fileserver &&
|
||||
mount -t nfs4 -o hard,timeo=50,retrans=2 nfs-server:/ /mnt/fileserver &&
|
||||
tail -f /dev/null"
|
||||
volumes:
|
||||
- ../../..:/workspace:ro
|
||||
networks:
|
||||
- fileserver-nfs
|
||||
|
||||
volumes:
|
||||
nfs-exports:
|
||||
|
||||
networks:
|
||||
fileserver-nfs:
|
||||
name: fileserver-nfs
|
||||
@@ -0,0 +1,67 @@
|
||||
# Fileserver front-proxy configuration.
|
||||
#
|
||||
# The application authorizes every download and then hands the transfer to Nginx with
|
||||
# X-Accel-Redirect. Two properties make that safe, and both are enforced here rather than assumed:
|
||||
#
|
||||
# 1. /__files/ is `internal`, so it is reachable ONLY through an internal redirect the application
|
||||
# issued. A direct request from a client returns 404 and never touches the content root.
|
||||
# 2. The application never emits an absolute path. It emits a relative URI below /__files/, and
|
||||
# the alias below is the only place that prefix becomes a filesystem location.
|
||||
#
|
||||
# Keep `alias` in sync with the storage root's content directory. A mismatch is a startup
|
||||
# misconfiguration, not a runtime fallback: the application's startup validator checks that the
|
||||
# internal mapping was proven before it accepts traffic.
|
||||
|
||||
worker_processes auto;
|
||||
|
||||
events {
|
||||
worker_connections 4096;
|
||||
}
|
||||
|
||||
http {
|
||||
include mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
sendfile on;
|
||||
sendfile_max_chunk 2m;
|
||||
tcp_nopush on;
|
||||
keepalive_timeout 65;
|
||||
|
||||
# Uploads stream through to the application; buffering a large body to disk here would double
|
||||
# the write and defeat the streaming upload path.
|
||||
proxy_request_buffering off;
|
||||
client_max_body_size 0;
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
|
||||
# Public API. Everything, including download authorization, is decided by the application.
|
||||
location / {
|
||||
proxy_pass http://app:8081;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# The application must never see a client-supplied delegation header: it would let a
|
||||
# caller name an arbitrary internal object.
|
||||
proxy_set_header X-Accel-Redirect "";
|
||||
}
|
||||
|
||||
# Internal transfer location. Not reachable from outside; see property (1) above.
|
||||
location /__files/ {
|
||||
internal;
|
||||
alias /srv/files/content/;
|
||||
|
||||
sendfile on;
|
||||
sendfile_max_chunk 2m;
|
||||
|
||||
# Uploaded content is never trusted to describe itself.
|
||||
add_header X-Content-Type-Options nosniff always;
|
||||
add_header Content-Disposition $upstream_http_content_disposition always;
|
||||
add_header Cache-Control $upstream_http_cache_control always;
|
||||
add_header ETag $upstream_http_etag always;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
# HTTP Client Platform — local test topology
|
||||
|
||||
The suites drive these dependencies through Testcontainers and in-process fixtures, so nothing here
|
||||
is required to run `./gradlew :adapter:outbound:httpclient:test`. These files exist for the nightly
|
||||
lane and for reproducing a failure locally with the same images and ports CI uses.
|
||||
|
||||
| Directory | Purpose | Used by |
|
||||
|---|---|---|
|
||||
| `toxiproxy/` | TCP fault injection (latency, reset, bandwidth) | `httpClientFailureInjectionTest` |
|
||||
| `tls/` | how the TLS and mTLS material is produced | TLS and mTLS suites |
|
||||
| `proxy/` | forward proxy with CONNECT and proxy authentication | proxy contract suite |
|
||||
| `oauth2/` | token endpoint behaviour under contention | OAuth2 suites |
|
||||
@@ -0,0 +1,12 @@
|
||||
# OAuth2 fixture
|
||||
|
||||
`OAuth2Fixture` exposes a token endpoint backed by the same deterministic fixture server as the rest
|
||||
of the suite.
|
||||
|
||||
It counts token requests, which is what makes design §20.3's single-flight guarantee provable rather
|
||||
than assumed: a hundred genuinely concurrent callers must produce exactly one token request. It can
|
||||
also issue rotating token values, so a stale cached token is detectable, and queue a failure status
|
||||
to exercise the refresh-failure path.
|
||||
|
||||
The token endpoint is configured as its own Named Client Profile, separate from the upstream it
|
||||
issues tokens for.
|
||||
@@ -0,0 +1,12 @@
|
||||
# Forward proxy fixture
|
||||
|
||||
`ProxyFixture` runs an in-process forward proxy so the proxy lane needs no external service.
|
||||
|
||||
| Factory | Behaviour |
|
||||
|---|---|
|
||||
| `ProxyFixture.openProxy()` | accepts CONNECT and tunnels to the target |
|
||||
| `ProxyFixture.authenticatingProxy(user, password)` | answers `407` until `Proxy-Authorization` matches |
|
||||
|
||||
The fixture records every request line and every `Proxy-Authorization` value it saw, which is what
|
||||
lets the suite prove design §24.3: proxy credentials never appear on the target request, and a proxy
|
||||
CONNECT failure is reported as `HttpProxyException` rather than as a target TLS failure.
|
||||
@@ -0,0 +1,16 @@
|
||||
# TLS fixtures
|
||||
|
||||
Certificates are generated **in process** by `TlsFixture`, not checked in. A committed private key
|
||||
is a private key that leaks, and design §21.2 forbids key material in the repository.
|
||||
|
||||
`TlsFixture` produces, from a throwaway CA created per test run:
|
||||
|
||||
| Fixture | Purpose |
|
||||
|---|---|
|
||||
| `TlsFixture.trusted()` | a server certificate valid for the loopback host |
|
||||
| `TlsFixture.hostnameMismatch()` | a certificate whose SAN does not match the connection host |
|
||||
| `TlsFixture.expired()` | an already-expired certificate |
|
||||
| `clientHandshake(true)` | client key material for the mTLS lane |
|
||||
|
||||
All three failure cases must classify as permanent (design §21.3) — never retried, never downgraded
|
||||
to plaintext.
|
||||
@@ -0,0 +1,17 @@
|
||||
# Fault-injection topology for the HTTP Client Platform failure suite (design §28.1, §28.3).
|
||||
#
|
||||
# The suite normally drives Toxiproxy through Testcontainers. This compose file exists for the
|
||||
# nightly lane and for reproducing a failure locally with the exact same image and ports.
|
||||
services:
|
||||
toxiproxy:
|
||||
image: ghcr.io/shopify/toxiproxy:2.9.0
|
||||
container_name: httpclient-toxiproxy
|
||||
ports:
|
||||
- "8474:8474" # control API
|
||||
- "18080:18080" # proxied upstream: plaintext
|
||||
- "18443:18443" # proxied upstream: TLS
|
||||
healthcheck:
|
||||
test: ["CMD", "/toxiproxy-cli", "list"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
@@ -0,0 +1,148 @@
|
||||
# Redis SDK topology lanes
|
||||
|
||||
These lanes exist to answer the questions the deterministic in-memory gateway cannot: how Lettuce
|
||||
actually behaves during a Sentinel promotion, what a Cluster resharding does to an in-flight
|
||||
command, and whether the ACL accounts grant exactly what the SDK issues.
|
||||
|
||||
All four have now run on Redis 7.4 and the evidence is recorded in
|
||||
`docs/redis/support-matrix.md`. `.github/workflows/redis-sdk-topology.yml` runs the standalone lane
|
||||
on any pull request that touches the Redis leaf, the full supported-version x topology matrix
|
||||
nightly, and the same matrix on demand for a release candidate.
|
||||
|
||||
TLS is a lane of that matrix rather than something to wire up by hand. It is `tls`, not a
|
||||
deployment mode: its shape is standalone and what it qualifies is the transport, so
|
||||
`redisTopologyTest` maps the lane name to `standalone` for the tests and keeps the tag filter and
|
||||
the required trust material on the lane.
|
||||
|
||||
## The TLS lane
|
||||
|
||||
`tls/compose.yml` is the standalone shape with the transport swapped. The plaintext port is turned
|
||||
off entirely (`--port 0`), which is the only configuration that proves anything: a lane accepting
|
||||
both would let a client that failed to negotiate TLS fall back silently and still pass.
|
||||
|
||||
Certificates are generated at start-up into a named volume rather than checked in — a private key
|
||||
in the repository is a private key in the repository, however the file is named — and they last a
|
||||
day, so a stale lane fails visibly instead of drifting.
|
||||
|
||||
```bash
|
||||
REDIS_VERSION=7.4 docker compose -f infra/redis-sdk/tls/compose.yml up -d --wait
|
||||
# The client needs the generated CA; copy it out of the volume first.
|
||||
docker compose -f infra/redis-sdk/tls/compose.yml cp redis:/tls/ca.crt /tmp/redis-lane-ca.pem
|
||||
cd src && ./gradlew :adapter:outbound:cache-redis:redisTopologyTest \
|
||||
-Predis.topology.host=127.0.0.1 -Predis.topology.port=6390 \
|
||||
-Predis.topology.mode=tls -Predis.topology.trust-material=/tmp/redis-lane-ca.pem
|
||||
```
|
||||
|
||||
The lane refuses to run without `redis.topology.trust-material`. A TLS lane that trusts anything
|
||||
qualifies nothing, so "no CA configured" is an error rather than a client with verification off.
|
||||
|
||||
## The ACL fixture
|
||||
|
||||
`acl/all-accounts.acl` provisions the accounts every lane uses. Two things about it matter, and
|
||||
neither can be written in the file itself — **Redis refuses to start if an `aclfile` contains a
|
||||
comment line**, so the whole file is directives and the explanation lives here.
|
||||
|
||||
`user default off` is the first line and is deliberate. Redis ships `default` enabled and
|
||||
passwordless; while it is on, every restriction in the remaining accounts can be bypassed by simply
|
||||
not authenticating, which makes the fixture decorative. Disabling it is what forces a client — and
|
||||
the compose healthchecks — to pick a named account.
|
||||
|
||||
Every named account carries a real password — `>fixture-application`, `>fixture-advanced`, and so
|
||||
on. They were `nopass`, which was the more dangerous kind of wrong: an account that accepts any
|
||||
password made every assertion about authentication pass for the same reason a typo would have, so
|
||||
the lane's coverage of AUTH, rotation and secret wiring was indistinguishable from no coverage.
|
||||
`LiveRedisCompositionTest` now presents a wrong password on purpose and requires `WRONGPASS`, which
|
||||
is only a meaningful assertion because the accounts enforce one.
|
||||
|
||||
The passwords are fixture values in a throwaway container and are **not** a deployment template: a
|
||||
real deployment resolves each account's credential through `secret://` and never writes one into
|
||||
configuration.
|
||||
|
||||
The accounts are also split by role, because that is how the SDK uses them. `ca-skeleton-application`
|
||||
runs ordinary data commands and cannot execute a script; `ca-skeleton-application-advanced` holds
|
||||
`SCRIPT LOAD` and `EVALSHA` and nothing else needs to. That separation is real rather than
|
||||
decorative: `LiveRedisSemanticPortsTest` runs the rate limiter without the advanced account and
|
||||
requires it to come back `Unavailable`.
|
||||
|
||||
## Running one
|
||||
|
||||
Each lane has its own endpoint, because the address a client is given is not the same kind of thing
|
||||
in each topology. Standalone declares a data node; Sentinel declares a *sentinel*, from which the
|
||||
primary is resolved and re-resolved when it is promoted; Cluster declares any node, from which the
|
||||
rest of the topology is discovered.
|
||||
|
||||
```bash
|
||||
# Standalone
|
||||
REDIS_VERSION=7.4 docker compose -f infra/redis-sdk/standalone/compose.yml up -d --wait
|
||||
cd src && ./gradlew :adapter:outbound:cache-redis:redisTopologyTest \
|
||||
-Predis.topology.host=localhost -Predis.topology.port=6379 -Predis.topology.mode=standalone
|
||||
|
||||
# Sentinel — the port is a sentinel, and the monitored primary has to be named
|
||||
REDIS_VERSION=7.4 docker compose -f infra/redis-sdk/sentinel/compose.yml up -d --wait
|
||||
cd src && ./gradlew :adapter:outbound:cache-redis:redisTopologyTest \
|
||||
-Predis.topology.host=localhost -Predis.topology.port=27010 \
|
||||
-Predis.topology.mode=sentinel -Predis.topology.master=skeleton
|
||||
|
||||
# Cluster — `up --wait` waits for the `ready` gate, not just for six servers that answer PING.
|
||||
# Slot assignment finishes after the nodes are healthy, and a client that connects in between sees
|
||||
# CLUSTERDOWN for reasons that have nothing to do with the SDK.
|
||||
REDIS_VERSION=7.4 docker compose -f infra/redis-sdk/cluster/compose.yml up -d --wait
|
||||
cd src && ./gradlew :adapter:outbound:cache-redis:redisTopologyTest \
|
||||
-Predis.topology.host=localhost -Predis.topology.port=7100 -Predis.topology.mode=cluster
|
||||
```
|
||||
|
||||
Tear a lane down with `docker compose -f infra/redis-sdk/<lane>/compose.yml down -v`.
|
||||
|
||||
| Lane | Ports | Notes |
|
||||
| --- | --- | --- |
|
||||
| standalone | 6379 | bridge network, published port |
|
||||
| sentinel | primary 7010, replica 7011, sentinels 27010–27012 | host network |
|
||||
| cluster | nodes 7100–7105, bus 17100–17105 | host network; `ready` gates on `cluster_state:ok` |
|
||||
| tls | 6390 | published port, no plaintext port at all; CA generated per run |
|
||||
|
||||
## Why the Sentinel and Cluster lanes use host networking
|
||||
|
||||
Neither topology proxies. Sentinel answers `SENTINEL get-master-addr-by-name` with the address it
|
||||
monitors and the client dials that itself; a cluster client reads `CLUSTER SHARDS` and connects to
|
||||
every node it names. On a bridge network those are container-internal addresses, so a client on the
|
||||
host resolves a topology it cannot reach — and after a promotion it resolves a *different* one it
|
||||
also cannot reach. Sharing the host network namespace makes the address the topology advertises the
|
||||
address the client can use, which is the difference between testing the SDK and testing Docker's
|
||||
network.
|
||||
|
||||
That is also why their ports are fixed rather than parameterised: the addresses are written into
|
||||
Sentinel's and the cluster's own configuration at creation time, and a lane whose two halves can
|
||||
disagree fails for reasons that are not the SDK's.
|
||||
|
||||
## Selection is by lane, not by hand
|
||||
|
||||
`redisTopologyTest` derives its JUnit tag expression from the declared mode: `redis-topology &
|
||||
lane-<mode>`. A promotion test is meaningless without sentinels and a cross-slot test is meaningless
|
||||
without a cluster, but expressing that as a runtime assumption would turn "the lane was never
|
||||
started" into a green skip. Selecting by tag keeps it fail-closed — what a mode cannot prove is not
|
||||
selected, and what is selected must pass.
|
||||
|
||||
The lane also fails closed on its endpoint: selecting `redisTopologyTest` without host, port, and
|
||||
mode (and `redis.topology.master` on the Sentinel lane) is an error, never a skip. A topology test
|
||||
that silently passes because it did not connect is worse than no topology test.
|
||||
|
||||
## `min-replicas-to-write` on the Sentinel lane
|
||||
|
||||
The Sentinel lane sets `min-replicas-to-write 1` and `min-replicas-max-lag 1`, and this is not
|
||||
incidental configuration. Without them the lane measured a promotion in which the superseded primary
|
||||
kept answering `+OK` for eleven seconds after it had been replaced: **2,086 writes acknowledged to
|
||||
the caller and then discarded**, with exactly one command failing. With them the same promotion lost
|
||||
one write and refused 2,020 with `NOREPLICAS`, which the SDK reports as a definite, non-ambiguous
|
||||
failure a caller can act on.
|
||||
|
||||
Any deployment where an acknowledgement is supposed to mean something has to set these. See
|
||||
`docs/redis/support-matrix.md` for the full record.
|
||||
|
||||
## ACL accounts
|
||||
|
||||
`acl/` holds one file per `CommandAccess` level. They are deliberately narrower than the SDK's own
|
||||
rules, so a mistake in the SDK is still refused by the server — the account is the last boundary and
|
||||
a permit never widens it.
|
||||
|
||||
Every lane loads the same file on every data node. Accounts are enforced per node, so "they exist on
|
||||
one node" is not evidence that a topology enforces them.
|
||||
@@ -0,0 +1,17 @@
|
||||
# ACL accounts
|
||||
|
||||
One file per `CommandAccess` level. Each account is deliberately narrower than the SDK's own rules:
|
||||
the account is the last enforcement boundary and a permit issued inside the process never widens it,
|
||||
so a mistake in the SDK is still refused by the server.
|
||||
|
||||
A Redis ACL file accepts nothing but complete `user` lines — no comments and no line continuations —
|
||||
which is why the rationale lives here instead of inline. Password material is supplied at deploy
|
||||
time; none of these files carries one.
|
||||
|
||||
| File | Account | Grants |
|
||||
| --- | --- | --- |
|
||||
| `application.acl` | `CommandAccess.APPLICATION` | the typed data-structure commands over `prod:*`, with the dangerous and deprecated names denied |
|
||||
| `application-advanced.acl` | `APPLICATION_ADVANCED` | the above plus pub/sub, transactions, and the registered-script path. `EVAL` is absent: only `EVALSHA` of an already loaded digest is reachable |
|
||||
| `raw-gateway.acl` | `RAW_GATEWAY` | exactly the commands the catalog classifies `RAW_ONLY`, and nothing else |
|
||||
| `all-accounts.acl` | – | the four accounts concatenated; Redis takes one `aclfile`, so this is what a deployment loads |
|
||||
| `admin-readonly.acl` | `ADMIN_READONLY` | read-only diagnostics. Every destructive counterpart is denied here and blocked in the command policy catalog — two independent controls for the same rule |
|
||||
@@ -0,0 +1 @@
|
||||
user ca-skeleton-admin-readonly on nopass ~* resetchannels -@all +info +dbsize +time +lastsave +memory|usage +memory|stats +slowlog|get +slowlog|len +latency|latest +latency|history +client|list +client|info +command|info +command|docs +command|count +command|getkeysandflags +config|get +acl|dryrun +acl|whoami +cluster|info +cluster|slots +cluster|shards +cluster|nodes +object|encoding +object|freq +object|idletime +pubsub|channels +pubsub|numsub +pubsub|shardchannels +xinfo|stream +xinfo|groups +xinfo|consumers +function|list +function|stats +cluster|keyslot
|
||||
@@ -0,0 +1,8 @@
|
||||
user default off
|
||||
user ca-skeleton-application on >fixture-application sanitize-payload ~prod:* resetchannels &prod:* -@all +@connection +@pubsub +@transaction +@read +@write +@string +@hash +@list +@set +@sortedset +@bitmap +@hyperloglog +@geo +@stream -keys -flushdb -flushall -shutdown -debug -sort -sort_ro -smembers -randomkey -migrate -swapdb -select +cluster|slots +cluster|shards +cluster|nodes +cluster|info +cluster|myid
|
||||
user ca-skeleton-application-advanced on >fixture-advanced sanitize-payload ~prod:* resetchannels &prod:* -@all +@read +@write +@string +@hash +@list +@set +@sortedset +@bitmap +@hyperloglog +@geo +@stream +@pubsub +@transaction +evalsha +evalsha_ro +script|load +script|exists +fcall +fcall_ro -keys -flushdb -flushall -shutdown -debug -eval -eval_ro -smembers -sort -sort_ro -randomkey -migrate -swapdb -select +cluster|slots +cluster|shards +cluster|nodes +cluster|info +cluster|myid
|
||||
user ca-skeleton-raw-gateway on >fixture-raw sanitize-payload ~prod:* resetchannels -@all +smembers +sort +sort_ro
|
||||
user ca-skeleton-admin-readonly on >fixture-admin ~* resetchannels -@all +info +dbsize +time +lastsave +memory|usage +memory|stats +slowlog|get +slowlog|len +latency|latest +latency|history +client|list +client|info +command|info +command|docs +command|count +command|getkeysandflags +config|get +acl|dryrun +acl|whoami +cluster|info +cluster|slots +cluster|shards +cluster|nodes +object|encoding +object|freq +object|idletime +pubsub|channels +pubsub|numsub +pubsub|shardchannels +xinfo|stream +xinfo|groups +xinfo|consumers +function|list +function|stats +cluster|keyslot +cluster|myid
|
||||
user ca-skeleton-replication on >fixture-replication ~* resetchannels -@all +psync +replconf +ping
|
||||
user ca-skeleton-sentinel on >fixture-sentinel ~* &* -@all +multi +slaveof +ping +exec +subscribe +config|rewrite +role +publish +info +client|setname +client|kill +script|kill +replconf +psync
|
||||
user ca-skeleton-cluster-bootstrap on >fixture-bootstrap ~* &* +@all
|
||||
@@ -0,0 +1 @@
|
||||
user ca-skeleton-application-advanced on nopass sanitize-payload ~prod:* resetchannels &prod:* -@all +@read +@write +@string +@hash +@list +@set +@sortedset +@bitmap +@hyperloglog +@geo +@stream +@pubsub +@transaction +evalsha +evalsha_ro +script|load +script|exists +fcall +fcall_ro -keys -flushdb -flushall -shutdown -debug -eval -eval_ro -smembers -sort -sort_ro -randomkey -migrate -swapdb -select
|
||||
@@ -0,0 +1 @@
|
||||
user ca-skeleton-application on nopass sanitize-payload ~prod:* resetchannels &prod:* -@all +@connection +@pubsub +@transaction +@read +@write +@string +@hash +@list +@set +@sortedset +@bitmap +@hyperloglog +@geo +@stream -keys -flushdb -flushall -shutdown -debug -sort -sort_ro -smembers -randomkey -migrate -swapdb -select
|
||||
@@ -0,0 +1 @@
|
||||
user ca-skeleton-raw-gateway on nopass sanitize-payload ~prod:* resetchannels -@all +smembers +sort +sort_ro
|
||||
@@ -0,0 +1,4 @@
|
||||
user default off
|
||||
user ca-skeleton-sentinel-client on >fixture-sentinel-client ~* &* -@all +auth +hello +ping +client|setname +client|id +subscribe +psubscribe +unsubscribe +punsubscribe +info +sentinel|get-master-addr-by-name +sentinel|master +sentinel|masters +sentinel|replicas +sentinel|slaves +sentinel|sentinels +sentinel|is-master-down-by-addr
|
||||
user ca-skeleton-sentinel-peer on >fixture-sentinel-peer ~* &* +@all
|
||||
user ca-skeleton-sentinel-operator on >fixture-sentinel-operator ~* &* -@all +auth +hello +ping +client|setname +info +subscribe +psubscribe +sentinel|get-master-addr-by-name +sentinel|master +sentinel|masters +sentinel|replicas +sentinel|slaves +sentinel|sentinels +sentinel|failover +sentinel|reset
|
||||
@@ -0,0 +1,134 @@
|
||||
# Cluster lane. Six nodes: three primaries so cross-slot behaviour is observable at all, and three
|
||||
# replicas so a promotion can be forced without losing a shard.
|
||||
#
|
||||
# Host networking for the same reason as the Sentinel lane, and a sharper one. A cluster client does
|
||||
# not talk to one address: it reads `CLUSTER SHARDS`, learns every node's address, and connects to
|
||||
# each of them itself. On a bridge those addresses are container-internal, so a client on the host
|
||||
# resolves a topology it cannot dial and every redirect points somewhere unreachable. Sharing the
|
||||
# host network namespace makes the addresses the cluster advertises the addresses the client can
|
||||
# use, which is the difference between testing the SDK and testing Docker's network.
|
||||
#
|
||||
# Ports are fixed because they are written into the cluster's own configuration at creation time:
|
||||
# the node identity a redirect names has to be an address the client can dial.
|
||||
#
|
||||
# nodes 7100..7105 · cluster bus 17100..17105
|
||||
#
|
||||
# The ACL file is loaded on every node. The accounts are the deployment's last enforcement boundary
|
||||
# and a cluster enforces them per node, so "they exist on one node" is not evidence.
|
||||
#
|
||||
# min-replicas-to-write is set here for the same reason as on the Sentinel lane. A cluster promotes
|
||||
# a replica without asking the client too, so a superseded primary keeps acknowledging writes it
|
||||
# will discard on resync — the Sentinel lane measured 2,086 of them in one eleven-second window.
|
||||
# Nothing about slot ownership changes that, and this lane was written without the setting at first
|
||||
# precisely because the failure mode is easy to think of as Sentinel-specific. It is not.
|
||||
x-node: &node
|
||||
image: "redis:${REDIS_VERSION:-7.4}"
|
||||
network_mode: host
|
||||
volumes:
|
||||
- ../acl:/etc/redis/acl:ro
|
||||
entrypoint:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
exec redis-server \
|
||||
--port $$NODE_PORT \
|
||||
--cluster-enabled yes \
|
||||
--cluster-config-file /tmp/nodes.conf \
|
||||
--cluster-node-timeout 2000 \
|
||||
--cluster-announce-ip 127.0.0.1 \
|
||||
--appendonly no \
|
||||
--save '' \
|
||||
--min-replicas-to-write 1 \
|
||||
--min-replicas-max-lag 1 \
|
||||
--masteruser ca-skeleton-replication \
|
||||
--masterauth fixture-replication \
|
||||
--aclfile /etc/redis/acl/all-accounts.acl
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "[ \"$$(redis-cli -p $$NODE_PORT --user ca-skeleton-application --pass fixture-application --no-auth-warning ping)\" = PONG ]"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
|
||||
services:
|
||||
node-1:
|
||||
<<: *node
|
||||
environment:
|
||||
NODE_PORT: "7100"
|
||||
|
||||
node-2:
|
||||
<<: *node
|
||||
environment:
|
||||
NODE_PORT: "7101"
|
||||
|
||||
node-3:
|
||||
<<: *node
|
||||
environment:
|
||||
NODE_PORT: "7102"
|
||||
|
||||
node-4:
|
||||
<<: *node
|
||||
environment:
|
||||
NODE_PORT: "7103"
|
||||
|
||||
node-5:
|
||||
<<: *node
|
||||
environment:
|
||||
NODE_PORT: "7104"
|
||||
|
||||
node-6:
|
||||
<<: *node
|
||||
environment:
|
||||
NODE_PORT: "7105"
|
||||
|
||||
# The cluster is created after every node reports healthy, and the lane is not "up" until every
|
||||
# slot is covered. A test that starts before slot assignment finishes sees MOVED and CLUSTERDOWN
|
||||
# for reasons that have nothing to do with the SDK.
|
||||
init:
|
||||
image: "redis:${REDIS_VERSION:-7.4}"
|
||||
network_mode: host
|
||||
depends_on:
|
||||
node-1: {condition: service_healthy}
|
||||
node-2: {condition: service_healthy}
|
||||
node-3: {condition: service_healthy}
|
||||
node-4: {condition: service_healthy}
|
||||
node-5: {condition: service_healthy}
|
||||
node-6: {condition: service_healthy}
|
||||
entrypoint:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
redis-cli --user ca-skeleton-cluster-bootstrap --pass fixture-bootstrap --no-auth-warning \
|
||||
--cluster create \
|
||||
127.0.0.1:7100 127.0.0.1:7101 127.0.0.1:7102 \
|
||||
127.0.0.1:7103 127.0.0.1:7104 127.0.0.1:7105 \
|
||||
--cluster-replicas 1 --cluster-yes
|
||||
# Authenticated, like every other command against this fixture. The `default` user is off,
|
||||
# so an unauthenticated CLUSTER INFO answers NOAUTH — which never matches, so this loop
|
||||
# never ended, the helper never exited, and `up --wait` returned on the nodes' own health
|
||||
# while slot assignment was still in flight. A lane that reports ready before it can serve
|
||||
# a key produces failures that look like SDK defects and are not.
|
||||
until redis-cli -p 7100 \
|
||||
--user ca-skeleton-cluster-bootstrap --pass fixture-bootstrap --no-auth-warning \
|
||||
cluster info | grep -q 'cluster_state:ok'; do sleep 1; done
|
||||
echo "cluster ready"
|
||||
|
||||
# `up --wait` returns when every service is running or healthy, and a one-shot helper is neither
|
||||
# for as long as it runs — so the wait ended while slots were still being assigned, and whichever
|
||||
# test connected first saw a cluster that could not serve its keys. This gate is a service the
|
||||
# wait can see: it cannot become healthy until the cluster reports a fully covered keyspace.
|
||||
ready:
|
||||
image: "redis:${REDIS_VERSION:-7.4}"
|
||||
network_mode: host
|
||||
depends_on:
|
||||
init: {condition: service_completed_successfully}
|
||||
command: ["sleep", "infinity"]
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- >-
|
||||
[ "$$(redis-cli -p 7100 --user ca-skeleton-cluster-bootstrap
|
||||
--pass fixture-bootstrap --no-auth-warning cluster info
|
||||
| tr -d '\r' | grep -c '^cluster_state:ok$$')" = 1 ]
|
||||
interval: 1s
|
||||
timeout: 3s
|
||||
retries: 60
|
||||
@@ -0,0 +1,126 @@
|
||||
# Sentinel lane. Three sentinels because a two-sentinel quorum cannot survive losing one, and a
|
||||
# failover test that cannot lose a sentinel is not testing failover.
|
||||
#
|
||||
# Host networking, not a bridge with published ports. Sentinel does not proxy: it answers
|
||||
# `SENTINEL get-master-addr-by-name` with the address it monitors, and the client then connects
|
||||
# there itself. On a bridge that address is the container's internal IP, which the client on the
|
||||
# host cannot reach, so the lane would resolve a primary it can never talk to — and after a
|
||||
# promotion it would resolve a different unreachable one. Sharing the host network namespace makes
|
||||
# the address Sentinel hands out the same address the client can dial, which is the only thing that
|
||||
# makes the promotion observable from outside.
|
||||
#
|
||||
# Ports are fixed rather than parameterised because Sentinel stores them in its own config: the
|
||||
# monitored address has to match what the client is told, and a lane whose two halves can disagree
|
||||
# is a lane that fails for reasons that are not the SDK's.
|
||||
#
|
||||
# primary 7010 · replica 7011 · sentinels 27010 27011 27012
|
||||
#
|
||||
# The ACL file is loaded on both data nodes. The accounts are the deployment's last enforcement
|
||||
# boundary, so "they exist in standalone" is not evidence that they exist in the topology that will
|
||||
# actually be run in production.
|
||||
#
|
||||
# Both data nodes take their entire configuration from one definition, and that is load-bearing
|
||||
# rather than tidiness. These two nodes swap roles on every failover, so a setting written only into
|
||||
# the one that happens to start as primary silently stops applying the moment the lane does the
|
||||
# thing it exists to do. The lane learned this the hard way: min-replicas-to-write was set on the
|
||||
# primary only, the first promotion passed, and the second promotion — now writing to the node that
|
||||
# never had the setting — discarded 2,099 acknowledged writes.
|
||||
x-data-node: &data-node
|
||||
image: "redis:${REDIS_VERSION:-7.4}"
|
||||
network_mode: host
|
||||
volumes:
|
||||
- ../acl:/etc/redis/acl:ro
|
||||
entrypoint:
|
||||
- /bin/sh
|
||||
- -c
|
||||
# REPLICA_OF is deliberately unquoted: it is either empty or a two-word --replicaof argument.
|
||||
#
|
||||
# min-replicas-to-write is what stops a superseded primary from acknowledging writes it cannot
|
||||
# keep. Without it a promotion silently destroys them — measured here at eleven seconds and two
|
||||
# thousand confirmed-then-discarded writes — because Sentinel does not demote the old primary
|
||||
# until well after it has promoted the new one. Requiring an in-sync replica turns that window
|
||||
# into an explicit NOREPLICAS refusal the caller can see and act on. Any deployment where an
|
||||
# acknowledgement is supposed to mean something has to set these.
|
||||
- |
|
||||
exec redis-server \
|
||||
--port $$NODE_PORT \
|
||||
$$REPLICA_OF \
|
||||
--appendonly no \
|
||||
--save '' \
|
||||
--min-replicas-to-write 1 \
|
||||
--min-replicas-max-lag 1 \
|
||||
--masteruser ca-skeleton-replication \
|
||||
--masterauth fixture-replication \
|
||||
--aclfile /etc/redis/acl/all-accounts.acl
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "[ \"$$(redis-cli -p $$NODE_PORT --user ca-skeleton-application --pass fixture-application --no-auth-warning ping)\" = PONG ]"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
|
||||
services:
|
||||
primary:
|
||||
<<: *data-node
|
||||
environment:
|
||||
NODE_PORT: "7010"
|
||||
REPLICA_OF: ""
|
||||
|
||||
replica:
|
||||
<<: *data-node
|
||||
environment:
|
||||
NODE_PORT: "7011"
|
||||
REPLICA_OF: "--replicaof 127.0.0.1 7010"
|
||||
depends_on:
|
||||
primary:
|
||||
condition: service_healthy
|
||||
|
||||
sentinel-1: &sentinel
|
||||
image: "redis:${REDIS_VERSION:-7.4}"
|
||||
network_mode: host
|
||||
# The config is written at start-up rather than mounted because Sentinel rewrites its own file
|
||||
# when it promotes. A read-only mount would make the first failover fail on a write error, and
|
||||
# a shared writable mount would have three sentinels rewriting one file.
|
||||
entrypoint:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
cat > /tmp/sentinel.conf <<CONF
|
||||
port $$SENTINEL_PORT
|
||||
sentinel monitor skeleton 127.0.0.1 7010 2
|
||||
sentinel auth-user skeleton ca-skeleton-sentinel
|
||||
sentinel auth-pass skeleton fixture-sentinel
|
||||
sentinel down-after-milliseconds skeleton 2000
|
||||
sentinel failover-timeout skeleton 10000
|
||||
sentinel parallel-syncs skeleton 1
|
||||
CONF
|
||||
exec redis-sentinel /tmp/sentinel.conf
|
||||
environment:
|
||||
SENTINEL_PORT: "27010"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "[ \"$$(redis-cli -p 27010 ping)\" = PONG ]"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
depends_on:
|
||||
primary:
|
||||
condition: service_healthy
|
||||
|
||||
sentinel-2:
|
||||
<<: *sentinel
|
||||
environment:
|
||||
SENTINEL_PORT: "27011"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "[ \"$$(redis-cli -p 27011 ping)\" = PONG ]"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
|
||||
sentinel-3:
|
||||
<<: *sentinel
|
||||
environment:
|
||||
SENTINEL_PORT: "27012"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "[ \"$$(redis-cli -p 27012 ping)\" = PONG ]"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
@@ -0,0 +1,26 @@
|
||||
# Standalone lane for the Redis SDK topology tests.
|
||||
#
|
||||
# The version is a build argument rather than a pinned image so the same file serves every row of
|
||||
# the support matrix. Nothing here is a production topology: no persistence, no TLS, no replication.
|
||||
# It exists to answer "does the driver behave the way the SDK claims", which is a question the
|
||||
# in-memory fixture cannot answer at all.
|
||||
services:
|
||||
redis:
|
||||
image: "redis:${REDIS_VERSION:-7.4}"
|
||||
command:
|
||||
- redis-server
|
||||
- --appendonly
|
||||
- "no"
|
||||
- --save
|
||||
- ""
|
||||
- --aclfile
|
||||
- /etc/redis/acl/all-accounts.acl
|
||||
volumes:
|
||||
- ../acl:/etc/redis/acl:ro
|
||||
ports:
|
||||
- "${REDIS_PORT:-6379}:6379"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "[ \"$$(redis-cli --user ca-skeleton-application --pass fixture-application --no-auth-warning ping)\" = PONG ]"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
@@ -0,0 +1,78 @@
|
||||
# TLS lane for the Redis SDK topology tests.
|
||||
#
|
||||
# Standalone in shape, but the point is the transport: the SDK's TLS settings — enabled, hostname
|
||||
# verification, trust material, client certificate — are configuration nothing else exercises, and
|
||||
# a TLS path that has never carried a command is a claim rather than a capability.
|
||||
#
|
||||
# The certificates are generated at start-up rather than checked in. A checked-in key is a secret in
|
||||
# the repository however loudly the file is named "test", and a lane that regenerates its own
|
||||
# material also proves the trust configuration actually matters: point the client at the wrong CA
|
||||
# and it fails, which is what the qualification has to show.
|
||||
services:
|
||||
certs:
|
||||
# The redis image carries no openssl, so certificate generation gets an image that does. The
|
||||
# alternative — checking the material in — puts a private key in the repository.
|
||||
image: alpine/openssl:latest
|
||||
user: root
|
||||
volumes:
|
||||
- tls:/tls
|
||||
entrypoint:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
if [ -f /tls/redis.crt ]; then exit 0; fi
|
||||
openssl genrsa -out /tls/ca.key 2048
|
||||
openssl req -x509 -new -nodes -key /tls/ca.key -sha256 -days 1 \
|
||||
-subj "/CN=ca-skeleton-test-ca" -out /tls/ca.crt
|
||||
openssl genrsa -out /tls/redis.key 2048
|
||||
openssl req -new -key /tls/redis.key -subj "/CN=localhost" -out /tls/redis.csr
|
||||
printf 'subjectAltName=DNS:localhost,IP:127.0.0.1' > /tls/redis.ext
|
||||
openssl x509 -req -in /tls/redis.csr -CA /tls/ca.crt -CAkey /tls/ca.key \
|
||||
-CAcreateserial -out /tls/redis.crt -days 1 -sha256 -extfile /tls/redis.ext
|
||||
chmod 644 /tls/redis.key /tls/ca.key
|
||||
|
||||
redis:
|
||||
image: "redis:${REDIS_VERSION:-7.4}"
|
||||
depends_on:
|
||||
certs:
|
||||
condition: service_completed_successfully
|
||||
volumes:
|
||||
- ../acl:/etc/redis/acl:ro
|
||||
- tls:/tls:ro
|
||||
command:
|
||||
- redis-server
|
||||
# Plaintext is off entirely. A lane that accepts both proves nothing about the TLS path,
|
||||
# because a misconfigured client would quietly fall back and still pass.
|
||||
- --port
|
||||
- "0"
|
||||
- --tls-port
|
||||
- "6379"
|
||||
- --tls-cert-file
|
||||
- /tls/redis.crt
|
||||
- --tls-key-file
|
||||
- /tls/redis.key
|
||||
- --tls-ca-cert-file
|
||||
- /tls/ca.crt
|
||||
- --tls-auth-clients
|
||||
- "no"
|
||||
- --appendonly
|
||||
- "no"
|
||||
- --save
|
||||
- ""
|
||||
- --aclfile
|
||||
- /etc/redis/acl/all-accounts.acl
|
||||
ports:
|
||||
- "${REDIS_PORT:-6390}:6379"
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- >-
|
||||
[ "$$(redis-cli --tls --cacert /tls/ca.crt
|
||||
--user ca-skeleton-application --pass fixture-application --no-auth-warning ping)" = PONG ]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
volumes:
|
||||
tls:
|
||||
Reference in New Issue
Block a user