# Toxiproxy fault injection for the notification delivery platform. # # Scope, stated up front: this is the *nightly and release* fault suite, not the PR gate. The PR # suite runs against a loopback socket harness in-process — deterministic, no Docker, no provider # sandbox — because a gate that needs infrastructure is a gate people learn to skip. What lives # here are the faults that harness cannot produce: real TCP behaviour under latency, bandwidth # starvation, and connection resets at a point the JVM's own socket layer decides. # # Usage: # docker compose -f infra/notification/toxiproxy/docker-compose.yml up -d # ./gradlew :adapter:outbound:notification:test -Dnotification.faultProxy=http://127.0.0.1:8474 # # The proxies below front *stub* upstreams, never a provider's real API. Pointing a toxic proxy at # a live provider sends real notifications to real people from a test run, and adds a rate-limit # incident on an account the team shares. services: toxiproxy: image: ghcr.io/shopify/toxiproxy:2.11.0 container_name: notification-toxiproxy ports: - "8474:8474" # control API - "18081:18081" # -> ses-stub - "18082:18082" # -> twilio-stub - "18083:18083" # -> push-stub (APNs / FCM / Web Push) networks: [notification-fault] healthcheck: test: ["CMD", "/toxiproxy-cli", "list"] interval: 5s timeout: 3s retries: 10 # Deterministic upstreams. Each returns the provider's success shape and nothing else; the # interesting behaviour is injected by the proxy in front of it, not by the stub. ses-stub: image: mendhak/http-https-echo:35 environment: HTTP_PORT: "8080" networks: [notification-fault] twilio-stub: image: mendhak/http-https-echo:35 environment: HTTP_PORT: "8080" networks: [notification-fault] push-stub: image: mendhak/http-https-echo:35 environment: HTTP_PORT: "8080" networks: [notification-fault] # Creates the proxies and the toxics once the control API is up. Kept as a job rather than a # README step so the topology is reproducible and reviewable rather than typed from memory. provision: image: ghcr.io/shopify/toxiproxy:2.11.0 depends_on: toxiproxy: condition: service_healthy networks: [notification-fault] entrypoint: - /bin/sh - -c - | set -e CLI="/toxiproxy-cli -h toxiproxy:8474" $$CLI create -l 0.0.0.0:18081 -u ses-stub:8080 ses $$CLI create -l 0.0.0.0:18082 -u twilio-stub:8080 twilio $$CLI create -l 0.0.0.0:18083 -u push-stub:8080 push # Response loss after the request was committed: the provider received and acted on the # message, and the answer never came back. This is the AMBIGUOUS case, and it is the one # fault no provider's documentation describes. $$CLI toxic add -t timeout -a timeout=0 -n response_loss --downstream --toxicity 0 ses $$CLI toxic add -t timeout -a timeout=0 -n response_loss --downstream --toxicity 0 twilio $$CLI toxic add -t timeout -a timeout=0 -n response_loss --downstream --toxicity 0 push # Latency past the adapter's own timeout, to prove the timeout is the adapter's decision # rather than the socket's. $$CLI toxic add -t latency -a latency=8000 -n slow --toxicity 0 ses $$CLI toxic add -t latency -a latency=8000 -n slow --toxicity 0 twilio $$CLI toxic add -t latency -a latency=8000 -n slow --toxicity 0 push # Partial write: the connection dies mid-body. Distinct from response loss, because the # provider never got a complete request and the attempt is genuinely retryable. $$CLI toxic add -t limit_data -a bytes=64 -n partial_write --upstream --toxicity 0 ses $$CLI toxic add -t limit_data -a bytes=64 -n partial_write --upstream --toxicity 0 twilio $$CLI toxic add -t limit_data -a bytes=64 -n partial_write --upstream --toxicity 0 push echo "proxies ready; toxics are registered at toxicity=0 and enabled per test" $$CLI list networks: notification-fault: driver: bridge