feat: jpa, messaging, notification, mongo, graphql 어댑터터 구현체 추가
This commit is contained in:
+26
-2
@@ -25,13 +25,37 @@ public class OutboundMessagePublisher implements MessagePublisher {
|
||||
|
||||
@Override
|
||||
public void publish(OutboundMessage message) {
|
||||
// The send and the observation are separate steps because they used to share a try block: a
|
||||
// logger that threw after a successful send was caught by the same catch and reported as a
|
||||
// publish failure. The broker had accepted the message; the only thing that failed was the
|
||||
// record of it, and the two must not be confusable.
|
||||
boolean sent = false;
|
||||
try {
|
||||
broker.send(message);
|
||||
dependencyLogger.logSuccess(broker.brokerId(), DEPENDENCY_TYPE, "publish");
|
||||
sent = true;
|
||||
} catch (Exception ex) {
|
||||
// fail-open: observe with correlationId, delegate durability to outbox/retry,
|
||||
// do NOT propagate — the core use case must still succeed.
|
||||
dependencyLogger.logFailure(broker.brokerId(), DEPENDENCY_TYPE, "publish", ex);
|
||||
observeQuietly(
|
||||
() -> dependencyLogger.logFailure(broker.brokerId(), DEPENDENCY_TYPE, "publish", ex));
|
||||
}
|
||||
if (sent) {
|
||||
observeQuietly(
|
||||
() -> dependencyLogger.logSuccess(broker.brokerId(), DEPENDENCY_TYPE, "publish"));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs an observation, absorbing whatever it throws.
|
||||
*
|
||||
* <p>Diagnostics are non-authoritative. An appender that is out of disk must not change what the
|
||||
* caller believes about the broker.
|
||||
*/
|
||||
private static void observeQuietly(Runnable observation) {
|
||||
try {
|
||||
observation.run();
|
||||
} catch (RuntimeException ignored) {
|
||||
// Nothing to report it to: the reporter is what failed.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
package dev.caskeleton.adapter.outbound.messaging.kafka;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* One broker endpoint, parsed rather than pattern-matched.
|
||||
*
|
||||
* <p>The regular expression it replaces accepted several things that are not addresses. It ran
|
||||
* against the <em>trimmed</em> value but the untrimmed original was what got stored, so {@code "
|
||||
* kafka:9092"} passed validation and was then handed to the client with its leading space. {@code
|
||||
* \\d{1,5}} accepts {@code 0} and {@code 99999}, neither of which is a port. And {@code [^:\\s]+}
|
||||
* cannot express a bracketed IPv6 literal at all, so {@code [::1]:9092} — the only correct way to
|
||||
* write an IPv6 endpoint — was rejected while {@code ::1:9092} was accepted and is ambiguous.
|
||||
*
|
||||
* @param host the host, without brackets for an IPv6 literal
|
||||
* @param port the port, between 1 and 65535
|
||||
* @param ipv6Literal whether the host was written as a bracketed IPv6 literal
|
||||
*/
|
||||
public record BrokerAddress(String host, int port, boolean ipv6Literal) {
|
||||
|
||||
private static final int MIN_PORT = 1;
|
||||
private static final int MAX_PORT = 65_535;
|
||||
|
||||
/** Validates the parsed components. */
|
||||
public BrokerAddress {
|
||||
Objects.requireNonNull(host, "host must not be null");
|
||||
if (host.isBlank()) {
|
||||
throw new IllegalArgumentException("broker host must not be blank");
|
||||
}
|
||||
if (port < MIN_PORT || port > MAX_PORT) {
|
||||
throw new IllegalArgumentException("broker port " + port + " is outside 1..65535");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses one {@code host:port} entry.
|
||||
*
|
||||
* @param raw the configured entry, possibly with surrounding whitespace
|
||||
* @return the canonical address
|
||||
* @throws IllegalArgumentException naming what is wrong with the entry
|
||||
*/
|
||||
public static BrokerAddress parse(String raw) {
|
||||
if (raw == null || raw.isBlank()) {
|
||||
throw new IllegalArgumentException("a broker entry must not be blank");
|
||||
}
|
||||
String entry = raw.trim();
|
||||
if (entry.startsWith("[")) {
|
||||
int closing = entry.indexOf(']');
|
||||
if (closing < 0 || closing + 1 >= entry.length() || entry.charAt(closing + 1) != ':') {
|
||||
throw new IllegalArgumentException(
|
||||
"broker entry '" + entry + "' is a bracketed host without a ':port' after the bracket");
|
||||
}
|
||||
String host = entry.substring(1, closing);
|
||||
if (host.isBlank()) {
|
||||
throw new IllegalArgumentException("broker entry '" + entry + "' has an empty host");
|
||||
}
|
||||
return new BrokerAddress(host, parsePort(entry, entry.substring(closing + 2)), true);
|
||||
}
|
||||
int separator = entry.lastIndexOf(':');
|
||||
if (separator < 0) {
|
||||
throw new IllegalArgumentException("broker entry '" + entry + "' is not host:port");
|
||||
}
|
||||
String host = entry.substring(0, separator);
|
||||
if (host.isBlank() || host.indexOf(':') >= 0) {
|
||||
// A bare IPv6 literal reaches here: it contains colons, and which one separates the port is
|
||||
// not decidable. Brackets are how the ambiguity is resolved, so require them.
|
||||
throw new IllegalArgumentException(
|
||||
"broker entry '"
|
||||
+ entry
|
||||
+ "' is not host:port; write an IPv6 address in brackets, as [::1]:9092");
|
||||
}
|
||||
if (host.chars().anyMatch(Character::isWhitespace)) {
|
||||
throw new IllegalArgumentException(
|
||||
"broker entry '" + entry + "' has whitespace inside the host");
|
||||
}
|
||||
return new BrokerAddress(host, parsePort(entry, entry.substring(separator + 1)), false);
|
||||
}
|
||||
|
||||
private static int parsePort(String entry, String port) {
|
||||
if (port.isEmpty() || !port.chars().allMatch(Character::isDigit)) {
|
||||
throw new IllegalArgumentException(
|
||||
"broker entry '" + entry + "' does not end in a numeric port");
|
||||
}
|
||||
int parsed;
|
||||
try {
|
||||
parsed = Integer.parseInt(port);
|
||||
} catch (NumberFormatException tooLong) {
|
||||
throw new IllegalArgumentException(
|
||||
"broker entry '" + entry + "' has a port outside 1..65535", tooLong);
|
||||
}
|
||||
if (parsed < MIN_PORT || parsed > MAX_PORT) {
|
||||
// Reported here rather than in the constructor so the message names the offending entry: an
|
||||
// operator reading "port 0 is invalid" against a list of nine brokers learns nothing.
|
||||
throw new IllegalArgumentException(
|
||||
"broker entry '" + entry + "' has port " + parsed + ", outside 1..65535");
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* The canonical {@code host:port} form, which is what the client is given.
|
||||
*
|
||||
* @return the canonical text
|
||||
*/
|
||||
public String canonical() {
|
||||
return ipv6Literal ? "[" + host + "]:" + port : host + ":" + port;
|
||||
}
|
||||
}
|
||||
+34
-14
@@ -1,28 +1,48 @@
|
||||
package dev.caskeleton.adapter.outbound.messaging.kafka;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
/**
|
||||
* Kafka broker tuning bound from {@code app.messaging.kafka.*}. Validation is format-only ({@code
|
||||
* host:port} per entry); the "Kafka selected ⇒ brokers required" cross-field rule is enforced in
|
||||
* {@code KafkaAdapterConfig}, so an empty list is valid at bind time.
|
||||
* Kafka broker tuning bound from {@code app.messaging.kafka.*}.
|
||||
*
|
||||
* @param brokers CSV of {@code host:port} broker endpoints (each entry format-validated)
|
||||
* <p>Each entry is parsed into a {@link BrokerAddress} and stored in its canonical form. The
|
||||
* previous binding validated the trimmed value with a regular expression and then stored the
|
||||
* untrimmed original, so a configured {@code " kafka:9092"} passed the check and reached the client
|
||||
* with its leading space; the same expression accepted port {@code 0} and port {@code 99999}, and
|
||||
* could not express a bracketed IPv6 literal.
|
||||
*
|
||||
* <p>The "Kafka selected implies brokers required" cross-field rule stays in {@code
|
||||
* KafkaAdapterConfig}, so an empty list is still valid at bind time.
|
||||
*
|
||||
* @param brokers CSV of {@code host:port} broker endpoints, canonicalised
|
||||
*/
|
||||
@ConfigurationProperties(prefix = "app.messaging.kafka")
|
||||
public record KafkaAdapterSettings(List<String> brokers) {
|
||||
|
||||
private static final Pattern HOST_PORT = Pattern.compile("^[^:\\s]+:\\d{1,5}$");
|
||||
|
||||
/** Parses and canonicalises every configured entry. */
|
||||
public KafkaAdapterSettings {
|
||||
brokers = (brokers == null) ? List.of() : List.copyOf(brokers);
|
||||
for (String broker : brokers) {
|
||||
if (!HOST_PORT.matcher(broker.trim()).matches()) {
|
||||
throw new IllegalArgumentException(
|
||||
"APP_MESSAGING_KAFKA_BROKERS entry '" + broker + "' is not host:port");
|
||||
}
|
||||
}
|
||||
List<String> configured = (brokers == null) ? List.of() : List.copyOf(brokers);
|
||||
brokers =
|
||||
configured.stream()
|
||||
.map(
|
||||
entry -> {
|
||||
try {
|
||||
return BrokerAddress.parse(entry).canonical();
|
||||
} catch (IllegalArgumentException invalid) {
|
||||
throw new IllegalArgumentException(
|
||||
"APP_MESSAGING_KAFKA_BROKERS " + invalid.getMessage(), invalid);
|
||||
}
|
||||
})
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* The parsed addresses, for callers that need the components rather than the text.
|
||||
*
|
||||
* @return one address per configured entry
|
||||
*/
|
||||
public List<BrokerAddress> addresses() {
|
||||
return brokers.stream().map(BrokerAddress::parse).toList();
|
||||
}
|
||||
}
|
||||
|
||||
+4
-1
@@ -37,8 +37,11 @@ public final class Slf4jOutboxRelayFailureReportAdapter implements OutboxRelayFa
|
||||
LoggingEventBuilder event =
|
||||
logger
|
||||
.atError()
|
||||
.setCause(report.cause())
|
||||
// No setCause: the encoder renders a Throwable's message and stack into the
|
||||
// operational JSON, and a driver's exception text carries endpoints, statements, and
|
||||
// occasionally credentials. The class name is a type; the code is bounded.
|
||||
.addKeyValue("error.code", report.code().code())
|
||||
.addKeyValue("error.cause_type", report.causeType())
|
||||
.addKeyValue("error.category", report.code().category().name())
|
||||
.addKeyValue("dependency_name", dependencyName)
|
||||
.addKeyValue("dependency_type", DEPENDENCY_TYPE)
|
||||
|
||||
+36
@@ -107,4 +107,40 @@ class OutboundMessagePublisherTest {
|
||||
.contains("dependency_type=\"messaging\"")
|
||||
.contains("operation=\"publish\"");
|
||||
}
|
||||
|
||||
@org.junit.jupiter.api.DisplayName(
|
||||
"a logger failure after a confirmed send is not a publish failure")
|
||||
@Test
|
||||
void aLoggerFailureAfterAConfirmedSendIsNotAPublishFailure() {
|
||||
// The send and the success log used to share one try block, so a logger that threw after the
|
||||
// broker had accepted the message was caught by the failure branch and recorded as a publish
|
||||
// failure. The broker's outcome and the record of it are different facts.
|
||||
FakeBroker broker = new FakeBroker();
|
||||
org.slf4j.Logger throwingLogger =
|
||||
(org.slf4j.Logger)
|
||||
java.lang.reflect.Proxy.newProxyInstance(
|
||||
getClass().getClassLoader(),
|
||||
new Class<?>[] {org.slf4j.Logger.class},
|
||||
(proxy, method, args) -> {
|
||||
if (method.getName().equals("debug")) {
|
||||
throw new IllegalStateException("the appender is out of disk");
|
||||
}
|
||||
if (method.getReturnType() == boolean.class) {
|
||||
return true;
|
||||
}
|
||||
if (method.getReturnType() == String.class) {
|
||||
return "test.messaging";
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
OutboundMessagePublisher publisher =
|
||||
new OutboundMessagePublisher(broker, new FailOpenDependencyLogger(throwingLogger));
|
||||
OutboundMessage message = new OutboundMessage("worklog-events", "wl-1", "{}");
|
||||
|
||||
assertThatCode(() -> publisher.publish(message)).doesNotThrowAnyException();
|
||||
assertThat(broker.sent)
|
||||
.as("the message reached the broker; only the record of it failed")
|
||||
.containsExactly(message);
|
||||
}
|
||||
}
|
||||
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
package dev.caskeleton.adapter.outbound.messaging.kafka;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* What counts as a broker endpoint.
|
||||
*
|
||||
* <p>The regular expression that decided this ran against the trimmed value and then stored the
|
||||
* untrimmed original, accepted port 0 and port 99999, and could not express a bracketed IPv6
|
||||
* literal at all — so the only correct way to write an IPv6 endpoint was rejected while an
|
||||
* ambiguous one was accepted.
|
||||
*/
|
||||
class BrokerAddressTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("an ordinary endpoint parses and canonicalises")
|
||||
void anOrdinaryEndpointParses() {
|
||||
BrokerAddress address = BrokerAddress.parse("kafka-1.internal:9092");
|
||||
|
||||
assertThat(address.host()).isEqualTo("kafka-1.internal");
|
||||
assertThat(address.port()).isEqualTo(9092);
|
||||
assertThat(address.canonical()).isEqualTo("kafka-1.internal:9092");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("surrounding whitespace is removed rather than validated away and kept")
|
||||
void surroundingWhitespaceIsRemoved() {
|
||||
assertThat(new KafkaAdapterSettings(List.of(" kafka:9092 ")).brokers())
|
||||
.as("the old binding validated the trimmed value and stored the untrimmed one")
|
||||
.containsExactly("kafka:9092");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("whitespace inside the host is refused")
|
||||
void whitespaceInsideTheHostIsRefused() {
|
||||
assertThatThrownBy(() -> BrokerAddress.parse("kaf ka:9092"))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("whitespace");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the port range is 1 to 65535")
|
||||
void thePortRangeIsOneToSixtyFiveThousand() {
|
||||
assertThatCode(() -> BrokerAddress.parse("kafka:1")).doesNotThrowAnyException();
|
||||
assertThatCode(() -> BrokerAddress.parse("kafka:65535")).doesNotThrowAnyException();
|
||||
|
||||
assertThatThrownBy(() -> BrokerAddress.parse("kafka:0"))
|
||||
.as("port 0 asks the operating system to choose, which a client cannot connect to")
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> BrokerAddress.parse("kafka:65536"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> BrokerAddress.parse("kafka:99999"))
|
||||
.as("the five-digit pattern accepted this")
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a bracketed IPv6 literal is accepted and stays bracketed")
|
||||
void aBracketedIpv6LiteralIsAccepted() {
|
||||
BrokerAddress address = BrokerAddress.parse("[2001:db8::1]:9092");
|
||||
|
||||
assertThat(address.host()).isEqualTo("2001:db8::1");
|
||||
assertThat(address.port()).isEqualTo(9092);
|
||||
assertThat(address.ipv6Literal()).isTrue();
|
||||
assertThat(address.canonical()).isEqualTo("[2001:db8::1]:9092");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a bare IPv6 literal is refused because it is ambiguous")
|
||||
void aBareIpv6LiteralIsRefused() {
|
||||
assertThatThrownBy(() -> BrokerAddress.parse("2001:db8::1:9092"))
|
||||
.as("which colon separates the port is not decidable, so brackets are required")
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("[::1]:9092");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a bracketed host with no port is refused")
|
||||
void aBracketedHostWithNoPortIsRefused() {
|
||||
assertThatThrownBy(() -> BrokerAddress.parse("[2001:db8::1]"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> BrokerAddress.parse("[2001:db8::1]9092"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an entry with no port at all is refused")
|
||||
void anEntryWithNoPortIsRefused() {
|
||||
assertThatThrownBy(() -> BrokerAddress.parse("kafka"))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("host:port");
|
||||
assertThatThrownBy(() -> BrokerAddress.parse("kafka:"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> BrokerAddress.parse("kafka:http"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> BrokerAddress.parse(":9092"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the settings binding reports which entry is wrong")
|
||||
void theSettingsBindingReportsWhichEntryIsWrong() {
|
||||
assertThatThrownBy(() -> new KafkaAdapterSettings(List.of("kafka-1:9092", "kafka-2:0")))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("APP_MESSAGING_KAFKA_BROKERS")
|
||||
.hasMessageContaining("kafka-2:0");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an absent broker list is empty, not a failure")
|
||||
void anAbsentBrokerListIsEmpty() {
|
||||
assertThat(new KafkaAdapterSettings(null).brokers()).isEmpty();
|
||||
}
|
||||
}
|
||||
+7
-3
@@ -7,7 +7,6 @@ import static org.mockito.Mockito.when;
|
||||
|
||||
import ch.qos.logback.classic.Level;
|
||||
import ch.qos.logback.classic.spi.ILoggingEvent;
|
||||
import ch.qos.logback.classic.spi.ThrowableProxy;
|
||||
import ch.qos.logback.core.read.ListAppender;
|
||||
import dev.caskeleton.application.outbox.OutboxRelayFailureReport;
|
||||
import java.time.Instant;
|
||||
@@ -70,9 +69,14 @@ class Slf4jOutboxRelayFailureReportAdapterTest {
|
||||
Map.entry("aggregate_id", "agg-1"),
|
||||
Map.entry("correlation_id", "corr-1"),
|
||||
Map.entry("attempt_count", 2),
|
||||
Map.entry("error.cause_type", "java.lang.RuntimeException"),
|
||||
Map.entry("runbook_link", "runbook://outbox/publish-failed"),
|
||||
Map.entry("next_attempt_at", "2026-07-25T01:02:03Z")));
|
||||
assertThat(((ThrowableProxy) event.getThrowableProxy()).getThrowable()).isSameAs(cause);
|
||||
assertThat(event.getThrowableProxy())
|
||||
.as(
|
||||
"the encoder renders a Throwable's message and stack into the operational JSON, and a"
|
||||
+ " driver's exception text carries endpoints, statements, and credentials")
|
||||
.isNull();
|
||||
assertThat(event.getFormattedMessage()).doesNotContain("unsafe-exception-derived-value");
|
||||
assertThat(keyValues(event).toString())
|
||||
.doesNotContain("payload-secret", "idempotency-secret", "unsafe-exception-derived-value");
|
||||
@@ -94,7 +98,7 @@ class Slf4jOutboxRelayFailureReportAdapterTest {
|
||||
.containsEntry("outcome", "DEAD")
|
||||
.containsEntry("runbook_link", "runbook://outbox/dead-letter")
|
||||
.doesNotContainKey("next_attempt_at");
|
||||
assertThat(((ThrowableProxy) event.getThrowableProxy()).getThrowable()).isSameAs(cause);
|
||||
assertThat(event.getThrowableProxy()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.admin;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.admin.AdminAccessDeniedException;
|
||||
import dev.caskeleton.application.notification.platform.admin.AdminActor;
|
||||
import dev.caskeleton.application.notification.platform.admin.NotificationAdminAuthority;
|
||||
import dev.caskeleton.application.notification.platform.api.TenantId;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Operator authority check.
|
||||
*
|
||||
* <p>Application authority never grants an operator authority. The two planes are separated so that
|
||||
* a compromised application credential cannot redrive a message or lift a suppression — the actions
|
||||
* whose whole purpose is to override the platform's own safety decisions.
|
||||
*/
|
||||
public final class AdminAuthorizationGuard {
|
||||
|
||||
/** Require an authority, or refuse. */
|
||||
public void require(AdminActor actor, NotificationAdminAuthority authority) {
|
||||
Objects.requireNonNull(actor, "actor");
|
||||
Objects.requireNonNull(authority, "authority");
|
||||
if (!actor.holds(authority)) {
|
||||
throw new AdminAccessDeniedException(authority);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Require that the actor may act on a tenant.
|
||||
*
|
||||
* <p>An actor with no tenant is a global operator; one bound to a tenant may only act inside it.
|
||||
*/
|
||||
public void requireTenant(AdminActor actor, TenantId tenantId) {
|
||||
Objects.requireNonNull(actor, "actor");
|
||||
Objects.requireNonNull(tenantId, "tenantId");
|
||||
Optional<TenantId> scope = actor.tenantId();
|
||||
if (scope.isPresent() && !scope.get().equals(tenantId)) {
|
||||
throw new AdminAccessDeniedException(NotificationAdminAuthority.SUPPRESS);
|
||||
}
|
||||
}
|
||||
}
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.admin;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.admin.DuplicateRiskApprovalRequiredException;
|
||||
import dev.caskeleton.application.notification.platform.api.delivery.AttemptConfirmation;
|
||||
import dev.caskeleton.application.notification.platform.callback.DeliveryAttemptSnapshot;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Blocks an unapproved redrive of an ambiguous attempt.
|
||||
*
|
||||
* <p>The platform cannot tell whether the first submission reached the user, so re-sending is a
|
||||
* decision with a real cost that only a human can accept. Requiring the approval flag makes that
|
||||
* acceptance an explicit, audited act rather than a default.
|
||||
*/
|
||||
public final class DuplicateRiskGuard {
|
||||
|
||||
/** Verify the operator accepted the duplicate risk when one exists. */
|
||||
public void verify(DeliveryAttemptSnapshot attempt, boolean approved) {
|
||||
Objects.requireNonNull(attempt, "attempt");
|
||||
boolean risky =
|
||||
attempt.confirmation() == AttemptConfirmation.AMBIGUOUS
|
||||
|| attempt.submissionOutcome()
|
||||
== dev.caskeleton.application.notification.platform.api.delivery.SubmissionOutcome
|
||||
.CONFIRMED_ACCEPTED;
|
||||
if (risky && !approved) {
|
||||
throw new DuplicateRiskApprovalRequiredException();
|
||||
}
|
||||
}
|
||||
}
|
||||
-328
@@ -1,328 +0,0 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.admin;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderRuntimeRegistry;
|
||||
import dev.caskeleton.application.notification.platform.admin.AdminActor;
|
||||
import dev.caskeleton.application.notification.platform.admin.AdminOperationResult;
|
||||
import dev.caskeleton.application.notification.platform.admin.AdminOperationStorePort;
|
||||
import dev.caskeleton.application.notification.platform.admin.NotificationAdminAuthority;
|
||||
import dev.caskeleton.application.notification.platform.admin.NotificationAdminService;
|
||||
import dev.caskeleton.application.notification.platform.admin.ReconcileCommand;
|
||||
import dev.caskeleton.application.notification.platform.admin.RedriveCommand;
|
||||
import dev.caskeleton.application.notification.platform.admin.SetProviderStateCommand;
|
||||
import dev.caskeleton.application.notification.platform.admin.SuppressCommand;
|
||||
import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId;
|
||||
import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState;
|
||||
import dev.caskeleton.application.notification.platform.callback.DeliveryAttemptSnapshot;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService;
|
||||
import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent;
|
||||
import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort;
|
||||
import dev.caskeleton.application.notification.platform.policy.SuppressionEntry;
|
||||
import dev.caskeleton.application.notification.platform.policy.SuppressionId;
|
||||
import dev.caskeleton.application.notification.platform.policy.SuppressionSource;
|
||||
import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState;
|
||||
import dev.caskeleton.application.transaction.TransactionPort;
|
||||
import java.time.Clock;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* N4 operator plane.
|
||||
*
|
||||
* <p>Four properties hold for every operation: a separate authority, an idempotent operation id, a
|
||||
* recorded reason, and an audit row. The idempotency matters more than it looks — an operator
|
||||
* retrying a redrive after a timeout must not send the message twice, which is exactly the failure
|
||||
* the operation is trying to repair.
|
||||
*
|
||||
* <p>A dry run reads and reports but writes nothing, so an operator can see the blast radius of a
|
||||
* bulk action before committing to it.
|
||||
*/
|
||||
public final class NotificationAdminServiceImpl implements NotificationAdminService {
|
||||
|
||||
private final AdminAuthorizationGuard authorization;
|
||||
private final DuplicateRiskGuard duplicateRiskGuard;
|
||||
private final DeliveryAttemptStorePort attempts;
|
||||
private final RecipientDeliveryStorePort recipients;
|
||||
private final ReconciliationService reconciliation;
|
||||
private final SuppressionStorePort suppressions;
|
||||
private final ProviderRuntimeRegistry runtimes;
|
||||
private final AdminOperationStorePort operations;
|
||||
private final NotificationAuditPort audit;
|
||||
private final TransactionPort transactions;
|
||||
private final Clock clock;
|
||||
|
||||
public NotificationAdminServiceImpl(
|
||||
AdminAuthorizationGuard authorization,
|
||||
DuplicateRiskGuard duplicateRiskGuard,
|
||||
DeliveryAttemptStorePort attempts,
|
||||
RecipientDeliveryStorePort recipients,
|
||||
ReconciliationService reconciliation,
|
||||
SuppressionStorePort suppressions,
|
||||
ProviderRuntimeRegistry runtimes,
|
||||
AdminOperationStorePort operations,
|
||||
NotificationAuditPort audit,
|
||||
TransactionPort transactions,
|
||||
Clock clock) {
|
||||
this.authorization = Objects.requireNonNull(authorization, "authorization");
|
||||
this.duplicateRiskGuard = Objects.requireNonNull(duplicateRiskGuard, "duplicateRiskGuard");
|
||||
this.attempts = Objects.requireNonNull(attempts, "attempts");
|
||||
this.recipients = Objects.requireNonNull(recipients, "recipients");
|
||||
this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation");
|
||||
this.suppressions = Objects.requireNonNull(suppressions, "suppressions");
|
||||
this.runtimes = Objects.requireNonNull(runtimes, "runtimes");
|
||||
this.operations = Objects.requireNonNull(operations, "operations");
|
||||
this.audit = Objects.requireNonNull(audit, "audit");
|
||||
this.transactions = Objects.requireNonNull(transactions, "transactions");
|
||||
this.clock = Objects.requireNonNull(clock, "clock");
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdminOperationResult redrive(RedriveCommand command, AdminActor actor) {
|
||||
Objects.requireNonNull(command, "command");
|
||||
authorization.require(actor, NotificationAdminAuthority.REDRIVE);
|
||||
|
||||
Optional<AdminOperationResult> replayed = operations.findByOperationId(command.operationId());
|
||||
if (replayed.isPresent()) {
|
||||
return replayed.get();
|
||||
}
|
||||
|
||||
DeliveryAttemptSnapshot original =
|
||||
attempts
|
||||
.snapshot(command.attemptId())
|
||||
.orElseThrow(() -> new IllegalStateException("delivery attempt is not available"));
|
||||
authorization.requireTenant(actor, original.tenantId());
|
||||
duplicateRiskGuard.verify(original, command.approveDuplicateRisk());
|
||||
|
||||
if (command.dryRun()) {
|
||||
return new AdminOperationResult(
|
||||
command.operationId(),
|
||||
true,
|
||||
1,
|
||||
Optional.of(original.notificationId()),
|
||||
Optional.of(original.recipientDeliveryId()),
|
||||
Optional.empty(),
|
||||
List.of("DRY_RUN"));
|
||||
}
|
||||
|
||||
return transactions.inWrite(
|
||||
() -> {
|
||||
// The logical identities are preserved and only the attempt is new, so the history stays
|
||||
// one story rather than becoming two unrelated notifications.
|
||||
recipients.transition(
|
||||
original.recipientDeliveryId(),
|
||||
RecipientDeliveryState.READY_TO_DISPATCH,
|
||||
Optional.of(clock.instant()));
|
||||
|
||||
AdminOperationResult result =
|
||||
new AdminOperationResult(
|
||||
command.operationId(),
|
||||
false,
|
||||
1,
|
||||
Optional.of(original.notificationId()),
|
||||
Optional.of(original.recipientDeliveryId()),
|
||||
Optional.empty(),
|
||||
List.of(command.reason()));
|
||||
audit.record(
|
||||
new NotificationAuditEvent(
|
||||
"ADMIN_REDRIVE",
|
||||
actor.actorRef(),
|
||||
Optional.of(command.reason()),
|
||||
Optional.of(command.operationId()),
|
||||
clock.instant(),
|
||||
Map.of(
|
||||
"provider", original.providerId().value(),
|
||||
"channel", original.channel().name())));
|
||||
return operations.save(result, actor, "ADMIN_REDRIVE");
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdminOperationResult reconcile(ReconcileCommand command, AdminActor actor) {
|
||||
Objects.requireNonNull(command, "command");
|
||||
authorization.require(actor, NotificationAdminAuthority.RECONCILE);
|
||||
|
||||
Optional<AdminOperationResult> replayed = operations.findByOperationId(command.operationId());
|
||||
if (replayed.isPresent()) {
|
||||
return replayed.get();
|
||||
}
|
||||
if (command.dryRun()) {
|
||||
return new AdminOperationResult(
|
||||
command.operationId(),
|
||||
true,
|
||||
command.attemptIds().size(),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
List.of("DRY_RUN"));
|
||||
}
|
||||
|
||||
List<String> reasons = new ArrayList<>();
|
||||
int reconciled = 0;
|
||||
for (DeliveryAttemptId attemptId : command.attemptIds()) {
|
||||
reconciliation.reconcile(attemptId);
|
||||
reconciled++;
|
||||
}
|
||||
reasons.add(command.reason());
|
||||
|
||||
AdminOperationResult result =
|
||||
new AdminOperationResult(
|
||||
command.operationId(),
|
||||
false,
|
||||
reconciled,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
List.copyOf(reasons));
|
||||
audit.record(
|
||||
new NotificationAuditEvent(
|
||||
"ADMIN_RECONCILE",
|
||||
actor.actorRef(),
|
||||
Optional.of(command.reason()),
|
||||
Optional.of(command.operationId()),
|
||||
clock.instant(),
|
||||
Map.of()));
|
||||
return operations.save(result, actor, "ADMIN_RECONCILE");
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdminOperationResult suppress(SuppressCommand command, AdminActor actor) {
|
||||
Objects.requireNonNull(command, "command");
|
||||
authorization.require(actor, NotificationAdminAuthority.SUPPRESS);
|
||||
authorization.requireTenant(actor, command.tenantId());
|
||||
|
||||
Optional<AdminOperationResult> replayed = operations.findByOperationId(command.operationId());
|
||||
if (replayed.isPresent()) {
|
||||
return replayed.get();
|
||||
}
|
||||
if (command.dryRun()) {
|
||||
return new AdminOperationResult(
|
||||
command.operationId(),
|
||||
true,
|
||||
1,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
List.of("DRY_RUN"));
|
||||
}
|
||||
|
||||
return transactions.inWrite(
|
||||
() -> {
|
||||
int affected;
|
||||
if (command.remove()) {
|
||||
// Removal is by fingerprint match rather than by id, because an operator lifting a
|
||||
// suppression knows the target, not the row identifier the platform assigned.
|
||||
affected =
|
||||
suppressions
|
||||
.activeFor(
|
||||
command.tenantId(), command.targetFingerprint(), clock.instant())
|
||||
.stream()
|
||||
.map(entry -> suppressions.remove(command.tenantId(), entry.id()))
|
||||
.filter(Optional::isPresent)
|
||||
.count()
|
||||
> 0
|
||||
? 1
|
||||
: 0;
|
||||
} else {
|
||||
suppressions.upsert(
|
||||
new SuppressionEntry(
|
||||
new SuppressionId(UUID.randomUUID()),
|
||||
command.tenantId(),
|
||||
command.scope(),
|
||||
command.reason(),
|
||||
command.targetFingerprint(),
|
||||
Optional.empty(),
|
||||
clock.instant(),
|
||||
command.expiresAt(),
|
||||
SuppressionSource.ADMIN));
|
||||
affected = 1;
|
||||
}
|
||||
|
||||
AdminOperationResult result =
|
||||
new AdminOperationResult(
|
||||
command.operationId(),
|
||||
false,
|
||||
affected,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
List.of(command.reasonText()));
|
||||
audit.record(
|
||||
new NotificationAuditEvent(
|
||||
command.remove() ? "ADMIN_SUPPRESSION_REMOVED" : "ADMIN_SUPPRESSION_ADDED",
|
||||
actor.actorRef(),
|
||||
Optional.of(command.reason().name()),
|
||||
Optional.of(command.operationId()),
|
||||
clock.instant(),
|
||||
Map.of()));
|
||||
return operations.save(
|
||||
result, actor, command.remove() ? "ADMIN_SUPPRESS_REMOVE" : "ADMIN_SUPPRESS_ADD");
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdminOperationResult setProviderState(SetProviderStateCommand command, AdminActor actor) {
|
||||
Objects.requireNonNull(command, "command");
|
||||
authorization.require(actor, NotificationAdminAuthority.PROVIDER_CONTROL);
|
||||
|
||||
Optional<AdminOperationResult> replayed = operations.findByOperationId(command.operationId());
|
||||
if (replayed.isPresent()) {
|
||||
return replayed.get();
|
||||
}
|
||||
if (command.dryRun()) {
|
||||
return new AdminOperationResult(
|
||||
command.operationId(),
|
||||
true,
|
||||
1,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
List.of("DRY_RUN"));
|
||||
}
|
||||
|
||||
var runtime = runtimes.current(command.profileId());
|
||||
switch (command.desiredState()) {
|
||||
case DISABLED -> runtime.markDisabled();
|
||||
case DRAINING -> runtime.markDraining();
|
||||
case HEALTHY -> runtime.markHealthy();
|
||||
case DEGRADED -> runtime.markDegraded(command.reason());
|
||||
case THROTTLED -> runtime.markThrottled();
|
||||
case AUTHENTICATION_FAILED -> runtime.markAuthenticationFailed(command.reason());
|
||||
// Unreachable while the enum is exhaustive; present so a state added later fails loudly here
|
||||
// rather than silently leaving the runtime in whatever state it was already in.
|
||||
default -> throw new IllegalStateException("unhandled provider runtime state");
|
||||
}
|
||||
|
||||
AdminOperationResult result =
|
||||
new AdminOperationResult(
|
||||
command.operationId(),
|
||||
false,
|
||||
1,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
List.of(command.reason()));
|
||||
audit.record(
|
||||
new NotificationAuditEvent(
|
||||
"ADMIN_PROVIDER_STATE",
|
||||
actor.actorRef(),
|
||||
Optional.of(command.reason()),
|
||||
Optional.of(command.operationId()),
|
||||
clock.instant(),
|
||||
Map.of(
|
||||
"providerProfile", command.profileId().value(),
|
||||
"status", command.desiredState().name())));
|
||||
return operations.save(result, actor, "ADMIN_PROVIDER_STATE");
|
||||
}
|
||||
|
||||
/** Current state of a provider runtime, for the health endpoint. */
|
||||
public ProviderRuntimeState providerState(
|
||||
dev.caskeleton.application.notification.platform.api.ProviderProfileId profileId) {
|
||||
return runtimes.state(profileId);
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderRuntime;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderCallbackAdapter;
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderEventProjector;
|
||||
import dev.caskeleton.application.notification.platform.provider.ReconciliationCapability;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* One provider profile, assembled into everything the platform needs from it.
|
||||
*
|
||||
* <p>The pieces used to be registered independently — a runtime here, a callback adapter there, a
|
||||
* projector in a third list — and nothing checked that a profile had contributed all of the ones it
|
||||
* needs. A callback-enabled profile with no callback adapter was a context that started and then
|
||||
* failed on the first provider event, which is hours after the mistake was made and in a component
|
||||
* that did not make it.
|
||||
*
|
||||
* <p>Returning them together makes the incomplete contribution unrepresentable: an assembler either
|
||||
* produces a working profile or fails, and it fails at startup.
|
||||
*
|
||||
* @param runtime the dispatch runtime, bound to its credential generation
|
||||
* @param channel the channel this profile serves
|
||||
* @param callback the callback adapter, when the family has one
|
||||
* @param projector the provider-event projector, when the family has one
|
||||
* @param reconciliation the status-query capability, when the family supports one
|
||||
*/
|
||||
public record AssembledProvider(
|
||||
ProviderRuntime runtime,
|
||||
Channel channel,
|
||||
Optional<ProviderCallbackAdapter> callback,
|
||||
Optional<ProviderEventProjector> projector,
|
||||
Optional<ReconciliationCapability> reconciliation) {
|
||||
|
||||
/** Validates the contribution. */
|
||||
public AssembledProvider {
|
||||
Objects.requireNonNull(runtime, "runtime");
|
||||
Objects.requireNonNull(channel, "channel");
|
||||
Objects.requireNonNull(callback, "callback");
|
||||
Objects.requireNonNull(projector, "projector");
|
||||
Objects.requireNonNull(reconciliation, "reconciliation");
|
||||
if (runtime.profile().channel() != channel) {
|
||||
throw new IllegalArgumentException(
|
||||
"the assembled runtime serves "
|
||||
+ runtime.profile().channel()
|
||||
+ " but the contribution claims "
|
||||
+ channel);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A profile that dispatches and nothing else.
|
||||
*
|
||||
* @param runtime the dispatch runtime
|
||||
* @param channel the channel it serves
|
||||
* @return the contribution
|
||||
*/
|
||||
public static AssembledProvider dispatchOnly(ProviderRuntime runtime, Channel channel) {
|
||||
return new AssembledProvider(
|
||||
runtime, channel, Optional.empty(), Optional.empty(), Optional.empty());
|
||||
}
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure;
|
||||
|
||||
/**
|
||||
* Whether this deployment can actually deliver anything.
|
||||
*
|
||||
* <p>A platform with no assembled provider used to look identical to one with providers: the same
|
||||
* beans, the same scheduler, the same readiness. Requests were accepted durably and then sat in the
|
||||
* queue with no eligible route. Naming the state makes it a decision an operator takes rather than
|
||||
* a situation they discover.
|
||||
*/
|
||||
public enum NotificationPlatformMode {
|
||||
|
||||
/** At least one provider assembled; the platform accepts and delivers. */
|
||||
SERVING,
|
||||
|
||||
/**
|
||||
* No provider configured. The platform accepts and stores requests, and readiness reports
|
||||
* non-serving so a load balancer does not route delivery traffic here.
|
||||
*
|
||||
* <p>Must be selected explicitly. A deployment that reaches zero providers by accident is a
|
||||
* misconfiguration, and the whole point of this enum is that the two are told apart.
|
||||
*/
|
||||
INGEST_ONLY
|
||||
}
|
||||
+31
-6
@@ -15,9 +15,17 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
*/
|
||||
@ConfigurationProperties("ca-skeleton.notification.platform")
|
||||
public record NotificationPlatformSettings(
|
||||
boolean enabled, Dispatch dispatch, Callbacks callbacks, Map<String, Provider> providers) {
|
||||
boolean enabled,
|
||||
NotificationPlatformMode mode,
|
||||
Dispatch dispatch,
|
||||
Callbacks callbacks,
|
||||
Map<String, Provider> providers) {
|
||||
|
||||
public NotificationPlatformSettings {
|
||||
// SERVING by default: a deployment that ends up with no provider is a misconfiguration unless
|
||||
// somebody said otherwise, and the assembly refuses it rather than accepting requests it cannot
|
||||
// deliver.
|
||||
mode = mode == null ? NotificationPlatformMode.SERVING : mode;
|
||||
dispatch = dispatch == null ? Dispatch.defaults() : dispatch;
|
||||
callbacks = callbacks == null ? Callbacks.defaults() : callbacks;
|
||||
providers = providers == null ? Map.of() : Map.copyOf(providers);
|
||||
@@ -75,12 +83,24 @@ public record NotificationPlatformSettings(
|
||||
/** Callback endpoint bounds. */
|
||||
public record Callbacks(boolean enabled, long maxBodyBytes, Duration replaySkew) {
|
||||
|
||||
private static final long MAX_BODY_CEILING = 1_048_576L;
|
||||
/**
|
||||
* The largest body the platform can retain, derived rather than chosen.
|
||||
*
|
||||
* <p>It was one mebibyte, while the ciphertext column holds 65,536 bytes and encryption adds a
|
||||
* 12-byte nonce and a 16-byte tag. Three layers each enforced a different number: configuration
|
||||
* allowed a mebibyte, the MVC controller hard-coded 65,536, and the database rejected anything
|
||||
* over 65,536 *after* encryption — so a body of exactly the configured maximum passed every
|
||||
* check above the database and failed the CHECK constraint, having already been acknowledged.
|
||||
*/
|
||||
private static final long MAX_BODY_CEILING = 65_536L - 28L;
|
||||
|
||||
public Callbacks {
|
||||
Objects.requireNonNull(replaySkew, "replaySkew");
|
||||
if (maxBodyBytes < 1 || maxBodyBytes > MAX_BODY_CEILING) {
|
||||
throw new IllegalArgumentException("max-body-bytes must be 1.." + MAX_BODY_CEILING);
|
||||
throw new IllegalArgumentException(
|
||||
"max-body-bytes must be 1.."
|
||||
+ MAX_BODY_CEILING
|
||||
+ "; the ciphertext column holds 65536 bytes and encryption adds 28");
|
||||
}
|
||||
if (replaySkew.isNegative()) {
|
||||
throw new IllegalArgumentException("replay-skew must not be negative");
|
||||
@@ -89,7 +109,9 @@ public record NotificationPlatformSettings(
|
||||
|
||||
/** Conservative defaults. */
|
||||
public static Callbacks defaults() {
|
||||
return new Callbacks(false, 65_536L, Duration.ofMinutes(5));
|
||||
// The storable maximum, not the column size: encryption adds 28 bytes, so a default of
|
||||
// 65,536 was a default that could not be stored.
|
||||
return new Callbacks(false, MAX_BODY_CEILING, Duration.ofMinutes(5));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,6 +119,7 @@ public record NotificationPlatformSettings(
|
||||
public record Provider(
|
||||
String type,
|
||||
boolean enabled,
|
||||
boolean primaryForChannel,
|
||||
String environment,
|
||||
String credentialProfile,
|
||||
String topic,
|
||||
@@ -112,7 +135,9 @@ public record NotificationPlatformSettings(
|
||||
if (!enabled) {
|
||||
return;
|
||||
}
|
||||
require(type != null && !type.isBlank(), profileId, "type is required");
|
||||
// Resolved against the closed enum here, so an unrecognised type is a binding failure rather
|
||||
// than a profile that binds successfully and assembles into nothing.
|
||||
ProviderType resolved = ProviderType.parse(profileId, type);
|
||||
require(environment != null && !environment.isBlank(), profileId, "environment is required");
|
||||
require(
|
||||
credentialProfile != null && !credentialProfile.isBlank(),
|
||||
@@ -125,7 +150,7 @@ public record NotificationPlatformSettings(
|
||||
require(maxConcurrency >= 1, profileId, "max-concurrency must be positive");
|
||||
require(ratePerSecond >= 1, profileId, "rate-limit-per-second must be positive");
|
||||
|
||||
switch (type == null ? "" : type.toUpperCase(java.util.Locale.ROOT)) {
|
||||
switch (resolved.name()) {
|
||||
case "APNS" ->
|
||||
require(topic != null && !topic.isBlank(), profileId, "APNs profiles require a topic");
|
||||
case "WEB_PUSH" ->
|
||||
|
||||
+239
@@ -0,0 +1,239 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderRuntimeRegistry;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderCallbackAdapter;
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderEventProjector;
|
||||
import dev.caskeleton.application.notification.platform.provider.ReconciliationCapability;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumMap;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.TreeMap;
|
||||
|
||||
/**
|
||||
* Assembles every configured profile, once, at startup — or refuses to start.
|
||||
*
|
||||
* <p>This is the step that did not exist. The registry was constructed empty, the route planner
|
||||
* with {@code Map.of()}, and the reconciliation gateway with {@code Map.of()}, so configuration and
|
||||
* runtime were two unrelated things that happened to be in the same application.
|
||||
*
|
||||
* <p>Everything it refuses, it refuses before the dispatch worker starts:
|
||||
*
|
||||
* <ul>
|
||||
* <li>an unknown provider type, which used to bind and then assemble into nothing;
|
||||
* <li>two enabled profiles claiming the same channel with no primary named, because "which
|
||||
* provider sends this" is not a question to answer by map iteration order;
|
||||
* <li>a profile whose family has no assembler — a transport that is a seam rather than an
|
||||
* implementation;
|
||||
* <li>zero providers without {@link NotificationPlatformMode#INGEST_ONLY}, because a platform
|
||||
* that cannot deliver should say so rather than accept and queue forever.
|
||||
* </ul>
|
||||
*/
|
||||
public final class NotificationProviderAssembly {
|
||||
|
||||
private final Map<ProviderType, ProviderRuntimeAssembler> assemblers;
|
||||
|
||||
/**
|
||||
* Creates the assembly over the available family assemblers.
|
||||
*
|
||||
* @param assemblers one assembler per supported family
|
||||
*/
|
||||
public NotificationProviderAssembly(List<ProviderRuntimeAssembler> assemblers) {
|
||||
Objects.requireNonNull(assemblers, "assemblers");
|
||||
Map<ProviderType, ProviderRuntimeAssembler> byType = new EnumMap<>(ProviderType.class);
|
||||
for (ProviderRuntimeAssembler assembler : assemblers) {
|
||||
ProviderRuntimeAssembler previous = byType.put(assembler.type(), assembler);
|
||||
if (previous != null) {
|
||||
throw new IllegalStateException(
|
||||
"two assemblers claim provider type "
|
||||
+ assembler.type()
|
||||
+ "; which one builds a profile must not depend on bean ordering");
|
||||
}
|
||||
}
|
||||
this.assemblers = Map.copyOf(byType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Assembles the configured platform.
|
||||
*
|
||||
* @param settings the bound configuration
|
||||
* @param mode the declared mode
|
||||
* @return the assembled platform
|
||||
* @throws IllegalStateException naming the profile and the reason, for any refusal
|
||||
*/
|
||||
public AssembledPlatform assemble(
|
||||
NotificationPlatformSettings settings, NotificationPlatformMode mode) {
|
||||
Objects.requireNonNull(settings, "settings");
|
||||
Objects.requireNonNull(mode, "mode");
|
||||
|
||||
// Sorted so a failure names the same profile on every boot: an assembly error that moves
|
||||
// between profiles run to run is an error nobody can act on.
|
||||
Map<String, NotificationPlatformSettings.Provider> configured =
|
||||
new TreeMap<>(settings.providers());
|
||||
|
||||
ProviderRuntimeRegistry runtimes = new ProviderRuntimeRegistry();
|
||||
Map<Channel, ProviderProfileId> routes = new EnumMap<>(Channel.class);
|
||||
Map<ProviderProfileId, ProviderCallbackAdapter> callbacks = new LinkedHashMap<>();
|
||||
Map<ProviderProfileId, ProviderEventProjector> projectors = new LinkedHashMap<>();
|
||||
Map<ProviderProfileId, ReconciliationCapability> reconciliations = new LinkedHashMap<>();
|
||||
Map<Channel, List<String>> claimants = new EnumMap<>(Channel.class);
|
||||
|
||||
for (Map.Entry<String, NotificationPlatformSettings.Provider> entry : configured.entrySet()) {
|
||||
String profileId = entry.getKey();
|
||||
NotificationPlatformSettings.Provider profile = entry.getValue();
|
||||
if (!profile.enabled()) {
|
||||
continue;
|
||||
}
|
||||
ProviderType type = ProviderType.parse(profileId, profile.type());
|
||||
ProviderRuntimeAssembler assembler = assemblers.get(type);
|
||||
if (assembler == null) {
|
||||
throw new IllegalStateException(
|
||||
"notification provider profile '"
|
||||
+ profileId
|
||||
+ "' is of type "
|
||||
+ type
|
||||
+ ", which has no assembler in this build. The transport is a seam, not an"
|
||||
+ " implementation; remove the profile or supply a ProviderRuntimeAssembler for"
|
||||
+ " that family.");
|
||||
}
|
||||
AssembledProvider assembled = assembler.assemble(profileId, profile);
|
||||
if (assembled.channel() != type.channel()) {
|
||||
throw new IllegalStateException(
|
||||
"the assembler for "
|
||||
+ type
|
||||
+ " produced a "
|
||||
+ assembled.channel()
|
||||
+ " provider; the channel a family serves is a property of the family");
|
||||
}
|
||||
|
||||
ProviderProfileId id = new ProviderProfileId(profileId);
|
||||
runtimes.register(assembled.runtime());
|
||||
claimants.computeIfAbsent(assembled.channel(), channel -> new ArrayList<>()).add(profileId);
|
||||
// The marked primary wins outright; otherwise the first (and, once the ambiguity check below
|
||||
// has run, only) claimant takes the channel. Falling back to putIfAbsent alone would let a
|
||||
// declared primary lose the route to whichever id sorts first.
|
||||
if (profile.primaryForChannel()) {
|
||||
routes.put(assembled.channel(), id);
|
||||
} else {
|
||||
routes.putIfAbsent(assembled.channel(), id);
|
||||
}
|
||||
assembled.callback().ifPresent(adapter -> callbacks.put(id, adapter));
|
||||
assembled.projector().ifPresent(projector -> projectors.put(id, projector));
|
||||
assembled.reconciliation().ifPresent(capability -> reconciliations.put(id, capability));
|
||||
}
|
||||
|
||||
refuseAmbiguousRoutes(claimants, settings);
|
||||
refuseEmptyPlatform(routes, mode);
|
||||
|
||||
return new AssembledPlatform(
|
||||
runtimes,
|
||||
Map.copyOf(routes),
|
||||
Map.copyOf(callbacks),
|
||||
Map.copyOf(projectors),
|
||||
Map.copyOf(reconciliations),
|
||||
routes.isEmpty() ? NotificationPlatformMode.INGEST_ONLY : NotificationPlatformMode.SERVING);
|
||||
}
|
||||
|
||||
private static void refuseAmbiguousRoutes(
|
||||
Map<Channel, List<String>> claimants, NotificationPlatformSettings settings) {
|
||||
Map<String, String> primaries = new HashMap<>();
|
||||
settings
|
||||
.providers()
|
||||
.forEach(
|
||||
(profileId, profile) -> {
|
||||
if (profile.primaryForChannel()) {
|
||||
String channel = ProviderType.parse(profileId, profile.type()).channel().name();
|
||||
String previous = primaries.put(channel, profileId);
|
||||
if (previous != null) {
|
||||
throw new IllegalStateException(
|
||||
"profiles '"
|
||||
+ previous
|
||||
+ "' and '"
|
||||
+ profileId
|
||||
+ "' are both marked primary for channel "
|
||||
+ channel);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
claimants.forEach(
|
||||
(channel, profiles) -> {
|
||||
if (profiles.size() > 1 && !primaries.containsKey(channel.name())) {
|
||||
throw new IllegalStateException(
|
||||
"profiles "
|
||||
+ profiles
|
||||
+ " all serve channel "
|
||||
+ channel
|
||||
+ " and none is marked primary-for-channel. Which provider sends a "
|
||||
+ channel.name().toLowerCase(Locale.ROOT)
|
||||
+ " notification must not depend on map iteration order.");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static void refuseEmptyPlatform(
|
||||
Map<Channel, ProviderProfileId> routes, NotificationPlatformMode mode) {
|
||||
if (routes.isEmpty() && mode != NotificationPlatformMode.INGEST_ONLY) {
|
||||
throw new IllegalStateException(
|
||||
"the notification platform is enabled with no assembled provider. Every request would be"
|
||||
+ " accepted durably and then find no eligible route. Configure a provider, or"
|
||||
+ " declare ca-skeleton.notification.platform.mode=INGEST_ONLY so readiness reports"
|
||||
+ " non-serving.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Everything the composition root needs, built from configuration in one place.
|
||||
*
|
||||
* @param runtimes the registry the dispatch gateway routes through
|
||||
* @param routes the channel-to-profile map the route planner uses
|
||||
* @param callbacks callback adapters by profile
|
||||
* @param projectors provider-event projectors by profile
|
||||
* @param reconciliations status-query capabilities by profile
|
||||
* @param mode whether this deployment can deliver
|
||||
*/
|
||||
public record AssembledPlatform(
|
||||
ProviderRuntimeRegistry runtimes,
|
||||
Map<Channel, ProviderProfileId> routes,
|
||||
Map<ProviderProfileId, ProviderCallbackAdapter> callbacks,
|
||||
Map<ProviderProfileId, ProviderEventProjector> projectors,
|
||||
Map<ProviderProfileId, ReconciliationCapability> reconciliations,
|
||||
NotificationPlatformMode mode) {
|
||||
|
||||
/** Validates the assembly. */
|
||||
public AssembledPlatform {
|
||||
Objects.requireNonNull(runtimes, "runtimes");
|
||||
routes = Map.copyOf(Objects.requireNonNull(routes, "routes"));
|
||||
callbacks = Map.copyOf(Objects.requireNonNull(callbacks, "callbacks"));
|
||||
projectors = Map.copyOf(Objects.requireNonNull(projectors, "projectors"));
|
||||
reconciliations = Map.copyOf(Objects.requireNonNull(reconciliations, "reconciliations"));
|
||||
Objects.requireNonNull(mode, "mode");
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether this deployment can deliver.
|
||||
*
|
||||
* @return true when at least one route was assembled
|
||||
*/
|
||||
public boolean serving() {
|
||||
return mode == NotificationPlatformMode.SERVING;
|
||||
}
|
||||
|
||||
/**
|
||||
* The profile serving a channel, if any.
|
||||
*
|
||||
* @param channel the channel
|
||||
* @return the profile
|
||||
*/
|
||||
public Optional<ProviderProfileId> routeFor(Channel channel) {
|
||||
return Optional.ofNullable(routes.get(channel));
|
||||
}
|
||||
}
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure;
|
||||
|
||||
/**
|
||||
* Turns one configured profile into one working provider.
|
||||
*
|
||||
* <p>Nothing did this. Configuration bound a map of profiles, validation checked that their fields
|
||||
* were present, and the runtime registry was constructed empty — so a fully configured provider
|
||||
* produced no runtime, no route, and no error. Requests reached durable acceptance and then found
|
||||
* no eligible route, which reads to an operator as "the platform is dropping my notifications".
|
||||
*
|
||||
* <p>One assembler per family, each returning a complete {@link AssembledProvider}: the adapter,
|
||||
* its transport, the credential generation, the limiter, and whichever callback, projector and
|
||||
* reconciliation pieces the family has. A family whose transport is not implemented fails here, by
|
||||
* name, rather than assembling into something that cannot send.
|
||||
*/
|
||||
public interface ProviderRuntimeAssembler {
|
||||
|
||||
/**
|
||||
* The family this assembler builds.
|
||||
*
|
||||
* @return the provider type
|
||||
*/
|
||||
ProviderType type();
|
||||
|
||||
/**
|
||||
* Assembles one profile.
|
||||
*
|
||||
* @param profileId the configured profile id
|
||||
* @param profile the bound settings for it
|
||||
* @return the complete contribution
|
||||
* @throws IllegalStateException when the profile cannot produce a working provider, naming what
|
||||
* is missing
|
||||
*/
|
||||
AssembledProvider assemble(String profileId, NotificationPlatformSettings.Provider profile);
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import java.util.Arrays;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* The provider families this platform can assemble.
|
||||
*
|
||||
* <p>Configuration carried the type as a free string, and the only thing that read it was a
|
||||
* validation {@code switch} whose {@code default} branch accepted everything. So a profile of type
|
||||
* {@code "sendgrid"} — or {@code "smpt"} — passed validation, was bound, counted, and then never
|
||||
* assembled into anything, because the assembly step did not exist either. The failure was silent
|
||||
* at every stage: no route, no runtime, no error.
|
||||
*
|
||||
* <p>A closed enum makes the unknown type a binding failure at startup, and makes the channel each
|
||||
* family serves a property of the family rather than something a deployment can disagree with.
|
||||
*/
|
||||
public enum ProviderType {
|
||||
|
||||
/** Apple Push Notification service. */
|
||||
APNS(Channel.PUSH),
|
||||
|
||||
/** Firebase Cloud Messaging. */
|
||||
FCM(Channel.PUSH),
|
||||
|
||||
/** Amazon Simple Email Service. */
|
||||
SES(Channel.EMAIL),
|
||||
|
||||
/** A directly-configured SMTP relay. */
|
||||
SMTP(Channel.EMAIL),
|
||||
|
||||
/** Twilio programmable messaging. */
|
||||
TWILIO(Channel.SMS),
|
||||
|
||||
/** RFC 8291 Web Push. */
|
||||
WEB_PUSH(Channel.WEB_PUSH),
|
||||
|
||||
/** An outbound HTTP webhook. */
|
||||
WEBHOOK(Channel.WEBHOOK);
|
||||
|
||||
private final Channel channel;
|
||||
|
||||
ProviderType(Channel channel) {
|
||||
this.channel = channel;
|
||||
}
|
||||
|
||||
/**
|
||||
* The channel this family serves.
|
||||
*
|
||||
* @return the channel
|
||||
*/
|
||||
public Channel channel() {
|
||||
return channel;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a configured type, case-insensitively.
|
||||
*
|
||||
* @param profileId the profile the value came from, named in the failure
|
||||
* @param value the configured type
|
||||
* @return the resolved family
|
||||
* @throws IllegalArgumentException listing every supported value
|
||||
*/
|
||||
public static ProviderType parse(String profileId, String value) {
|
||||
Objects.requireNonNull(profileId, "profileId");
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException(
|
||||
"notification provider profile '" + profileId + "': type is required");
|
||||
}
|
||||
String normalized = value.trim().toUpperCase(Locale.ROOT).replace('-', '_');
|
||||
for (ProviderType candidate : values()) {
|
||||
if (candidate.name().equals(normalized)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException(
|
||||
"notification provider profile '"
|
||||
+ profileId
|
||||
+ "': unknown provider type '"
|
||||
+ value
|
||||
+ "'; supported types are "
|
||||
+ Arrays.toString(values())
|
||||
+ ". An unrecognised type used to bind successfully and then assemble into nothing.");
|
||||
}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.ProviderProfileCatalogPort;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* The configured channel-to-profile map, and nothing else.
|
||||
*
|
||||
* <p>What is left of {@code ConfiguredRoutePlanner} after NTF-020. It also decided the recipient's
|
||||
* effective channel order, applied blocked channels, and walked the strategy's fallback — all
|
||||
* product policy, all of it in the layer that speaks provider protocols. Those moved to {@code
|
||||
* PolicyRoutePlanner}; this answers the one question the adapter is actually the authority on.
|
||||
*/
|
||||
public final class ConfiguredProfileCatalog implements ProviderProfileCatalogPort {
|
||||
|
||||
private final Map<Channel, ProviderProfileId> profilesByChannel;
|
||||
|
||||
public ConfiguredProfileCatalog(Map<Channel, ProviderProfileId> profilesByChannel) {
|
||||
this.profilesByChannel =
|
||||
Map.copyOf(Objects.requireNonNull(profilesByChannel, "profilesByChannel"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<ProviderProfileId> profileFor(Channel channel) {
|
||||
return Optional.ofNullable(profilesByChannel.get(Objects.requireNonNull(channel, "channel")));
|
||||
}
|
||||
}
|
||||
-72
@@ -1,72 +0,0 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.api.RecipientSpec;
|
||||
import dev.caskeleton.application.notification.platform.api.TenantId;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.DeliveryStrategy;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.ExplicitChannel;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.OrderedFallback;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.NotificationRoutePlannerPort;
|
||||
import dev.caskeleton.application.notification.platform.policy.RouteCandidate;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Turns a strategy into an ordered route plan using the configured channel-to-profile map.
|
||||
*
|
||||
* <p>A channel with no configured provider, or a recipient with no contact point for it, simply
|
||||
* produces no candidate. The routing engine then reports {@code NO_ELIGIBLE_ROUTE} rather than the
|
||||
* dispatcher failing on a null, which is the difference between a diagnosable state and a stack
|
||||
* trace.
|
||||
*/
|
||||
public final class ConfiguredRoutePlanner implements NotificationRoutePlannerPort {
|
||||
|
||||
private final Map<Channel, ProviderProfileId> profilesByChannel;
|
||||
|
||||
public ConfiguredRoutePlanner(Map<Channel, ProviderProfileId> profilesByChannel) {
|
||||
this.profilesByChannel =
|
||||
Map.copyOf(Objects.requireNonNull(profilesByChannel, "profilesByChannel"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<RouteCandidate> plan(
|
||||
TenantId tenantId, RecipientSpec recipient, DeliveryStrategy strategy) {
|
||||
Objects.requireNonNull(tenantId, "tenantId");
|
||||
Objects.requireNonNull(recipient, "recipient");
|
||||
Objects.requireNonNull(strategy, "strategy");
|
||||
|
||||
List<Channel> ordered =
|
||||
switch (strategy) {
|
||||
case ExplicitChannel explicit -> List.of(explicit.channel());
|
||||
case OrderedFallback fallback -> fallback.channels();
|
||||
};
|
||||
|
||||
List<RouteCandidate> routes = new ArrayList<>(ordered.size());
|
||||
int index = 0;
|
||||
for (Channel channel : ordered) {
|
||||
ProviderProfileId profileId = profilesByChannel.get(channel);
|
||||
if (profileId == null) {
|
||||
continue;
|
||||
}
|
||||
var selector =
|
||||
recipient.contactPoints().stream()
|
||||
.filter(candidate -> candidate.channel() == channel)
|
||||
.findFirst();
|
||||
if (selector.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
boolean blocked =
|
||||
recipient
|
||||
.channelOverride()
|
||||
.map(override -> override.blockedChannels().contains(channel))
|
||||
.orElse(false);
|
||||
routes.add(
|
||||
new RouteCandidate(
|
||||
index++, channel, selector.get().contactPointId(), profileId, !blocked, true));
|
||||
}
|
||||
return List.copyOf(routes);
|
||||
}
|
||||
}
|
||||
+92
-9
@@ -11,26 +11,46 @@ import java.util.Objects;
|
||||
/**
|
||||
* Recovers deliveries a dead worker left in flight.
|
||||
*
|
||||
* <p>An expired lease on a {@code DISPATCHING} delivery is the crash case: the attempt row exists,
|
||||
* so a provider call may have happened. Recovery therefore reconciles rather than re-dispatching —
|
||||
* re-dispatching would be the platform choosing to duplicate rather than to ask.
|
||||
* <p>An expired lease on a {@code DISPATCHING} delivery is the crash case, and which recovery is
|
||||
* correct depends on how far the worker got:
|
||||
*
|
||||
* <ul>
|
||||
* <li><strong>No attempt row.</strong> The worker died between claiming the delivery and
|
||||
* recording that it was about to call the provider, which is proof no provider call happened.
|
||||
* Requeue is safe, and it is the only case where it is.
|
||||
* <li><strong>An attempt row with no completion.</strong> A provider call may have happened.
|
||||
* Recovery reconciles rather than re-dispatching — re-dispatching here would be the platform
|
||||
* choosing to duplicate rather than to ask.
|
||||
* </ul>
|
||||
*
|
||||
* <p>The first case used to be unhandled: recovery iterated attempts, and a delivery with none had
|
||||
* nothing to iterate. Those rows stayed {@code DISPATCHING} forever, holding a delivery nobody had
|
||||
* even tried to send.
|
||||
*/
|
||||
public final class LeaseRecoveryService {
|
||||
|
||||
private final RecipientLeaseStorePort leases;
|
||||
private final DeliveryAttemptStorePort attempts;
|
||||
private final ReconciliationService reconciliation;
|
||||
private final dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort
|
||||
deliveries;
|
||||
private final java.time.Clock clock;
|
||||
private final AttemptReconciler reconciliation;
|
||||
private final Duration staleAfter;
|
||||
private final int batchSize;
|
||||
|
||||
public LeaseRecoveryService(
|
||||
RecipientLeaseStorePort leases,
|
||||
DeliveryAttemptStorePort attempts,
|
||||
ReconciliationService reconciliation,
|
||||
dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort
|
||||
deliveries,
|
||||
AttemptReconciler reconciliation,
|
||||
java.time.Clock clock,
|
||||
Duration staleAfter,
|
||||
int batchSize) {
|
||||
this.leases = Objects.requireNonNull(leases, "leases");
|
||||
this.attempts = Objects.requireNonNull(attempts, "attempts");
|
||||
this.deliveries = Objects.requireNonNull(deliveries, "deliveries");
|
||||
this.clock = Objects.requireNonNull(clock, "clock");
|
||||
this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation");
|
||||
this.staleAfter = Objects.requireNonNull(staleAfter, "staleAfter");
|
||||
this.batchSize = batchSize;
|
||||
@@ -42,16 +62,79 @@ public final class LeaseRecoveryService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Asks what happened to one attempt.
|
||||
*
|
||||
* <p>A narrow seam over {@link ReconciliationService#reconcile}, which recovery is the only
|
||||
* caller of. Depending on the concrete service would drag its seven collaborators into every test
|
||||
* of the two-case split below, and the split is the part that was wrong.
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface AttemptReconciler {
|
||||
|
||||
/**
|
||||
* Reconciles one attempt whose outcome is unknown.
|
||||
*
|
||||
* @param attemptId the attempt
|
||||
*/
|
||||
void reconcile(DeliveryAttemptId attemptId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the stored evidence proves no request ever began.
|
||||
*
|
||||
* <p>Reads the persisted certainty and infers nothing from it. A fact that is merely {@code
|
||||
* INFERRED} or {@code UNKNOWN} is not proof, and the difference between "we know it did not
|
||||
* start" and "we do not know whether it started" is the difference between a safe requeue and a
|
||||
* duplicate notification.
|
||||
*
|
||||
* @param attempt the abandoned attempt
|
||||
* @return true only when the adapter proved the request never started
|
||||
*/
|
||||
private static boolean provablyNeverStarted(
|
||||
dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptRecord attempt) {
|
||||
var started = attempt.executionEvidence().requestStarted();
|
||||
return started.certainty()
|
||||
== dev.caskeleton.application.notification.platform.provider.EvidenceCertainty.PROVEN
|
||||
&& !started.value();
|
||||
}
|
||||
|
||||
/** Recover one batch of abandoned deliveries; returns how many were handled. */
|
||||
public int recoverOnce() {
|
||||
// Retiring expired deliveries rides along with recovery because both answer the same operator
|
||||
// question — "why is this row still here?" — and neither needs a scheduler of its own.
|
||||
int handled = leases.expireOverdue(batchSize);
|
||||
List<dev.caskeleton.application.notification.platform.api.RecipientDeliveryId> abandoned =
|
||||
leases.expiredDispatching(batchSize, staleAfter);
|
||||
int handled = 0;
|
||||
for (var recipientDeliveryId : abandoned) {
|
||||
for (var attempt : attempts.attemptsOf(recipientDeliveryId)) {
|
||||
var attemptsOfDelivery = attempts.attemptsOf(recipientDeliveryId);
|
||||
if (attemptsOfDelivery.isEmpty()) {
|
||||
// Crash before the attempt row: no provider call can have happened, so this is the one
|
||||
// safe requeue. Leaving it DISPATCHING stranded the delivery permanently.
|
||||
deliveries.transition(
|
||||
recipientDeliveryId,
|
||||
dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState
|
||||
.READY_TO_DISPATCH,
|
||||
java.util.Optional.of(clock.instant()));
|
||||
handled++;
|
||||
continue;
|
||||
}
|
||||
for (var attempt : attemptsOfDelivery) {
|
||||
if (attempt.completedAt().isEmpty()) {
|
||||
DeliveryAttemptId attemptId = attempt.id();
|
||||
reconciliation.reconcile(attemptId);
|
||||
if (provablyNeverStarted(attempt)) {
|
||||
// The row exists and the adapter recorded, with certainty, that no request began. There
|
||||
// is nothing for a provider to tell us, so asking costs a round trip to learn what is
|
||||
// already written down. This branch only became possible once the certainty survived
|
||||
// persistence: before, "proven not started" and "unknown whether started" read back
|
||||
// identically, so every abandoned attempt had to be reconciled.
|
||||
deliveries.transition(
|
||||
recipientDeliveryId,
|
||||
dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState
|
||||
.READY_TO_DISPATCH,
|
||||
java.util.Optional.of(clock.instant()));
|
||||
} else {
|
||||
reconciliation.reconcile(attempt.id());
|
||||
}
|
||||
handled++;
|
||||
}
|
||||
}
|
||||
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Runs the recovery and replay passes, and stops them on shutdown.
|
||||
*
|
||||
* <p>Both existed as classes with no caller. {@code LeaseRecoveryService} was written to recover
|
||||
* deliveries a dead worker left in flight and was never scheduled; the ledger's replay queries were
|
||||
* implemented in persistence and never read. The dispatch scheduler's own error path assumed
|
||||
* recovery existed — it leaves a lease to expire rather than releasing it optimistically, on the
|
||||
* grounds that recovery will decide — so the absence turned a deliberate design into a leak.
|
||||
*
|
||||
* <p>One executor for both, because they are cheap, periodic, and must stop together. A failure in
|
||||
* one pass is logged and the schedule continues: an exception escaping a scheduled task cancels it
|
||||
* silently, which is how a background worker stops running without anybody being told.
|
||||
*/
|
||||
public final class NotificationBackgroundWorkers implements AutoCloseable {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(NotificationBackgroundWorkers.class);
|
||||
|
||||
private final LeaseRecoveryService recovery;
|
||||
private final ProviderEventReplayWorker replay;
|
||||
private final ReconciliationJobWorker reconciliation;
|
||||
private final Duration interval;
|
||||
private final Duration shutdownGrace;
|
||||
private final ScheduledExecutorService scheduler;
|
||||
private final AtomicBoolean started = new AtomicBoolean();
|
||||
|
||||
private final java.util.List<java.util.concurrent.ScheduledFuture<?>> passes =
|
||||
new java.util.concurrent.CopyOnWriteArrayList<>();
|
||||
|
||||
/**
|
||||
* Creates the workers.
|
||||
*
|
||||
* @param recovery recovers abandoned deliveries
|
||||
* @param replay projects stored provider events that were never applied
|
||||
* @param reconciliation asks providers about attempts whose outcome is unknown
|
||||
* @param interval how often each pass runs
|
||||
* @param shutdownGrace how long shutdown waits for a pass in flight
|
||||
*/
|
||||
public NotificationBackgroundWorkers(
|
||||
LeaseRecoveryService recovery,
|
||||
ProviderEventReplayWorker replay,
|
||||
ReconciliationJobWorker reconciliation,
|
||||
Duration interval,
|
||||
Duration shutdownGrace) {
|
||||
this.recovery = Objects.requireNonNull(recovery, "recovery");
|
||||
this.replay = Objects.requireNonNull(replay, "replay");
|
||||
this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation");
|
||||
this.interval = requirePositive(interval, "interval");
|
||||
this.shutdownGrace = requirePositive(shutdownGrace, "shutdownGrace");
|
||||
this.scheduler =
|
||||
Executors.newScheduledThreadPool(
|
||||
1,
|
||||
runnable -> {
|
||||
Thread thread = new Thread(runnable, "notification-background");
|
||||
// A daemon thread: this executor must never be the reason a JVM refuses to exit.
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
}
|
||||
|
||||
/** Starts both passes. Idempotent. */
|
||||
public void start() {
|
||||
if (!started.compareAndSet(false, true)) {
|
||||
return;
|
||||
}
|
||||
schedule("recovery", recovery::recoverOnce);
|
||||
schedule("provider-event-replay", replay::replayOnce);
|
||||
schedule("reconciliation", reconciliation::reconcileOnce);
|
||||
}
|
||||
|
||||
private void schedule(String name, java.util.function.IntSupplier pass) {
|
||||
// The handle is kept so cancellation is possible and so the ignored-future check has an
|
||||
// answer: the task swallows its own exceptions, so the future never completes exceptionally
|
||||
// and there is nothing for a caller to observe on it.
|
||||
java.util.concurrent.ScheduledFuture<?> scheduled =
|
||||
scheduler.scheduleWithFixedDelay(
|
||||
() -> {
|
||||
try {
|
||||
int handled = pass.getAsInt();
|
||||
if (handled > 0) {
|
||||
log.debug("notification {} pass handled {}", name, handled);
|
||||
}
|
||||
} catch (RuntimeException failure) {
|
||||
// Swallowed on purpose: an exception that escapes here cancels the schedule for the
|
||||
// lifetime of the process, and a recovery worker that stopped silently is worse
|
||||
// than
|
||||
// one that fails a pass.
|
||||
log.warn(
|
||||
"notification {} pass failed reason={}",
|
||||
name,
|
||||
failure.getClass().getSimpleName());
|
||||
}
|
||||
},
|
||||
interval.toMillis(),
|
||||
interval.toMillis(),
|
||||
TimeUnit.MILLISECONDS);
|
||||
passes.add(scheduled);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
passes.forEach(pass -> pass.cancel(false));
|
||||
passes.clear();
|
||||
scheduler.shutdown();
|
||||
try {
|
||||
if (!scheduler.awaitTermination(shutdownGrace.toMillis(), TimeUnit.MILLISECONDS)) {
|
||||
scheduler.shutdownNow();
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
scheduler.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the passes are scheduled.
|
||||
*
|
||||
* @return true after start
|
||||
*/
|
||||
public boolean started() {
|
||||
return started.get();
|
||||
}
|
||||
|
||||
private static Duration requirePositive(Duration value, String name) {
|
||||
Objects.requireNonNull(value, name);
|
||||
if (value.isNegative() || value.isZero()) {
|
||||
throw new IllegalArgumentException(name + " must be positive and finite");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
+59
-22
@@ -5,6 +5,7 @@ import dev.caskeleton.application.notification.platform.dispatch.RecipientLease;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort;
|
||||
import dev.caskeleton.application.notification.platform.observation.NotificationMetricName;
|
||||
import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort;
|
||||
import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
@@ -35,22 +36,28 @@ public final class NotificationSchedulerWorker implements AutoCloseable {
|
||||
private final NotificationDispatchService dispatcher;
|
||||
private final NotificationMetricsPort metrics;
|
||||
private final NotificationDispatchProperties properties;
|
||||
private final NotificationServingStatePort servingState;
|
||||
private final String workerId;
|
||||
private final ExecutorService dispatchExecutor;
|
||||
private final Semaphore globalConcurrency;
|
||||
private final AtomicBoolean running = new AtomicBoolean();
|
||||
private final AtomicBoolean shuttingDown = new AtomicBoolean();
|
||||
|
||||
/** The polling thread, kept so shutdown can actually stop it. */
|
||||
private volatile Thread pollingThread;
|
||||
|
||||
public NotificationSchedulerWorker(
|
||||
RecipientLeaseStorePort leases,
|
||||
NotificationDispatchService dispatcher,
|
||||
NotificationMetricsPort metrics,
|
||||
NotificationDispatchProperties properties,
|
||||
NotificationServingStatePort servingState,
|
||||
String workerId) {
|
||||
this.leases = Objects.requireNonNull(leases, "leases");
|
||||
this.dispatcher = Objects.requireNonNull(dispatcher, "dispatcher");
|
||||
this.metrics = Objects.requireNonNull(metrics, "metrics");
|
||||
this.properties = Objects.requireNonNull(properties, "properties");
|
||||
this.servingState = Objects.requireNonNull(servingState, "servingState");
|
||||
this.workerId = Objects.requireNonNull(workerId, "workerId");
|
||||
this.dispatchExecutor = Executors.newVirtualThreadPerTaskExecutor();
|
||||
this.globalConcurrency = new Semaphore(properties.maxGlobalConcurrency());
|
||||
@@ -61,9 +68,23 @@ public final class NotificationSchedulerWorker implements AutoCloseable {
|
||||
if (shuttingDown.get()) {
|
||||
return 0;
|
||||
}
|
||||
List<RecipientLease> claimed =
|
||||
leases.claim(workerId, properties.claimBatchSize(), properties.leaseDuration());
|
||||
metrics.gauge(NotificationMetricName.QUEUE_DEPTH, Map.of(), claimed.size());
|
||||
// Only as many as can start now. The batch used to be claimed in full and then queued behind
|
||||
// the semaphore, so a batch larger than the concurrency limit held leases on deliveries nobody
|
||||
// was working on — and with a short lease those expired before their turn came, letting another
|
||||
// worker claim a delivery this one still had queued.
|
||||
int executable = Math.min(properties.claimBatchSize(), globalConcurrency.availablePermits());
|
||||
if (executable < 1) {
|
||||
return 0;
|
||||
}
|
||||
List<RecipientLease> claimed = leases.claim(workerId, executable, properties.leaseDuration());
|
||||
// The backlog, not the batch. This gauge used to report claimed.size(), which is bounded above
|
||||
// by the claim batch size — so a queue of ten and a queue of ten million published the same
|
||||
// number, and the one metric named "queue depth" was the one that could not show a queue
|
||||
// growing.
|
||||
metrics.gauge(
|
||||
NotificationMetricName.QUEUE_DEPTH,
|
||||
Map.of(),
|
||||
(double) servingState.currentState().backlogDepth());
|
||||
|
||||
for (RecipientLease lease : claimed) {
|
||||
globalConcurrency.acquireUninterruptibly();
|
||||
@@ -92,32 +113,48 @@ public final class NotificationSchedulerWorker implements AutoCloseable {
|
||||
if (!running.compareAndSet(false, true)) {
|
||||
return;
|
||||
}
|
||||
Thread.ofVirtual()
|
||||
.name("notification-scheduler-" + workerId)
|
||||
.start(
|
||||
() -> {
|
||||
while (running.get() && !shuttingDown.get()) {
|
||||
try {
|
||||
if (runOnce() == 0) {
|
||||
Thread.sleep(properties.pollInterval().toMillis());
|
||||
pollingThread =
|
||||
Thread.ofVirtual()
|
||||
.name("notification-scheduler-" + workerId)
|
||||
.unstarted(
|
||||
() -> {
|
||||
while (running.get() && !shuttingDown.get()) {
|
||||
try {
|
||||
if (runOnce() == 0) {
|
||||
Thread.sleep(properties.pollInterval().toMillis());
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
return;
|
||||
} catch (RuntimeException failure) {
|
||||
log.warn(
|
||||
"notification scheduler tick failed worker={} reason={}",
|
||||
workerId,
|
||||
failure.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
return;
|
||||
} catch (RuntimeException failure) {
|
||||
log.warn(
|
||||
"notification scheduler tick failed worker={} reason={}",
|
||||
workerId,
|
||||
failure.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
pollingThread.start();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
shuttingDown.set(true);
|
||||
running.set(false);
|
||||
// The polling thread was started and forgotten: close() shut the dispatch executor down and
|
||||
// returned while the loop was still free to claim another batch, so shutdown could leave leases
|
||||
// held by a process that was already gone. Interrupting it breaks the poll-interval sleep, and
|
||||
// joining it means "closed" is a fact rather than a request.
|
||||
Thread poller = pollingThread;
|
||||
if (poller != null) {
|
||||
poller.interrupt();
|
||||
try {
|
||||
poller.join(properties.shutdownGrace().toMillis());
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
pollingThread = null;
|
||||
}
|
||||
dispatchExecutor.shutdown();
|
||||
try {
|
||||
if (!dispatchExecutor.awaitTermination(
|
||||
|
||||
+39
-10
@@ -7,7 +7,7 @@ import dev.caskeleton.application.notification.platform.api.error.ProviderUnavai
|
||||
import java.time.Clock;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
/**
|
||||
* Per-provider rate and concurrency guard.
|
||||
@@ -22,8 +22,25 @@ public final class ProviderAttemptLimiter {
|
||||
private final int maxConcurrency;
|
||||
private final int ratePerSecond;
|
||||
private final Clock clock;
|
||||
private final AtomicLong windowStartSecond = new AtomicLong();
|
||||
private final AtomicLong issuedInWindow = new AtomicLong();
|
||||
|
||||
/**
|
||||
* The rate window and its count, as one value.
|
||||
*
|
||||
* <p>They were two atomics. A thread crossing a second boundary would CAS the window start and
|
||||
* then reset the count in a separate operation, so every increment another thread made between
|
||||
* those two steps was discarded — the limiter let more through than configured at exactly the
|
||||
* moment traffic rolls over. One reference makes "which window, and how many so far" a single
|
||||
* observable fact.
|
||||
*/
|
||||
private final AtomicReference<RateWindow> window = new AtomicReference<>();
|
||||
|
||||
/**
|
||||
* One rate window.
|
||||
*
|
||||
* @param epochSecond the second this window covers
|
||||
* @param used how many attempts it has admitted
|
||||
*/
|
||||
private record RateWindow(long epochSecond, int used) {}
|
||||
|
||||
public ProviderAttemptLimiter(int maxConcurrency, int ratePerSecond, Clock clock) {
|
||||
if (maxConcurrency < 1) {
|
||||
@@ -36,21 +53,33 @@ public final class ProviderAttemptLimiter {
|
||||
this.maxConcurrency = maxConcurrency;
|
||||
this.ratePerSecond = ratePerSecond;
|
||||
this.clock = Objects.requireNonNull(clock, "clock");
|
||||
this.windowStartSecond.set(clock.instant().getEpochSecond());
|
||||
this.window.set(new RateWindow(clock.instant().getEpochSecond(), 0));
|
||||
}
|
||||
|
||||
/** Acquire one attempt slot, or fail fast when the provider budget is spent. */
|
||||
public void acquire() {
|
||||
long second = clock.instant().getEpochSecond();
|
||||
long windowStart = windowStartSecond.get();
|
||||
if (second != windowStart && windowStartSecond.compareAndSet(windowStart, second)) {
|
||||
issuedInWindow.set(0L);
|
||||
}
|
||||
if (issuedInWindow.incrementAndGet() > ratePerSecond) {
|
||||
// One CAS decides both the window and the count. accumulateAndGet retries until it wins, so a
|
||||
// rollover cannot lose an increment another thread made.
|
||||
RateWindow admitted =
|
||||
window.accumulateAndGet(
|
||||
new RateWindow(second, 1),
|
||||
(current, attempt) ->
|
||||
current.epochSecond() == attempt.epochSecond()
|
||||
? new RateWindow(current.epochSecond(), current.used() + 1)
|
||||
: new RateWindow(attempt.epochSecond(), 1));
|
||||
if (admitted.used() > ratePerSecond) {
|
||||
throw unavailable();
|
||||
}
|
||||
if (!concurrency.tryAcquire()) {
|
||||
issuedInWindow.decrementAndGet();
|
||||
// Give the rate slot back, but only within the window that granted it: decrementing a window
|
||||
// that has since rolled over would credit the new one.
|
||||
window.accumulateAndGet(
|
||||
new RateWindow(second, 0),
|
||||
(current, refund) ->
|
||||
current.epochSecond() == refund.epochSecond()
|
||||
? new RateWindow(current.epochSecond(), Math.max(0, current.used() - 1))
|
||||
: current);
|
||||
throw unavailable();
|
||||
}
|
||||
}
|
||||
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderEventLedger;
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderEventProjectionService;
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderEventRecord;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Replays provider events that were stored but never projected.
|
||||
*
|
||||
* <p>{@code ProviderEventLedger} has always been able to list them — {@code pendingProjection} for
|
||||
* events whose attempt was not resolvable yet, {@code unmatched} for events that arrived before the
|
||||
* submitting process wrote the provider request id. Both were implemented in persistence and
|
||||
* neither had a caller.
|
||||
*
|
||||
* <p>That is not a small omission, because the callback path <em>depends</em> on the retry. A
|
||||
* provider that delivers its callback before the submitting transaction commits is normal, and the
|
||||
* ingestion path deliberately stores such an event as {@code PENDING} rather than dropping it. With
|
||||
* nothing replaying it, "we will match it later" was true only in the comment: the delivery stayed
|
||||
* unconfirmed forever and the callback that would have confirmed it sat in a table.
|
||||
*/
|
||||
public final class ProviderEventReplayWorker {
|
||||
|
||||
private final ProviderEventLedger ledger;
|
||||
private final ProviderEventProjectionService projection;
|
||||
private final int batchSize;
|
||||
|
||||
/**
|
||||
* Creates the replay worker.
|
||||
*
|
||||
* @param ledger the stored events
|
||||
* @param projection the projection the events feed
|
||||
* @param batchSize how many events one pass handles
|
||||
*/
|
||||
public ProviderEventReplayWorker(
|
||||
ProviderEventLedger ledger, ProviderEventProjectionService projection, int batchSize) {
|
||||
this.ledger = Objects.requireNonNull(ledger, "ledger");
|
||||
this.projection = Objects.requireNonNull(projection, "projection");
|
||||
if (batchSize < 1) {
|
||||
throw new IllegalArgumentException("batchSize");
|
||||
}
|
||||
this.batchSize = batchSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replays one batch.
|
||||
*
|
||||
* <p>Unmatched events are attempted first: they are the ones a late-arriving attempt row has most
|
||||
* likely just made matchable, and projecting them promptly is what keeps a delivery's outcome
|
||||
* honest rather than eventually correct.
|
||||
*
|
||||
* @return how many events projected successfully
|
||||
*/
|
||||
public int replayOnce() {
|
||||
int projected = 0;
|
||||
projected += replay(ledger.unmatched(batchSize));
|
||||
projected += replay(ledger.pendingProjection(batchSize));
|
||||
return projected;
|
||||
}
|
||||
|
||||
private int replay(List<ProviderEventRecord> events) {
|
||||
int projected = 0;
|
||||
for (ProviderEventRecord event : events) {
|
||||
// One event's failure is not the batch's. An event whose projector is missing is marked
|
||||
// FAILED by the projection service and would otherwise stop every event behind it.
|
||||
try {
|
||||
if (projection.project(event).isPresent()) {
|
||||
projected++;
|
||||
}
|
||||
} catch (RuntimeException failure) {
|
||||
ledger.markFailed(event.id(), failure.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
return projected;
|
||||
}
|
||||
}
|
||||
+170
-30
@@ -27,8 +27,30 @@ public final class ProviderRuntime {
|
||||
private final ProviderProfileSnapshot profile;
|
||||
private final NotificationProviderAdapter adapter;
|
||||
private final ProviderAttemptLimiter limiter;
|
||||
private final AtomicReference<ProviderRuntimeState> state;
|
||||
private final AtomicReference<String> unhealthyReason = new AtomicReference<>();
|
||||
|
||||
/**
|
||||
* Health and its reason, as one value.
|
||||
*
|
||||
* <p>They were two references, so a reader could observe a state and a reason that never held
|
||||
* together — {@code AUTHENTICATION_FAILED} with the previous failure's reason, or {@code HEALTHY}
|
||||
* with a stale one. An operator reading that snapshot is being told something the runtime never
|
||||
* believed.
|
||||
*/
|
||||
private final AtomicReference<RuntimeHealth> health;
|
||||
|
||||
/**
|
||||
* One consistent health observation.
|
||||
*
|
||||
* @param state the runtime state
|
||||
* @param reason why it is unhealthy, when it is
|
||||
*/
|
||||
public record RuntimeHealth(ProviderRuntimeState state, Optional<String> reason) {
|
||||
|
||||
public RuntimeHealth {
|
||||
Objects.requireNonNull(state, "state");
|
||||
Objects.requireNonNull(reason, "reason");
|
||||
}
|
||||
}
|
||||
|
||||
public ProviderRuntime(
|
||||
ProviderProfileSnapshot profile,
|
||||
@@ -37,7 +59,8 @@ public final class ProviderRuntime {
|
||||
this.profile = Objects.requireNonNull(profile, "profile");
|
||||
this.adapter = Objects.requireNonNull(adapter, "adapter");
|
||||
this.limiter = Objects.requireNonNull(limiter, "limiter");
|
||||
this.state = new AtomicReference<>(ProviderRuntimeState.HEALTHY);
|
||||
this.health =
|
||||
new AtomicReference<>(new RuntimeHealth(ProviderRuntimeState.HEALTHY, Optional.empty()));
|
||||
}
|
||||
|
||||
/** Profile snapshot including the credential generation. */
|
||||
@@ -55,14 +78,24 @@ public final class ProviderRuntime {
|
||||
return adapter;
|
||||
}
|
||||
|
||||
/**
|
||||
* State and reason as one observation.
|
||||
*
|
||||
* <p>Prefer this to calling {@link #state()} and {@link #unhealthyReason()} in turn: two reads
|
||||
* can straddle a transition and produce a pairing the runtime never held.
|
||||
*/
|
||||
public RuntimeHealth health() {
|
||||
return health.get();
|
||||
}
|
||||
|
||||
/** Current health. */
|
||||
public ProviderRuntimeState state() {
|
||||
return state.get();
|
||||
return health.get().state();
|
||||
}
|
||||
|
||||
/** Why the runtime is unhealthy, if it is. */
|
||||
public Optional<String> unhealthyReason() {
|
||||
return Optional.ofNullable(unhealthyReason.get());
|
||||
return health.get().reason();
|
||||
}
|
||||
|
||||
/** Attempts currently in flight on this generation. */
|
||||
@@ -77,7 +110,7 @@ public final class ProviderRuntime {
|
||||
* a token it cannot use.
|
||||
*/
|
||||
public AttemptPermit acquireAttempt() {
|
||||
ProviderRuntimeState current = state.get();
|
||||
ProviderRuntimeState current = health.get().state();
|
||||
if (!current.admitsNewAttempts()) {
|
||||
throw new ProviderUnavailableException(
|
||||
NotificationFailureDescriptor.preDispatch(
|
||||
@@ -92,45 +125,152 @@ public final class ProviderRuntime {
|
||||
|
||||
/** Mark the credential as rejected by the provider. */
|
||||
public void markAuthenticationFailed(String reasonCode) {
|
||||
unhealthyReason.set(Objects.requireNonNull(reasonCode, "reasonCode"));
|
||||
state.set(ProviderRuntimeState.AUTHENTICATION_FAILED);
|
||||
Objects.requireNonNull(reasonCode, "reasonCode");
|
||||
// One write, so the state and the reason it carries are never observed apart.
|
||||
health.set(
|
||||
new RuntimeHealth(ProviderRuntimeState.AUTHENTICATION_FAILED, Optional.of(reasonCode)));
|
||||
}
|
||||
|
||||
/** Mark the provider as rate limited. */
|
||||
public void markThrottled() {
|
||||
state.compareAndSet(ProviderRuntimeState.HEALTHY, ProviderRuntimeState.THROTTLED);
|
||||
/**
|
||||
* Mark the provider as rate limited.
|
||||
*
|
||||
* @return whether it is now throttled
|
||||
*/
|
||||
public boolean markThrottled() {
|
||||
return health
|
||||
.updateAndGet(
|
||||
current ->
|
||||
current.state() == ProviderRuntimeState.HEALTHY
|
||||
? new RuntimeHealth(
|
||||
ProviderRuntimeState.THROTTLED, Optional.of("THROTTLED"))
|
||||
: current)
|
||||
.state()
|
||||
== ProviderRuntimeState.THROTTLED;
|
||||
}
|
||||
|
||||
/** Mark the provider as degraded but still usable. */
|
||||
public void markDegraded(String reasonCode) {
|
||||
unhealthyReason.set(reasonCode);
|
||||
state.compareAndSet(ProviderRuntimeState.HEALTHY, ProviderRuntimeState.DEGRADED);
|
||||
/**
|
||||
* Mark the provider as degraded but still usable.
|
||||
*
|
||||
* @return whether it is now degraded
|
||||
*/
|
||||
public boolean markDegraded(String reasonCode) {
|
||||
Objects.requireNonNull(reasonCode, "reasonCode");
|
||||
return health
|
||||
.updateAndGet(
|
||||
current ->
|
||||
current.state() == ProviderRuntimeState.HEALTHY
|
||||
? new RuntimeHealth(ProviderRuntimeState.DEGRADED, Optional.of(reasonCode))
|
||||
: current)
|
||||
.state()
|
||||
== ProviderRuntimeState.DEGRADED;
|
||||
}
|
||||
|
||||
/** Return to healthy after a successful attempt. */
|
||||
public void markHealthy() {
|
||||
unhealthyReason.set(null);
|
||||
state.compareAndSet(ProviderRuntimeState.THROTTLED, ProviderRuntimeState.HEALTHY);
|
||||
state.compareAndSet(ProviderRuntimeState.DEGRADED, ProviderRuntimeState.HEALTHY);
|
||||
/**
|
||||
* Return to healthy after a successful attempt.
|
||||
*
|
||||
* <p>A success clears throttling and degradation and nothing else. It does not clear an
|
||||
* authentication failure — the credential the provider rejected is still the credential in use,
|
||||
* and only a rotation replaces it. It does not resume a draining or disabled runtime either;
|
||||
* those states are decisions, not symptoms.
|
||||
*
|
||||
* <p>The reason is cleared only when the state actually changes. Clearing it unconditionally left
|
||||
* {@code AUTHENTICATION_FAILED} with no reason attached, so an operator reading the runtime was
|
||||
* shown a failure the platform could no longer explain.
|
||||
*
|
||||
* @return whether the runtime is now healthy
|
||||
*/
|
||||
public boolean markHealthy() {
|
||||
return health
|
||||
.updateAndGet(
|
||||
current ->
|
||||
switch (current.state()) {
|
||||
case THROTTLED, DEGRADED ->
|
||||
new RuntimeHealth(ProviderRuntimeState.HEALTHY, Optional.empty());
|
||||
default -> current;
|
||||
})
|
||||
.state()
|
||||
== ProviderRuntimeState.HEALTHY;
|
||||
}
|
||||
|
||||
/** Stop admitting new attempts; in-flight attempts finish. */
|
||||
public void markDraining() {
|
||||
state.set(ProviderRuntimeState.DRAINING);
|
||||
/**
|
||||
* Clear an operator-imposed state.
|
||||
*
|
||||
* <p>This is the admin counterpart of {@link #markHealthy()}: it resumes a runtime that an
|
||||
* operator drained or disabled. It still refuses {@code AUTHENTICATION_FAILED}, because declaring
|
||||
* a provider healthy does not give it a credential the provider will accept — the caller is told
|
||||
* so rather than being handed a runtime that will fail on its first attempt.
|
||||
*
|
||||
* @return whether the runtime is now healthy
|
||||
*/
|
||||
public boolean resumeHealthy() {
|
||||
return health
|
||||
.updateAndGet(
|
||||
current ->
|
||||
current.state() == ProviderRuntimeState.AUTHENTICATION_FAILED
|
||||
? current
|
||||
: new RuntimeHealth(ProviderRuntimeState.HEALTHY, Optional.empty()))
|
||||
.state()
|
||||
== ProviderRuntimeState.HEALTHY;
|
||||
}
|
||||
|
||||
/** Operator disable. */
|
||||
public void markDisabled() {
|
||||
state.set(ProviderRuntimeState.DISABLED);
|
||||
/**
|
||||
* Stop admitting new attempts; in-flight attempts finish.
|
||||
*
|
||||
* @return whether it is now draining
|
||||
*/
|
||||
public boolean markDraining() {
|
||||
return health
|
||||
.updateAndGet(
|
||||
current ->
|
||||
new RuntimeHealth(ProviderRuntimeState.DRAINING, Optional.of("DRAINING")))
|
||||
.state()
|
||||
== ProviderRuntimeState.DRAINING;
|
||||
}
|
||||
|
||||
/** A permit that releases exactly one limiter slot. */
|
||||
private record LimiterPermit(long generation, ProviderAttemptLimiter limiter)
|
||||
implements AttemptPermit {
|
||||
/**
|
||||
* Operator disable.
|
||||
*
|
||||
* @return whether it is now disabled
|
||||
*/
|
||||
public boolean markDisabled() {
|
||||
return health
|
||||
.updateAndGet(
|
||||
current ->
|
||||
new RuntimeHealth(ProviderRuntimeState.DISABLED, Optional.of("DISABLED")))
|
||||
.state()
|
||||
== ProviderRuntimeState.DISABLED;
|
||||
}
|
||||
|
||||
/**
|
||||
* A permit that releases exactly one limiter slot, however many times it is closed.
|
||||
*
|
||||
* <p>{@code close} released unconditionally. A permit closed twice — a {@code finally} plus an
|
||||
* explicit close, or a retry wrapper — released two slots for one acquisition, and a {@code
|
||||
* Semaphore} grows when you release more than you took. The concurrency ceiling would then be
|
||||
* permanently higher than configured, silently, in the direction of overloading the provider.
|
||||
*/
|
||||
private static final class LimiterPermit implements AttemptPermit {
|
||||
|
||||
private final long generation;
|
||||
private final ProviderAttemptLimiter limiter;
|
||||
private final java.util.concurrent.atomic.AtomicBoolean released =
|
||||
new java.util.concurrent.atomic.AtomicBoolean();
|
||||
|
||||
private LimiterPermit(long generation, ProviderAttemptLimiter limiter) {
|
||||
this.generation = generation;
|
||||
this.limiter = limiter;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long generation() {
|
||||
return generation;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
limiter.release();
|
||||
if (released.compareAndSet(false, true)) {
|
||||
limiter.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+82
-20
@@ -2,24 +2,45 @@ package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
/** Holds the current generation of every provider profile plus the generations still draining. */
|
||||
public final class ProviderRuntimeRegistry {
|
||||
|
||||
private final Map<ProviderProfileId, ProviderRuntime> current = new ConcurrentHashMap<>();
|
||||
private final Map<ProviderProfileId, CopyOnWriteArrayList<ProviderRuntime>> draining =
|
||||
new ConcurrentHashMap<>();
|
||||
|
||||
/** Register the first generation of a profile. */
|
||||
/**
|
||||
* Draining generations, held as immutable lists.
|
||||
*
|
||||
* <p>The value was a {@code CopyOnWriteArrayList} mutated outside any lock, so adding a
|
||||
* generation and sweeping drained ones were separate operations on the same list. Replacing the
|
||||
* whole list inside {@code compute} makes "add this generation" and "forget the drained ones"
|
||||
* mutually exclusive, and a reader always sees a list that some single writer actually produced.
|
||||
*/
|
||||
private final Map<ProviderProfileId, List<ProviderRuntime>> draining = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* Register the first generation of a profile.
|
||||
*
|
||||
* <p>Registering twice was a silent overwrite: the displaced runtime kept whatever attempts it
|
||||
* had in flight, but nothing was draining it and nothing could reach it to observe them. Two
|
||||
* configuration sources claiming one profile is a wiring bug, so it is reported as one.
|
||||
*
|
||||
* @throws IllegalStateException if the profile already has a current generation
|
||||
*/
|
||||
public void register(ProviderRuntime runtime) {
|
||||
Objects.requireNonNull(runtime, "runtime");
|
||||
current.put(runtime.profile().profileId(), runtime);
|
||||
ProviderRuntime existing = current.putIfAbsent(runtime.profile().profileId(), runtime);
|
||||
if (existing != null) {
|
||||
throw new IllegalStateException(
|
||||
"provider runtime already registered for the profile; use replace to rotate");
|
||||
}
|
||||
}
|
||||
|
||||
/** Current generation, or a configuration failure when the profile is unknown. */
|
||||
@@ -41,23 +62,58 @@ public final class ProviderRuntimeRegistry {
|
||||
*
|
||||
* <p>New dispatches immediately use the new generation while the previous one finishes what it
|
||||
* already started, which is what makes a credential rotation invisible to callers.
|
||||
*
|
||||
* <p>The swap, the drain and the enrolment happen as one operation, and a generation that does
|
||||
* not supersede the current one is refused. Nothing serialises two rotations of the same profile,
|
||||
* so as three separate steps they could interleave into an older generation ending up current —
|
||||
* the registry would then be serving credentials a later rotation had already retired.
|
||||
*
|
||||
* @throws IllegalArgumentException if the replacement does not supersede the current generation
|
||||
*/
|
||||
public Optional<ProviderRuntime> replace(ProviderRuntime replacement) {
|
||||
Objects.requireNonNull(replacement, "replacement");
|
||||
ProviderProfileId profileId = replacement.profile().profileId();
|
||||
ProviderRuntime previous = current.put(profileId, replacement);
|
||||
if (previous != null) {
|
||||
previous.markDraining();
|
||||
draining.computeIfAbsent(profileId, key -> new CopyOnWriteArrayList<>()).add(previous);
|
||||
forgetIfDrained(profileId);
|
||||
}
|
||||
return Optional.ofNullable(previous);
|
||||
AtomicReference<ProviderRuntime> displaced = new AtomicReference<>();
|
||||
current.compute(
|
||||
profileId,
|
||||
(key, existing) -> {
|
||||
if (existing == null) {
|
||||
return replacement;
|
||||
}
|
||||
if (replacement.generation() <= existing.generation()) {
|
||||
throw new IllegalArgumentException(
|
||||
"replacement generation does not supersede the current one");
|
||||
}
|
||||
// Drain before enrolling: a runtime added to the draining list while it still admits
|
||||
// attempts can pass the "nothing in flight" sweep in the instant before the next one
|
||||
// starts.
|
||||
existing.markDraining();
|
||||
draining.compute(profileId, (id, generations) -> enrol(generations, existing));
|
||||
displaced.set(existing);
|
||||
return replacement;
|
||||
});
|
||||
return Optional.ofNullable(displaced.get());
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a generation to the draining set and sweep the ones that have finished.
|
||||
*
|
||||
* @param generations the current draining set, possibly null
|
||||
* @param enrolling the generation being retired
|
||||
* @return the new draining set, or null when nothing is left draining
|
||||
*/
|
||||
private static List<ProviderRuntime> enrol(
|
||||
List<ProviderRuntime> generations, ProviderRuntime enrolling) {
|
||||
List<ProviderRuntime> next =
|
||||
generations == null ? new ArrayList<>() : new ArrayList<>(generations);
|
||||
next.add(enrolling);
|
||||
return sweep(next);
|
||||
}
|
||||
|
||||
/** Generations that are draining and still have work in flight. */
|
||||
public List<ProviderRuntime> drainingGenerations(ProviderProfileId profileId) {
|
||||
forgetIfDrained(profileId);
|
||||
return List.copyOf(draining.getOrDefault(profileId, new CopyOnWriteArrayList<>()));
|
||||
return draining.getOrDefault(profileId, List.of());
|
||||
}
|
||||
|
||||
/** Health of the current generation. */
|
||||
@@ -66,13 +122,19 @@ public final class ProviderRuntimeRegistry {
|
||||
}
|
||||
|
||||
private void forgetIfDrained(ProviderProfileId profileId) {
|
||||
CopyOnWriteArrayList<ProviderRuntime> generations = draining.get(profileId);
|
||||
if (generations == null) {
|
||||
return;
|
||||
}
|
||||
// compute, not removeIf: sweeping under the same lock as replace is what stops a generation
|
||||
// enrolled mid-sweep from being dropped along with the ones that had genuinely finished.
|
||||
draining.computeIfPresent(profileId, (key, generations) -> sweep(new ArrayList<>(generations)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the generations that have finished, keeping the list immutable.
|
||||
*
|
||||
* @param generations a private copy the caller owns
|
||||
* @return the surviving generations, or null to remove the entry entirely
|
||||
*/
|
||||
private static List<ProviderRuntime> sweep(List<ProviderRuntime> generations) {
|
||||
generations.removeIf(runtime -> runtime.activeAttempts() == 0);
|
||||
if (generations.isEmpty()) {
|
||||
draining.remove(profileId);
|
||||
}
|
||||
return generations.isEmpty() ? null : List.copyOf(generations);
|
||||
}
|
||||
}
|
||||
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJob;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJobStorePort;
|
||||
import dev.caskeleton.application.notification.platform.provider.ReconciliationResult;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* Asks the provider what happened to attempts whose outcome is unknown.
|
||||
*
|
||||
* <p>Reconciliation only ever ran when a lease expired and recovery walked past an incomplete
|
||||
* attempt. An ambiguous submission whose worker exited cleanly — the common case, because a worker
|
||||
* that records AMBIGUOUS and then finishes its shift has not crashed — was never asked about again.
|
||||
* The delivery sat {@code RECONCILIATION_REQUIRED} indefinitely, which reads as a queue that
|
||||
* stopped rather than as an outcome nobody knows.
|
||||
*
|
||||
* <p>An unsupported provider is not retried in a loop. Without a status-query capability the answer
|
||||
* will not change, so the job is completed and the attempt stays visibly ambiguous for an operator
|
||||
* — a busy loop against a capability that does not exist is how a background worker burns a
|
||||
* connection pool while achieving nothing.
|
||||
*/
|
||||
public final class ReconciliationJobWorker {
|
||||
|
||||
private final ReconciliationJobStorePort jobs;
|
||||
private final Function<
|
||||
dev.caskeleton.application.notification.platform.api.DeliveryAttemptId,
|
||||
ReconciliationResult>
|
||||
reconciler;
|
||||
private final Clock clock;
|
||||
private final Duration retryBackoff;
|
||||
private final int batchSize;
|
||||
private final int maxAttempts;
|
||||
|
||||
/**
|
||||
* Creates the worker.
|
||||
*
|
||||
* @param jobs the outstanding questions
|
||||
* @param reconciler asks one attempt's provider
|
||||
* @param clock the clock
|
||||
* @param retryBackoff how long to wait before asking again
|
||||
* @param batchSize how many jobs one pass handles
|
||||
* @param maxAttempts how many times one job may ask before it is left to an operator
|
||||
*/
|
||||
public ReconciliationJobWorker(
|
||||
ReconciliationJobStorePort jobs,
|
||||
Function<
|
||||
dev.caskeleton.application.notification.platform.api.DeliveryAttemptId,
|
||||
ReconciliationResult>
|
||||
reconciler,
|
||||
Clock clock,
|
||||
Duration retryBackoff,
|
||||
int batchSize,
|
||||
int maxAttempts) {
|
||||
this.jobs = Objects.requireNonNull(jobs, "jobs");
|
||||
this.reconciler = Objects.requireNonNull(reconciler, "reconciler");
|
||||
this.clock = Objects.requireNonNull(clock, "clock");
|
||||
this.retryBackoff = Objects.requireNonNull(retryBackoff, "retryBackoff");
|
||||
if (retryBackoff.isNegative() || retryBackoff.isZero()) {
|
||||
throw new IllegalArgumentException("retryBackoff must be positive and finite");
|
||||
}
|
||||
if (batchSize < 1) {
|
||||
throw new IllegalArgumentException("batchSize");
|
||||
}
|
||||
if (maxAttempts < 1) {
|
||||
throw new IllegalArgumentException("maxAttempts");
|
||||
}
|
||||
this.batchSize = batchSize;
|
||||
this.maxAttempts = maxAttempts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one pass.
|
||||
*
|
||||
* @return how many jobs reached a terminal decision
|
||||
*/
|
||||
public int reconcileOnce() {
|
||||
List<ReconciliationJob> due = jobs.claimDue(batchSize, clock.instant());
|
||||
int settled = 0;
|
||||
for (ReconciliationJob job : due) {
|
||||
// One job's failure is not the pass's: a provider that is refusing connections would
|
||||
// otherwise stop every other provider's jobs behind it.
|
||||
try {
|
||||
if (handle(job)) {
|
||||
settled++;
|
||||
}
|
||||
} catch (RuntimeException failure) {
|
||||
jobs.reschedule(
|
||||
job, failure.getClass().getSimpleName(), clock.instant().plus(retryBackoff));
|
||||
}
|
||||
}
|
||||
return settled;
|
||||
}
|
||||
|
||||
private boolean handle(ReconciliationJob job) {
|
||||
if (job.attempts() >= maxAttempts) {
|
||||
// Asked enough times. Completing the job leaves the attempt ambiguous and visible rather
|
||||
// than asking forever; an outcome that has not arrived after this many tries is an
|
||||
// operator's decision, not a scheduler's.
|
||||
jobs.reschedule(job, "MAX_ATTEMPTS", clock.instant().plus(retryBackoff));
|
||||
jobs.complete(job);
|
||||
return true;
|
||||
}
|
||||
ReconciliationResult result = reconciler.apply(job.attemptId());
|
||||
return switch (result) {
|
||||
case ReconciliationResult.Confirmed ignored -> {
|
||||
jobs.complete(job);
|
||||
yield true;
|
||||
}
|
||||
case ReconciliationResult.Unsupported ignored -> {
|
||||
// The capability does not exist; asking again cannot change that.
|
||||
jobs.complete(job);
|
||||
yield true;
|
||||
}
|
||||
case ReconciliationResult.StillUnknown stillUnknown -> {
|
||||
jobs.reschedule(job, "STILL_UNKNOWN", stillUnknown.nextCheckAt());
|
||||
yield false;
|
||||
}
|
||||
case ReconciliationResult.Failed failed -> {
|
||||
if (failed.retryable()) {
|
||||
jobs.reschedule(job, failed.code(), clock.instant().plus(retryBackoff));
|
||||
yield false;
|
||||
}
|
||||
jobs.complete(job);
|
||||
yield true;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.admin.ProviderRuntimeControlPort;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* The registry, behind the two operations the admin plane actually needs.
|
||||
*
|
||||
* <p>What is left in the outbound adapter after NTF-020: a translation from an application request
|
||||
* to a runtime transition. No actor, no authority, no tenant scope, no transaction — those are
|
||||
* decisions about who may do what, and they now live where the rest of the platform's policy lives.
|
||||
*/
|
||||
public final class RegistryProviderRuntimeControl implements ProviderRuntimeControlPort {
|
||||
|
||||
private final ProviderRuntimeRegistry runtimes;
|
||||
|
||||
public RegistryProviderRuntimeControl(ProviderRuntimeRegistry runtimes) {
|
||||
this.runtimes = Objects.requireNonNull(runtimes, "runtimes");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean setState(
|
||||
ProviderProfileId profileId, ProviderRuntimeState desiredState, String reason) {
|
||||
Objects.requireNonNull(profileId, "profileId");
|
||||
Objects.requireNonNull(desiredState, "desiredState");
|
||||
Objects.requireNonNull(reason, "reason");
|
||||
ProviderRuntime runtime = runtimes.current(profileId);
|
||||
return switch (desiredState) {
|
||||
case DISABLED -> runtime.markDisabled();
|
||||
case DRAINING -> runtime.markDraining();
|
||||
case HEALTHY -> runtime.resumeHealthy();
|
||||
case DEGRADED -> runtime.markDegraded(reason);
|
||||
case THROTTLED -> runtime.markThrottled();
|
||||
case AUTHENTICATION_FAILED -> {
|
||||
runtime.markAuthenticationFailed(reason);
|
||||
yield true;
|
||||
}
|
||||
// Unreachable while the enum is exhaustive; present so a state added later fails loudly here
|
||||
// rather than silently leaving the runtime in whatever state it was already in.
|
||||
default -> throw new IllegalStateException("unhandled provider runtime state");
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProviderRuntimeState state(ProviderProfileId profileId) {
|
||||
return runtimes.state(Objects.requireNonNull(profileId, "profileId"));
|
||||
}
|
||||
}
|
||||
+13
-6
@@ -30,20 +30,27 @@ public final class LoggingNotificationMetrics implements NotificationMetricsPort
|
||||
|
||||
@Override
|
||||
public void increment(String metricName, Map<String, String> tags) {
|
||||
guard.validate(tags);
|
||||
log.info("metric={} kind=counter tags={}", metricName, ordered(tags));
|
||||
// bound, not validate: the keys were checked and the values never were, so one metric could
|
||||
// become one series per caller-supplied category.
|
||||
Map<String, String> bounded = guard.bound(tags);
|
||||
log.info("metric={} kind=counter tags={}", metricName, ordered(bounded));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void record(String metricName, Map<String, String> tags, Duration value) {
|
||||
guard.validate(tags);
|
||||
log.info("metric={} kind=timer millis={} tags={}", metricName, value.toMillis(), ordered(tags));
|
||||
// bound, not validate: the keys were checked and the values never were, so one metric could
|
||||
// become one series per caller-supplied category.
|
||||
Map<String, String> bounded = guard.bound(tags);
|
||||
log.info(
|
||||
"metric={} kind=timer millis={} tags={}", metricName, value.toMillis(), ordered(bounded));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void gauge(String metricName, Map<String, String> tags, double value) {
|
||||
guard.validate(tags);
|
||||
log.info("metric={} kind=gauge value={} tags={}", metricName, value, ordered(tags));
|
||||
// bound, not validate: the keys were checked and the values never were, so one metric could
|
||||
// become one series per caller-supplied category.
|
||||
Map<String, String> bounded = guard.bound(tags);
|
||||
log.info("metric={} kind=gauge value={} tags={}", metricName, value, ordered(bounded));
|
||||
}
|
||||
|
||||
private static Map<String, String> ordered(Map<String, String> tags) {
|
||||
|
||||
+54
-2
@@ -2,11 +2,16 @@ package dev.caskeleton.adapter.outbound.notification.platform.observation;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderRuntimeRegistry;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.observation.NotificationServingState;
|
||||
import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Builds the operational snapshot.
|
||||
@@ -14,16 +19,32 @@ import java.util.Objects;
|
||||
* <p>A provider whose credentials were rejected reports unhealthy even though the process is fine:
|
||||
* that is exactly the condition an operator needs paged on, and it is invisible from process-level
|
||||
* health.
|
||||
*
|
||||
* <p>It used to report provider states and an empty queue map, so the platform was "healthy"
|
||||
* whenever the runtimes were — with a backlog of any size, leases stuck on a dead worker, and
|
||||
* provider events piling up unapplied. Readiness now includes the numbers that describe whether
|
||||
* anything is actually being delivered, each measured against a declared threshold rather than
|
||||
* eyeballed by whoever is reading the endpoint.
|
||||
*/
|
||||
public final class NotificationHealthReporter {
|
||||
|
||||
private final ProviderRuntimeRegistry runtimes;
|
||||
private final List<ProviderProfileId> monitoredProfiles;
|
||||
private final NotificationServingStatePort servingState;
|
||||
private final Set<Channel> routedChannels;
|
||||
private final NotificationServingThresholds thresholds;
|
||||
|
||||
public NotificationHealthReporter(
|
||||
ProviderRuntimeRegistry runtimes, List<ProviderProfileId> monitoredProfiles) {
|
||||
ProviderRuntimeRegistry runtimes,
|
||||
List<ProviderProfileId> monitoredProfiles,
|
||||
NotificationServingStatePort servingState,
|
||||
Set<Channel> routedChannels,
|
||||
NotificationServingThresholds thresholds) {
|
||||
this.runtimes = Objects.requireNonNull(runtimes, "runtimes");
|
||||
this.monitoredProfiles = List.copyOf(Objects.requireNonNull(monitoredProfiles, "profiles"));
|
||||
this.servingState = Objects.requireNonNull(servingState, "servingState");
|
||||
this.routedChannels = Set.copyOf(Objects.requireNonNull(routedChannels, "routedChannels"));
|
||||
this.thresholds = Objects.requireNonNull(thresholds, "thresholds");
|
||||
}
|
||||
|
||||
/** Current snapshot. */
|
||||
@@ -52,6 +73,37 @@ public final class NotificationHealthReporter {
|
||||
runtime.get().activeAttempts()));
|
||||
}
|
||||
|
||||
return new NotificationHealthSnapshot(healthy, providers, Map.of());
|
||||
// A platform with providers but no route accepts every request and delivers none. It was
|
||||
// reported healthy because every runtime was healthy — which was true and beside the point.
|
||||
if (!monitoredProfiles.isEmpty() && routedChannels.isEmpty()) {
|
||||
healthy = false;
|
||||
}
|
||||
|
||||
NotificationServingState serving = servingState.currentState();
|
||||
if (thresholds.exceededBy(serving)) {
|
||||
healthy = false;
|
||||
}
|
||||
return new NotificationHealthSnapshot(healthy, providers, queue(serving));
|
||||
}
|
||||
|
||||
/**
|
||||
* The serving state as the endpoint's queue map.
|
||||
*
|
||||
* @param serving the measured state
|
||||
* @return counts and ages, all of them numbers with no identifiers in them
|
||||
*/
|
||||
private static Map<String, Long> queue(NotificationServingState serving) {
|
||||
Map<String, Long> values = new LinkedHashMap<>();
|
||||
values.put("backlogDepth", serving.backlogDepth());
|
||||
values.put("oldestDueAgeSeconds", serving.oldestDueAge().toSeconds());
|
||||
values.put("stuckLeases", serving.stuckLeases());
|
||||
values.put("pendingProjections", serving.pendingProjections());
|
||||
values.put("failedProjections", serving.failedProjections());
|
||||
values.put("unmatchedProjections", serving.unmatchedProjections());
|
||||
values.put(
|
||||
"oldestPendingProjectionAgeSeconds", serving.oldestPendingProjectionAge().toSeconds());
|
||||
values.put("reconciliationDue", serving.reconciliationDue());
|
||||
values.put("oldestReconciliationAgeSeconds", serving.oldestReconciliationAge().toSeconds());
|
||||
return Map.copyOf(values);
|
||||
}
|
||||
}
|
||||
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.observation;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.observation.NotificationServingState;
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* The point at which a backlog stops being normal operation.
|
||||
*
|
||||
* <p>Written down rather than judged by whoever reads the endpoint. A number with no threshold
|
||||
* beside it is a number nobody can act on at three in the morning: a backlog of 4,000 is either
|
||||
* routine or an incident depending on a deployment's throughput, and only the deployment knows
|
||||
* which.
|
||||
*
|
||||
* @param maxBacklogDepth deliveries due and unclaimed before readiness fails
|
||||
* @param maxOldestDueAge how long the oldest due delivery may wait
|
||||
* @param maxStuckLeases leases whose holder died, before readiness fails
|
||||
* @param maxPendingProjections provider events stored and unapplied
|
||||
* @param maxFailedProjections provider events whose projection failed
|
||||
* @param maxOldestPendingProjectionAge how long an unapplied event may wait
|
||||
* @param maxOldestReconciliationAge how long an unanswered provider question may wait
|
||||
*/
|
||||
public record NotificationServingThresholds(
|
||||
long maxBacklogDepth,
|
||||
Duration maxOldestDueAge,
|
||||
long maxStuckLeases,
|
||||
long maxPendingProjections,
|
||||
long maxFailedProjections,
|
||||
Duration maxOldestPendingProjectionAge,
|
||||
Duration maxOldestReconciliationAge) {
|
||||
|
||||
/**
|
||||
* Defaults chosen so that a healthy deployment never trips them and a stopped one always does.
|
||||
*
|
||||
* <p>The ages are the operative checks. A depth threshold has to be guessed from throughput; an
|
||||
* age does not — a delivery that has been due for ten minutes is behind whatever the throughput
|
||||
* is.
|
||||
*/
|
||||
public static final NotificationServingThresholds DEFAULT =
|
||||
new NotificationServingThresholds(
|
||||
100_000,
|
||||
Duration.ofMinutes(10),
|
||||
50,
|
||||
50_000,
|
||||
1_000,
|
||||
Duration.ofMinutes(15),
|
||||
Duration.ofMinutes(30));
|
||||
|
||||
public NotificationServingThresholds {
|
||||
Objects.requireNonNull(maxOldestDueAge, "maxOldestDueAge");
|
||||
Objects.requireNonNull(maxOldestPendingProjectionAge, "maxOldestPendingProjectionAge");
|
||||
Objects.requireNonNull(maxOldestReconciliationAge, "maxOldestReconciliationAge");
|
||||
if (maxBacklogDepth < 0
|
||||
|| maxStuckLeases < 0
|
||||
|| maxPendingProjections < 0
|
||||
|| maxFailedProjections < 0) {
|
||||
throw new IllegalArgumentException("thresholds must not be negative");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the measured state has passed any threshold.
|
||||
*
|
||||
* @param state the measured serving state
|
||||
* @return true when at least one threshold is exceeded
|
||||
*/
|
||||
public boolean exceededBy(NotificationServingState state) {
|
||||
Objects.requireNonNull(state, "state");
|
||||
return state.backlogDepth() > maxBacklogDepth
|
||||
|| state.oldestDueAge().compareTo(maxOldestDueAge) > 0
|
||||
|| state.stuckLeases() > maxStuckLeases
|
||||
|| state.pendingProjections() > maxPendingProjections
|
||||
|| state.failedProjections() > maxFailedProjections
|
||||
|| state.oldestPendingProjectionAge().compareTo(maxOldestPendingProjectionAge) > 0
|
||||
|| state.oldestReconciliationAge().compareTo(maxOldestReconciliationAge) > 0;
|
||||
}
|
||||
}
|
||||
+26
@@ -75,6 +75,11 @@ public final class ApnsRequestMapper {
|
||||
NotificationJsonMapper.mapper()
|
||||
.writeValueAsString(payload)
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
// Applied to the bytes that will actually be sent. The capability declared a 4096-byte ceiling
|
||||
// and nothing compared anything to it, so an oversized payload reached APNs and came back as a
|
||||
// rejection with a provider-specific reason — a round trip and a failed attempt to learn
|
||||
// something the sender already knew.
|
||||
requireWithinPayloadLimit(body.length);
|
||||
return new NotificationHttpRequest(
|
||||
"POST",
|
||||
URI.create(properties.endpoint() + "/3/device/" + token.value()),
|
||||
@@ -83,6 +88,27 @@ public final class ApnsRequestMapper {
|
||||
properties.timeout());
|
||||
}
|
||||
|
||||
/** The APNs payload ceiling, in bytes of the serialized JSON. */
|
||||
public static final int MAX_PAYLOAD_BYTES = 4096;
|
||||
|
||||
/**
|
||||
* Refuses a payload the provider will refuse.
|
||||
*
|
||||
* @param size the serialized payload size
|
||||
*/
|
||||
public static void requireWithinPayloadLimit(int size) {
|
||||
if (size > MAX_PAYLOAD_BYTES) {
|
||||
throw new dev.caskeleton.application.notification.platform.provider
|
||||
.ProviderCallNotStartedException(
|
||||
dev.caskeleton.application.notification.platform.api.error.NotificationFailureCode
|
||||
.PROVIDER_PAYLOAD_LIMIT,
|
||||
dev.caskeleton.application.notification.platform.api.error.FailureCategory
|
||||
.INVALID_PAYLOAD,
|
||||
false,
|
||||
"the rendered payload is " + size + " bytes and the APNs limit is " + MAX_PAYLOAD_BYTES);
|
||||
}
|
||||
}
|
||||
|
||||
/** Current time, exposed so expiry mapping stays testable. */
|
||||
public java.time.Instant now() {
|
||||
return clock.instant();
|
||||
|
||||
+10
-1
@@ -68,7 +68,16 @@ public final class FcmBatchCoordinator {
|
||||
protector.reveal(
|
||||
submission.contactPoint(),
|
||||
AccessContext.dispatch(submission.profile().profileId().value()));
|
||||
messages.add(messageMapper.map(submission, targetMapper.map(value)));
|
||||
Map<String, Object> message = messageMapper.map(submission, targetMapper.map(value));
|
||||
// The bytes that will be sent, measured before sending them. The 4096-byte ceiling was
|
||||
// declared by the capability model and compared to nothing.
|
||||
messageMapper.requireWithinPayloadLimit(
|
||||
dev.caskeleton.adapter.outbound.notification.platform.template.NotificationJsonMapper
|
||||
.mapper()
|
||||
.writeValueAsString(message)
|
||||
.getBytes(java.nio.charset.StandardCharsets.UTF_8)
|
||||
.length);
|
||||
messages.add(message);
|
||||
}
|
||||
|
||||
FcmBatchResult batch = gateway.sendBatch(messages);
|
||||
|
||||
+53
-5
@@ -55,18 +55,66 @@ public final class FcmMessageMapper {
|
||||
return Map.of("message", message);
|
||||
}
|
||||
|
||||
/** Effective TTL for a submission. */
|
||||
/**
|
||||
* Effective TTL for a submission, in three distinct cases.
|
||||
*
|
||||
* <p>The previous version filtered out a negative remaining duration and then fell through to
|
||||
* {@code orElse(maxTtl)} — so a notification that had <em>already expired</em> was sent with the
|
||||
* provider's <em>maximum</em> lifetime. The one input that means "do not deliver this" produced
|
||||
* the longest possible delivery window, and FCM would retry it for as long as the maximum
|
||||
* allowed.
|
||||
*
|
||||
* <ul>
|
||||
* <li>No expiry: the provider maximum, because the caller set no deadline.
|
||||
* <li>Expiry already passed: refused here, before the call. There is no TTL that expresses "too
|
||||
* late" to FCM, so the honest answer is not to send.
|
||||
* <li>Expiry ahead: the smaller of what remains and the provider maximum.
|
||||
* </ul>
|
||||
*/
|
||||
public Duration ttl(ProviderSubmission submission) {
|
||||
Optional<Duration> remaining =
|
||||
submission.expiresAt().map(expiry -> Duration.between(clock.instant(), expiry));
|
||||
return remaining
|
||||
.filter(value -> value.compareTo(properties.maxTtl()) < 0)
|
||||
.filter(value -> !value.isNegative())
|
||||
.orElse(properties.maxTtl());
|
||||
if (remaining.isEmpty()) {
|
||||
return properties.maxTtl();
|
||||
}
|
||||
Duration left = remaining.get();
|
||||
if (left.isNegative() || left.isZero()) {
|
||||
throw new dev.caskeleton.application.notification.platform.provider
|
||||
.ProviderCallNotStartedException(
|
||||
dev.caskeleton.application.notification.platform.api.error.NotificationFailureCode
|
||||
.VALIDATION_FAILED,
|
||||
dev.caskeleton.application.notification.platform.api.error.FailureCategory
|
||||
.INVALID_PAYLOAD,
|
||||
false,
|
||||
"the notification expired before it reached the provider");
|
||||
}
|
||||
return left.compareTo(properties.maxTtl()) < 0 ? left : properties.maxTtl();
|
||||
}
|
||||
|
||||
/** Payload ceiling enforced before the provider call. */
|
||||
public int maxPayloadBytes() {
|
||||
return MAX_PAYLOAD_BYTES;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a payload FCM will refuse.
|
||||
*
|
||||
* <p>Applied to the serialized bytes rather than to a field count. The ceiling was declared by
|
||||
* the capability and compared to nothing, so an oversized message reached FCM and returned a
|
||||
* provider-specific rejection — a round trip to learn what the sender could have known.
|
||||
*
|
||||
* @param size the serialized payload size
|
||||
*/
|
||||
public void requireWithinPayloadLimit(int size) {
|
||||
if (size > MAX_PAYLOAD_BYTES) {
|
||||
throw new dev.caskeleton.application.notification.platform.provider
|
||||
.ProviderCallNotStartedException(
|
||||
dev.caskeleton.application.notification.platform.api.error.NotificationFailureCode
|
||||
.PROVIDER_PAYLOAD_LIMIT,
|
||||
dev.caskeleton.application.notification.platform.api.error.FailureCategory
|
||||
.INVALID_PAYLOAD,
|
||||
false,
|
||||
"the rendered payload is " + size + " bytes and the FCM limit is " + MAX_PAYLOAD_BYTES);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+49
-15
@@ -57,8 +57,10 @@ public final class JdkNotificationHttpGateway implements NotificationHttpGateway
|
||||
});
|
||||
|
||||
try {
|
||||
HttpResponse<byte[]> response =
|
||||
client.send(builder.build(), HttpResponse.BodyHandlers.ofByteArray());
|
||||
// Bounded, not ofByteArray(). A provider response is diagnostic — a status, some headers, an
|
||||
// error document — and reading it without a cap makes the sender's heap a function of what
|
||||
// the far end chooses to send. A chunked response with no end is a single-request outage.
|
||||
HttpResponse<byte[]> response = client.send(builder.build(), boundedBody(MAX_RESPONSE_BYTES));
|
||||
return new NotificationHttpResponse(
|
||||
response.statusCode(), Map.copyOf(response.headers().map()), response.body());
|
||||
} catch (HttpTimeoutException timeout) {
|
||||
@@ -74,23 +76,55 @@ public final class JdkNotificationHttpGateway implements NotificationHttpGateway
|
||||
}
|
||||
|
||||
/**
|
||||
* A connect failure happens before anything is written; anything else may have written the body.
|
||||
* Whether the request body may have reached the provider.
|
||||
*
|
||||
* <p>The default is deliberately the pessimistic one: guessing "not committed" would turn an
|
||||
* unknown into an automatic resend.
|
||||
* <p>Decided from the exception's <em>type</em>, not from its message. The previous version
|
||||
* lower-cased {@code getMessage()} and looked for "connection refused", "unresolved", "no route
|
||||
* to host" and "connect timed out" — none of which is a contract. Those strings come from the
|
||||
* platform's C library and the JDK's own wording; they are localised on some platforms, they
|
||||
* changed between JDK releases, and a proxy that reports a refused connection in its own words
|
||||
* would be read as "the body was sent".
|
||||
*
|
||||
* <p>The JDK does expose the distinction as types. {@link ConnectException} and {@link
|
||||
* UnknownHostException} are raised while establishing the connection, so no request byte can have
|
||||
* been written. Everything else stays committed — the default is deliberately the pessimistic
|
||||
* one, because guessing "not committed" turns an unknown into an automatic resend.
|
||||
*/
|
||||
private static boolean bodyWasLikelyCommitted(IOException failure) {
|
||||
String message = failure.getMessage();
|
||||
if (message == null) {
|
||||
return true;
|
||||
// Depth-bounded rather than cycle-detecting: a cause chain can be circular (two exceptions
|
||||
// each initCause'd to the other), and an unbounded walk over one hangs the dispatch thread.
|
||||
// Ten is far deeper than any real transport wrapping.
|
||||
Throwable cause = failure;
|
||||
for (int depth = 0; cause != null && depth < 10; depth++, cause = cause.getCause()) {
|
||||
if (cause instanceof java.net.ConnectException
|
||||
|| cause instanceof java.net.UnknownHostException
|
||||
|| cause instanceof java.net.NoRouteToHostException) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
String normalized = message.toLowerCase(java.util.Locale.ROOT);
|
||||
boolean beforeSend =
|
||||
normalized.contains("connection refused")
|
||||
|| normalized.contains("unresolved")
|
||||
|| normalized.contains("no route to host")
|
||||
|| normalized.contains("connect timed out");
|
||||
return !beforeSend;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* The largest provider response body this gateway retains.
|
||||
*
|
||||
* <p>64 KiB: far more than any provider's acknowledgement or error document, and small enough
|
||||
* that a hostile or broken endpoint cannot make it interesting.
|
||||
*/
|
||||
public static final int MAX_RESPONSE_BYTES = 65_536;
|
||||
|
||||
/**
|
||||
* A body handler that stops reading at the cap.
|
||||
*
|
||||
* <p>Truncating rather than failing: the status code is the part that decides the outcome, and a
|
||||
* provider that accepted the message and then wrote a large body should not turn into an
|
||||
* ambiguous submission.
|
||||
*/
|
||||
private static HttpResponse.BodyHandler<byte[]> boundedBody(int maxBytes) {
|
||||
return responseInfo ->
|
||||
HttpResponse.BodySubscribers.mapping(
|
||||
HttpResponse.BodySubscribers.ofByteArray(),
|
||||
body -> body.length <= maxBytes ? body : java.util.Arrays.copyOf(body, maxBytes));
|
||||
}
|
||||
|
||||
/** Header map helper for adapters. */
|
||||
|
||||
+86
@@ -40,4 +40,90 @@ public final class NotificationEndpoints {
|
||||
String host = endpoint.getHost() == null ? "" : endpoint.getHost().toLowerCase(Locale.ROOT);
|
||||
return LOOPBACK_HOSTS.contains(host);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses an endpoint that resolves into the deployment's own network.
|
||||
*
|
||||
* <p>{@link #requireSecureOrLoopback} checks the scheme and nothing else, so any HTTPS URL was
|
||||
* accepted — including {@code https://169.254.169.254/}, the cloud metadata service, and any RFC
|
||||
* 1918 address. Web Push endpoints and webhook targets are supplied by clients, which makes this
|
||||
* a server-side request forgery primitive: the platform will happily fetch an internal address
|
||||
* and, for a webhook, deliver the message body there.
|
||||
*
|
||||
* <p>Resolution happens here rather than being left to the HTTP client because the check has to
|
||||
* see the addresses. A name that resolves to a public address in one lookup and a private one in
|
||||
* the next — DNS rebinding — is refused by checking every address the name currently returns; the
|
||||
* client re-resolves independently, so this narrows the window rather than closing it, and that
|
||||
* limitation is real rather than papered over.
|
||||
*
|
||||
* @param endpoint the endpoint to check
|
||||
* @param name what to call it in the failure
|
||||
* @param allowLoopback whether a loopback target is acceptable, for local and contract profiles
|
||||
* @return the endpoint
|
||||
* @throws IllegalArgumentException when the endpoint is not externally routable
|
||||
*/
|
||||
public static URI requireExternallyRoutable(URI endpoint, String name, boolean allowLoopback) {
|
||||
Objects.requireNonNull(endpoint, name);
|
||||
requireSecureOrLoopback(endpoint, name);
|
||||
if (endpoint.getUserInfo() != null) {
|
||||
// user:password@host is how a target is disguised: many parsers, and many humans reading a
|
||||
// log line, take the text before the '@' for the host.
|
||||
throw new IllegalArgumentException(name + " must not carry userinfo");
|
||||
}
|
||||
String host = endpoint.getHost();
|
||||
if (host == null || host.isBlank()) {
|
||||
throw new IllegalArgumentException(name + " has no host");
|
||||
}
|
||||
if (allowLoopback && isLoopback(endpoint)) {
|
||||
return endpoint;
|
||||
}
|
||||
|
||||
java.net.InetAddress[] resolved;
|
||||
try {
|
||||
resolved = java.net.InetAddress.getAllByName(host);
|
||||
} catch (java.net.UnknownHostException unresolvable) {
|
||||
throw new IllegalArgumentException(name + " does not resolve", unresolvable);
|
||||
}
|
||||
if (resolved.length == 0) {
|
||||
throw new IllegalArgumentException(name + " does not resolve");
|
||||
}
|
||||
for (java.net.InetAddress address : resolved) {
|
||||
// Every answer, not the first: a name that returns one public and one private address is the
|
||||
// ordinary shape of a rebinding attack, and taking the first answer would accept it half the
|
||||
// time.
|
||||
if (isInternal(address)) {
|
||||
throw new IllegalArgumentException(
|
||||
name + " resolves to an address inside the deployment's own network");
|
||||
}
|
||||
}
|
||||
return endpoint;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an address belongs to the deployment rather than to the internet.
|
||||
*
|
||||
* @param address the resolved address
|
||||
* @return true when the address must not be fetched
|
||||
*/
|
||||
public static boolean isInternal(java.net.InetAddress address) {
|
||||
Objects.requireNonNull(address, "address");
|
||||
if (address.isLoopbackAddress()
|
||||
|| address.isLinkLocalAddress()
|
||||
|| address.isSiteLocalAddress()
|
||||
|| address.isAnyLocalAddress()
|
||||
|| address.isMulticastAddress()) {
|
||||
return true;
|
||||
}
|
||||
byte[] octets = address.getAddress();
|
||||
if (octets.length == 4) {
|
||||
int first = octets[0] & 0xFF;
|
||||
int second = octets[1] & 0xFF;
|
||||
// 169.254.169.254 is link-local and already covered; 100.64/10 (carrier NAT) and 192.0.0/24
|
||||
// are not, and both routinely reach infrastructure the application should not talk to.
|
||||
return (first == 100 && second >= 64 && second <= 127)
|
||||
|| (first == 192 && second == 0 && (octets[2] & 0xFF) == 0);
|
||||
}
|
||||
// IPv6 unique local addresses: fc00::/7.
|
||||
return (octets[0] & 0xFE) == 0xFC;
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,8 +1,8 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.provider.ses;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.template.NotificationJsonMapper;
|
||||
import dev.caskeleton.application.notification.platform.api.CallbackRequest;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderId;
|
||||
import dev.caskeleton.application.notification.platform.callback.CallbackRequest;
|
||||
import dev.caskeleton.application.notification.platform.callback.CallbackVerificationResult;
|
||||
import dev.caskeleton.application.notification.platform.callback.NormalizedProviderEvent;
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderCallbackAdapter;
|
||||
|
||||
+38
-3
@@ -46,10 +46,16 @@ public final class SnsSignatureVerifier {
|
||||
Objects.requireNonNull(envelope, "envelope");
|
||||
String certificateUrl = envelope.get("SigningCertURL");
|
||||
String signature = envelope.get("Signature");
|
||||
String version = envelope.getOrDefault("SignatureVersion", "1");
|
||||
String version = envelope.get("SignatureVersion");
|
||||
if (certificateUrl == null || signature == null) {
|
||||
return false;
|
||||
}
|
||||
// Exactly the versions this verifier implements. The default was "1", so an envelope with the
|
||||
// field missing — or set to anything unrecognised — silently downgraded to SHA-1, and an
|
||||
// attacker chooses that field.
|
||||
if (!"1".equals(version) && !"2".equals(version)) {
|
||||
return false;
|
||||
}
|
||||
if (!isTrustedCertificateUrl(certificateUrl)) {
|
||||
return false;
|
||||
}
|
||||
@@ -66,12 +72,41 @@ public final class SnsSignatureVerifier {
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the certificate URL is on an Amazon host over TLS. */
|
||||
/**
|
||||
* Whether the certificate URL is one this verifier will fetch.
|
||||
*
|
||||
* <p>A suffix match is not a host check. {@code evilamazonaws.com} ends with {@code
|
||||
* amazonaws.com}, so the previous version would fetch a signing certificate from an
|
||||
* attacker-owned domain and then verify the envelope against it — which makes the whole signature
|
||||
* check decorative. The suffix must begin at a label boundary, and the rest of the URL has to
|
||||
* look like what SNS actually publishes.
|
||||
*/
|
||||
public boolean isTrustedCertificateUrl(String certificateUrl) {
|
||||
try {
|
||||
URI uri = URI.create(certificateUrl);
|
||||
if (!"https".equalsIgnoreCase(uri.getScheme())) {
|
||||
return false;
|
||||
}
|
||||
if (uri.getUserInfo() != null || uri.getQuery() != null || uri.getFragment() != null) {
|
||||
// None of these appear in an SNS certificate URL, and each is a way to make one URL read as
|
||||
// another to a human or to a lenient parser.
|
||||
return false;
|
||||
}
|
||||
if (uri.getPort() != -1 && uri.getPort() != 443) {
|
||||
return false;
|
||||
}
|
||||
String host = uri.getHost() == null ? "" : uri.getHost().toLowerCase(Locale.ROOT);
|
||||
return "https".equalsIgnoreCase(uri.getScheme()) && host.endsWith(certificateHostSuffix);
|
||||
// At a label boundary, or the suffix itself. "evilamazonaws.com".endsWith("amazonaws.com")
|
||||
// is true; "evil.amazonaws.com" is the only shape that should pass.
|
||||
boolean onTheSuffix =
|
||||
host.equals(certificateHostSuffix) || host.endsWith("." + certificateHostSuffix);
|
||||
if (!onTheSuffix) {
|
||||
return false;
|
||||
}
|
||||
String path = uri.getPath() == null ? "" : uri.getPath();
|
||||
// SNS publishes its certificates under /SimpleNotificationService-<id>.pem. Constraining the
|
||||
// path stops the same host being used to serve an attacker-chosen document.
|
||||
return path.startsWith("/SimpleNotificationService-") && path.endsWith(".pem");
|
||||
} catch (IllegalArgumentException malformed) {
|
||||
return false;
|
||||
}
|
||||
|
||||
+59
-2
@@ -1,6 +1,7 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.provider.smtp;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderId;
|
||||
import dev.caskeleton.application.notification.platform.api.content.EmailContent;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.contact.ContactPointValue;
|
||||
import dev.caskeleton.application.notification.platform.contact.EmailAddress;
|
||||
@@ -8,6 +9,7 @@ import dev.caskeleton.application.notification.platform.provider.NotificationPro
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderCapabilities;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmission;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmissionResult;
|
||||
import dev.caskeleton.application.notification.platform.provider.ResolvedAttachment;
|
||||
import dev.caskeleton.application.notification.platform.security.AccessContext;
|
||||
import dev.caskeleton.application.notification.platform.security.ContactPointProtector;
|
||||
import java.time.Duration;
|
||||
@@ -36,6 +38,8 @@ public final class SmtpNotificationProviderAdapter implements NotificationProvid
|
||||
private final ContactPointProtector protector;
|
||||
private final SmtpProviderProperties properties;
|
||||
private final Executor executor;
|
||||
private final dev.caskeleton.application.notification.platform.dispatch.AttachmentIntegrityGuard
|
||||
attachmentGuard;
|
||||
|
||||
public SmtpNotificationProviderAdapter(
|
||||
SmtpDispatch dispatch,
|
||||
@@ -43,13 +47,16 @@ public final class SmtpNotificationProviderAdapter implements NotificationProvid
|
||||
SmtpFailureClassifier classifier,
|
||||
ContactPointProtector protector,
|
||||
SmtpProviderProperties properties,
|
||||
Executor executor) {
|
||||
Executor executor,
|
||||
dev.caskeleton.application.notification.platform.dispatch.AttachmentIntegrityGuard
|
||||
attachmentGuard) {
|
||||
this.dispatch = Objects.requireNonNull(dispatch, "dispatch");
|
||||
this.mimeFactory = Objects.requireNonNull(mimeFactory, "mimeFactory");
|
||||
this.classifier = Objects.requireNonNull(classifier, "classifier");
|
||||
this.protector = Objects.requireNonNull(protector, "protector");
|
||||
this.properties = Objects.requireNonNull(properties, "properties");
|
||||
this.executor = Objects.requireNonNull(executor, "executor");
|
||||
this.attachmentGuard = Objects.requireNonNull(attachmentGuard, "attachmentGuard");
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -86,13 +93,63 @@ public final class SmtpNotificationProviderAdapter implements NotificationProvid
|
||||
throw new IllegalArgumentException("SMTP requires an email contact point");
|
||||
}
|
||||
|
||||
// Resolved, verified and closed around the send. The factory was handed List.of() whatever the
|
||||
// content asked for, so an email with attachments went out without them — the caller was told
|
||||
// it was accepted, and the recipient received a message missing the thing it was about.
|
||||
List<ResolvedAttachment> opened = resolve(submission);
|
||||
try {
|
||||
dispatch.send(
|
||||
mimeFactory.create(
|
||||
submission, address.normalized(), properties.senderIdentity(), List.of()));
|
||||
submission, address.normalized(), properties.senderIdentity(), opened));
|
||||
return ProviderSubmissionResult.accepted(null, "250", elapsedSince(startedNanos));
|
||||
} catch (SmtpDispatchException failure) {
|
||||
return classifier.classify(failure, elapsedSince(startedNanos));
|
||||
} finally {
|
||||
// Closed on every path. A resolver hands back an open stream, and a failed send is exactly
|
||||
// when a leaked one goes unnoticed.
|
||||
opened.forEach(SmtpNotificationProviderAdapter::closeQuietly);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves and verifies every attachment the content declares.
|
||||
*
|
||||
* <p>The integrity guard runs before the provider call, not after: a digest or size that does not
|
||||
* match what the caller declared means the bytes are not the bytes that were approved, and
|
||||
* discovering that after the mail has left is discovering it too late.
|
||||
*/
|
||||
private List<ResolvedAttachment> resolve(ProviderSubmission submission) {
|
||||
if (!(submission.content().content() instanceof EmailContent email)
|
||||
|| email.attachments().isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<ResolvedAttachment> resolved = new java.util.ArrayList<>(email.attachments().size());
|
||||
try {
|
||||
for (var reference : email.attachments()) {
|
||||
// The guard resolves and verifies size and digest in one step, so an attachment whose
|
||||
// bytes are not the approved bytes never reaches the MIME factory.
|
||||
resolved.add(
|
||||
attachmentGuard.resolve(
|
||||
reference,
|
||||
new dev.caskeleton.application.notification.platform.provider
|
||||
.AttachmentAccessContext(
|
||||
new dev.caskeleton.application.notification.platform.api.TenantId(
|
||||
submission.profile().environment()),
|
||||
submission.attemptId())));
|
||||
}
|
||||
return List.copyOf(resolved);
|
||||
} catch (RuntimeException failure) {
|
||||
// Everything already opened is closed before the failure propagates.
|
||||
resolved.forEach(SmtpNotificationProviderAdapter::closeQuietly);
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
|
||||
private static void closeQuietly(ResolvedAttachment attachment) {
|
||||
try {
|
||||
attachment.close();
|
||||
} catch (Exception ignored) {
|
||||
// A stream that will not close is not a reason to change the send's outcome.
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.provider.twilio;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.CallbackRequest;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderId;
|
||||
import dev.caskeleton.application.notification.platform.callback.CallbackRequest;
|
||||
import dev.caskeleton.application.notification.platform.callback.CallbackVerificationResult;
|
||||
import dev.caskeleton.application.notification.platform.callback.NormalizedProviderEvent;
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderCallbackAdapter;
|
||||
|
||||
+29
-4
@@ -94,16 +94,41 @@ public final class WebhookNotificationProviderAdapter implements NotificationPro
|
||||
return CompletableFuture.completedFuture(send(submission));
|
||||
}
|
||||
|
||||
/**
|
||||
* The webhook envelope version.
|
||||
*
|
||||
* <p>Present so a receiver can distinguish an added field from a changed meaning. The previous
|
||||
* body had no version and no content, so there was nothing to version.
|
||||
*/
|
||||
public static final int WEBHOOK_SCHEMA_VERSION = 1;
|
||||
|
||||
private ProviderSubmissionResult send(ProviderSubmission submission) {
|
||||
long startedNanos = System.nanoTime();
|
||||
WebhookSubscription subscription = subscriptionResolver.apply(submission);
|
||||
|
||||
// The rendered notification, not just its digest. The body carried an attempt id and a content
|
||||
// hash and nothing else, so a receiver got a webhook that said a notification had happened and
|
||||
// could not tell what it said — the one thing a webhook exists to deliver.
|
||||
var rendered = submission.content().content();
|
||||
Map<String, Object> envelope = new LinkedHashMap<>();
|
||||
// A schema version, so a receiver can tell an added field from a changed meaning.
|
||||
envelope.put("schemaVersion", WEBHOOK_SCHEMA_VERSION);
|
||||
envelope.put("attemptId", submission.attemptId().value().toString());
|
||||
envelope.put("channel", submission.channel().name());
|
||||
envelope.put("contentDigest", submission.content().contentDigest());
|
||||
submission.expiresAt().ifPresent(expiry -> envelope.put("expiresAt", expiry.toString()));
|
||||
submission.providerIdempotencyKey().ifPresent(key -> envelope.put("idempotencyKey", key));
|
||||
if (rendered
|
||||
instanceof
|
||||
dev.caskeleton.application.notification.platform.api.content.InAppContent content) {
|
||||
envelope.put("title", content.title());
|
||||
envelope.put("body", content.body());
|
||||
content.deepLink().ifPresent(link -> envelope.put("deepLink", link.toString()));
|
||||
envelope.put("category", content.category());
|
||||
}
|
||||
byte[] body =
|
||||
NotificationJsonMapper.mapper()
|
||||
.writeValueAsString(
|
||||
Map.of(
|
||||
"attemptId", submission.attemptId().value().toString(),
|
||||
"contentDigest", submission.content().contentDigest()))
|
||||
.writeValueAsString(envelope)
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
Map<String, String> headers = new LinkedHashMap<>();
|
||||
|
||||
+10
-7
@@ -12,8 +12,6 @@ import dev.caskeleton.application.notification.platform.api.error.ProviderConfig
|
||||
import dev.caskeleton.application.notification.platform.api.error.ProviderPayloadLimitException;
|
||||
import dev.caskeleton.application.notification.platform.contact.WebPushSubscriptionValue;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmission;
|
||||
import dev.caskeleton.application.notification.platform.security.SecretMaterialProvider;
|
||||
import dev.caskeleton.application.notification.platform.security.SecretPurpose;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
@@ -32,19 +30,19 @@ public final class WebPushRequestMapper {
|
||||
|
||||
private final Rfc8291Aes128GcmEncryptor encryptor;
|
||||
private final VapidAuthorizationProvider signer;
|
||||
private final SecretMaterialProvider secrets;
|
||||
private final VapidKeyRegistry keys;
|
||||
private final WebPushProviderProperties properties;
|
||||
private final Clock clock;
|
||||
|
||||
public WebPushRequestMapper(
|
||||
Rfc8291Aes128GcmEncryptor encryptor,
|
||||
VapidAuthorizationProvider signer,
|
||||
SecretMaterialProvider secrets,
|
||||
VapidKeyRegistry keys,
|
||||
WebPushProviderProperties properties,
|
||||
Clock clock) {
|
||||
this.encryptor = Objects.requireNonNull(encryptor, "encryptor");
|
||||
this.signer = Objects.requireNonNull(signer, "signer");
|
||||
this.secrets = Objects.requireNonNull(secrets, "secrets");
|
||||
this.keys = Objects.requireNonNull(keys, "keys");
|
||||
this.properties = Objects.requireNonNull(properties, "properties");
|
||||
this.clock = Objects.requireNonNull(clock, "clock");
|
||||
}
|
||||
@@ -104,8 +102,13 @@ public final class WebPushRequestMapper {
|
||||
"authorization",
|
||||
signer.authorization(
|
||||
subscription.endpoint(),
|
||||
secrets.activeKey(SecretPurpose.VAPID_SIGNING),
|
||||
properties.vapidPublicKeyBase64Url()));
|
||||
// The key this subscription was created against, not whichever key is active now. A
|
||||
// browser stores the application server key at subscribe time and rejects a push
|
||||
// signed by any other; after a VAPID rotation, every pre-rotation subscription would
|
||||
// have been signed with the new key and silently refused. VapidKeyRegistry already
|
||||
// resolved the historical key per subscription — nothing called it.
|
||||
keys.signingKeyFor(subscription),
|
||||
keys.publicKeyFor(subscription)));
|
||||
|
||||
return new NotificationHttpRequest(
|
||||
"POST",
|
||||
|
||||
+34
-2
@@ -46,12 +46,41 @@ public final class AesGcmCallbackPayloadProtection implements CallbackPayloadPro
|
||||
SecretMaterialProvider secrets, SecureRandom random, int maxRetainedBytes) {
|
||||
this.secrets = Objects.requireNonNull(secrets, "secrets");
|
||||
this.random = Objects.requireNonNull(random, "random");
|
||||
this.maxRetainedBytes = maxRetainedBytes;
|
||||
if (maxRetainedBytes < 1) {
|
||||
throw new IllegalArgumentException("maxRetainedBytes");
|
||||
}
|
||||
if (maxRetainedBytes > MAX_PLAINTEXT_BYTES) {
|
||||
// The database check constrains the *ciphertext*, and encryption adds a 12-byte nonce and a
|
||||
// 16-byte GCM tag. Truncating the plaintext to the ciphertext bound produced a value 28 bytes
|
||||
// over it, so a callback of exactly the configured maximum was accepted by every layer above
|
||||
// and then rejected by a CHECK constraint after the provider had been told it was stored.
|
||||
throw new IllegalArgumentException(
|
||||
"callback retention of "
|
||||
+ maxRetainedBytes
|
||||
+ " plaintext bytes cannot be stored: the ciphertext column holds "
|
||||
+ MAX_CIPHERTEXT_BYTES
|
||||
+ " bytes and encryption adds "
|
||||
+ ENVELOPE_OVERHEAD_BYTES
|
||||
+ ", so the plaintext ceiling is "
|
||||
+ MAX_PLAINTEXT_BYTES);
|
||||
}
|
||||
this.maxRetainedBytes = maxRetainedBytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* The largest ciphertext the {@code notification_provider_event} check constraint accepts.
|
||||
*
|
||||
* <p>Named here because this class is what has to fit inside it. The constraint was written
|
||||
* against the plaintext bound, and nothing reconciled the two.
|
||||
*/
|
||||
public static final int MAX_CIPHERTEXT_BYTES = 65_536;
|
||||
|
||||
/** The nonce and GCM tag every encryption adds. */
|
||||
public static final int ENVELOPE_OVERHEAD_BYTES = NONCE_BYTES + TAG_BITS / 8;
|
||||
|
||||
/** The largest plaintext that still fits the column once encrypted. */
|
||||
public static final int MAX_PLAINTEXT_BYTES = MAX_CIPHERTEXT_BYTES - ENVELOPE_OVERHEAD_BYTES;
|
||||
|
||||
@Override
|
||||
public byte[] protectRawPayload(byte[] rawBody) {
|
||||
Objects.requireNonNull(rawBody, "rawBody");
|
||||
@@ -109,7 +138,10 @@ public final class AesGcmCallbackPayloadProtection implements CallbackPayloadPro
|
||||
Mac mac = Mac.getInstance("HmacSHA256");
|
||||
mac.init(
|
||||
new SecretKeySpec(
|
||||
secrets.activeKey(SecretPurpose.CONTACT_LOOKUP_HMAC).material(), "HmacSHA256"));
|
||||
// A fingerprint key and a contact-lookup key protect different things and must not
|
||||
// fall
|
||||
// together.
|
||||
secrets.activeKey(SecretPurpose.CALLBACK_FINGERPRINT_HMAC).material(), "HmacSHA256"));
|
||||
return HexFormat.of().formatHex(mac.doFinal(seed.getBytes(StandardCharsets.UTF_8)));
|
||||
} catch (GeneralSecurityException failure) {
|
||||
throw new IllegalStateException("callback fingerprinting failed", failure);
|
||||
|
||||
+35
-19
@@ -69,7 +69,9 @@ public final class AesGcmContactPointProtector implements ContactPointProtector
|
||||
|
||||
byte[] nonce = new byte[NONCE_BYTES];
|
||||
random.nextBytes(nonce);
|
||||
byte[] plaintext = value.normalized().getBytes(StandardCharsets.UTF_8);
|
||||
// The complete form is what gets encrypted; the identity form is what gets fingerprinted. They
|
||||
// used to be the same string, so every field outside the identity was simply not stored.
|
||||
byte[] plaintext = value.serialized().getBytes(StandardCharsets.UTF_8);
|
||||
byte[] ciphertext = encrypt(encryption, nonce, associatedData(value.type()), plaintext);
|
||||
|
||||
return new ProtectedContactPoint(
|
||||
@@ -166,15 +168,23 @@ public final class AesGcmContactPointProtector implements ContactPointProtector
|
||||
}
|
||||
}
|
||||
|
||||
private static ContactPointValue parse(ContactPointType type, String normalized) {
|
||||
/**
|
||||
* Reads back what {@link ContactPointValue#serialized()} wrote.
|
||||
*
|
||||
* <p>The version prefix is stripped here rather than in every subtype: the default serialized
|
||||
* form is {@code "v1:" + normalized()}, and a stored row written before this change has neither
|
||||
* prefix nor the fields it introduced. Accepting both is what keeps existing rows readable.
|
||||
*/
|
||||
private static ContactPointValue parse(ContactPointType type, String stored) {
|
||||
String body = stored.startsWith("v1:") ? stored.substring("v1:".length()) : stored;
|
||||
return switch (type) {
|
||||
case EMAIL -> EmailAddress.parse(normalized);
|
||||
case PHONE -> new PhoneNumber(normalized);
|
||||
case FCM_FID -> new FcmInstallationId(normalized);
|
||||
case FCM_REGISTRATION_TOKEN_LEGACY -> new LegacyFcmRegistrationToken(normalized);
|
||||
case APNS_DEVICE_TOKEN -> parseApns(normalized);
|
||||
case WEB_PUSH_SUBSCRIPTION -> parseWebPush(normalized);
|
||||
case IN_APP_RECIPIENT -> new InAppRecipientRef(normalized);
|
||||
case EMAIL -> EmailAddress.parse(body);
|
||||
case PHONE -> new PhoneNumber(body);
|
||||
case FCM_FID -> new FcmInstallationId(body);
|
||||
case FCM_REGISTRATION_TOKEN_LEGACY -> new LegacyFcmRegistrationToken(body);
|
||||
case APNS_DEVICE_TOKEN -> parseApns(body);
|
||||
case WEB_PUSH_SUBSCRIPTION -> parseWebPush(stored);
|
||||
case IN_APP_RECIPIENT -> new InAppRecipientRef(body);
|
||||
};
|
||||
}
|
||||
|
||||
@@ -188,17 +198,23 @@ public final class AesGcmContactPointProtector implements ContactPointProtector
|
||||
ApnsEnvironment.valueOf(normalized.substring(0, separator)));
|
||||
}
|
||||
|
||||
private static WebPushSubscriptionValue parseWebPush(String normalized) {
|
||||
int separator = normalized.indexOf('|');
|
||||
if (separator <= 0) {
|
||||
throw new IllegalStateException("stored Web Push subscription is malformed");
|
||||
/**
|
||||
* Reads a stored Web Push subscription back with every field it was saved with.
|
||||
*
|
||||
* <p>This method used to return sixteen zero bytes for the auth secret and the literal string
|
||||
* {@code "restored"} for the VAPID key id, on the stated grounds that both lived in their own
|
||||
* encrypted columns. Those columns do not exist — not in the migration, not on the entity. So a
|
||||
* subscription came back from storage unable to produce an RFC 8291 payload its browser could
|
||||
* decrypt, and unable to say which VAPID key had signed for it.
|
||||
*/
|
||||
private static WebPushSubscriptionValue parseWebPush(String stored) {
|
||||
String[] parts = stored.split("\\|", -1);
|
||||
if (parts.length != 5 || !"v1".equals(parts[0])) {
|
||||
throw new IllegalStateException(
|
||||
"stored Web Push subscription is malformed or predates the versioned envelope");
|
||||
}
|
||||
// The auth secret and VAPID key id are stored in their own encrypted columns; the normalized
|
||||
// form only has to round-trip the equality-relevant parts.
|
||||
Base64.Decoder decoder = Base64.getUrlDecoder();
|
||||
return new WebPushSubscriptionValue(
|
||||
URI.create(normalized.substring(0, separator)),
|
||||
Base64.getUrlDecoder().decode(normalized.substring(separator + 1)),
|
||||
new byte[16],
|
||||
"restored");
|
||||
URI.create(parts[1]), decoder.decode(parts[2]), decoder.decode(parts[3]), parts[4]);
|
||||
}
|
||||
}
|
||||
|
||||
+7
-1
@@ -36,7 +36,13 @@ public final class HmacProviderRequestIdHasher implements ProviderRequestIdHashe
|
||||
Mac mac = Mac.getInstance("HmacSHA256");
|
||||
mac.init(
|
||||
new SecretKeySpec(
|
||||
secrets.activeKey(SecretPurpose.CONTACT_LOOKUP_HMAC).material(), "HmacSHA256"));
|
||||
// Its own purpose. This shared the contact-lookup key, so one compromised key forged
|
||||
// both
|
||||
// the contact index and the provider-request index — purpose separation exists to
|
||||
// stop
|
||||
// exactly that.
|
||||
secrets.activeKey(SecretPurpose.PROVIDER_REQUEST_LOOKUP_HMAC).material(),
|
||||
"HmacSHA256"));
|
||||
mac.update((profileId.value() + ":").getBytes(StandardCharsets.UTF_8));
|
||||
return HexFormat.of()
|
||||
.formatHex(mac.doFinal(providerRequestId.getBytes(StandardCharsets.UTF_8)));
|
||||
|
||||
+24
-9
@@ -5,6 +5,7 @@ import dev.caskeleton.application.notification.platform.security.SecretKeyMateri
|
||||
import dev.caskeleton.application.notification.platform.security.SecretMaterialProvider;
|
||||
import dev.caskeleton.application.notification.platform.security.SecretPurpose;
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
@@ -37,19 +38,33 @@ public final class ProviderCredentialManager {
|
||||
this.clock = Objects.requireNonNull(clock, "clock");
|
||||
}
|
||||
|
||||
/** Record the generation a profile starts on. */
|
||||
/**
|
||||
* Record the generation a profile starts on.
|
||||
*
|
||||
* <p>The supersede check and the store are one operation. They were a {@code get}, a check and a
|
||||
* {@code put}: two rotations racing both read the same predecessor, both concluded they
|
||||
* superseded it, and whichever wrote last won — so generation 2 could land after generation 3 and
|
||||
* the profile would run on credentials that had already been retired. "Strictly increasing" only
|
||||
* means anything if nothing can intervene between reading the current value and replacing it.
|
||||
*
|
||||
* @throws IllegalArgumentException if the generation does not supersede whatever is active at the
|
||||
* moment it is stored
|
||||
*/
|
||||
public CredentialGeneration activate(CredentialGeneration generation) {
|
||||
Objects.requireNonNull(generation, "generation");
|
||||
CredentialGeneration existing = current.get(generation.profileId());
|
||||
if (existing != null && !generation.supersedes(existing)) {
|
||||
throw new IllegalArgumentException("generation does not supersede the active one");
|
||||
}
|
||||
// Fetch once at activation so a key id that does not resolve fails the rotation instead of
|
||||
// failing the first notification that happens to use the profile.
|
||||
// failing the first notification that happens to use the profile. Resolution reads an external
|
||||
// provider, so it stays outside the map's compute lock; it is a read and repeating it is safe.
|
||||
requireResolvable(generation);
|
||||
CredentialGeneration activated = generation.activatedAt(clock.instant());
|
||||
current.put(activated.profileId(), activated);
|
||||
return activated;
|
||||
Instant activatedAt = clock.instant();
|
||||
return current.compute(
|
||||
generation.profileId(),
|
||||
(profileId, existing) -> {
|
||||
if (existing != null && !generation.supersedes(existing)) {
|
||||
throw new IllegalArgumentException("generation does not supersede the active one");
|
||||
}
|
||||
return generation.activatedAt(activatedAt);
|
||||
});
|
||||
}
|
||||
|
||||
/** Current generation of a profile. */
|
||||
|
||||
+21
-2
@@ -122,12 +122,31 @@ public final class CanonicalNotificationRenderer implements NotificationTemplate
|
||||
|
||||
private String slot(
|
||||
NotificationTemplateVersion template, TemplateSlot slot, Map<String, Object> variables) {
|
||||
return engine.render(template.content().requireSlot(slot), variables);
|
||||
return engine.render(modeOf(slot), template.content().requireSlot(slot), variables);
|
||||
}
|
||||
|
||||
private Optional<String> optionalSlot(
|
||||
NotificationTemplateVersion template, TemplateSlot slot, Map<String, Object> variables) {
|
||||
return template.content().slot(slot).map(source -> engine.render(source, variables));
|
||||
return template
|
||||
.content()
|
||||
.slot(slot)
|
||||
.map(source -> engine.render(modeOf(slot), source, variables));
|
||||
}
|
||||
|
||||
/**
|
||||
* How a slot's substituted values have to be escaped.
|
||||
*
|
||||
* <p>Every slot used to render through one raw-substitution path, so a caller's value became
|
||||
* active markup in an HTML body and could split a header in a subject. Escaping belongs to the
|
||||
* destination, and this is where the destination is known.
|
||||
*/
|
||||
private static TemplateSlotMode modeOf(TemplateSlot slot) {
|
||||
return switch (slot) {
|
||||
case SUBJECT -> TemplateSlotMode.SUBJECT;
|
||||
case HTML_BODY -> TemplateSlotMode.HTML_TEXT;
|
||||
case DEEP_LINK -> TemplateSlotMode.URI;
|
||||
case TEXT_BODY, TITLE, BODY, CATEGORY -> TemplateSlotMode.TEXT;
|
||||
};
|
||||
}
|
||||
|
||||
private static String canonicalForm(NotificationContent content) {
|
||||
|
||||
+15
@@ -20,4 +20,19 @@ public interface NotificationTemplateEngine {
|
||||
* when a referenced variable is absent — never rendered as an empty string
|
||||
*/
|
||||
String render(String source, Map<String, Object> variables);
|
||||
|
||||
/**
|
||||
* Render one slot, escaping for what the slot is.
|
||||
*
|
||||
* <p>The mode is required rather than inferred: the same template text is safe in a text part and
|
||||
* dangerous in an HTML one, and only the caller knows which it is filling.
|
||||
*
|
||||
* @param mode what the rendered value will become
|
||||
* @param source the template text
|
||||
* @param variables the values to substitute
|
||||
* @return the rendered slot
|
||||
*/
|
||||
default String render(TemplateSlotMode mode, String source, Map<String, Object> variables) {
|
||||
return render(source, variables);
|
||||
}
|
||||
}
|
||||
|
||||
+94
-2
@@ -23,9 +23,22 @@ public final class PlaceholderTemplateEngine implements NotificationTemplateEngi
|
||||
|
||||
private static final Pattern PLACEHOLDER = Pattern.compile("\\{([a-zA-Z0-9_.-]{1,64})\\}");
|
||||
|
||||
/** Render one slot. */
|
||||
/** Render one slot as plain text, which is what the single-argument contract means. */
|
||||
@Override
|
||||
public String render(String source, Map<String, Object> variables) {
|
||||
return render(TemplateSlotMode.TEXT, source, variables);
|
||||
}
|
||||
|
||||
/**
|
||||
* Render one slot, escaping each substituted value for the slot it lands in.
|
||||
*
|
||||
* <p>The template text itself is trusted — an operator published it — and the substituted values
|
||||
* are not. So escaping is applied to the value, never to the surrounding template, which is what
|
||||
* lets an HTML template keep its markup while a caller's {@code <script>} becomes text.
|
||||
*/
|
||||
@Override
|
||||
public String render(TemplateSlotMode mode, String source, Map<String, Object> variables) {
|
||||
Objects.requireNonNull(mode, "mode");
|
||||
Objects.requireNonNull(source, "source");
|
||||
Objects.requireNonNull(variables, "variables");
|
||||
|
||||
@@ -39,9 +52,88 @@ public final class PlaceholderTemplateEngine implements NotificationTemplateEngi
|
||||
NotificationFailureCode.TEMPLATE_RENDERING_FAILED,
|
||||
FailureCategory.TEMPLATE_FAILURE));
|
||||
}
|
||||
matcher.appendReplacement(rendered, Matcher.quoteReplacement(String.valueOf(value)));
|
||||
matcher.appendReplacement(
|
||||
rendered, Matcher.quoteReplacement(escape(mode, String.valueOf(value))));
|
||||
}
|
||||
matcher.appendTail(rendered);
|
||||
return rendered.toString();
|
||||
}
|
||||
|
||||
/** Escapes one substituted value for its destination. */
|
||||
private static String escape(TemplateSlotMode mode, String value) {
|
||||
return switch (mode) {
|
||||
case TEXT -> value;
|
||||
case SUBJECT -> requireSingleLine(value);
|
||||
case HTML_TEXT -> escapeHtml(value);
|
||||
case URI -> requireAllowedScheme(value);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a value that would split a header.
|
||||
*
|
||||
* <p>A carriage return or newline in a subject is header injection: everything after it is read
|
||||
* as a new header by the receiving agent.
|
||||
*/
|
||||
private static String requireSingleLine(String value) {
|
||||
for (int index = 0; index < value.length(); index++) {
|
||||
if (value.charAt(index) < 0x20) {
|
||||
throw new TemplateRenderingException(
|
||||
NotificationFailureDescriptor.preDispatch(
|
||||
NotificationFailureCode.TEMPLATE_RENDERING_FAILED,
|
||||
FailureCategory.TEMPLATE_FAILURE));
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Escapes for HTML text and attribute content.
|
||||
*
|
||||
* <p>Quotes included, because a value substituted inside an attribute can otherwise close it and
|
||||
* start an event handler — {@code " onerror="} needs no angle bracket at all.
|
||||
*/
|
||||
private static String escapeHtml(String value) {
|
||||
StringBuilder escaped = new StringBuilder(value.length() + 16);
|
||||
for (int index = 0; index < value.length(); index++) {
|
||||
char character = value.charAt(index);
|
||||
switch (character) {
|
||||
case '&' -> escaped.append("&");
|
||||
case '<' -> escaped.append("<");
|
||||
case '>' -> escaped.append(">");
|
||||
case '"' -> escaped.append(""");
|
||||
case '\'' -> escaped.append("'");
|
||||
default -> escaped.append(character);
|
||||
}
|
||||
}
|
||||
return escaped.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Allows only schemes a notification may legitimately link to.
|
||||
*
|
||||
* <p>{@code javascript:} in a link is script execution; {@code data:} is an arbitrary document
|
||||
* the platform vouches for; {@code file:} points at the reader's own machine. The slot used to be
|
||||
* parsed as a URI and otherwise accepted, and parsing succeeds for all three.
|
||||
*/
|
||||
private static String requireAllowedScheme(String value) {
|
||||
String normalized = value.trim().toLowerCase(java.util.Locale.ROOT);
|
||||
boolean allowed =
|
||||
ALLOWED_URI_SCHEMES.stream().anyMatch(scheme -> normalized.startsWith(scheme + ":"));
|
||||
if (!allowed) {
|
||||
throw new TemplateRenderingException(
|
||||
NotificationFailureDescriptor.preDispatch(
|
||||
NotificationFailureCode.TEMPLATE_RENDERING_FAILED, FailureCategory.TEMPLATE_FAILURE));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* The schemes a rendered link may use.
|
||||
*
|
||||
* <p>HTTPS, and the application's own deep-link scheme. Plain HTTP is absent deliberately: a link
|
||||
* in a notification is followed by a person who has no way to check it.
|
||||
*/
|
||||
private static final java.util.Set<String> ALLOWED_URI_SCHEMES =
|
||||
java.util.Set.of("https", "caskeleton");
|
||||
}
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.template;
|
||||
|
||||
/**
|
||||
* What a rendered slot is, and therefore how a caller's value has to be escaped.
|
||||
*
|
||||
* <p>One engine rendered every slot by raw substitution: subject lines, plain text, HTML bodies,
|
||||
* SMS, and deep links all went through the same code path. In an HTML body that means a caller's
|
||||
* value becomes active markup — the value {@code <img src=x onerror=...>} is a script the platform
|
||||
* put in its own email. In a subject line it means a caller's newline splits the header. And the
|
||||
* Thymeleaf alternative had the mirror-image fault: rendering plain text in HTML mode turns {@code
|
||||
* a & b} into an escaped entity in an SMS, where there is no markup to decode it.
|
||||
*
|
||||
* <p>Escaping is a property of the destination, not of the value, so the destination has to be
|
||||
* named.
|
||||
*/
|
||||
public enum TemplateSlotMode {
|
||||
|
||||
/** An email subject or equivalent single-line header. Control characters are refused. */
|
||||
SUBJECT,
|
||||
|
||||
/** Plain text: SMS, a text email part, a push body. Nothing is escaped, nothing is decoded. */
|
||||
TEXT,
|
||||
|
||||
/** An HTML body. Values are escaped so a caller cannot contribute markup. */
|
||||
HTML_TEXT,
|
||||
|
||||
/** A link. Only an allowlisted scheme survives. */
|
||||
URI
|
||||
}
|
||||
+15
-1
@@ -97,6 +97,7 @@ class NotificationPlatformSettingsTest {
|
||||
NotificationPlatformSettings.Provider provider) {
|
||||
return new NotificationPlatformSettings(
|
||||
true,
|
||||
NotificationPlatformMode.SERVING,
|
||||
NotificationPlatformSettings.Dispatch.defaults(),
|
||||
NotificationPlatformSettings.Callbacks.defaults(),
|
||||
Map.of("profile", provider));
|
||||
@@ -106,6 +107,7 @@ class NotificationPlatformSettingsTest {
|
||||
return new NotificationPlatformSettings.Provider(
|
||||
"WEB_PUSH",
|
||||
true,
|
||||
false,
|
||||
"PRODUCTION",
|
||||
"webpush-main",
|
||||
null,
|
||||
@@ -118,13 +120,24 @@ class NotificationPlatformSettingsTest {
|
||||
|
||||
private static NotificationPlatformSettings.Provider apnsWithoutTopic() {
|
||||
return new NotificationPlatformSettings.Provider(
|
||||
"APNS", true, "PRODUCTION", "apns-main", null, null, null, Duration.ofSeconds(3), 8, 20);
|
||||
"APNS",
|
||||
true,
|
||||
false,
|
||||
"PRODUCTION",
|
||||
"apns-main",
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
Duration.ofSeconds(3),
|
||||
8,
|
||||
20);
|
||||
}
|
||||
|
||||
private static NotificationPlatformSettings.Provider twilioWithoutSigningSecret() {
|
||||
return new NotificationPlatformSettings.Provider(
|
||||
"TWILIO",
|
||||
true,
|
||||
false,
|
||||
"PRODUCTION",
|
||||
"twilio-main",
|
||||
null,
|
||||
@@ -140,6 +153,7 @@ class NotificationPlatformSettingsTest {
|
||||
return new NotificationPlatformSettings.Provider(
|
||||
provider.type(),
|
||||
false,
|
||||
provider.primaryForChannel(),
|
||||
provider.environment(),
|
||||
provider.credentialProfile(),
|
||||
provider.topic(),
|
||||
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderAttemptLimiter;
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderRuntime;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderId;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.provider.NotificationProviderAdapter;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderCapabilities;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderProfileSnapshot;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmission;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmissionResult;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletionStage;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Whether configuring a provider produces one.
|
||||
*
|
||||
* <p>It did not. The registry was constructed empty, the route planner with {@code Map.of()} and
|
||||
* the reconciliation gateway with {@code Map.of()}, so a fully configured, fully validated provider
|
||||
* profile assembled into nothing. A request reached durable acceptance and then found no eligible
|
||||
* route — which reads to an operator as the platform silently dropping notifications, and reads in
|
||||
* the code as three collaborators that were never connected.
|
||||
*/
|
||||
class NotificationProviderAssemblyTest {
|
||||
|
||||
private static final ProviderCapabilities NO_CAPABILITIES =
|
||||
new ProviderCapabilities(
|
||||
false, false, false, false, false, false, false, false, 1, 1_024L, Duration.ofMinutes(1));
|
||||
|
||||
@Test
|
||||
@DisplayName("a configured profile becomes a runtime and a route")
|
||||
void aConfiguredProfileBecomesARuntimeAndARoute() {
|
||||
NotificationProviderAssembly.AssembledPlatform platform =
|
||||
assemble(
|
||||
Map.of("email-main", profile("SES", true, false)), fakeAssembler(ProviderType.SES));
|
||||
|
||||
assertThat(platform.serving()).isTrue();
|
||||
assertThat(platform.routeFor(Channel.EMAIL)).hasValue(new ProviderProfileId("email-main"));
|
||||
assertThat(platform.runtimes().find(new ProviderProfileId("email-main")))
|
||||
.as("the registry used to be constructed empty however many profiles were configured")
|
||||
.isPresent();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an unknown provider type fails the boot instead of binding to nothing")
|
||||
void anUnknownProviderTypeFailsTheBoot() {
|
||||
// Binding is where it has to fail: the profile record itself is just data, and the settings
|
||||
// constructor is what the boot invokes.
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
assemble(
|
||||
Map.of("mystery", profile("SENDGRID", true, false)),
|
||||
fakeAssembler(ProviderType.SES)))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("unknown provider type")
|
||||
.hasMessageContaining("mystery");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a family with no assembler is refused by name")
|
||||
void aFamilyWithNoAssemblerIsRefused() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
assemble(
|
||||
Map.of("push-main", profile("FCM", true, false)),
|
||||
fakeAssembler(ProviderType.SES)))
|
||||
.as("FCM's transport is a seam; assembling it into a provider that cannot send is worse")
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("push-main")
|
||||
.hasMessageContaining("no assembler");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("two profiles on one channel need one of them marked primary")
|
||||
void twoProfilesOnOneChannelNeedAPrimary() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
assemble(
|
||||
Map.of(
|
||||
"email-a", profile("SES", true, false),
|
||||
"email-b", profile("SES", true, false)),
|
||||
fakeAssembler(ProviderType.SES)))
|
||||
.as("which provider sends an email must not depend on map iteration order")
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("primary");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a marked primary resolves the ambiguity")
|
||||
void aMarkedPrimaryResolvesTheAmbiguity() {
|
||||
// "z-email" sorts after "email-b", so a route chosen by iteration order would pick the wrong
|
||||
// one and the assertion below is what makes the primary mean something.
|
||||
NotificationProviderAssembly.AssembledPlatform platform =
|
||||
assemble(
|
||||
Map.of(
|
||||
"email-b", profile("SES", true, false),
|
||||
"z-email", profile("SES", true, true)),
|
||||
fakeAssembler(ProviderType.SES));
|
||||
|
||||
assertThat(platform.routeFor(Channel.EMAIL)).hasValue(new ProviderProfileId("z-email"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("two profiles both marked primary for one channel are refused")
|
||||
void twoPrimariesAreRefused() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
assemble(
|
||||
Map.of(
|
||||
"email-a", profile("SES", true, true),
|
||||
"email-b", profile("SES", true, true)),
|
||||
fakeAssembler(ProviderType.SES)))
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("both marked primary");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a disabled profile assembles nothing")
|
||||
void aDisabledProfileAssemblesNothing() {
|
||||
NotificationProviderAssembly.AssembledPlatform platform =
|
||||
assemble(
|
||||
Map.of("email-main", profile("SES", false, false)),
|
||||
NotificationPlatformMode.INGEST_ONLY,
|
||||
fakeAssembler(ProviderType.SES));
|
||||
|
||||
assertThat(platform.routes()).isEmpty();
|
||||
assertThat(platform.serving()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an enabled platform with no provider refuses to start")
|
||||
void anEnabledPlatformWithNoProviderRefusesToStart() {
|
||||
assertThatThrownBy(() -> assemble(Map.of(), fakeAssembler(ProviderType.SES)))
|
||||
.as("every request would be accepted durably and then find no route")
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("INGEST_ONLY");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("INGEST_ONLY is a declaration, and it reports non-serving")
|
||||
void ingestOnlyIsADeclaration() {
|
||||
NotificationProviderAssembly.AssembledPlatform platform =
|
||||
assemble(Map.of(), NotificationPlatformMode.INGEST_ONLY, fakeAssembler(ProviderType.SES));
|
||||
|
||||
assertThat(platform.mode()).isEqualTo(NotificationPlatformMode.INGEST_ONLY);
|
||||
assertThat(platform.serving()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("two assemblers for one family are refused at construction")
|
||||
void twoAssemblersForOneFamilyAreRefused() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
new NotificationProviderAssembly(
|
||||
List.of(fakeAssembler(ProviderType.SES), fakeAssembler(ProviderType.SES))))
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("bean ordering");
|
||||
}
|
||||
|
||||
private static NotificationProviderAssembly.AssembledPlatform assemble(
|
||||
Map<String, NotificationPlatformSettings.Provider> providers,
|
||||
ProviderRuntimeAssembler... assemblers) {
|
||||
return assemble(providers, NotificationPlatformMode.SERVING, assemblers);
|
||||
}
|
||||
|
||||
private static NotificationProviderAssembly.AssembledPlatform assemble(
|
||||
Map<String, NotificationPlatformSettings.Provider> providers,
|
||||
NotificationPlatformMode mode,
|
||||
ProviderRuntimeAssembler... assemblers) {
|
||||
NotificationPlatformSettings settings =
|
||||
new NotificationPlatformSettings(
|
||||
true,
|
||||
mode,
|
||||
NotificationPlatformSettings.Dispatch.defaults(),
|
||||
NotificationPlatformSettings.Callbacks.defaults(),
|
||||
providers);
|
||||
return new NotificationProviderAssembly(List.of(assemblers)).assemble(settings, mode);
|
||||
}
|
||||
|
||||
private static NotificationPlatformSettings.Provider profile(
|
||||
String type, boolean enabled, boolean primary) {
|
||||
return new NotificationPlatformSettings.Provider(
|
||||
type,
|
||||
enabled,
|
||||
primary,
|
||||
"PRODUCTION",
|
||||
"credential-main",
|
||||
"topic",
|
||||
"vapid-public-key",
|
||||
"callback-secret",
|
||||
Duration.ofSeconds(3),
|
||||
8,
|
||||
20);
|
||||
}
|
||||
|
||||
/**
|
||||
* An adapter that is never asked to submit.
|
||||
*
|
||||
* <p>These tests are about whether configuration reaches the runtime at all. A real transport
|
||||
* would make them slower and would not make them stronger — the property under test is the
|
||||
* wiring, and the wiring was entirely absent.
|
||||
*/
|
||||
private record UnusedAdapter(ProviderType type) implements NotificationProviderAdapter {
|
||||
|
||||
@Override
|
||||
public ProviderId providerId() {
|
||||
return new ProviderId(type.name().toLowerCase(java.util.Locale.ROOT));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Channel> channels() {
|
||||
return Set.of(type.channel());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProviderCapabilities capabilities() {
|
||||
return NO_CAPABILITIES;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletionStage<ProviderSubmissionResult> submit(ProviderSubmission submission) {
|
||||
throw new UnsupportedOperationException("the assembly contract never submits");
|
||||
}
|
||||
}
|
||||
|
||||
/** An assembler that produces a runtime without a transport, so the wiring is what is tested. */
|
||||
private static ProviderRuntimeAssembler fakeAssembler(ProviderType type) {
|
||||
return new ProviderRuntimeAssembler() {
|
||||
|
||||
@Override
|
||||
public ProviderType type() {
|
||||
return type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AssembledProvider assemble(
|
||||
String profileId, NotificationPlatformSettings.Provider profile) {
|
||||
ProviderProfileSnapshot snapshot =
|
||||
new ProviderProfileSnapshot(
|
||||
new ProviderProfileId(profileId),
|
||||
new ProviderId(type.name().toLowerCase(java.util.Locale.ROOT)),
|
||||
type.channel(),
|
||||
profile.environment(),
|
||||
1L,
|
||||
NO_CAPABILITIES,
|
||||
Map.of());
|
||||
return AssembledProvider.dispatchOnly(
|
||||
new ProviderRuntime(
|
||||
snapshot,
|
||||
new UnusedAdapter(type),
|
||||
new ProviderAttemptLimiter(
|
||||
profile.maxConcurrency(), profile.ratePerSecond(), Clock.systemUTC())),
|
||||
type.channel());
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
+305
@@ -0,0 +1,305 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId;
|
||||
import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId;
|
||||
import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptRecord;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryRecord;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.RecipientLease;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* What recovery does with a delivery a dead worker left behind.
|
||||
*
|
||||
* <p>It iterated the delivery's attempts and reconciled the incomplete ones. A delivery whose
|
||||
* worker died <em>before</em> writing the attempt row has no attempts, so the loop did nothing and
|
||||
* the row stayed {@code DISPATCHING} forever — the one case where nothing can have reached the
|
||||
* provider, and the one case recovery could not handle.
|
||||
*/
|
||||
class LeaseRecoveryServiceTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-15T09:00:00Z");
|
||||
private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC);
|
||||
|
||||
private final RecordingLeases leases = new RecordingLeases();
|
||||
private final RecordingAttempts attempts = new RecordingAttempts();
|
||||
private final RecordingDeliveries deliveries = new RecordingDeliveries();
|
||||
private final List<DeliveryAttemptId> reconciled = new ArrayList<>();
|
||||
|
||||
private final LeaseRecoveryService recovery =
|
||||
new LeaseRecoveryService(
|
||||
leases, attempts, deliveries, reconciled::add, CLOCK, Duration.ofMinutes(1), 10);
|
||||
|
||||
@Test
|
||||
@DisplayName("a delivery with no attempt is requeued, because nothing can have been sent")
|
||||
void aDeliveryWithNoAttemptIsRequeued() {
|
||||
RecipientDeliveryId id = new RecipientDeliveryId(UUID.randomUUID());
|
||||
leases.abandoned.add(id);
|
||||
|
||||
assertThat(recovery.recoverOnce()).isEqualTo(1);
|
||||
|
||||
assertThat(deliveries.transitions)
|
||||
.as("the worker died between claiming and recording an attempt: no provider call happened")
|
||||
.containsExactly(Map.entry(id, RecipientDeliveryState.READY_TO_DISPATCH));
|
||||
assertThat(reconciled).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a delivery with an incomplete attempt is reconciled, never requeued")
|
||||
void aDeliveryWithAnIncompleteAttemptIsReconciled() {
|
||||
RecipientDeliveryId id = new RecipientDeliveryId(UUID.randomUUID());
|
||||
DeliveryAttemptId attemptId = new DeliveryAttemptId(UUID.randomUUID());
|
||||
leases.abandoned.add(id);
|
||||
attempts.byDelivery.put(id, List.of(incompleteAttempt(attemptId)));
|
||||
|
||||
assertThat(recovery.recoverOnce()).isEqualTo(1);
|
||||
|
||||
assertThat(reconciled)
|
||||
.as("a provider call may have happened; re-dispatching would choose to duplicate")
|
||||
.containsExactly(attemptId);
|
||||
assertThat(deliveries.transitions).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an attempt proven never to have started is requeued rather than reconciled")
|
||||
void anAttemptProvenNeverStartedIsRequeued() {
|
||||
RecipientDeliveryId id = new RecipientDeliveryId(UUID.randomUUID());
|
||||
leases.abandoned.add(id);
|
||||
attempts.byDelivery.put(
|
||||
id, List.of(attemptWith(new DeliveryAttemptId(UUID.randomUUID()), notStarted())));
|
||||
|
||||
assertThat(recovery.recoverOnce()).isEqualTo(1);
|
||||
|
||||
assertThat(reconciled)
|
||||
.as("the row says, with certainty, that no request began; a provider has nothing to add")
|
||||
.isEmpty();
|
||||
assertThat(deliveries.transitions)
|
||||
.containsExactly(Map.entry(id, RecipientDeliveryState.READY_TO_DISPATCH));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an attempt that may have started is reconciled, never requeued")
|
||||
void anAttemptThatMayHaveStartedIsReconciled() {
|
||||
RecipientDeliveryId id = new RecipientDeliveryId(UUID.randomUUID());
|
||||
DeliveryAttemptId attemptId = new DeliveryAttemptId(UUID.randomUUID());
|
||||
leases.abandoned.add(id);
|
||||
attempts.byDelivery.put(id, List.of(attemptWith(attemptId, unknownWhetherStarted())));
|
||||
|
||||
assertThat(recovery.recoverOnce()).isEqualTo(1);
|
||||
|
||||
assertThat(reconciled)
|
||||
.as("'we do not know whether it started' must never become a requeue")
|
||||
.containsExactly(attemptId);
|
||||
assertThat(deliveries.transitions).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the decision depends on certainty, not on the value alone")
|
||||
void theDecisionDependsOnCertaintyNotValueAlone() {
|
||||
// Both evidences carry requestStarted().value() == false. Under the old three-boolean row they
|
||||
// were the same stored fact, so a restart could not have made this distinction at all.
|
||||
assertThat(notStarted().requestStarted().value())
|
||||
.isEqualTo(unknownWhetherStarted().requestStarted().value());
|
||||
assertThat(notStarted().requestStarted().certainty())
|
||||
.isNotEqualTo(unknownWhetherStarted().requestStarted().certainty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("recovery also retires deliveries whose window closed")
|
||||
void recoveryAlsoRetiresExpiredDeliveries() {
|
||||
leases.expired = 3;
|
||||
|
||||
assertThat(recovery.recoverOnce()).isEqualTo(3);
|
||||
}
|
||||
|
||||
private static dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence
|
||||
notStarted() {
|
||||
return dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence
|
||||
.notStarted();
|
||||
}
|
||||
|
||||
private static dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence
|
||||
unknownWhetherStarted() {
|
||||
return new dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence(
|
||||
dev.caskeleton.application.notification.platform.provider.EvidenceFact.UNKNOWN,
|
||||
dev.caskeleton.application.notification.platform.provider.EvidenceFact.UNKNOWN,
|
||||
dev.caskeleton.application.notification.platform.provider.EvidenceFact.UNKNOWN,
|
||||
dev.caskeleton.application.notification.platform.provider.EvidenceFact.UNKNOWN);
|
||||
}
|
||||
|
||||
private static DeliveryAttemptRecord attemptWith(
|
||||
DeliveryAttemptId id,
|
||||
dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence
|
||||
evidence) {
|
||||
DeliveryAttemptRecord base = incompleteAttempt(id);
|
||||
return new DeliveryAttemptRecord(
|
||||
base.id(),
|
||||
base.recipientDeliveryId(),
|
||||
base.contactPointId(),
|
||||
base.attemptNo(),
|
||||
base.channel(),
|
||||
base.providerId(),
|
||||
base.providerProfileId(),
|
||||
base.providerRequestId(),
|
||||
evidence,
|
||||
base.submissionOutcome(),
|
||||
base.deliveryOutcome(),
|
||||
base.confirmation(),
|
||||
base.evidenceLevel(),
|
||||
base.failureCategory(),
|
||||
base.failureCode(),
|
||||
base.nativeStatus(),
|
||||
base.providerOccurredAt(),
|
||||
base.startedAt(),
|
||||
base.completedAt(),
|
||||
base.elapsed(),
|
||||
base.credentialGeneration(),
|
||||
base.renderedContentDigest());
|
||||
}
|
||||
|
||||
/** An attempt row that exists with no completion: the provider may or may not have seen it. */
|
||||
private static DeliveryAttemptRecord incompleteAttempt(DeliveryAttemptId id) {
|
||||
return new DeliveryAttemptRecord(
|
||||
id,
|
||||
new RecipientDeliveryId(UUID.randomUUID()),
|
||||
new dev.caskeleton.application.notification.platform.api.ContactPointId(UUID.randomUUID()),
|
||||
1,
|
||||
dev.caskeleton.application.notification.platform.api.routing.Channel.EMAIL,
|
||||
new dev.caskeleton.application.notification.platform.api.ProviderId("ses"),
|
||||
new dev.caskeleton.application.notification.platform.api.ProviderProfileId("email-main"),
|
||||
Optional.empty(),
|
||||
// Started, and then nothing: whether the body reached the provider is genuinely unknown,
|
||||
// which is the state this recovery test exists to describe.
|
||||
new dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence(
|
||||
dev.caskeleton.application.notification.platform.provider.EvidenceFact.proven(),
|
||||
dev.caskeleton.application.notification.platform.provider.EvidenceFact.UNKNOWN,
|
||||
dev.caskeleton.application.notification.platform.provider.EvidenceFact.UNKNOWN,
|
||||
dev.caskeleton.application.notification.platform.provider.EvidenceFact.UNKNOWN),
|
||||
dev.caskeleton.application.notification.platform.api.delivery.SubmissionOutcome
|
||||
.NOT_SUBMITTED,
|
||||
dev.caskeleton.application.notification.platform.api.delivery.DeliveryOutcome.UNKNOWN,
|
||||
dev.caskeleton.application.notification.platform.api.delivery.AttemptConfirmation.AMBIGUOUS,
|
||||
dev.caskeleton.application.notification.platform.api.delivery.EvidenceLevel.PLATFORM_QUEUED,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
NOW,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
1L,
|
||||
"0".repeat(64));
|
||||
}
|
||||
|
||||
private static final class RecordingLeases implements RecipientLeaseStorePort {
|
||||
|
||||
private final List<RecipientDeliveryId> abandoned = new ArrayList<>();
|
||||
private int expired;
|
||||
|
||||
@Override
|
||||
public List<RecipientLease> claim(String workerId, int limit, Duration leaseDuration) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<RecipientLease> renew(RecipientLease lease, Duration leaseDuration) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void release(RecipientLease lease) {
|
||||
// Not part of this contract.
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean stillHeld(RecipientLease lease) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int expireOverdue(int limit) {
|
||||
return expired;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<RecipientDeliveryId> expiredDispatching(int limit, Duration olderThan) {
|
||||
return List.copyOf(abandoned);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class RecordingAttempts implements DeliveryAttemptStorePort {
|
||||
|
||||
private final Map<RecipientDeliveryId, List<DeliveryAttemptRecord>> byDelivery =
|
||||
new java.util.HashMap<>();
|
||||
|
||||
@Override
|
||||
public DeliveryAttemptRecord insertDispatching(DeliveryAttemptRecord attempt) {
|
||||
return attempt;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DeliveryAttemptRecord recordOutcome(DeliveryAttemptRecord attempt) {
|
||||
return attempt;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<DeliveryAttemptRecord> find(DeliveryAttemptId id) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int nextAttemptNo(RecipientDeliveryId recipientDeliveryId) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DeliveryAttemptRecord> attemptsOf(RecipientDeliveryId recipientDeliveryId) {
|
||||
return byDelivery.getOrDefault(recipientDeliveryId, List.of());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<
|
||||
dev.caskeleton.application.notification.platform.callback.DeliveryAttemptSnapshot>
|
||||
snapshot(DeliveryAttemptId id) {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
private static final class RecordingDeliveries implements RecipientDeliveryStorePort {
|
||||
|
||||
private final List<Map.Entry<RecipientDeliveryId, RecipientDeliveryState>> transitions =
|
||||
new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public Optional<RecipientDeliveryRecord> find(RecipientDeliveryId id) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecipientDeliveryRecord save(RecipientDeliveryRecord record) {
|
||||
return record;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecipientDeliveryRecord transition(
|
||||
RecipientDeliveryId id, RecipientDeliveryState state, Optional<Instant> nextDispatchAt) {
|
||||
transitions.add(Map.entry(id, state));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.error.ProviderUnavailableException;
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Whether the configured rate is the rate.
|
||||
*
|
||||
* <p>The window and its count were two atomics. A thread that noticed a new second won the CAS on
|
||||
* the window start and then reset the count as a second step, so every increment other threads made
|
||||
* in between was thrown away — and the threads that made them had already been admitted. The
|
||||
* limiter therefore leaked exactly at a second boundary, which under steady load is every second.
|
||||
*
|
||||
* <p>The refund had the same shape. A rate slot granted in one window but rejected by the
|
||||
* concurrency semaphore was decremented unconditionally, so if the window had rolled the refund
|
||||
* credited the new window with an attempt it never admitted.
|
||||
*/
|
||||
class ProviderAttemptLimiterConcurrencyTest {
|
||||
|
||||
private static final Instant START = Instant.parse("2026-08-15T09:00:00Z");
|
||||
|
||||
/** A clock the test advances by whole seconds. */
|
||||
private static final class SteppingClock extends Clock {
|
||||
|
||||
private final AtomicLong epochSecond = new AtomicLong(START.getEpochSecond());
|
||||
|
||||
@Override
|
||||
public Instant instant() {
|
||||
return Instant.ofEpochSecond(epochSecond.get());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ZoneId getZone() {
|
||||
return ZoneOffset.UTC;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Clock withZone(ZoneId zone) {
|
||||
return this;
|
||||
}
|
||||
|
||||
void advanceOneSecond() {
|
||||
epochSecond.incrementAndGet();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("concurrent acquires inside one second admit exactly the configured rate")
|
||||
void oneSecondAdmitsExactlyTheRate() throws InterruptedException {
|
||||
SteppingClock clock = new SteppingClock();
|
||||
ProviderAttemptLimiter limiter = new ProviderAttemptLimiter(1_000, 100, clock);
|
||||
|
||||
int admitted = race(128, 1, limiter);
|
||||
|
||||
assertThat(admitted).isEqualTo(100);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the window after a rollover admits the rate, not the rate twice")
|
||||
void theWindowAfterARolloverAdmitsTheRateOnce() throws InterruptedException {
|
||||
// The previous window is left partly spent on purpose. That is the case the two-atomic version
|
||||
// got wrong: threads arriving in the new window were first measured against the old window's
|
||||
// count, and only then did the reset land — so the same second admitted the leftover budget and
|
||||
// then a whole fresh budget on top of it. A saturated previous window hides the defect, because
|
||||
// the pre-reset arrivals are all refused; an unsaturated one exposes it.
|
||||
for (int round = 0; round < 100; round++) {
|
||||
SteppingClock clock = new SteppingClock();
|
||||
ProviderAttemptLimiter limiter = new ProviderAttemptLimiter(1_000, 8, clock);
|
||||
for (int spent = 0; spent < 3; spent++) {
|
||||
limiter.acquire();
|
||||
limiter.release();
|
||||
}
|
||||
|
||||
clock.advanceOneSecond();
|
||||
// Attempts are spread over a loop rather than fired all at once. A single simultaneous burst
|
||||
// is over before a delayed reset could land, so it cannot observe the gap it is looking for.
|
||||
int admitted = race(32, 100, limiter);
|
||||
|
||||
assertThat(admitted)
|
||||
.as("round %d: one second, one budget of 8", round)
|
||||
.isLessThanOrEqualTo(8);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"a slot refused by the concurrency ceiling is returned to the window that granted it")
|
||||
void aRefusedSlotIsReturnedToItsOwnWindow() {
|
||||
SteppingClock clock = new SteppingClock();
|
||||
ProviderAttemptLimiter limiter = new ProviderAttemptLimiter(1, 4, clock);
|
||||
|
||||
limiter.acquire();
|
||||
// The single concurrency slot is taken, so this consumes a rate token and gives it back.
|
||||
assertThatThrownBy(limiter::acquire).isInstanceOf(ProviderUnavailableException.class);
|
||||
limiter.release();
|
||||
|
||||
// Three tokens are left in this window, not two: the refused attempt returned the one it took.
|
||||
limiter.acquire();
|
||||
limiter.release();
|
||||
limiter.acquire();
|
||||
limiter.release();
|
||||
limiter.acquire();
|
||||
limiter.release();
|
||||
|
||||
assertThatThrownBy(limiter::acquire)
|
||||
.as("the fifth attempt in a window of four is refused")
|
||||
.isInstanceOf(ProviderUnavailableException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a refund does not credit the window that follows it")
|
||||
void aRefundDoesNotCreditTheNextWindow() {
|
||||
SteppingClock clock = new SteppingClock();
|
||||
ProviderAttemptLimiter limiter = new ProviderAttemptLimiter(1, 2, clock);
|
||||
|
||||
limiter.acquire();
|
||||
assertThatThrownBy(limiter::acquire).isInstanceOf(ProviderUnavailableException.class);
|
||||
limiter.release();
|
||||
clock.advanceOneSecond();
|
||||
|
||||
limiter.acquire();
|
||||
limiter.release();
|
||||
limiter.acquire();
|
||||
limiter.release();
|
||||
|
||||
assertThatThrownBy(limiter::acquire)
|
||||
.as("a refund from the previous window must not raise this one's budget")
|
||||
.isInstanceOf(ProviderUnavailableException.class);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start every caller at once and count how many acquires were admitted.
|
||||
*
|
||||
* @param threads how many callers contend
|
||||
* @param attemptsEach how many acquires each caller makes
|
||||
* @param limiter the limiter under test
|
||||
* @return the number of successful acquires
|
||||
*/
|
||||
private static int race(int threads, int attemptsEach, ProviderAttemptLimiter limiter)
|
||||
throws InterruptedException {
|
||||
AtomicInteger admitted = new AtomicInteger();
|
||||
CountDownLatch ready = new CountDownLatch(threads);
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
CountDownLatch done = new CountDownLatch(threads);
|
||||
|
||||
try (ExecutorService pool = Executors.newFixedThreadPool(threads)) {
|
||||
for (int i = 0; i < threads; i++) {
|
||||
pool.execute(
|
||||
() -> {
|
||||
ready.countDown();
|
||||
try {
|
||||
start.await();
|
||||
for (int attempt = 0; attempt < attemptsEach; attempt++) {
|
||||
try {
|
||||
limiter.acquire();
|
||||
admitted.incrementAndGet();
|
||||
limiter.release();
|
||||
} catch (ProviderUnavailableException expected) {
|
||||
// The budget was spent; that is the outcome being counted.
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
} finally {
|
||||
done.countDown();
|
||||
}
|
||||
});
|
||||
}
|
||||
if (!ready.await(10, TimeUnit.SECONDS)) {
|
||||
throw new IllegalStateException("threads did not reach the starting line");
|
||||
}
|
||||
start.countDown();
|
||||
if (!done.await(10, TimeUnit.SECONDS)) {
|
||||
throw new IllegalStateException("acquires did not finish");
|
||||
}
|
||||
}
|
||||
return admitted.get();
|
||||
}
|
||||
}
|
||||
+194
@@ -0,0 +1,194 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderId;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.provider.NotificationProviderAdapter;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderCapabilities;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderProfileSnapshot;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmission;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmissionResult;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionStage;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* What happens when rotation and cleanup run at the same time.
|
||||
*
|
||||
* <p>Three separate steps made up a replace — put the new runtime, mark the old one draining, add
|
||||
* it to the draining list — and a fourth, the sweep, ran on that list from anywhere. A generation
|
||||
* could therefore be enrolled by one thread and swept by another before it was marked draining,
|
||||
* which is how a generation with attempts still in flight disappears from the set that exists to
|
||||
* track exactly that. Nothing ordered two rotations either, so the older of two concurrent
|
||||
* generations could end up current — the registry serving credentials a later rotation had already
|
||||
* retired.
|
||||
*/
|
||||
class ProviderRuntimeRegistryConcurrencyTest {
|
||||
|
||||
private static final ProviderProfileId PROFILE = new ProviderProfileId("apns-main");
|
||||
private static final Clock CLOCK =
|
||||
Clock.fixed(Instant.parse("2026-08-15T09:00:00Z"), ZoneOffset.UTC);
|
||||
|
||||
@Test
|
||||
@DisplayName("registering the same profile twice is reported, not silently overwritten")
|
||||
void registeringTwiceIsReported() {
|
||||
var registry = new ProviderRuntimeRegistry();
|
||||
registry.register(runtime(1));
|
||||
|
||||
assertThatThrownBy(() -> registry.register(runtime(2)))
|
||||
.as("the displaced runtime would keep its in-flight attempts with nothing tracking them")
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("already registered");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a generation that does not supersede the current one is refused")
|
||||
void anOlderGenerationIsRefused() {
|
||||
var registry = new ProviderRuntimeRegistry();
|
||||
registry.register(runtime(5));
|
||||
|
||||
assertThatThrownBy(() -> registry.replace(runtime(4)))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("supersede");
|
||||
assertThatThrownBy(() -> registry.replace(runtime(5)))
|
||||
.as("re-installing the same generation is not a rotation")
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThat(registry.current(PROFILE).generation()).isEqualTo(5);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("concurrent rotations leave the highest generation current")
|
||||
void concurrentRotationsLeaveTheHighestGenerationCurrent() throws InterruptedException {
|
||||
for (int round = 0; round < 50; round++) {
|
||||
var registry = new ProviderRuntimeRegistry();
|
||||
registry.register(runtime(1));
|
||||
int rotations = 16;
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
CountDownLatch done = new CountDownLatch(rotations);
|
||||
|
||||
try (ExecutorService pool = Executors.newFixedThreadPool(8)) {
|
||||
for (int generation = 2; generation <= rotations + 1; generation++) {
|
||||
long candidate = generation;
|
||||
pool.execute(
|
||||
() -> {
|
||||
try {
|
||||
start.await();
|
||||
registry.replace(runtime(candidate));
|
||||
} catch (IllegalArgumentException superseded) {
|
||||
// A rotation that lost the race is refused, which is the point.
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
} finally {
|
||||
done.countDown();
|
||||
}
|
||||
});
|
||||
}
|
||||
start.countDown();
|
||||
assertThat(done.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
}
|
||||
|
||||
assertThat(registry.current(PROFILE).generation())
|
||||
.as("round %d: no rotation may install a generation an earlier one already passed", round)
|
||||
.isEqualTo(rotations + 1);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a draining generation with work in flight survives a concurrent sweep")
|
||||
void aDrainingGenerationWithWorkInFlightSurvivesASweep() throws InterruptedException {
|
||||
var registry = new ProviderRuntimeRegistry();
|
||||
var first = runtime(1);
|
||||
registry.register(first);
|
||||
var permit = first.acquireAttempt();
|
||||
|
||||
AtomicBoolean sweeping = new AtomicBoolean(true);
|
||||
boolean lost = false;
|
||||
CountDownLatch sweeperRunning = new CountDownLatch(1);
|
||||
|
||||
try (ExecutorService pool = Executors.newSingleThreadExecutor()) {
|
||||
pool.execute(
|
||||
() -> {
|
||||
sweeperRunning.countDown();
|
||||
while (sweeping.get()) {
|
||||
registry.drainingGenerations(PROFILE);
|
||||
}
|
||||
});
|
||||
assertThat(sweeperRunning.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
|
||||
for (long generation = 2; generation <= 200; generation++) {
|
||||
registry.replace(runtime(generation));
|
||||
List<ProviderRuntime> draining = registry.drainingGenerations(PROFILE);
|
||||
if (draining.stream().noneMatch(runtime -> runtime == first)) {
|
||||
lost = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
sweeping.set(false);
|
||||
}
|
||||
|
||||
assertThat(lost).as("generation 1 still holds a permit, so it is still draining").isFalse();
|
||||
permit.close();
|
||||
assertThat(registry.drainingGenerations(PROFILE))
|
||||
.as("once the permit closes there is nothing left to drain")
|
||||
.isEmpty();
|
||||
}
|
||||
|
||||
private static ProviderRuntime runtime(long generation) {
|
||||
return new ProviderRuntime(
|
||||
new ProviderProfileSnapshot(
|
||||
PROFILE,
|
||||
new ProviderId("apns"),
|
||||
Channel.PUSH,
|
||||
"PRODUCTION",
|
||||
generation,
|
||||
capabilities(),
|
||||
Map.of("topic", "com.example.app")),
|
||||
new StubAdapter(),
|
||||
new ProviderAttemptLimiter(4, 1_000_000, CLOCK));
|
||||
}
|
||||
|
||||
private static ProviderCapabilities capabilities() {
|
||||
return new ProviderCapabilities(
|
||||
false, false, false, false, false, false, false, true, 1, 4096L, Duration.ofHours(1));
|
||||
}
|
||||
|
||||
/** Adapter that is never actually invoked by these registry tests. */
|
||||
private static final class StubAdapter implements NotificationProviderAdapter {
|
||||
|
||||
@Override
|
||||
public ProviderId providerId() {
|
||||
return new ProviderId("apns");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Channel> channels() {
|
||||
return Set.of(Channel.PUSH);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProviderCapabilities capabilities() {
|
||||
return ProviderRuntimeRegistryConcurrencyTest.capabilities();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletionStage<ProviderSubmissionResult> submit(ProviderSubmission submission) {
|
||||
return CompletableFuture.failedFuture(new UnsupportedOperationException("not submitted"));
|
||||
}
|
||||
}
|
||||
}
|
||||
+241
@@ -0,0 +1,241 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderId;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.api.error.ProviderUnavailableException;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.provider.NotificationProviderAdapter;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderCapabilities;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderProfileSnapshot;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmission;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmissionResult;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionStage;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* What a runtime's health means, and what closing a permit twice costs.
|
||||
*
|
||||
* <p>State and reason lived in two references, so a reader could see a state paired with a reason
|
||||
* that belonged to a different one. Recovery cleared the reason unconditionally while transitioning
|
||||
* only some states, which left an authentication failure in place with nothing left to explain it.
|
||||
* And the permit released its semaphore slot on every close, so a permit closed twice raised the
|
||||
* concurrency ceiling above the configured maximum — permanently, and in the direction of
|
||||
* overloading the provider it was meant to protect.
|
||||
*/
|
||||
class ProviderRuntimeStateTest {
|
||||
|
||||
private static final ProviderProfileId PROFILE = new ProviderProfileId("apns-main");
|
||||
private static final Clock CLOCK =
|
||||
Clock.fixed(Instant.parse("2026-08-15T09:00:00Z"), ZoneOffset.UTC);
|
||||
|
||||
@Test
|
||||
@DisplayName("closing a permit twice does not raise the concurrency ceiling")
|
||||
void closingAPermitTwiceDoesNotRaiseTheCeiling() {
|
||||
ProviderRuntime runtime = runtime(1, 1);
|
||||
|
||||
var permit = runtime.acquireAttempt();
|
||||
permit.close();
|
||||
permit.close();
|
||||
|
||||
var next = runtime.acquireAttempt();
|
||||
assertThatThrownBy(runtime::acquireAttempt)
|
||||
.as("a double close used to leave two slots where the configuration allows one")
|
||||
.isInstanceOf(ProviderUnavailableException.class);
|
||||
next.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("concurrent double closes still release exactly one slot each")
|
||||
void concurrentDoubleClosesReleaseOneSlotEach() throws InterruptedException {
|
||||
ProviderRuntime runtime = runtime(1, 10_000);
|
||||
|
||||
for (int round = 0; round < 500; round++) {
|
||||
var permit = runtime.acquireAttempt();
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
CountDownLatch done = new CountDownLatch(2);
|
||||
try (ExecutorService pool = Executors.newFixedThreadPool(2)) {
|
||||
for (int closer = 0; closer < 2; closer++) {
|
||||
pool.execute(
|
||||
() -> {
|
||||
try {
|
||||
start.await();
|
||||
permit.close();
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
} finally {
|
||||
done.countDown();
|
||||
}
|
||||
});
|
||||
}
|
||||
start.countDown();
|
||||
assertThat(done.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
}
|
||||
assertThat(runtime.activeAttempts()).as("round %d", round).isZero();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a success clears throttling and degradation")
|
||||
void aSuccessClearsThrottlingAndDegradation() {
|
||||
ProviderRuntime throttled = runtime(1, 1);
|
||||
throttled.markThrottled();
|
||||
assertThat(throttled.markHealthy()).isTrue();
|
||||
assertThat(throttled.health().state()).isEqualTo(ProviderRuntimeState.HEALTHY);
|
||||
assertThat(throttled.health().reason()).isEmpty();
|
||||
|
||||
ProviderRuntime degraded = runtime(1, 1);
|
||||
degraded.markDegraded("UPSTREAM_5XX");
|
||||
assertThat(degraded.markHealthy()).isTrue();
|
||||
assertThat(degraded.health().reason()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a success does not clear an authentication failure, and says so")
|
||||
void aSuccessDoesNotClearAnAuthenticationFailure() {
|
||||
ProviderRuntime runtime = runtime(1, 1);
|
||||
runtime.markAuthenticationFailed("INVALID_CREDENTIAL");
|
||||
|
||||
assertThat(runtime.markHealthy())
|
||||
.as("the rejected credential is still the credential in use")
|
||||
.isFalse();
|
||||
assertThat(runtime.health().state()).isEqualTo(ProviderRuntimeState.AUTHENTICATION_FAILED);
|
||||
assertThat(runtime.health().reason())
|
||||
.as("the reason used to be cleared even when the state was not")
|
||||
.contains("INVALID_CREDENTIAL");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an operator can resume a drained or disabled runtime but not a failed credential")
|
||||
void resumeHealthyClearsOperatorStatesOnly() {
|
||||
ProviderRuntime drained = runtime(1, 1);
|
||||
drained.markDraining();
|
||||
assertThat(drained.resumeHealthy()).isTrue();
|
||||
|
||||
ProviderRuntime disabled = runtime(1, 1);
|
||||
disabled.markDisabled();
|
||||
assertThat(disabled.resumeHealthy()).isTrue();
|
||||
|
||||
ProviderRuntime failed = runtime(1, 1);
|
||||
failed.markAuthenticationFailed("INVALID_CREDENTIAL");
|
||||
assertThat(failed.resumeHealthy())
|
||||
.as("declaring it healthy does not give it a credential the provider accepts")
|
||||
.isFalse();
|
||||
assertThat(failed.state()).isEqualTo(ProviderRuntimeState.AUTHENTICATION_FAILED);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("degrading an already failed runtime is refused rather than silently ignored")
|
||||
void degradingAFailedRuntimeIsRefused() {
|
||||
ProviderRuntime runtime = runtime(1, 1);
|
||||
runtime.markAuthenticationFailed("INVALID_CREDENTIAL");
|
||||
|
||||
assertThat(runtime.markDegraded("UPSTREAM_5XX")).isFalse();
|
||||
assertThat(runtime.health().reason()).contains("INVALID_CREDENTIAL");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("state and reason are never observed as a pair the runtime did not hold")
|
||||
void stateAndReasonAreNeverObservedApart() throws InterruptedException {
|
||||
ProviderRuntime runtime = runtime(1, 1);
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
CountDownLatch done = new CountDownLatch(2);
|
||||
|
||||
try (ExecutorService pool = Executors.newFixedThreadPool(2)) {
|
||||
pool.execute(
|
||||
() -> {
|
||||
try {
|
||||
start.await();
|
||||
for (int i = 0; i < 20_000; i++) {
|
||||
runtime.markAuthenticationFailed("INVALID_CREDENTIAL");
|
||||
runtime.resumeHealthy();
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
} finally {
|
||||
done.countDown();
|
||||
}
|
||||
});
|
||||
pool.execute(
|
||||
() -> {
|
||||
try {
|
||||
start.await();
|
||||
for (int i = 0; i < 20_000; i++) {
|
||||
var health = runtime.health();
|
||||
boolean consistent =
|
||||
health.state() == ProviderRuntimeState.HEALTHY
|
||||
? health.reason().isEmpty()
|
||||
: health.reason().isPresent();
|
||||
assertThat(consistent)
|
||||
.as("observed %s with reason %s", health.state(), health.reason())
|
||||
.isTrue();
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
} finally {
|
||||
done.countDown();
|
||||
}
|
||||
});
|
||||
start.countDown();
|
||||
assertThat(done.await(30, TimeUnit.SECONDS)).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
private static ProviderRuntime runtime(long generation, int maxConcurrency) {
|
||||
return new ProviderRuntime(
|
||||
new ProviderProfileSnapshot(
|
||||
PROFILE,
|
||||
new ProviderId("apns"),
|
||||
Channel.PUSH,
|
||||
"PRODUCTION",
|
||||
generation,
|
||||
capabilities(),
|
||||
Map.of("topic", "com.example.app")),
|
||||
new StubAdapter(),
|
||||
new ProviderAttemptLimiter(maxConcurrency, 1_000_000, CLOCK));
|
||||
}
|
||||
|
||||
private static ProviderCapabilities capabilities() {
|
||||
return new ProviderCapabilities(
|
||||
false, false, false, false, false, false, false, true, 1, 4096L, Duration.ofHours(1));
|
||||
}
|
||||
|
||||
/** Adapter that is never actually invoked by these runtime tests. */
|
||||
private static final class StubAdapter implements NotificationProviderAdapter {
|
||||
|
||||
@Override
|
||||
public ProviderId providerId() {
|
||||
return new ProviderId("apns");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Channel> channels() {
|
||||
return Set.of(Channel.PUSH);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProviderCapabilities capabilities() {
|
||||
return ProviderRuntimeStateTest.capabilities();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletionStage<ProviderSubmissionResult> submit(ProviderSubmission submission) {
|
||||
return CompletableFuture.failedFuture(new UnsupportedOperationException("not submitted"));
|
||||
}
|
||||
}
|
||||
}
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.callback.NormalizedEventType;
|
||||
import dev.caskeleton.application.notification.platform.callback.NormalizedProviderEvent;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJob;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJobStorePort;
|
||||
import dev.caskeleton.application.notification.platform.provider.ReconciliationResult;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Whether an ambiguous attempt is ever asked about again.
|
||||
*
|
||||
* <p>It was not. Reconciliation ran only when a lease expired and recovery walked past an
|
||||
* incomplete attempt, so a worker that recorded AMBIGUOUS and then exited cleanly left its delivery
|
||||
* in {@code RECONCILIATION_REQUIRED} permanently. The job table for exactly this has existed since
|
||||
* V3 with no entity, no store and no worker.
|
||||
*/
|
||||
class ReconciliationJobWorkerTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-15T09:00:00Z");
|
||||
private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC);
|
||||
|
||||
private final RecordingJobs jobs = new RecordingJobs();
|
||||
private final Map<DeliveryAttemptId, ReconciliationResult> answers = new java.util.HashMap<>();
|
||||
|
||||
private final ReconciliationJobWorker worker =
|
||||
new ReconciliationJobWorker(jobs, answers::get, CLOCK, Duration.ofMinutes(5), 10, 3);
|
||||
|
||||
@Test
|
||||
@DisplayName("a confirmed outcome completes the job")
|
||||
void aConfirmedOutcomeCompletesTheJob() {
|
||||
ReconciliationJob job = due(0);
|
||||
answers.put(job.attemptId(), new ReconciliationResult.Confirmed(event()));
|
||||
|
||||
assertThat(worker.reconcileOnce()).isEqualTo(1);
|
||||
assertThat(jobs.completed).containsExactly(job.id());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an unsupported provider is not asked in a loop")
|
||||
void anUnsupportedProviderIsNotAskedInALoop() {
|
||||
ReconciliationJob job = due(0);
|
||||
answers.put(job.attemptId(), new ReconciliationResult.Unsupported());
|
||||
|
||||
assertThat(worker.reconcileOnce()).isEqualTo(1);
|
||||
assertThat(jobs.completed)
|
||||
.as(
|
||||
"without a status-query capability the answer will not change; the attempt stays"
|
||||
+ " visibly ambiguous for an operator instead")
|
||||
.containsExactly(job.id());
|
||||
assertThat(jobs.rescheduled).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a still-unknown outcome is asked again at the provider's own next time")
|
||||
void aStillUnknownOutcomeIsAskedAgain() {
|
||||
ReconciliationJob job = due(0);
|
||||
Instant providerSuggested = NOW.plus(Duration.ofHours(2));
|
||||
answers.put(job.attemptId(), new ReconciliationResult.StillUnknown(providerSuggested));
|
||||
|
||||
assertThat(worker.reconcileOnce()).isZero();
|
||||
assertThat(jobs.rescheduled).containsExactly(Map.entry(job.id(), providerSuggested));
|
||||
assertThat(jobs.completed).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a retryable failure is asked again after the backoff")
|
||||
void aRetryableFailureIsAskedAgain() {
|
||||
ReconciliationJob job = due(0);
|
||||
answers.put(job.attemptId(), new ReconciliationResult.Failed("PROVIDER_TIMEOUT", true));
|
||||
|
||||
assertThat(worker.reconcileOnce()).isZero();
|
||||
assertThat(jobs.rescheduled)
|
||||
.containsExactly(Map.entry(job.id(), NOW.plus(Duration.ofMinutes(5))));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a non-retryable failure completes the job")
|
||||
void aNonRetryableFailureCompletesTheJob() {
|
||||
ReconciliationJob job = due(0);
|
||||
answers.put(job.attemptId(), new ReconciliationResult.Failed("ATTEMPT_UNKNOWN", false));
|
||||
|
||||
assertThat(worker.reconcileOnce()).isEqualTo(1);
|
||||
assertThat(jobs.completed).containsExactly(job.id());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a job that has asked enough times stops asking")
|
||||
void aJobThatHasAskedEnoughTimesStopsAsking() {
|
||||
ReconciliationJob job = due(3);
|
||||
|
||||
assertThat(worker.reconcileOnce()).isEqualTo(1);
|
||||
assertThat(jobs.completed)
|
||||
.as("an outcome that has not arrived after this many tries is an operator's decision")
|
||||
.containsExactly(job.id());
|
||||
assertThat(answers).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("one job throwing does not stop the rest of the pass")
|
||||
void oneJobThrowingDoesNotStopTheRest() {
|
||||
ReconciliationJob failing = due(0);
|
||||
ReconciliationJob healthy = due(0);
|
||||
answers.put(healthy.attemptId(), new ReconciliationResult.Confirmed(event()));
|
||||
|
||||
// The failing job has no answer registered, so the reconciler returns null and the switch
|
||||
// throws — which is exactly the shape of a provider refusing connections.
|
||||
assertThat(worker.reconcileOnce()).isEqualTo(1);
|
||||
assertThat(jobs.completed).containsExactly(healthy.id());
|
||||
assertThat(jobs.rescheduled).hasSize(1);
|
||||
assertThat(jobs.rescheduled.get(0).getKey()).isEqualTo(failing.id());
|
||||
}
|
||||
|
||||
private ReconciliationJob due(int attempts) {
|
||||
ReconciliationJob job =
|
||||
new ReconciliationJob(
|
||||
UUID.randomUUID(),
|
||||
new DeliveryAttemptId(UUID.randomUUID()),
|
||||
new ProviderProfileId("email-main"),
|
||||
NOW,
|
||||
attempts,
|
||||
Optional.empty(),
|
||||
NOW,
|
||||
NOW);
|
||||
jobs.due.add(job);
|
||||
return job;
|
||||
}
|
||||
|
||||
private static NormalizedProviderEvent event() {
|
||||
return new NormalizedProviderEvent(
|
||||
NormalizedEventType.DELIVERY_CONFIRMED,
|
||||
"delivered",
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
Optional.of(NOW),
|
||||
Map.of());
|
||||
}
|
||||
|
||||
private static final class RecordingJobs implements ReconciliationJobStorePort {
|
||||
|
||||
private final List<ReconciliationJob> due = new ArrayList<>();
|
||||
private final List<UUID> completed = new ArrayList<>();
|
||||
private final List<Map.Entry<UUID, Instant>> rescheduled = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public ReconciliationJob schedule(
|
||||
DeliveryAttemptId attemptId, ProviderProfileId providerProfileId, Instant dueAt) {
|
||||
throw new UnsupportedOperationException("the worker never schedules");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ReconciliationJob> claimDue(int limit, Instant now) {
|
||||
return List.copyOf(due);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reschedule(ReconciliationJob job, String result, Instant nextCheckAt) {
|
||||
rescheduled.add(Map.entry(job.id(), nextCheckAt));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void complete(ReconciliationJob job) {
|
||||
completed.add(job.id());
|
||||
}
|
||||
}
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.provider;
|
||||
|
||||
import static org.assertj.core.api.Assertions.fail;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
/**
|
||||
* Where a real provider call goes, and why the job cannot report green until one is here.
|
||||
*
|
||||
* <p>The sandbox job's entire body used to be two {@code echo} statements behind {@code
|
||||
* continue-on-error}. It reported success on every run, and the support matrix graded five channels
|
||||
* {@code Stable} partly on the strength of a job that had never opened a socket.
|
||||
*
|
||||
* <p>This class is deliberately a failure rather than a placeholder that passes. A test that exists
|
||||
* and asserts nothing is the same green light with more ceremony — and the whole point of the
|
||||
* evidence manifest is that a claim without an artifact behind it must be visible. Until a real
|
||||
* sandbox call is implemented here, {@code provider-wire-qualified} stays unsatisfied in {@code
|
||||
* docs/notification/evidence-manifest.json} and no channel may be graded {@code Stable}.
|
||||
*
|
||||
* <p>It runs only when the sandbox environment asks for it, so an ordinary build is unaffected.
|
||||
*/
|
||||
@EnabledIfEnvironmentVariable(named = "NOTIFICATION_SANDBOX_ENABLED", matches = "true")
|
||||
class ProviderSandboxSmokeTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("a real provider sandbox call is not implemented, and the job says so")
|
||||
void aRealProviderSandboxCallIsNotImplemented() {
|
||||
fail(
|
||||
"provider-wire-qualified has no implementation. A qualifying test sends one message per "
|
||||
+ "configured provider profile to that provider's sandbox endpoint, records the "
|
||||
+ "provider's correlation id, and writes the request/response pair as an immutable "
|
||||
+ "artifact. Implement it here, then mark the claim satisfied in "
|
||||
+ "docs/notification/evidence-manifest.json — in that order.");
|
||||
}
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.provider;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.security.AesGcmContactPointProtector;
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.security.SecurityFixtures;
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.testkit.ProviderFixtures;
|
||||
import dev.caskeleton.application.notification.platform.api.content.NotificationContent;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.contact.EmailAddress;
|
||||
import dev.caskeleton.application.notification.platform.security.ContactPointProtector;
|
||||
import java.util.Optional;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Whether a submission's channel, profile and content have to agree.
|
||||
*
|
||||
* <p>Nothing checked, so a WEBHOOK submission could carry Web Push content — and one did, in the
|
||||
* webhook adapter's own fixtures. A suite built on a shape no dispatch can emit proves things about
|
||||
* a system that does not exist; adding this invariant made those fixtures fail immediately, which
|
||||
* is how the problem became visible.
|
||||
*/
|
||||
class SubmissionCompatibilityTest {
|
||||
|
||||
private final ContactPointProtector protector =
|
||||
new AesGcmContactPointProtector(SecurityFixtures.keys());
|
||||
|
||||
@Test
|
||||
@DisplayName("an email channel with email content and an email profile is accepted")
|
||||
void aConsistentSubmissionIsAccepted() {
|
||||
assertThatCode(() -> submission(Channel.EMAIL, Channel.EMAIL, ProviderFixtures.email()))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("content of the wrong kind for the channel is refused")
|
||||
void contentOfTheWrongKindIsRefused() {
|
||||
assertThatThrownBy(() -> submission(Channel.EMAIL, Channel.EMAIL, ProviderFixtures.sms()))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("cannot carry");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a profile serving another channel is refused")
|
||||
void aProfileServingAnotherChannelIsRefused() {
|
||||
assertThatThrownBy(() -> submission(Channel.EMAIL, Channel.SMS, ProviderFixtures.email()))
|
||||
.as("a profile is bound to one channel; using it for another is a wiring mistake")
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("serves");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a webhook submission cannot carry Web Push content")
|
||||
void aWebhookSubmissionCannotCarryWebPushContent() {
|
||||
assertThatThrownBy(
|
||||
() -> submission(Channel.WEBHOOK, Channel.WEBHOOK, ProviderFixtures.webPush()))
|
||||
.as("this is the exact combination the webhook fixtures were built on")
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
private dev.caskeleton.application.notification.platform.provider.ProviderSubmission submission(
|
||||
Channel channel, Channel profileChannel, NotificationContent content) {
|
||||
return ProviderFixtures.submission(
|
||||
ProviderFixtures.profile("profile-1", "ses", profileChannel),
|
||||
channel,
|
||||
content,
|
||||
protector,
|
||||
EmailAddress.parse("someone@example.com"),
|
||||
Optional.empty());
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.provider.fcm;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.testkit.ProviderFixtures;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderCallNotStartedException;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmission;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.Optional;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* What TTL an expiring notification is sent with.
|
||||
*
|
||||
* <p>The mapper filtered a negative remaining duration out of the {@code Optional} and then fell
|
||||
* through to {@code orElse(maxTtl)} — so a notification that had <em>already expired</em> went to
|
||||
* FCM with the provider's <em>maximum</em> lifetime. The one input meaning "do not deliver this"
|
||||
* produced the longest possible delivery window, and FCM retries for the whole of it.
|
||||
*/
|
||||
class FcmExpiryTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-15T09:00:00Z");
|
||||
private static final Duration MAX_TTL = Duration.ofHours(4);
|
||||
|
||||
private final FcmMessageMapper mapper =
|
||||
new FcmMessageMapper(
|
||||
new FcmProviderProperties(
|
||||
java.net.URI.create("https://fcm.example"),
|
||||
"example-prod",
|
||||
"mobile-main",
|
||||
500,
|
||||
MAX_TTL,
|
||||
Duration.ofSeconds(3)),
|
||||
Clock.fixed(NOW, ZoneOffset.UTC));
|
||||
|
||||
@Test
|
||||
@DisplayName("no expiry uses the provider maximum")
|
||||
void noExpiryUsesTheProviderMaximum() {
|
||||
assertThat(mapper.ttl(submission(Optional.empty()))).isEqualTo(MAX_TTL);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an expiry ahead of now uses what remains")
|
||||
void anExpiryAheadUsesWhatRemains() {
|
||||
assertThat(mapper.ttl(submission(Optional.of(NOW.plus(Duration.ofMinutes(90))))))
|
||||
.isEqualTo(Duration.ofMinutes(90));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an expiry beyond the provider maximum is capped")
|
||||
void anExpiryBeyondTheMaximumIsCapped() {
|
||||
assertThat(mapper.ttl(submission(Optional.of(NOW.plus(Duration.ofDays(2))))))
|
||||
.isEqualTo(MAX_TTL);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an already-expired notification is refused rather than revived")
|
||||
void anAlreadyExpiredNotificationIsRefused() {
|
||||
assertThatThrownBy(() -> mapper.ttl(submission(Optional.of(NOW.minusSeconds(1)))))
|
||||
.as("this used to send the provider maximum, the longest window available")
|
||||
.isInstanceOf(ProviderCallNotStartedException.class)
|
||||
.hasMessageContaining("expired");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an expiry exactly now is refused too")
|
||||
void anExpiryExactlyNowIsRefused() {
|
||||
assertThatThrownBy(() -> mapper.ttl(submission(Optional.of(NOW))))
|
||||
.isInstanceOf(ProviderCallNotStartedException.class);
|
||||
}
|
||||
|
||||
private static ProviderSubmission submission(Optional<Instant> expiresAt) {
|
||||
return ProviderFixtures.submission(
|
||||
ProviderFixtures.profile("push-main", "fcm", Channel.PUSH),
|
||||
Channel.PUSH,
|
||||
ProviderFixtures.push(),
|
||||
new dev.caskeleton.adapter.outbound.notification.platform.security
|
||||
.AesGcmContactPointProtector(
|
||||
dev.caskeleton.adapter.outbound.notification.platform.security.SecurityFixtures.keys()),
|
||||
new dev.caskeleton.application.notification.platform.contact.FcmInstallationId(
|
||||
"installation-1"),
|
||||
expiresAt);
|
||||
}
|
||||
}
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.provider.http;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.UnknownHostException;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Which endpoints the platform will fetch.
|
||||
*
|
||||
* <p>The only check was the scheme, so every HTTPS URL was accepted — including {@code
|
||||
* https://169.254.169.254/}, the cloud metadata service, and any RFC 1918 address. Web Push
|
||||
* endpoints and webhook targets come from clients, which makes that a server-side request forgery
|
||||
* primitive: the platform fetches an internal address on request and, for a webhook, delivers the
|
||||
* message body there.
|
||||
*/
|
||||
class EndpointRoutabilityTest {
|
||||
|
||||
/**
|
||||
* Parses a literal address without a name lookup.
|
||||
*
|
||||
* <p>Every value here is already an IP literal, so there is nothing to resolve; going through the
|
||||
* name-based API would make these tests depend on the machine's resolver.
|
||||
*/
|
||||
private static InetAddress literal(String address) throws UnknownHostException {
|
||||
return InetAddress.getAllByName(address)[0];
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the cloud metadata address is refused")
|
||||
void theCloudMetadataAddressIsRefused() throws UnknownHostException {
|
||||
assertThat(NotificationEndpoints.isInternal(literal("169.254.169.254")))
|
||||
.as("the single most useful address to an attacker who can choose a webhook target")
|
||||
.isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("private ranges are refused")
|
||||
void privateRangesAreRefused() throws UnknownHostException {
|
||||
for (String address : new String[] {"10.0.0.1", "172.16.0.1", "192.168.1.1", "127.0.0.1"}) {
|
||||
assertThat(NotificationEndpoints.isInternal(literal(address))).as("%s", address).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("carrier-grade NAT and IETF protocol assignment ranges are refused")
|
||||
void carrierNatAndProtocolRangesAreRefused() throws UnknownHostException {
|
||||
assertThat(NotificationEndpoints.isInternal(literal("100.64.0.1")))
|
||||
.as("100.64/10 routinely reaches infrastructure an application should not talk to")
|
||||
.isTrue();
|
||||
assertThat(NotificationEndpoints.isInternal(literal("192.0.0.1"))).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("IPv6 loopback and unique-local addresses are refused")
|
||||
void ipv6LocalAddressesAreRefused() throws UnknownHostException {
|
||||
assertThat(NotificationEndpoints.isInternal(literal("::1"))).isTrue();
|
||||
assertThat(NotificationEndpoints.isInternal(literal("fd00::1")))
|
||||
.as("fc00::/7 is the IPv6 equivalent of RFC 1918")
|
||||
.isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a public address is allowed")
|
||||
void aPublicAddressIsAllowed() throws UnknownHostException {
|
||||
assertThat(NotificationEndpoints.isInternal(literal("93.184.216.34"))).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an endpoint pointing at the loopback interface is refused unless allowed")
|
||||
void aLoopbackEndpointIsRefusedUnlessAllowed() {
|
||||
URI loopback = URI.create("https://127.0.0.1/push");
|
||||
|
||||
assertThatThrownBy(
|
||||
() -> NotificationEndpoints.requireExternallyRoutable(loopback, "endpoint", false))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatCode(
|
||||
() -> NotificationEndpoints.requireExternallyRoutable(loopback, "endpoint", true))
|
||||
.as("the contract suite needs a real socket, and that exception is explicit")
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("userinfo is refused, because it disguises the host")
|
||||
void userInfoIsRefused() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
NotificationEndpoints.requireExternallyRoutable(
|
||||
URI.create("https://push.example@169.254.169.254/"), "endpoint", false))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("userinfo");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a name that does not resolve is refused rather than attempted")
|
||||
void anUnresolvableNameIsRefused() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
NotificationEndpoints.requireExternallyRoutable(
|
||||
URI.create("https://no-such-host.invalid/push"), "endpoint", false))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("resolve");
|
||||
}
|
||||
}
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.provider.http;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.net.ConnectException;
|
||||
import java.net.NoRouteToHostException;
|
||||
import java.net.SocketException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.Locale;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Whether the request body reached the provider — decided from a type, not from prose.
|
||||
*
|
||||
* <p>The classifier lower-cased {@code getMessage()} and searched for "connection refused",
|
||||
* "unresolved", "no route to host" and "connect timed out". None of those is a contract. They come
|
||||
* from the platform's C library and the JDK's own wording: they are localised on some systems, they
|
||||
* have changed between JDK releases, and a proxy phrasing a refused connection differently was read
|
||||
* as "the body was sent".
|
||||
*
|
||||
* <p>Getting this wrong is expensive in one direction. A body wrongly judged committed makes the
|
||||
* attempt ambiguous, and an ambiguous attempt is deliberately never retried and never falls back.
|
||||
*/
|
||||
class HttpCommitmentClassificationTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("a connect failure is classified before the write, whatever it says")
|
||||
void aConnectFailureIsClassifiedBeforeTheWrite() {
|
||||
// The message is deliberately not the English the old classifier looked for.
|
||||
assertThat(committed(new ConnectException("Verbindung abgelehnt"))).isFalse();
|
||||
assertThat(committed(new UnknownHostException("nom d'hôte inconnu"))).isFalse();
|
||||
assertThat(committed(new NoRouteToHostException(""))).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a connect failure wrapped in another IOException is still before the write")
|
||||
void aWrappedConnectFailureIsStillBeforeTheWrite() {
|
||||
assertThat(committed(new IOException("send failed", new ConnectException("refused"))))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a reset after the body is committed, because it may already be there")
|
||||
void aResetAfterTheBodyIsCommitted() {
|
||||
assertThat(committed(new SocketException("Connection reset")))
|
||||
.as("guessing 'not committed' here turns an unknown into an automatic resend")
|
||||
.isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a message that merely mentions a refusal does not decide anything")
|
||||
void aMessageThatMentionsARefusalDecidesNothing() {
|
||||
assertThat(committed(new SocketException("upstream said: connection refused")))
|
||||
.as("the old classifier read this as a pre-write failure on the strength of the wording")
|
||||
.isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a null message is committed rather than a NullPointerException")
|
||||
void aNullMessageIsCommitted() {
|
||||
assertThat(committed(new IOException())).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a circular cause chain terminates instead of hanging the dispatch thread")
|
||||
void aCircularCauseChainTerminates() {
|
||||
// The JDK refuses self-causation but permits a two-node cycle, and an unbounded walk over one
|
||||
// never returns.
|
||||
IOException first = new IOException("first");
|
||||
IOException second = new IOException("second");
|
||||
first.initCause(second);
|
||||
second.initCause(first);
|
||||
|
||||
assertThat(committed(first)).isTrue();
|
||||
}
|
||||
|
||||
private static boolean committed(IOException failure) {
|
||||
try {
|
||||
Method classifier =
|
||||
JdkNotificationHttpGateway.class.getDeclaredMethod(
|
||||
"bodyWasLikelyCommitted", IOException.class);
|
||||
classifier.setAccessible(true);
|
||||
return (boolean) classifier.invoke(null, failure);
|
||||
} catch (ReflectiveOperationException unreachable) {
|
||||
throw new IllegalStateException(
|
||||
"the classifier moved; this test exists to pin it".toLowerCase(Locale.ROOT), unreachable);
|
||||
}
|
||||
}
|
||||
}
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.provider.ses;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Which host the verifier will fetch a signing certificate from.
|
||||
*
|
||||
* <p>It matched a suffix. {@code evilamazonaws.com} ends with {@code amazonaws.com}, so an
|
||||
* attacker-owned domain passed — and the envelope was then verified against a certificate that
|
||||
* attacker served, which makes the signature check decorative rather than wrong.
|
||||
*/
|
||||
class SnsCertificateUrlTest {
|
||||
|
||||
private final SnsSignatureVerifier verifier =
|
||||
new SnsSignatureVerifier(certificateUrl -> null, "amazonaws.com");
|
||||
|
||||
@Test
|
||||
@DisplayName("a genuine SNS certificate URL is accepted")
|
||||
void aGenuineUrlIsAccepted() {
|
||||
assertThat(
|
||||
verifier.isTrustedCertificateUrl(
|
||||
"https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-abc123.pem"))
|
||||
.isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a host that merely ends with the suffix is refused")
|
||||
void aHostThatMerelyEndsWithTheSuffixIsRefused() {
|
||||
assertThat(
|
||||
verifier.isTrustedCertificateUrl(
|
||||
"https://evilamazonaws.com/SimpleNotificationService-abc123.pem"))
|
||||
.as("the suffix has to begin at a label boundary; this is the whole finding")
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("userinfo, query and fragment are refused")
|
||||
void userInfoQueryAndFragmentAreRefused() {
|
||||
assertThat(
|
||||
verifier.isTrustedCertificateUrl(
|
||||
"https://sns.amazonaws.com@evil.example/SimpleNotificationService-a.pem"))
|
||||
.isFalse();
|
||||
assertThat(
|
||||
verifier.isTrustedCertificateUrl(
|
||||
"https://sns.amazonaws.com/SimpleNotificationService-a.pem?x=1"))
|
||||
.isFalse();
|
||||
assertThat(
|
||||
verifier.isTrustedCertificateUrl(
|
||||
"https://sns.amazonaws.com/SimpleNotificationService-a.pem#f"))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a non-default port is refused")
|
||||
void aNonDefaultPortIsRefused() {
|
||||
assertThat(
|
||||
verifier.isTrustedCertificateUrl(
|
||||
"https://sns.amazonaws.com:8443/SimpleNotificationService-a.pem"))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a path SNS does not publish is refused")
|
||||
void anUnexpectedPathIsRefused() {
|
||||
assertThat(verifier.isTrustedCertificateUrl("https://sns.amazonaws.com/anything.pem"))
|
||||
.isFalse();
|
||||
assertThat(
|
||||
verifier.isTrustedCertificateUrl(
|
||||
"https://sns.amazonaws.com/SimpleNotificationService-a.txt"))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("plaintext is refused")
|
||||
void plaintextIsRefused() {
|
||||
assertThat(
|
||||
verifier.isTrustedCertificateUrl(
|
||||
"http://sns.amazonaws.com/SimpleNotificationService-a.pem"))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an unknown or absent signature version does not downgrade to SHA-1")
|
||||
void anUnknownSignatureVersionIsRefused() {
|
||||
assertThat(
|
||||
verifier.isValid(
|
||||
java.util.Map.of(
|
||||
"SigningCertURL",
|
||||
"https://sns.amazonaws.com/SimpleNotificationService-a.pem",
|
||||
"Signature",
|
||||
"AA==",
|
||||
"SignatureVersion",
|
||||
"3")))
|
||||
.as("the default was version one, so a missing or unrecognised field chose SHA-1 for us")
|
||||
.isFalse();
|
||||
assertThat(
|
||||
verifier.isValid(
|
||||
java.util.Map.of(
|
||||
"SigningCertURL",
|
||||
"https://sns.amazonaws.com/SimpleNotificationService-a.pem",
|
||||
"Signature",
|
||||
"AA==")))
|
||||
.isFalse();
|
||||
}
|
||||
}
|
||||
+6
-1
@@ -126,7 +126,12 @@ class SmtpNotificationProviderAdapterTest {
|
||||
new SmtpFailureClassifier(),
|
||||
protector,
|
||||
properties,
|
||||
Executors.newSingleThreadExecutor());
|
||||
Executors.newSingleThreadExecutor(),
|
||||
// The unconfigured resolver refuses on use, which is the honest default: an attachment the
|
||||
// deployment has no way to fetch must fail before the send, not silently vanish from it.
|
||||
new dev.caskeleton.application.notification.platform.dispatch.AttachmentIntegrityGuard(
|
||||
new dev.caskeleton.adapter.outbound.notification.platform.provider
|
||||
.UnconfiguredAttachmentResolver()));
|
||||
}
|
||||
|
||||
private ProviderSubmission submission() {
|
||||
|
||||
+1
-1
@@ -3,9 +3,9 @@ package dev.caskeleton.adapter.outbound.notification.platform.provider.twilio;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.security.SecurityFixtures;
|
||||
import dev.caskeleton.application.notification.platform.api.CallbackRequest;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderId;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.callback.CallbackRequest;
|
||||
import dev.caskeleton.application.notification.platform.callback.NormalizedEventType;
|
||||
import dev.caskeleton.application.notification.platform.security.SecretPurpose;
|
||||
import java.net.URLEncoder;
|
||||
|
||||
+1
-1
@@ -2,9 +2,9 @@ package dev.caskeleton.adapter.outbound.notification.platform.provider.twilio;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.security.SecurityFixtures;
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.testkit.CallbackContract;
|
||||
import dev.caskeleton.application.notification.platform.api.CallbackRequest;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderId;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.callback.CallbackRequest;
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderCallbackAdapter;
|
||||
import dev.caskeleton.application.notification.platform.security.SecretPurpose;
|
||||
import java.net.URLEncoder;
|
||||
|
||||
+1
-1
@@ -104,7 +104,7 @@ class WebhookNotificationProviderAdapterTest {
|
||||
return ProviderFixtures.submission(
|
||||
ProviderFixtures.profile("webhook-main", "webhook", Channel.WEBHOOK),
|
||||
Channel.WEBHOOK,
|
||||
ProviderFixtures.webPush(),
|
||||
ProviderFixtures.webhook(),
|
||||
protector,
|
||||
new InAppRecipientRef("user-1"),
|
||||
Optional.empty());
|
||||
|
||||
+5
-2
@@ -143,7 +143,8 @@ class WebPushProviderAdapterTest {
|
||||
// Signing needs a PKCS#8 EC key, which VapidJwtSignerTest covers; this keeps the
|
||||
// transport test about TTL, encryption and status mapping.
|
||||
(endpoint, signingKey, publicKey) -> "vapid t=stub-token, k=" + publicKey,
|
||||
SecurityFixtures.keys(),
|
||||
new VapidKeyRegistry(
|
||||
SecurityFixtures.keys(), "vapid-1", java.util.Map.of("vapid-1", "BStubPublicKey")),
|
||||
properties,
|
||||
CLOCK),
|
||||
new WebPushFailureClassifier(),
|
||||
@@ -172,7 +173,9 @@ class WebPushProviderAdapterTest {
|
||||
URI.create(harness.baseUri() + "/push/subscription-1"),
|
||||
Rfc8291Aes128GcmEncryptor.encodePoint((ECPublicKey) pair.getPublic()),
|
||||
authSecret,
|
||||
"vapid-key-1");
|
||||
// The key id the registry knows: signing now resolves the key this subscription was
|
||||
// created against rather than whichever key happens to be active.
|
||||
"vapid-1");
|
||||
} catch (java.security.GeneralSecurityException failure) {
|
||||
throw new IllegalStateException(failure);
|
||||
}
|
||||
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.security;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Three layers, three different numbers, and a body that satisfied all of them but one.
|
||||
*
|
||||
* <p>Configuration permitted a one-mebibyte callback body. The MVC controller hard-coded 65,536.
|
||||
* The database check constrains the <em>ciphertext</em> to 65,536, and AES-GCM adds a 12-byte nonce
|
||||
* and a 16-byte tag. So a body of exactly 65,536 bytes passed configuration, passed the controller,
|
||||
* was truncated to 65,536 by the protector, became 65,564 bytes of ciphertext, and was rejected by
|
||||
* a CHECK constraint — after the provider had been answered.
|
||||
*
|
||||
* <p>The bound is now derived from the column in one place and everything else is checked against
|
||||
* it. This test is what makes the derivation stay true: it reads the constraint out of the
|
||||
* migration rather than restating it.
|
||||
*/
|
||||
class CallbackPayloadBoundTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("the configured ceiling and the derived ceiling are the same number")
|
||||
void theConfiguredCeilingMatchesTheDerivedOne() {
|
||||
assertThat(
|
||||
dev.caskeleton.adapter.outbound.notification.platform.autoconfigure
|
||||
.NotificationPlatformSettings.Callbacks.defaults()
|
||||
.maxBodyBytes())
|
||||
.as("configuration permitted a mebibyte while the column held 65,536")
|
||||
.isEqualTo(AesGcmCallbackPayloadProtection.MAX_PLAINTEXT_BYTES);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the plaintext ceiling leaves room for the envelope")
|
||||
void thePlaintextCeilingLeavesRoomForTheEnvelope() {
|
||||
assertThat(
|
||||
AesGcmCallbackPayloadProtection.MAX_PLAINTEXT_BYTES
|
||||
+ AesGcmCallbackPayloadProtection.ENVELOPE_OVERHEAD_BYTES)
|
||||
.isEqualTo(AesGcmCallbackPayloadProtection.MAX_CIPHERTEXT_BYTES);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a retention setting the column cannot hold is refused at construction")
|
||||
void aRetentionTheColumnCannotHoldIsRefused() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
new AesGcmCallbackPayloadProtection(
|
||||
SecurityFixtures.keys(),
|
||||
new java.security.SecureRandom(),
|
||||
AesGcmCallbackPayloadProtection.MAX_CIPHERTEXT_BYTES))
|
||||
.as("this is exactly the configuration that produced 65,564 bytes of ciphertext")
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("plaintext ceiling");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a payload at the ceiling encrypts to something the column accepts")
|
||||
void aPayloadAtTheCeilingFitsTheColumn() {
|
||||
AesGcmCallbackPayloadProtection protection =
|
||||
new AesGcmCallbackPayloadProtection(
|
||||
SecurityFixtures.keys(),
|
||||
new java.security.SecureRandom(),
|
||||
AesGcmCallbackPayloadProtection.MAX_PLAINTEXT_BYTES);
|
||||
|
||||
byte[] stored =
|
||||
protection.protectRawPayload(new byte[AesGcmCallbackPayloadProtection.MAX_PLAINTEXT_BYTES]);
|
||||
|
||||
assertThat(stored.length)
|
||||
.isLessThanOrEqualTo(AesGcmCallbackPayloadProtection.MAX_CIPHERTEXT_BYTES);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a larger payload is truncated to the ceiling, still inside the column")
|
||||
void aLargerPayloadIsTruncatedAndStillFits() {
|
||||
AesGcmCallbackPayloadProtection protection =
|
||||
new AesGcmCallbackPayloadProtection(
|
||||
SecurityFixtures.keys(),
|
||||
new java.security.SecureRandom(),
|
||||
AesGcmCallbackPayloadProtection.MAX_PLAINTEXT_BYTES);
|
||||
|
||||
byte[] stored =
|
||||
protection.protectRawPayload(
|
||||
new byte[AesGcmCallbackPayloadProtection.MAX_CIPHERTEXT_BYTES * 2]);
|
||||
|
||||
assertThat(stored.length)
|
||||
.isLessThanOrEqualTo(AesGcmCallbackPayloadProtection.MAX_CIPHERTEXT_BYTES);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a retention inside the ceiling is accepted")
|
||||
void aRetentionInsideTheCeilingIsAccepted() {
|
||||
assertThatCode(
|
||||
() ->
|
||||
new AesGcmCallbackPayloadProtection(
|
||||
SecurityFixtures.keys(),
|
||||
new java.security.SecureRandom(),
|
||||
AesGcmCallbackPayloadProtection.MAX_PLAINTEXT_BYTES))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.security;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Whether a rotation can go backwards.
|
||||
*
|
||||
* <p>Activation read the current generation, checked that the candidate superseded it, and then
|
||||
* stored — three steps with nothing holding the value still in between. Two rotations racing both
|
||||
* read the same predecessor, both concluded they superseded it, and whichever wrote last won. That
|
||||
* ordering is decided by thread scheduling, so generation 2 could land after generation 3 and the
|
||||
* profile would go on using credentials a later rotation had already retired — while every attempt
|
||||
* record attributes the work to a generation number that no longer means what it says.
|
||||
*/
|
||||
class ProviderCredentialManagerConcurrencyTest {
|
||||
|
||||
private static final ProviderProfileId PROFILE = new ProviderProfileId("apns-main");
|
||||
private static final Clock CLOCK =
|
||||
Clock.fixed(Instant.parse("2026-08-15T09:00:00Z"), ZoneOffset.UTC);
|
||||
|
||||
@Test
|
||||
@DisplayName("a lower generation never survives a concurrent activation")
|
||||
void aLowerGenerationNeverSurvives() throws InterruptedException {
|
||||
for (int round = 0; round < 100; round++) {
|
||||
var manager = new ProviderCredentialManager(SecurityFixtures.keys(), CLOCK);
|
||||
int rotations = 16;
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
CountDownLatch done = new CountDownLatch(rotations);
|
||||
|
||||
try (ExecutorService pool = Executors.newFixedThreadPool(8)) {
|
||||
for (int number = 1; number <= rotations; number++) {
|
||||
long generation = number;
|
||||
pool.execute(
|
||||
() -> {
|
||||
try {
|
||||
start.await();
|
||||
manager.activate(
|
||||
new CredentialGeneration(PROFILE, generation, "cred-1", Optional.empty()));
|
||||
} catch (IllegalArgumentException superseded) {
|
||||
// Losing the race is refused rather than applied out of order.
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
} finally {
|
||||
done.countDown();
|
||||
}
|
||||
});
|
||||
}
|
||||
start.countDown();
|
||||
assertThat(done.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
}
|
||||
|
||||
assertThat(manager.current(PROFILE).orElseThrow().generation())
|
||||
.as("round %d: whatever won, no later activation may be undone by an earlier one", round)
|
||||
.isEqualTo(rotations);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("activation is idempotent about nothing: the same number twice is refused")
|
||||
void theSameNumberTwiceIsRefused() {
|
||||
var manager = new ProviderCredentialManager(SecurityFixtures.keys(), CLOCK);
|
||||
manager.activate(new CredentialGeneration(PROFILE, 3, "cred-1", Optional.empty()));
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
manager.activate(new CredentialGeneration(PROFILE, 3, "cred-1", Optional.empty())))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("supersede");
|
||||
assertThat(manager.current(PROFILE).orElseThrow().generation()).isEqualTo(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an unresolvable key id fails the rotation and leaves the active generation alone")
|
||||
void anUnresolvableKeyLeavesTheActiveGenerationAlone() {
|
||||
var manager = new ProviderCredentialManager(SecurityFixtures.keys(), CLOCK);
|
||||
manager.activate(new CredentialGeneration(PROFILE, 1, "cred-1", Optional.empty()));
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
manager.activate(
|
||||
new CredentialGeneration(PROFILE, 2, "no-such-key", Optional.empty())))
|
||||
.isInstanceOf(RuntimeException.class);
|
||||
assertThat(manager.current(PROFILE).orElseThrow().generation())
|
||||
.as("a rotation that cannot resolve its key must not displace one that can")
|
||||
.isEqualTo(1);
|
||||
}
|
||||
}
|
||||
+6
@@ -27,6 +27,12 @@ public final class SecurityFixtures {
|
||||
SecretPurpose.PAYLOAD_ENCRYPTION,
|
||||
new SecretKeyMaterial(
|
||||
"payload-1", SecretPurpose.PAYLOAD_ENCRYPTION, filled((byte) 0x55, 32)),
|
||||
SecretPurpose.PROVIDER_REQUEST_LOOKUP_HMAC,
|
||||
new SecretKeyMaterial(
|
||||
"req-1", SecretPurpose.PROVIDER_REQUEST_LOOKUP_HMAC, filled((byte) 0x77, 32)),
|
||||
SecretPurpose.CALLBACK_FINGERPRINT_HMAC,
|
||||
new SecretKeyMaterial(
|
||||
"fp-1", SecretPurpose.CALLBACK_FINGERPRINT_HMAC, filled((byte) 0x88, 32)),
|
||||
SecretPurpose.VAPID_SIGNING,
|
||||
new SecretKeyMaterial("vapid-1", SecretPurpose.VAPID_SIGNING, filled((byte) 0x66, 32))),
|
||||
Map.of());
|
||||
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.template;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.api.error.TemplateRenderingException;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Whether a caller's value can become something other than text.
|
||||
*
|
||||
* <p>One engine rendered every slot by raw substitution — subject lines, plain text, HTML bodies,
|
||||
* SMS and deep links all through the same path. In an HTML body that makes a caller's value active
|
||||
* markup: the platform puts the caller's script in its own email. In a subject it lets a newline
|
||||
* split the header. And a deep link was parsed as a URI and otherwise accepted, which {@code
|
||||
* javascript:} satisfies.
|
||||
*
|
||||
* <p>The Thymeleaf alternative had the mirror-image fault — rendering plain text in HTML mode turns
|
||||
* {@code a & b} into an escaped entity in an SMS, where nothing will decode it.
|
||||
*/
|
||||
class SlotAwareRenderingTest {
|
||||
|
||||
private final PlaceholderTemplateEngine engine = new PlaceholderTemplateEngine();
|
||||
|
||||
@Test
|
||||
@DisplayName("an HTML body escapes a value that would otherwise be markup")
|
||||
void anHtmlBodyEscapesMarkup() {
|
||||
String rendered =
|
||||
engine.render(
|
||||
TemplateSlotMode.HTML_TEXT,
|
||||
"<p>Hello {name}</p>",
|
||||
Map.of("name", "<img src=x onerror=alert(1)>"));
|
||||
|
||||
assertThat(rendered)
|
||||
.as("the template's own markup survives; the caller's does not")
|
||||
.startsWith("<p>Hello ")
|
||||
.doesNotContain("<img")
|
||||
.contains("<img");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an HTML attribute cannot be broken out of")
|
||||
void anHtmlAttributeCannotBeBrokenOutOf() {
|
||||
String rendered =
|
||||
engine.render(
|
||||
TemplateSlotMode.HTML_TEXT,
|
||||
"<a title=\"{name}\">link</a>",
|
||||
Map.of("name", "\" onmouseover=\"steal()"));
|
||||
|
||||
assertThat(rendered)
|
||||
.as("closing the quote needs no angle bracket, so quotes are escaped too")
|
||||
.doesNotContain("onmouseover=\"steal")
|
||||
.contains(""");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("plain text is left exactly as the caller wrote it")
|
||||
void plainTextIsLeftAlone() {
|
||||
String rendered = engine.render(TemplateSlotMode.TEXT, "{value}", Map.of("value", "a & b < c"));
|
||||
|
||||
assertThat(rendered)
|
||||
.as("an SMS has no markup to decode an entity, so escaping there corrupts the message")
|
||||
.isEqualTo("a & b < c");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a subject refuses a value containing a newline")
|
||||
void aSubjectRefusesANewline() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
engine.render(
|
||||
TemplateSlotMode.SUBJECT,
|
||||
"Receipt for {order}",
|
||||
Map.of("order", "1\r\nBcc: attacker@example.com")))
|
||||
.as("everything after the newline is read as a new header")
|
||||
.isInstanceOf(TemplateRenderingException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a subject accepts ordinary text")
|
||||
void aSubjectAcceptsOrdinaryText() {
|
||||
assertThat(engine.render(TemplateSlotMode.SUBJECT, "Receipt {order}", Map.of("order", "1042")))
|
||||
.isEqualTo("Receipt 1042");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a link refuses script, data and file schemes")
|
||||
void aLinkRefusesDangerousSchemes() {
|
||||
for (String scheme : new String[] {"javascript:alert(1)", "data:text/html,x", "file:///etc"}) {
|
||||
assertThatThrownBy(
|
||||
() -> engine.render(TemplateSlotMode.URI, "{link}", Map.of("link", scheme)))
|
||||
.as("%s", scheme)
|
||||
.isInstanceOf(TemplateRenderingException.class);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a link accepts https and the application scheme")
|
||||
void aLinkAcceptsHttpsAndTheApplicationScheme() {
|
||||
assertThatCode(
|
||||
() -> {
|
||||
engine.render(
|
||||
TemplateSlotMode.URI, "{link}", Map.of("link", "https://example.com/order/1"));
|
||||
engine.render(TemplateSlotMode.URI, "{link}", Map.of("link", "caskeleton://order/1"));
|
||||
})
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("plain http is refused, because a reader cannot check the link")
|
||||
void plainHttpIsRefused() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
engine.render(
|
||||
TemplateSlotMode.URI, "{link}", Map.of("link", "http://example.com/order/1")))
|
||||
.isInstanceOf(TemplateRenderingException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the single-argument contract still renders plain text")
|
||||
void theSingleArgumentContractRendersPlainText() {
|
||||
assertThat(engine.render("{value}", Map.of("value", "a & b"))).isEqualTo("a & b");
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -2,7 +2,7 @@ package dev.caskeleton.adapter.outbound.notification.platform.testkit;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.application.notification.platform.callback.CallbackRequest;
|
||||
import dev.caskeleton.application.notification.platform.api.CallbackRequest;
|
||||
import dev.caskeleton.application.notification.platform.callback.CallbackVerificationResult;
|
||||
import dev.caskeleton.application.notification.platform.callback.NormalizedProviderEvent;
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderCallbackAdapter;
|
||||
|
||||
+9
-3
@@ -194,7 +194,11 @@ public final class ContractAdapters {
|
||||
// Signing needs a PKCS#8 EC key, which WebPushCryptoTest covers; the suites here
|
||||
// are about transport and evidence semantics.
|
||||
(endpoint, signingKey, publicKey) -> "vapid t=stub-token, k=" + publicKey,
|
||||
SecurityFixtures.keys(),
|
||||
new dev.caskeleton.adapter.outbound.notification.platform.provider.webpush
|
||||
.VapidKeyRegistry(
|
||||
SecurityFixtures.keys(),
|
||||
"vapid-1",
|
||||
java.util.Map.of("vapid-1", "BStubPublicKey")),
|
||||
properties,
|
||||
CLOCK),
|
||||
new WebPushFailureClassifier(),
|
||||
@@ -232,7 +236,7 @@ public final class ContractAdapters {
|
||||
ProviderFixtures.submission(
|
||||
ProviderFixtures.profile("webhook-main", "webhook", Channel.WEBHOOK),
|
||||
Channel.WEBHOOK,
|
||||
ProviderFixtures.webPush(),
|
||||
ProviderFixtures.webhook(),
|
||||
protector,
|
||||
new InAppRecipientRef("user-1"),
|
||||
Optional.empty()));
|
||||
@@ -249,7 +253,9 @@ public final class ContractAdapters {
|
||||
URI.create(harness.baseUri() + "/push/subscription-1"),
|
||||
Rfc8291Aes128GcmEncryptor.encodePoint((ECPublicKey) pair.getPublic()),
|
||||
authSecret,
|
||||
"vapid-key-1");
|
||||
// The key id the registry knows: signing now resolves the key this subscription was
|
||||
// created against rather than whichever key happens to be active.
|
||||
"vapid-1");
|
||||
} catch (java.security.GeneralSecurityException failure) {
|
||||
throw new IllegalStateException(failure);
|
||||
}
|
||||
|
||||
+17
@@ -98,6 +98,23 @@ public final class ProviderFixtures {
|
||||
}
|
||||
|
||||
/** Web Push content. */
|
||||
/**
|
||||
* The content a webhook actually carries.
|
||||
*
|
||||
* <p>The webhook fixtures used {@link #webPush()}, which the platform can never produce for a
|
||||
* WEBHOOK channel — {@code ProviderSubmission} now refuses the combination, and the suite that
|
||||
* used it was proving things about a shape no dispatch emits.
|
||||
*/
|
||||
public static dev.caskeleton.application.notification.platform.api.content.InAppContent
|
||||
webhook() {
|
||||
return new dev.caskeleton.application.notification.platform.api.content.InAppContent(
|
||||
"Order shipped",
|
||||
"Your order is on its way.",
|
||||
java.util.Optional.empty(),
|
||||
java.util.List.of(),
|
||||
"order");
|
||||
}
|
||||
|
||||
public static WebPushContent webPush() {
|
||||
return new WebPushContent(
|
||||
"Contract title", "Contract body", Optional.empty(), Map.of(), WebPushOptions.DEFAULT);
|
||||
|
||||
@@ -58,7 +58,7 @@ cd src
|
||||
./gradlew :adapter:outbound:persistence-jpa:jpaPlatformFailureTest # deadlock, commit ambiguity
|
||||
./gradlew :adapter:outbound:persistence-jpa:jpaPlatformQueryPlanTest # EXPLAIN structure
|
||||
./gradlew :adapter:outbound:persistence-jpa:jpaPlatformSecurityTest # runtime role privileges
|
||||
./gradlew :adapter:outbound:persistence-jpa:jpaPlatformPerformanceTest # pool pressure
|
||||
./gradlew :adapter:outbound:persistence-jpa:jpaPlatformPoolContractTest # pool behaviour
|
||||
./gradlew jpaReleaseGate # every gate, from the root
|
||||
```
|
||||
|
||||
|
||||
@@ -42,6 +42,29 @@ configurations {
|
||||
jpaPlatformPerformanceTestRuntimeOnly.extendsFrom testRuntimeOnly
|
||||
}
|
||||
|
||||
// The testkit as a consumable artifact.
|
||||
//
|
||||
// The rule pack in `testkit` was only ever exercised by its own fixture tests: nothing imported the
|
||||
// production graph and ran the rules against it, so "controller must not expose an entity" and
|
||||
// "domain must not depend on Hibernate" were verified as library code and applied to nothing. The
|
||||
// composition root is the only place that can see every runtime leaf at once, so it is where the
|
||||
// production suite belongs — and it needs the rules.
|
||||
tasks.register('testkitJar', Jar) {
|
||||
archiveClassifier = 'testkit'
|
||||
from sourceSets.testkit.output
|
||||
}
|
||||
|
||||
configurations {
|
||||
jpaTestkit {
|
||||
canBeConsumed = true
|
||||
canBeResolved = false
|
||||
}
|
||||
}
|
||||
|
||||
artifacts {
|
||||
jpaTestkit(tasks.named('testkitJar', Jar))
|
||||
}
|
||||
|
||||
// Every test lane compiles and runs against the testkit.
|
||||
sourceSets.test {
|
||||
compileClasspath += sourceSets.testkit.output
|
||||
@@ -172,6 +195,12 @@ def postgresqlFileserverMetadataIntegrationTest = registerPostgreSqlReadinessTes
|
||||
def postgresqlFileserverReclamationIntegrationTest = registerPostgreSqlReadinessTest(
|
||||
'postgresqlFileserverReclamationIntegrationTest',
|
||||
'dev.caskeleton.adapter.outbound.persistence.readiness.PostgreSqlFileserverReclamationIntegrationTest')
|
||||
// The notification stream is opt-in and lives outside the default Flyway location, so "is it
|
||||
// applied and promoted" is a real deployment question with a real wrong answer. This lane asks it
|
||||
// against a real server; the entity-scan half is a unit test.
|
||||
def postgresqlNotificationSchemaActivationIntegrationTest = registerPostgreSqlReadinessTest(
|
||||
'postgresqlNotificationSchemaActivationIntegrationTest',
|
||||
'dev.caskeleton.adapter.outbound.persistence.readiness.PostgreSqlNotificationSchemaActivationIntegrationTest')
|
||||
|
||||
def verifyJpaSqlConstructionSafety = tasks.register('verifyJpaSqlConstructionSafety') {
|
||||
group = 'verification'
|
||||
@@ -281,19 +310,29 @@ def jpaPlatformSecurityTest = registerJpaPlatformLane(
|
||||
'jpa-security',
|
||||
'Verifies runtime role privileges and search_path safety (design §36).')
|
||||
|
||||
// Machine-dependent bounds live in their own source set and never gate an ordinary build: attaching
|
||||
// them to `check` would make a laptop's `check` fail for reasons that are not about the code.
|
||||
def jpaPlatformPerformanceTest = tasks.register('jpaPlatformPerformanceTest', Test) {
|
||||
// The pool behaviour contract. Named for what it does.
|
||||
//
|
||||
// It was `jpaPlatformPerformanceTest`, described as certifying pool and REQUIRES_NEW pressure, and
|
||||
// gated by `performance.assertions.enabled` — which defaulted to false everywhere, including in the
|
||||
// nightly workflow that set it explicitly to false. So the release gate depended on a lane whose
|
||||
// only threshold assertion was that thresholds were not being asserted. "Certified" described a
|
||||
// run in which no latency or throughput bound was ever compared to anything.
|
||||
//
|
||||
// What the lane genuinely verifies is a behaviour contract: a REQUIRES_NEW depth of one needs two
|
||||
// connections per concurrent thread, a saturated pool reports its pending count, and a caller waits
|
||||
// rather than proceeding without a connection. Those are true on any machine, so they need no flag
|
||||
// — and this name does not promise a number nobody measured. A real performance gate needs a
|
||||
// dedicated runner, warmup and sample counts, and recorded thresholds; when that exists it belongs
|
||||
// in a lane of its own rather than behind a boolean on this one.
|
||||
def jpaPlatformPoolContractTest = tasks.register('jpaPlatformPoolContractTest', Test) {
|
||||
group = 'verification'
|
||||
description = 'Certifies Hikari pool and REQUIRES_NEW connection pressure (design §38).'
|
||||
description = 'Verifies Hikari pool and REQUIRES_NEW connection behaviour (design §38).'
|
||||
testClassesDirs = sourceSets.jpaPlatformPerformanceTest.output.classesDirs
|
||||
classpath = sourceSets.jpaPlatformPerformanceTest.runtimeClasspath
|
||||
useJUnitPlatform()
|
||||
failOnNoDiscoveredTests = true
|
||||
outputs.upToDateWhen { false }
|
||||
jvmArgs('-Duser.timezone=UTC')
|
||||
systemProperty 'performance.assertions.enabled',
|
||||
(project.findProperty('performance.assertions.enabled') ?: 'false').toString()
|
||||
}
|
||||
|
||||
// The JPA release gate (design §41). Aggregates every lane whose absence would let one of the
|
||||
@@ -307,7 +346,7 @@ tasks.register('jpaPlatformReleaseGate') {
|
||||
dependsOn jpaPlatformFailureTest
|
||||
dependsOn jpaPlatformQueryPlanTest
|
||||
dependsOn jpaPlatformSecurityTest
|
||||
dependsOn jpaPlatformPerformanceTest
|
||||
dependsOn jpaPlatformPoolContractTest
|
||||
}
|
||||
|
||||
apply from: rootProject.file('gradle/jpa-evidence.gradle')
|
||||
|
||||
+3
-8
@@ -21,9 +21,6 @@ import org.junit.jupiter.api.Test;
|
||||
*/
|
||||
class PoolPressureContractTest {
|
||||
|
||||
private static final boolean ASSERTIONS_ENABLED =
|
||||
Boolean.parseBoolean(System.getProperty("performance.assertions.enabled", "false"));
|
||||
|
||||
@Test
|
||||
@DisplayName("pending count and acquire latency are reported together")
|
||||
void reportsPendingAndAcquireLatencyTogether() {
|
||||
@@ -43,10 +40,8 @@ class PoolPressureContractTest {
|
||||
|
||||
int required = concurrentThreads * (1 + maxRequiresNewDepth) + 1;
|
||||
|
||||
assertThat(required).isEqualTo(17);
|
||||
if (!ASSERTIONS_ENABLED) {
|
||||
// Machine-dependent bounds are not asserted in this run; the arithmetic above is.
|
||||
assertThat(ASSERTIONS_ENABLED).isFalse();
|
||||
}
|
||||
assertThat(required)
|
||||
.as("this is a property of REQUIRES_NEW, not of the machine, so it needs no opt-in")
|
||||
.isEqualTo(17);
|
||||
}
|
||||
}
|
||||
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.api.error;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName;
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Turns a vendor's raw failure into a classified one.
|
||||
*
|
||||
* <p>Declared in {@code api} so the vendor-neutral transaction package can call a vendor translator
|
||||
* without naming one. Depending on {@code postgresql} directly from {@code transaction} closes a
|
||||
* cycle — the PostgreSQL configuration already depends on the transaction package for its SPI — and
|
||||
* a cycle between two packages the module map describes as layered has to be unpicked from both
|
||||
* ends before either can move.
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface VendorFailureTranslator {
|
||||
|
||||
/**
|
||||
* Classifies one failure.
|
||||
*
|
||||
* @param failure what the attempt threw
|
||||
* @param operation the registered operation the attempt served
|
||||
* @param attempt the 1-based attempt number
|
||||
* @param elapsed how long the attempt ran
|
||||
* @param traceId the current trace identifier, or {@code null}
|
||||
*/
|
||||
JpaPersistenceException translate(
|
||||
Throwable failure,
|
||||
PersistenceOperationName operation,
|
||||
int attempt,
|
||||
Duration elapsed,
|
||||
String traceId);
|
||||
}
|
||||
+48
-3
@@ -50,13 +50,33 @@ public final class SignedJsonCursorCodec<C> implements CursorCodec<C> {
|
||||
this.key = key.clone();
|
||||
}
|
||||
|
||||
/** Longest token this codec will look at, in characters. */
|
||||
public static final int MAX_ENCODED_LENGTH = 4096;
|
||||
|
||||
/** Largest payload this codec will decode, in bytes. */
|
||||
public static final int MAX_PAYLOAD_BYTES = 2048;
|
||||
|
||||
/** HMAC-SHA256 produces exactly this many bytes. */
|
||||
private static final int MAC_LENGTH_BYTES = 32;
|
||||
|
||||
@Override
|
||||
public String encode(C cursor) {
|
||||
Objects.requireNonNull(cursor, "cursor");
|
||||
String payload =
|
||||
encoder.encodeToString(payloadCodec.toJson(cursor).getBytes(StandardCharsets.UTF_8));
|
||||
byte[] json = payloadCodec.toJson(cursor).getBytes(StandardCharsets.UTF_8);
|
||||
if (json.length > MAX_PAYLOAD_BYTES) {
|
||||
// The application's own bug, not a caller's: a cursor this large is a payload the codec
|
||||
// cannot hand back, so producing it would mint a token that fails on the next page.
|
||||
throw new IllegalStateException(
|
||||
"cursor payload is " + json.length + " bytes; the bound is " + MAX_PAYLOAD_BYTES);
|
||||
}
|
||||
String payload = encoder.encodeToString(json);
|
||||
String signed = VERSION + SEPARATOR + payload;
|
||||
return signed + SEPARATOR + encoder.encodeToString(mac(signed));
|
||||
String token = signed + SEPARATOR + encoder.encodeToString(mac(signed));
|
||||
if (token.length() > MAX_ENCODED_LENGTH) {
|
||||
throw new IllegalStateException(
|
||||
"cursor token is " + token.length() + " characters; the bound is " + MAX_ENCODED_LENGTH);
|
||||
}
|
||||
return token;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -64,6 +84,13 @@ public final class SignedJsonCursorCodec<C> implements CursorCodec<C> {
|
||||
if (encoded == null || encoded.isBlank()) {
|
||||
throw new IllegalArgumentException("cursor must not be blank");
|
||||
}
|
||||
// First line, before any substring, decode or MAC. A paging endpoint is public, and everything
|
||||
// below this point allocates in proportion to what the caller sent: repeatedly posting a very
|
||||
// large token made the server build strings, byte arrays and a MAC input before it had any
|
||||
// reason to believe the token was real. A page-size bound does not bound the token.
|
||||
if (encoded.length() > MAX_ENCODED_LENGTH) {
|
||||
throw new IllegalArgumentException("cursor exceeds the maximum token length");
|
||||
}
|
||||
int payloadSeparator = encoded.indexOf(SEPARATOR);
|
||||
int macSeparator = encoded.lastIndexOf(SEPARATOR);
|
||||
if (payloadSeparator <= 0 || macSeparator <= payloadSeparator) {
|
||||
@@ -73,15 +100,33 @@ public final class SignedJsonCursorCodec<C> implements CursorCodec<C> {
|
||||
if (!VERSION.equals(version)) {
|
||||
throw new IllegalArgumentException("unknown cursor version");
|
||||
}
|
||||
// Base64 expands by 4/3, so the encoded payload segment's length bounds the decoded size
|
||||
// exactly. Checking it here refuses an oversized payload without allocating it first.
|
||||
int encodedPayloadLength = macSeparator - payloadSeparator - 1;
|
||||
if (decodedLengthOf(encodedPayloadLength) > MAX_PAYLOAD_BYTES) {
|
||||
throw new IllegalArgumentException("cursor payload exceeds the maximum size");
|
||||
}
|
||||
String signed = encoded.substring(0, macSeparator);
|
||||
byte[] presented = decodeBase64(encoded.substring(macSeparator + 1));
|
||||
if (presented.length != MAC_LENGTH_BYTES) {
|
||||
// Checked before the comparison. `MessageDigest.isEqual` is constant time for equal-length
|
||||
// inputs; feeding it a differently-sized array asks it a question it was not built to answer.
|
||||
throw new IllegalArgumentException("malformed cursor");
|
||||
}
|
||||
if (!MessageDigest.isEqual(mac(signed), presented)) {
|
||||
throw new IllegalArgumentException("cursor signature does not verify");
|
||||
}
|
||||
// The payload decoder runs only after the signature verified, so an unsigned token never
|
||||
// reaches the application's JSON parsing at all.
|
||||
byte[] payload = decodeBase64(encoded.substring(payloadSeparator + 1, macSeparator));
|
||||
return payloadCodec.fromJson(new String(payload, StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
/** The number of bytes a Base64 segment of this many characters decodes to, at most. */
|
||||
private static int decodedLengthOf(int encodedLength) {
|
||||
return encodedLength / 4 * 3 + 3;
|
||||
}
|
||||
|
||||
private byte[] decodeBase64(String value) {
|
||||
try {
|
||||
return decoder.decode(value);
|
||||
|
||||
+7
-3
@@ -1,9 +1,7 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.transaction;
|
||||
package dev.caskeleton.adapter.outbound.persistence.api.transaction;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName;
|
||||
import dev.caskeleton.adapter.outbound.persistence.api.error.JpaPersistenceException;
|
||||
import dev.caskeleton.adapter.outbound.persistence.api.transaction.RetryDecision;
|
||||
import dev.caskeleton.adapter.outbound.persistence.api.transaction.TransactionAttempt;
|
||||
|
||||
/**
|
||||
* Attempt-level events from the retry coordinator (design §19, §37).
|
||||
@@ -11,6 +9,12 @@ import dev.caskeleton.adapter.outbound.persistence.api.transaction.TransactionAt
|
||||
* <p>The split between attempt events and the one terminal event is deliberate. Logging every
|
||||
* retried attempt at WARN turns a healthy, self-correcting contention pattern into a page; the
|
||||
* attempt events belong in metrics, and only the final outcome is worth a log line.
|
||||
*
|
||||
* <p>Declared in {@code api} because both sides need it and neither should depend on the other.
|
||||
* While it lived in {@code transaction}, the observation package imported it — so the package whose
|
||||
* whole job is to watch the platform was a dependency of nothing and a dependent of the engine,
|
||||
* which is the wrong way round for a cross-cutting SPI and closes a loop the moment the engine
|
||||
* wants to report anything.
|
||||
*/
|
||||
public interface RetryEventListener {
|
||||
|
||||
+66
-6
@@ -5,11 +5,71 @@ import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
|
||||
|
||||
/**
|
||||
* Registers this module's JPA entities and Spring Data repositories. The explicit scans are
|
||||
* required because the Boot main class lives in another package; the simple name avoids {@code
|
||||
* JpaConfig} to dodge a bean-name collision with the sample module. See README "config".
|
||||
* Registers this module's always-installed JPA entities and Spring Data repositories.
|
||||
*
|
||||
* <p>The explicit scans are required because the Boot main class lives in another package; the
|
||||
* simple name avoids {@code JpaConfig} to dodge a bean-name collision with the sample module. See
|
||||
* README "config".
|
||||
*
|
||||
* <p>The package list is enumerated rather than given as the persistence root, and the omission is
|
||||
* deliberate: {@code ...persistence.notification} is an <em>opt-in</em> capability whose schema
|
||||
* stream is not in the default Flyway location. Scanning the whole root put its entities into the
|
||||
* persistence unit unconditionally, so a deployment that never enabled notification still had
|
||||
* {@code ddl-auto=validate} looking for {@code notification_request} — and failed to boot over a
|
||||
* capability it had switched off. {@code NotificationJpaPersistenceConfig} owns that scan and only
|
||||
* when the capability is on.
|
||||
*
|
||||
* <p>A new always-installed sub-package must be added here; leaving it out is a silent omission
|
||||
* rather than a compile error, which is what {@code PersistenceEntityScanCoverageTest} checks.
|
||||
*/
|
||||
@Configuration
|
||||
@EntityScan(basePackages = "dev.caskeleton.adapter.outbound.persistence")
|
||||
@EnableJpaRepositories(basePackages = "dev.caskeleton.adapter.outbound.persistence")
|
||||
public class PersistenceJpaConfig {}
|
||||
@EntityScan(
|
||||
basePackages = {
|
||||
"dev.caskeleton.adapter.outbound.persistence.api",
|
||||
"dev.caskeleton.adapter.outbound.persistence.audit",
|
||||
"dev.caskeleton.adapter.outbound.persistence.auditing",
|
||||
"dev.caskeleton.adapter.outbound.persistence.cache",
|
||||
"dev.caskeleton.adapter.outbound.persistence.envers",
|
||||
"dev.caskeleton.adapter.outbound.persistence.experimental",
|
||||
"dev.caskeleton.adapter.outbound.persistence.failure",
|
||||
"dev.caskeleton.adapter.outbound.persistence.fileserver",
|
||||
"dev.caskeleton.adapter.outbound.persistence.hibernate",
|
||||
"dev.caskeleton.adapter.outbound.persistence.idempotency",
|
||||
"dev.caskeleton.adapter.outbound.persistence.lock",
|
||||
"dev.caskeleton.adapter.outbound.persistence.migration",
|
||||
"dev.caskeleton.adapter.outbound.persistence.observation",
|
||||
"dev.caskeleton.adapter.outbound.persistence.outbox",
|
||||
"dev.caskeleton.adapter.outbound.persistence.postgresql",
|
||||
"dev.caskeleton.adapter.outbound.persistence.querydsl",
|
||||
"dev.caskeleton.adapter.outbound.persistence.security",
|
||||
"dev.caskeleton.adapter.outbound.persistence.springdata",
|
||||
"dev.caskeleton.adapter.outbound.persistence.transaction"
|
||||
})
|
||||
@EnableJpaRepositories(
|
||||
basePackages = {
|
||||
"dev.caskeleton.adapter.outbound.persistence.api",
|
||||
"dev.caskeleton.adapter.outbound.persistence.audit",
|
||||
"dev.caskeleton.adapter.outbound.persistence.auditing",
|
||||
"dev.caskeleton.adapter.outbound.persistence.cache",
|
||||
"dev.caskeleton.adapter.outbound.persistence.envers",
|
||||
"dev.caskeleton.adapter.outbound.persistence.experimental",
|
||||
"dev.caskeleton.adapter.outbound.persistence.failure",
|
||||
"dev.caskeleton.adapter.outbound.persistence.fileserver",
|
||||
"dev.caskeleton.adapter.outbound.persistence.hibernate",
|
||||
"dev.caskeleton.adapter.outbound.persistence.idempotency",
|
||||
"dev.caskeleton.adapter.outbound.persistence.lock",
|
||||
"dev.caskeleton.adapter.outbound.persistence.migration",
|
||||
"dev.caskeleton.adapter.outbound.persistence.observation",
|
||||
"dev.caskeleton.adapter.outbound.persistence.outbox",
|
||||
"dev.caskeleton.adapter.outbound.persistence.postgresql",
|
||||
"dev.caskeleton.adapter.outbound.persistence.querydsl",
|
||||
"dev.caskeleton.adapter.outbound.persistence.security",
|
||||
"dev.caskeleton.adapter.outbound.persistence.springdata",
|
||||
"dev.caskeleton.adapter.outbound.persistence.transaction"
|
||||
})
|
||||
public class PersistenceJpaConfig {
|
||||
|
||||
/** The one sub-package deliberately excluded above, named so a test can assert the exclusion. */
|
||||
public static final String OPT_IN_NOTIFICATION_PACKAGE =
|
||||
"dev.caskeleton.adapter.outbound.persistence.notification";
|
||||
}
|
||||
|
||||
+30
-1
@@ -1,5 +1,7 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.experimental.replica;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.experimental.ExperimentalFeature;
|
||||
import dev.caskeleton.adapter.outbound.persistence.experimental.ExperimentalFeatureGate;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
@@ -19,12 +21,39 @@ import java.util.Objects;
|
||||
* <p>The decision is made once and holds for the whole transaction. Switching mid-transaction would
|
||||
* mean two connections to two databases inside one unit of work, with no relationship between what
|
||||
* each of them sees.
|
||||
*
|
||||
* <p>Constructed only through {@code enabledBy}, and the constructor is package-private so that is
|
||||
* the only way. The gate's own documentation says that having this class on the classpath is not
|
||||
* consent to run it — but nothing enforced that: the public constructor meant one {@code new} was a
|
||||
* complete bypass, and replica routing is exactly the kind of behaviour whose accidental activation
|
||||
* is invisible until it routes a read somewhere wrong.
|
||||
*/
|
||||
public final class ConsistencyAwareDataSourceRouter {
|
||||
|
||||
private final ReplicaLagMonitor lagMonitor;
|
||||
|
||||
public ConsistencyAwareDataSourceRouter(ReplicaLagMonitor lagMonitor) {
|
||||
/**
|
||||
* The only way to obtain one.
|
||||
*
|
||||
* <p>The gate is asked before anything is constructed, so a deployment that never set the flag
|
||||
* cannot end up holding an instance. Taking the gate as a parameter rather than consulting a
|
||||
* static makes the requirement part of the signature: a caller cannot forget an argument the
|
||||
* compiler insists on.
|
||||
*
|
||||
* @param gate the experimental consent gate
|
||||
* @param flags the deployment's experimental flags
|
||||
* @throws IllegalStateException naming the property that must be set
|
||||
*/
|
||||
public static ConsistencyAwareDataSourceRouter enabledBy(
|
||||
ExperimentalFeatureGate gate,
|
||||
java.util.Map<String, Boolean> flags,
|
||||
ReplicaLagMonitor lagMonitor) {
|
||||
java.util.Objects.requireNonNull(gate, "gate")
|
||||
.requireEnabled(ExperimentalFeature.READ_REPLICA, flags);
|
||||
return new ConsistencyAwareDataSourceRouter(lagMonitor);
|
||||
}
|
||||
|
||||
ConsistencyAwareDataSourceRouter(ReplicaLagMonitor lagMonitor) {
|
||||
this.lagMonitor = Objects.requireNonNull(lagMonitor, "lagMonitor");
|
||||
}
|
||||
|
||||
|
||||
+6
@@ -16,6 +16,12 @@ import java.util.Objects;
|
||||
*
|
||||
* <p>The tenant is bound as a parameter, never concatenated. A tenant id is externally influenced
|
||||
* data, and {@code set_config} takes a string.
|
||||
*
|
||||
* <p>Constructed only through {@code enabledBy}, and the constructor is package-private so that is
|
||||
* the only way. The gate's own documentation says that having this class on the classpath is not
|
||||
* consent to run it — but nothing enforced that: the public constructor meant one {@code new} was a
|
||||
* complete bypass, and row-level security tenant binding is exactly the kind of behaviour whose
|
||||
* accidental activation is invisible until it routes a read somewhere wrong.
|
||||
*/
|
||||
public final class RlsTenantSessionBinder {
|
||||
|
||||
|
||||
+32
-1
@@ -1,5 +1,7 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.experimental.schema;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.experimental.ExperimentalFeature;
|
||||
import dev.caskeleton.adapter.outbound.persistence.experimental.ExperimentalFeatureGate;
|
||||
import dev.caskeleton.adapter.outbound.persistence.experimental.tenant.TenantId;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
@@ -22,6 +24,12 @@ import org.flywaydb.core.api.output.MigrateResult;
|
||||
* <p>Failures are recorded and the run continues, but nothing is repaired automatically: a checksum
|
||||
* mismatch on one tenant is the same evidence it is anywhere else, and repairing it here would
|
||||
* erase it N times over.
|
||||
*
|
||||
* <p>Constructed only through {@code enabledBy}, and the constructor is package-private so that is
|
||||
* the only way. The gate's own documentation says that having this class on the classpath is not
|
||||
* consent to run it — but nothing enforced that: the public constructor meant one {@code new} was a
|
||||
* complete bypass, and schema-per-tenant migration is exactly the kind of behaviour whose
|
||||
* accidental activation is invisible until it routes a read somewhere wrong.
|
||||
*/
|
||||
public final class SchemaTenantMigrationOrchestrator {
|
||||
|
||||
@@ -30,7 +38,30 @@ public final class SchemaTenantMigrationOrchestrator {
|
||||
private final String migrationLocation;
|
||||
private final Map<TenantId, TenantMigrationStatus> status = new LinkedHashMap<>();
|
||||
|
||||
public SchemaTenantMigrationOrchestrator(
|
||||
/**
|
||||
* The only way to obtain one.
|
||||
*
|
||||
* <p>The gate is asked before anything is constructed, so a deployment that never set the flag
|
||||
* cannot end up holding an instance. Taking the gate as a parameter rather than consulting a
|
||||
* static makes the requirement part of the signature: a caller cannot forget an argument the
|
||||
* compiler insists on.
|
||||
*
|
||||
* @param gate the experimental consent gate
|
||||
* @param flags the deployment's experimental flags
|
||||
* @throws IllegalStateException naming the property that must be set
|
||||
*/
|
||||
public static SchemaTenantMigrationOrchestrator enabledBy(
|
||||
ExperimentalFeatureGate gate,
|
||||
java.util.Map<String, Boolean> flags,
|
||||
DataSource dataSource,
|
||||
SchemaTenantRegistry registry,
|
||||
String migrationLocation) {
|
||||
java.util.Objects.requireNonNull(gate, "gate")
|
||||
.requireEnabled(ExperimentalFeature.MULTITENANCY_SCHEMA, flags);
|
||||
return new SchemaTenantMigrationOrchestrator(dataSource, registry, migrationLocation);
|
||||
}
|
||||
|
||||
SchemaTenantMigrationOrchestrator(
|
||||
DataSource dataSource, SchemaTenantRegistry registry, String migrationLocation) {
|
||||
this.dataSource = Objects.requireNonNull(dataSource, "dataSource");
|
||||
this.registry = Objects.requireNonNull(registry, "registry");
|
||||
|
||||
+40
-3
@@ -67,6 +67,38 @@ public class JpaCleanupQueue implements CleanupQueue {
|
||||
now));
|
||||
}
|
||||
|
||||
/** How long a claim is valid before the reaper may take the item over. */
|
||||
private static final java.time.Duration LEASE = java.time.Duration.ofMinutes(10);
|
||||
|
||||
/**
|
||||
* Tokens for claims this process holds.
|
||||
*
|
||||
* <p>Not persisted anywhere but the row. A settlement whose token is gone — because the process
|
||||
* restarted, or because the reaper handed the item to someone else — updates zero rows, which is
|
||||
* the correct answer rather than an error: this worker no longer owns the item.
|
||||
*/
|
||||
private final java.util.Map<UUID, UUID> heldClaims =
|
||||
new java.util.concurrent.ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* The owner recorded on a claim, distinct per process incarnation.
|
||||
*
|
||||
* <p>A configured name alone would make a restarted process indistinguishable from the one it
|
||||
* replaced, which is the same defect the notification dispatcher had.
|
||||
*/
|
||||
private final String owner = "fileserver-cleanup:" + UUID.randomUUID();
|
||||
|
||||
private void settle(
|
||||
UUID cleanupId, String status, Instant nextAttemptAt, String reasonCode, Instant now) {
|
||||
UUID token = heldClaims.remove(cleanupId);
|
||||
if (token == null) {
|
||||
// Nothing to settle: this process does not hold the claim. Writing anyway is exactly the
|
||||
// stale-worker overwrite the token exists to prevent.
|
||||
return;
|
||||
}
|
||||
items.recordAttempt(cleanupId, token, status, nextAttemptAt, reasonCode, now);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<CleanupItem> claimDue(Instant now, int limit) {
|
||||
if (limit < 1) {
|
||||
@@ -74,7 +106,12 @@ public class JpaCleanupQueue implements CleanupQueue {
|
||||
}
|
||||
List<CleanupItem> claimed = new ArrayList<>();
|
||||
for (CleanupItemEntity due : items.findDue(now, Limit.of(limit))) {
|
||||
if (items.claim(due.getCleanupId(), now) == 1) {
|
||||
UUID token = UUID.randomUUID();
|
||||
if (items.claim(due.getCleanupId(), owner, token, now.plus(LEASE), now) == 1) {
|
||||
// The token stays in this process. A restart loses it, and that is the point: a worker
|
||||
// that restarted no longer holds the claim, so its later completion must not apply. The
|
||||
// reaper takes the item over once the lease expires.
|
||||
heldClaims.put(due.getCleanupId(), token);
|
||||
claimed.add(toItem(due));
|
||||
}
|
||||
}
|
||||
@@ -84,14 +121,14 @@ public class JpaCleanupQueue implements CleanupQueue {
|
||||
@Override
|
||||
public void markDone(CleanupItem item) {
|
||||
Instant now = clock.instant();
|
||||
items.recordAttempt(item.cleanupId(), STATUS_DONE, now, null, now);
|
||||
settle(item.cleanupId(), STATUS_DONE, now, null, now);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markFailed(CleanupItem item, String reasonCode, Instant nextAttemptAt) {
|
||||
Instant now = clock.instant();
|
||||
boolean exhausted = item.attempt() + 1 >= MAXIMUM_ATTEMPTS;
|
||||
items.recordAttempt(
|
||||
settle(
|
||||
item.cleanupId(),
|
||||
exhausted ? STATUS_ABANDONED : STATUS_FAILED,
|
||||
nextAttemptAt,
|
||||
|
||||
+21
@@ -61,6 +61,27 @@ public class CleanupItemEntity {
|
||||
@Column(name = "created_at", nullable = false, updatable = false)
|
||||
private Instant createdAt;
|
||||
|
||||
/**
|
||||
* Who holds the claim, until when, and which claim this is.
|
||||
*
|
||||
* <p>The claim used to record only a status. A worker that performed the physical delete and then
|
||||
* died left the row IN_PROGRESS with nothing distinguishing it from work in progress, so nothing
|
||||
* reclaimed it — and completion matched on the item id alone, so a worker paused past its lease
|
||||
* could write over the one that replaced it.
|
||||
*/
|
||||
@Column(name = "claim_owner", length = 120)
|
||||
private String claimOwner;
|
||||
|
||||
@JdbcTypeCode(SqlTypes.UUID)
|
||||
@Column(name = "claim_token")
|
||||
private UUID claimToken;
|
||||
|
||||
@Column(name = "lease_until")
|
||||
private Instant leaseUntil;
|
||||
|
||||
@Column(name = "claim_fence", nullable = false)
|
||||
private long claimFence;
|
||||
|
||||
@Column(name = "updated_at", nullable = false)
|
||||
private Instant updatedAt;
|
||||
|
||||
|
||||
+45
-2
@@ -23,22 +23,43 @@ public interface FileserverCleanupRepository extends JpaRepository<CleanupItemEn
|
||||
List<CleanupItemEntity> findDue(@Param("now") Instant now, Limit limit);
|
||||
|
||||
/**
|
||||
* Takes ownership of one due item.
|
||||
* Takes ownership of one due item, recording who took it and until when.
|
||||
*
|
||||
* <p>The conditional status keeps two workers from running the same delete: whoever loses the
|
||||
* race updates zero rows and skips the item rather than deleting behind the winner.
|
||||
*
|
||||
* <p>The owner, token and lease are what make that ownership recoverable. Without them a worker
|
||||
* that performed the physical delete and then died left the row IN_PROGRESS with nothing to
|
||||
* distinguish it from an item a live worker is actively deleting — so nothing reclaimed it, and
|
||||
* the file's quota and lifecycle stayed unsettled indefinitely.
|
||||
*/
|
||||
@Modifying(clearAutomatically = true, flushAutomatically = true)
|
||||
@Query(
|
||||
"""
|
||||
update CleanupItemEntity c
|
||||
set c.status = 'IN_PROGRESS',
|
||||
c.claimOwner = :owner,
|
||||
c.claimToken = :token,
|
||||
c.leaseUntil = :leaseUntil,
|
||||
c.claimFence = c.claimFence + 1,
|
||||
c.updatedAt = :now
|
||||
where c.cleanupId = :cleanupId
|
||||
and c.status in ('PENDING', 'FAILED')
|
||||
""")
|
||||
int claim(@Param("cleanupId") UUID cleanupId, @Param("now") Instant now);
|
||||
int claim(
|
||||
@Param("cleanupId") UUID cleanupId,
|
||||
@Param("owner") String owner,
|
||||
@Param("token") UUID token,
|
||||
@Param("leaseUntil") Instant leaseUntil,
|
||||
@Param("now") Instant now);
|
||||
|
||||
/**
|
||||
* Records the outcome of an attempt, for the worker that still holds the claim.
|
||||
*
|
||||
* <p>Matched on the claim token. It used to match on {@code cleanup_id} alone, so a worker paused
|
||||
* past any reasonable lease could write DONE over an item another worker had since claimed and
|
||||
* was part-way through.
|
||||
*/
|
||||
@Modifying(clearAutomatically = true, flushAutomatically = true)
|
||||
@Query(
|
||||
"""
|
||||
@@ -47,13 +68,35 @@ public interface FileserverCleanupRepository extends JpaRepository<CleanupItemEn
|
||||
c.attempt = c.attempt + 1,
|
||||
c.nextAttemptAt = :nextAttemptAt,
|
||||
c.lastErrorCode = :lastErrorCode,
|
||||
c.claimOwner = null,
|
||||
c.claimToken = null,
|
||||
c.leaseUntil = null,
|
||||
c.updatedAt = :now
|
||||
where c.cleanupId = :cleanupId
|
||||
and c.claimToken = :token
|
||||
""")
|
||||
int recordAttempt(
|
||||
@Param("cleanupId") UUID cleanupId,
|
||||
@Param("token") UUID token,
|
||||
@Param("status") String status,
|
||||
@Param("nextAttemptAt") Instant nextAttemptAt,
|
||||
@Param("lastErrorCode") String lastErrorCode,
|
||||
@Param("now") Instant now);
|
||||
|
||||
/**
|
||||
* Returns items whose claim has expired, for a reaper to take over.
|
||||
*
|
||||
* <p>A {@code NULL} lease is excluded on purpose: it means the item was claimed before fencing
|
||||
* existed, and taking it over automatically would be guessing about work whose state nobody
|
||||
* recorded. That case needs an operator.
|
||||
*/
|
||||
@Query(
|
||||
"""
|
||||
select c from CleanupItemEntity c
|
||||
where c.status = 'IN_PROGRESS'
|
||||
and c.leaseUntil is not null
|
||||
and c.leaseUntil < :now
|
||||
order by c.leaseUntil
|
||||
""")
|
||||
List<CleanupItemEntity> findExpiredClaims(@Param("now") Instant now, Limit limit);
|
||||
}
|
||||
|
||||
+12
-2
@@ -49,10 +49,20 @@ public final class HibernateJpaBatchExecutor implements JpaBatchExecutor {
|
||||
persister.accept(item);
|
||||
processed++;
|
||||
maxManagedEntities = Math.max(maxManagedEntities, managedEntityCount());
|
||||
if (processed % profile.flushSize() == 0L) {
|
||||
boolean flushDue = processed % profile.flushSize() == 0L;
|
||||
boolean clearDue = processed % profile.clearSize() == 0L;
|
||||
// Clearing detaches every managed entity, and a detached entity that was never flushed is
|
||||
// simply gone: no INSERT is issued and nothing reports a problem. With flushSize=100 and
|
||||
// clearSize=150, the clear at 150 arrives when the flush condition is false, so rows 101–150
|
||||
// were dropped while the executor returned processed=300 and the table held 250.
|
||||
//
|
||||
// So a clear always flushes first. The two knobs stay independent — which is what the API
|
||||
// offers — and the invariant that makes independence safe is enforced here rather than left
|
||||
// to a caller choosing multiples.
|
||||
if (flushDue || clearDue) {
|
||||
entityManager.flush();
|
||||
}
|
||||
if (processed % profile.clearSize() == 0L) {
|
||||
if (clearDue) {
|
||||
entityManager.clear();
|
||||
}
|
||||
}
|
||||
|
||||
+13
-3
@@ -32,7 +32,10 @@ public final class HibernateStatelessSessionRunner implements StatelessSessionRu
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T execute(StatelessWorkName name, long maxRows, Function<StatelessSession, T> work) {
|
||||
public <T> T execute(
|
||||
StatelessWorkName name,
|
||||
long maxRows,
|
||||
Function<StatelessSession, StatelessWorkResult<T>> work) {
|
||||
Objects.requireNonNull(name, "name");
|
||||
Objects.requireNonNull(work, "work");
|
||||
if (!registeredWork.contains(name)) {
|
||||
@@ -45,9 +48,16 @@ public final class HibernateStatelessSessionRunner implements StatelessSessionRu
|
||||
try (StatelessSession session = sessionFactory.openStatelessSession()) {
|
||||
Transaction transaction = session.beginTransaction();
|
||||
try {
|
||||
T result = work.apply(session);
|
||||
StatelessWorkResult<T> result = work.apply(session);
|
||||
if (result.affectedRows() > maxRows) {
|
||||
// Rolled back before the check reports anything. Keeping an over-large write and
|
||||
// complaining about it afterwards is the worst of both: the caller declared the bound
|
||||
// because exceeding it means the query is wrong.
|
||||
transaction.rollback();
|
||||
throw new StatelessRowCapExceededException(name, maxRows, result.affectedRows());
|
||||
}
|
||||
transaction.commit();
|
||||
return result;
|
||||
return result.value();
|
||||
} catch (RuntimeException failure) {
|
||||
if (transaction.isActive()) {
|
||||
transaction.rollback();
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.hibernate.stateless;
|
||||
|
||||
/**
|
||||
* A stateless unit of work touched more rows than it declared.
|
||||
*
|
||||
* <p>Thrown after the transaction has been rolled back, so the over-large write is not partially
|
||||
* applied. Reporting it and keeping the rows would be worse than either alternative: the caller
|
||||
* declared a bound because exceeding it means something is wrong with the query, not with the
|
||||
* bound.
|
||||
*/
|
||||
public final class StatelessRowCapExceededException extends RuntimeException {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* Creates the exception.
|
||||
*
|
||||
* @param work the registered work name
|
||||
* @param maxRows the declared cap
|
||||
* @param affectedRows what the work actually touched
|
||||
*/
|
||||
public StatelessRowCapExceededException(StatelessWorkName work, long maxRows, long affectedRows) {
|
||||
super(
|
||||
"stateless work "
|
||||
+ work.value()
|
||||
+ " touched "
|
||||
+ affectedRows
|
||||
+ " rows against a declared cap of "
|
||||
+ maxRows
|
||||
+ "; the transaction was rolled back");
|
||||
}
|
||||
}
|
||||
+12
-2
@@ -20,7 +20,17 @@ public interface StatelessSessionRunner {
|
||||
/**
|
||||
* Runs {@code work} in its own stateless session and transaction.
|
||||
*
|
||||
* @param maxRows the declared row cap for this unit of work
|
||||
* <p>The work reports how many rows it touched, and exceeding {@code maxRows} rolls the
|
||||
* transaction back. The cap used to be a positive-number check and nothing else: a caller
|
||||
* declared a bound, the runner validated the number, and the work then touched however many rows
|
||||
* it liked. On the one session type chosen for work that can be enormous, that is the bound
|
||||
* nobody was applying.
|
||||
*
|
||||
* @param maxRows the row cap for this unit of work
|
||||
* @throws StatelessRowCapExceededException when the work touched more, after rolling back
|
||||
*/
|
||||
<T> T execute(StatelessWorkName name, long maxRows, Function<StatelessSession, T> work);
|
||||
<T> T execute(
|
||||
StatelessWorkName name,
|
||||
long maxRows,
|
||||
Function<StatelessSession, StatelessWorkResult<T>> work);
|
||||
}
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.hibernate.stateless;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* What a stateless unit of work produced, and how many rows it touched.
|
||||
*
|
||||
* <p>The row count is the point. The runner took a {@code maxRows} argument, checked it was
|
||||
* positive, and then never compared anything to it — so the cap was a parameter the caller supplied
|
||||
* and nobody enforced. A stateless session exists precisely for the work that could touch far more
|
||||
* rows than a Persistence Context would tolerate, which makes "how many did it touch" the one
|
||||
* question the bound is about.
|
||||
*
|
||||
* @param <T> the work's own result type
|
||||
* @param value what the work produced
|
||||
* @param affectedRows how many rows the work read or wrote
|
||||
*/
|
||||
public record StatelessWorkResult<T>(T value, long affectedRows) {
|
||||
|
||||
public StatelessWorkResult {
|
||||
if (affectedRows < 0L) {
|
||||
throw new IllegalArgumentException("affected rows cannot be negative");
|
||||
}
|
||||
}
|
||||
|
||||
/** A result that touched no rows. */
|
||||
public static <T> StatelessWorkResult<T> none(T value) {
|
||||
return new StatelessWorkResult<>(value, 0L);
|
||||
}
|
||||
|
||||
/** A result that touched a counted number of rows. */
|
||||
public static <T> StatelessWorkResult<T> of(T value, long affectedRows) {
|
||||
return new StatelessWorkResult<>(value, affectedRows);
|
||||
}
|
||||
|
||||
/** The value, for a caller that has already checked the count. */
|
||||
public T orThrow() {
|
||||
return Objects.requireNonNull(value, "value");
|
||||
}
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.notification;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.persistence.autoconfigure.EntityScan;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
|
||||
|
||||
/**
|
||||
* Scans the notification capability's entities and repositories, and only when it is enabled.
|
||||
*
|
||||
* <p>Separate from {@code PersistenceJpaConfig} because the capability is genuinely optional: its
|
||||
* schema stream is not in the default Flyway location, so its tables do not exist in a deployment
|
||||
* that never asked for them. An unconditional scan put the entities into the persistence unit
|
||||
* regardless, which made {@code ddl-auto=validate} fail on tables the deployment had deliberately
|
||||
* not created — the switch was off and the capability still decided whether the application could
|
||||
* start.
|
||||
*
|
||||
* <p>The condition is the same master switch the adapter beans use, so "disabled" means one thing
|
||||
* everywhere: no entity metadata, no repository beans, no schema expectation.
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@ConditionalOnProperty(
|
||||
prefix = "ca-skeleton.notification.platform",
|
||||
name = "enabled",
|
||||
havingValue = "true")
|
||||
@EntityScan(basePackages = "dev.caskeleton.adapter.outbound.persistence.notification")
|
||||
@EnableJpaRepositories(basePackages = "dev.caskeleton.adapter.outbound.persistence.notification")
|
||||
public class NotificationJpaPersistenceConfig {}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.notification;
|
||||
|
||||
import org.springframework.jdbc.core.JdbcOperations;
|
||||
|
||||
/**
|
||||
* Proves the notification schema stream was applied and promoted before the capability serves a
|
||||
* request.
|
||||
*
|
||||
* <p>The stream is operator-applied, like the Fileserver one: the application migrates {@code
|
||||
* db/migration/postgresql} only, and {@code db/migration/jpa/notification-platform} is applied and
|
||||
* promoted to {@code ACTIVE} deliberately. Until then the {@code notification_*} tables either do
|
||||
* not exist or are not sanctioned for use.
|
||||
*
|
||||
* <p>Without this check the capability had two failure shapes and neither named the cause. With
|
||||
* {@code ddl-auto=validate}, boot failed over a missing {@code notification_request} in deployments
|
||||
* that had the feature switched <em>off</em>, because the entities were scanned unconditionally.
|
||||
* With {@code ddl-auto=none} and the feature on, boot succeeded and the first send returned a raw
|
||||
* {@code relation "notification_request" does not exist} to whoever happened to send first.
|
||||
*
|
||||
* <p>Startup, not per call: an unpromoted stream is a deployment state, and re-asking the registry
|
||||
* on every notification would put a round trip on the send path to answer a question that cannot
|
||||
* change while the process runs.
|
||||
*/
|
||||
public final class NotificationSchemaActivation {
|
||||
|
||||
static final String CAPABILITY_ID = "jpa-notification-platform-v4";
|
||||
|
||||
private static final String ACTIVE_CAPABILITY_SQL =
|
||||
"""
|
||||
select count(*)
|
||||
from capability_schema_registry
|
||||
where capability_id = 'jpa-notification-platform-v4'
|
||||
and core_epoch = 1
|
||||
and feature_revision >= 4
|
||||
and lifecycle_state = 'ACTIVE'
|
||||
""";
|
||||
|
||||
private final JdbcOperations jdbc;
|
||||
|
||||
/**
|
||||
* Creates the check.
|
||||
*
|
||||
* @param jdbc a connection to the schema the capability will use
|
||||
*/
|
||||
public NotificationSchemaActivation(JdbcOperations jdbc) {
|
||||
this.jdbc = jdbc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fails closed unless the stream is applied and promoted.
|
||||
*
|
||||
* <p>An unreadable registry is treated as "not promoted" rather than "assume fine": the registry
|
||||
* table is created by the core stream, so its absence means the prerequisite chain was never
|
||||
* established at all.
|
||||
*
|
||||
* @throws IllegalStateException naming the stream, the history table and the promotion step
|
||||
*/
|
||||
public void requireActive() {
|
||||
Integer active;
|
||||
try {
|
||||
active = jdbc.queryForObject(ACTIVE_CAPABILITY_SQL, Integer.class);
|
||||
} catch (RuntimeException unreadable) {
|
||||
throw new IllegalStateException(
|
||||
CAPABILITY_ID
|
||||
+ " could not be verified: the capability schema registry is unreadable, so the "
|
||||
+ "notification schema stream cannot be confirmed as applied",
|
||||
unreadable);
|
||||
}
|
||||
if (active == null || active != 1) {
|
||||
throw new IllegalStateException(
|
||||
CAPABILITY_ID
|
||||
+ " is not ACTIVE at core epoch 1 revision 4. Apply "
|
||||
+ "db/migration/jpa/notification-platform against history table "
|
||||
+ "flyway_jpa_notification_history and promote the capability before enabling "
|
||||
+ "ca-skeleton.notification.platform.enabled");
|
||||
}
|
||||
}
|
||||
}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.notification;
|
||||
|
||||
import java.util.Objects;
|
||||
import javax.sql.DataSource;
|
||||
import org.flywaydb.core.Flyway;
|
||||
|
||||
/**
|
||||
* The notification schema as its own Flyway stream, with its own history table.
|
||||
*
|
||||
* <p>The migrations live in {@code db/migration/jpa/notification-platform} while the primary Flyway
|
||||
* location is {@code db/migration/postgresql}, so nothing applied them. The obvious fix — adding
|
||||
* the directory to the primary location list — is the wrong one: both trees number from V1, and a
|
||||
* shared history table would make {@code V1__notification_platform_core} and {@code
|
||||
* V1__initial_schema} the same version. Flyway would either refuse the second or, worse, record one
|
||||
* and skip the other depending on resolution order.
|
||||
*
|
||||
* <p>A separate stream with {@code flyway_jpa_notification_history} keeps the two version series
|
||||
* independent, which is what makes the capability genuinely optional: a deployment that never
|
||||
* enables notifications has no notification history table and no notification tables, and a
|
||||
* deployment that enables it later applies a stream that starts at its own V1.
|
||||
*/
|
||||
public final class NotificationSchemaStream {
|
||||
|
||||
/** Where the notification migrations live. */
|
||||
public static final String LOCATION = "classpath:db/migration/jpa/notification-platform";
|
||||
|
||||
/** The history table this stream records into, separate from the primary one. */
|
||||
public static final String HISTORY_TABLE = "flyway_jpa_notification_history";
|
||||
|
||||
private final DataSource dataSource;
|
||||
|
||||
/**
|
||||
* Creates the stream over a data source.
|
||||
*
|
||||
* @param dataSource the database to migrate
|
||||
*/
|
||||
public NotificationSchemaStream(DataSource dataSource) {
|
||||
this.dataSource = Objects.requireNonNull(dataSource, "dataSource");
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies every outstanding notification migration.
|
||||
*
|
||||
* <p>Baselined at version zero, which is required rather than lax: this stream has its own
|
||||
* history table on a schema the core stream already populated, so Flyway would otherwise refuse
|
||||
* to start against a "non-empty schema with no history table". Baselining at zero means every
|
||||
* notification migration still runs — a baseline at any higher version would skip them, which is
|
||||
* the failure mode the setting is usually feared for.
|
||||
*
|
||||
* <p>Forward-only otherwise: {@code cleanDisabled} so no path here can drop the schema, and
|
||||
* checksums validated, because a migration edited after it was applied is a schema that differs
|
||||
* between environments in a way nothing else reports.
|
||||
*
|
||||
* @return how many migrations were applied
|
||||
*/
|
||||
public int migrate() {
|
||||
Flyway flyway =
|
||||
Flyway.configure()
|
||||
.dataSource(dataSource)
|
||||
.locations(LOCATION)
|
||||
.table(HISTORY_TABLE)
|
||||
.baselineVersion("0")
|
||||
.baselineDescription("notification platform baseline")
|
||||
.baselineOnMigrate(true)
|
||||
.outOfOrder(false)
|
||||
.cleanDisabled(true)
|
||||
.validateOnMigrate(true)
|
||||
.load();
|
||||
return flyway.migrate().migrationsExecuted;
|
||||
}
|
||||
}
|
||||
+281
@@ -0,0 +1,281 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.notification.configuration;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.NotificationSchemaActivation;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.AdminAuditJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.ConsentJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.ContactPointJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.DeduplicationClaimJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.DeliveryAttemptJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaAdminOperationStore;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaContactPointStore;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaDeliveryAttemptStore;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaNotificationRequestStore;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaNotificationSideEffectStore;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaPolicyStores;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaProviderEventLedger;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaRecipientDeliveryStore;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaRecipientLeaseStore;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaSuppressionStore;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaTemplateRegistry;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.NotificationRecordMapper;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.NotificationRequestJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.PreferenceJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.ProviderEventJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.RecipientDeliveryJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.SuppressionJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.TemplateVersionJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.inbox.InboxCommitEventPublisher;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.inbox.InboxItemJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.inbox.JpaNotificationInbox;
|
||||
import dev.caskeleton.application.notification.platform.admin.AdminOperationStorePort;
|
||||
import dev.caskeleton.application.notification.platform.callback.DeliveryAttemptResolverPort;
|
||||
import dev.caskeleton.application.notification.platform.callback.DeliveryProjectionStorePort;
|
||||
import dev.caskeleton.application.notification.platform.callback.NotificationSideEffectPort;
|
||||
import dev.caskeleton.application.notification.platform.callback.ProviderEventLedger;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.ContactPointStorePort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.NotificationIdGeneratorPort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.NotificationRequestStorePort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.NotificationRoutingPlanCodecPort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.NotificationVariablesCodecPort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.ProviderRequestIdHasherPort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.TenantContextPort;
|
||||
import dev.caskeleton.application.notification.platform.inbox.InboxContentCodecPort;
|
||||
import dev.caskeleton.application.notification.platform.inbox.NotificationInbox;
|
||||
import dev.caskeleton.application.notification.platform.inbox.NotificationInboxSignalPort;
|
||||
import dev.caskeleton.application.notification.platform.policy.ConsentStorePort;
|
||||
import dev.caskeleton.application.notification.platform.policy.DeduplicationStorePort;
|
||||
import dev.caskeleton.application.notification.platform.policy.PreferenceStorePort;
|
||||
import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort;
|
||||
import dev.caskeleton.application.notification.platform.template.TemplateContentCodecPort;
|
||||
import dev.caskeleton.application.notification.platform.template.TemplateRegistry;
|
||||
import java.time.Clock;
|
||||
import java.util.Locale;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.jdbc.core.JdbcOperations;
|
||||
|
||||
/**
|
||||
* Binds the notification application ports to their JPA implementations.
|
||||
*
|
||||
* <p>This is the only place that sees both the application ports and the persistence adapter, which
|
||||
* is what keeps application-core free of JPA and the persistence leaf free of use-case policy.
|
||||
*
|
||||
* <p>It lives in the persistence leaf, not in the composition root. As a bootstrap class it
|
||||
* imported twenty-two persistence implementation types by name — every store, every repository, the
|
||||
* record mapper — which made all of them cross-leaf source contracts: moving one to another
|
||||
* package, or reducing its visibility, broke the composition root's compile. The root now imports
|
||||
* this one facade, and what it assembles stays the leaf's own business.
|
||||
*
|
||||
* <p>The master switch used to gate these beans and nothing else. The entities were scanned
|
||||
* unconditionally by {@code PersistenceJpaConfig}, and the schema stream — which lives outside the
|
||||
* default Flyway location — had no activation check at all. So a deployment with the feature off
|
||||
* still failed {@code ddl-auto=validate} on tables it had deliberately not created, and one with
|
||||
* the feature on but the stream unapplied started cleanly and returned a raw {@code relation
|
||||
* "notification_request" does not exist} to whoever sent the first notification. Both halves are
|
||||
* closed now: the scan moved to {@code NotificationJpaPersistenceConfig} behind the same switch,
|
||||
* and {@link NotificationSchemaActivation} refuses startup until the stream is promoted.
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@ConditionalOnProperty(
|
||||
prefix = "ca-skeleton.notification.platform",
|
||||
name = "enabled",
|
||||
havingValue = "true")
|
||||
public class NotificationJpaPersistenceFacade {
|
||||
|
||||
/**
|
||||
* Refuses to start the capability until its schema stream is applied and promoted.
|
||||
*
|
||||
* <p>Declared before the stores so the failure names the deployment step that is missing, rather
|
||||
* than surfacing later as a missing relation on somebody's first send.
|
||||
*/
|
||||
@Bean
|
||||
public NotificationSchemaActivation notificationSchemaActivation(JdbcOperations jdbc) {
|
||||
NotificationSchemaActivation activation = new NotificationSchemaActivation(jdbc);
|
||||
activation.requireActive();
|
||||
return activation;
|
||||
}
|
||||
|
||||
/**
|
||||
* The outstanding reconciliation questions.
|
||||
*
|
||||
* <p>Exported from here rather than assembled in the composition root: the store is a persistence
|
||||
* type, and app-bootstrap imports only this leaf's exported packages.
|
||||
*
|
||||
* @param jdbc the database
|
||||
* @return the job store
|
||||
*/
|
||||
@Bean
|
||||
public dev.caskeleton.application.notification.platform.dispatch.ReconciliationJobStorePort
|
||||
reconciliationJobStore(JdbcOperations jdbc) {
|
||||
return new dev.caskeleton.adapter.outbound.persistence.notification.platform
|
||||
.JdbcReconciliationJobStore(jdbc);
|
||||
}
|
||||
|
||||
/**
|
||||
* What the platform is actually doing, measured from the durable rows.
|
||||
*
|
||||
* <p>Exported here for the same reason as the job store: the queries read persistence tables, so
|
||||
* they belong to this leaf, and app-bootstrap sees only the port.
|
||||
*
|
||||
* @param jdbc the database
|
||||
* @param clock the clock the ages are measured against
|
||||
* @return the serving state reader
|
||||
*/
|
||||
@Bean
|
||||
public dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort
|
||||
notificationServingState(JdbcOperations jdbc, java.time.Clock clock) {
|
||||
return new dev.caskeleton.adapter.outbound.persistence.notification.platform
|
||||
.JdbcNotificationServingState(jdbc, clock);
|
||||
}
|
||||
|
||||
/** Entity/record mapping. */
|
||||
@Bean
|
||||
public NotificationRecordMapper notificationRecordMapper(
|
||||
NotificationRoutingPlanCodecPort routingPlans, NotificationVariablesCodecPort variables) {
|
||||
return new NotificationRecordMapper(routingPlans, variables);
|
||||
}
|
||||
|
||||
/** Request and recipient persistence. */
|
||||
@Bean
|
||||
public NotificationRequestStorePort notificationRequestStore(
|
||||
NotificationRequestJpaRepository requests,
|
||||
RecipientDeliveryJpaRepository recipients,
|
||||
NotificationRecordMapper mapper,
|
||||
Clock clock) {
|
||||
return new JpaNotificationRequestStore(requests, recipients, mapper, clock);
|
||||
}
|
||||
|
||||
/** Single recipient job persistence. */
|
||||
@Bean
|
||||
public RecipientDeliveryStorePort recipientDeliveryStore(
|
||||
RecipientDeliveryJpaRepository recipients, NotificationRecordMapper mapper, Clock clock) {
|
||||
return new JpaRecipientDeliveryStore(recipients, mapper, clock);
|
||||
}
|
||||
|
||||
/** Durable queue claim. */
|
||||
@Bean
|
||||
public RecipientLeaseStorePort recipientLeaseStore(
|
||||
RecipientDeliveryJpaRepository recipients, Clock clock) {
|
||||
return new JpaRecipientLeaseStore(recipients, clock);
|
||||
}
|
||||
|
||||
/** Attempt persistence, which also serves the resolver and projection ports. */
|
||||
@Bean
|
||||
public JpaDeliveryAttemptStore deliveryAttemptStore(
|
||||
DeliveryAttemptJpaRepository attempts,
|
||||
RecipientDeliveryJpaRepository recipients,
|
||||
ProviderRequestIdHasherPort hasher,
|
||||
Clock clock) {
|
||||
return new JpaDeliveryAttemptStore(attempts, recipients, hasher, clock);
|
||||
}
|
||||
|
||||
/** Attempt store as its application port. */
|
||||
@Bean
|
||||
public DeliveryAttemptStorePort deliveryAttemptStorePort(JpaDeliveryAttemptStore store) {
|
||||
return store;
|
||||
}
|
||||
|
||||
/** Attempt resolution for incoming provider events. */
|
||||
@Bean
|
||||
public DeliveryAttemptResolverPort deliveryAttemptResolverPort(JpaDeliveryAttemptStore store) {
|
||||
return store;
|
||||
}
|
||||
|
||||
/** Projection persistence. */
|
||||
@Bean
|
||||
public DeliveryProjectionStorePort deliveryProjectionStorePort(JpaDeliveryAttemptStore store) {
|
||||
return store;
|
||||
}
|
||||
|
||||
/** Append-only provider event ledger. */
|
||||
@Bean
|
||||
public ProviderEventLedger providerEventLedger(
|
||||
ProviderEventJpaRepository events,
|
||||
DeliveryAttemptResolverPort resolver,
|
||||
ProviderRequestIdHasherPort hasher,
|
||||
NotificationVariablesCodecPort attributes,
|
||||
NotificationIdGeneratorPort ids) {
|
||||
return new JpaProviderEventLedger(events, resolver, hasher, attributes, ids);
|
||||
}
|
||||
|
||||
/** Contact point persistence. */
|
||||
@Bean
|
||||
public ContactPointStorePort contactPointStore(
|
||||
ContactPointJpaRepository contactPoints, Clock clock) {
|
||||
return new JpaContactPointStore(contactPoints, clock);
|
||||
}
|
||||
|
||||
/** Suppression persistence. */
|
||||
@Bean
|
||||
public SuppressionStorePort suppressionStore(
|
||||
SuppressionJpaRepository suppressions, NotificationIdGeneratorPort ids, Clock clock) {
|
||||
return new JpaSuppressionStore(suppressions, ids, clock);
|
||||
}
|
||||
|
||||
/** Contact-point and suppression side effects of a projected event. */
|
||||
@Bean
|
||||
public NotificationSideEffectPort notificationSideEffectPort(
|
||||
ContactPointStorePort contactPoints, SuppressionStorePort suppressions, Clock clock) {
|
||||
return new JpaNotificationSideEffectStore(contactPoints, suppressions, clock);
|
||||
}
|
||||
|
||||
/** Preference persistence. */
|
||||
@Bean
|
||||
public PreferenceStorePort preferenceStore(
|
||||
PreferenceJpaRepository preferences, NotificationIdGeneratorPort ids, Clock clock) {
|
||||
return new JpaPolicyStores.Preferences(preferences, ids, clock);
|
||||
}
|
||||
|
||||
/** Consent persistence. */
|
||||
@Bean
|
||||
public ConsentStorePort consentStore(
|
||||
ConsentJpaRepository consents, NotificationIdGeneratorPort ids) {
|
||||
return new JpaPolicyStores.Consents(consents, ids);
|
||||
}
|
||||
|
||||
/** Deduplication window claims. */
|
||||
@Bean
|
||||
public DeduplicationStorePort deduplicationStore(
|
||||
DeduplicationClaimJpaRepository claims, NotificationIdGeneratorPort ids, Clock clock) {
|
||||
return new JpaPolicyStores.DeduplicationClaims(claims, ids, clock);
|
||||
}
|
||||
|
||||
/** Operator action idempotency and audit. */
|
||||
@Bean
|
||||
public AdminOperationStorePort adminOperationStore(
|
||||
AdminAuditJpaRepository audits, NotificationIdGeneratorPort ids, Clock clock) {
|
||||
return new JpaAdminOperationStore(audits, ids, clock);
|
||||
}
|
||||
|
||||
/** Durable template registry. */
|
||||
@Bean
|
||||
public TemplateRegistry templateRegistry(
|
||||
TemplateVersionJpaRepository versions,
|
||||
TemplateContentCodecPort contentCodec,
|
||||
TenantContextPort tenants,
|
||||
NotificationIdGeneratorPort ids,
|
||||
Clock clock) {
|
||||
return new JpaTemplateRegistry(versions, contentCodec, tenants, ids, clock, Locale.ENGLISH);
|
||||
}
|
||||
|
||||
/** Post-commit inbox signal. */
|
||||
@Bean
|
||||
public InboxCommitEventPublisher inboxCommitEventPublisher(NotificationInboxSignalPort signals) {
|
||||
return new InboxCommitEventPublisher(signals);
|
||||
}
|
||||
|
||||
/** Durable in-app inbox. */
|
||||
@Bean
|
||||
public NotificationInbox notificationInbox(
|
||||
InboxItemJpaRepository items,
|
||||
InboxContentCodecPort contentCodec,
|
||||
InboxCommitEventPublisher publisher,
|
||||
NotificationIdGeneratorPort ids,
|
||||
Clock clock) {
|
||||
return new JpaNotificationInbox(items, contentCodec, publisher, ids, clock);
|
||||
}
|
||||
}
|
||||
+9
@@ -34,6 +34,15 @@ public class AdminAuditEntity {
|
||||
@Column(name = "tenant_id", length = 100, updatable = false)
|
||||
private String tenantId;
|
||||
|
||||
/**
|
||||
* Declared as JSON, because the column is {@code jsonb} and the field is a {@code String}.
|
||||
*
|
||||
* <p>Without the type code Hibernate binds a {@code varchar}, and PostgreSQL refuses to put
|
||||
* {@code character varying} into a {@code jsonb} column — so the mismatch surfaces on the first
|
||||
* write rather than at startup. The UUID columns on these same entities already declare their
|
||||
* type this way; the JSON ones were the omission.
|
||||
*/
|
||||
@JdbcTypeCode(SqlTypes.JSON)
|
||||
@Column(name = "attributes", nullable = false, updatable = false)
|
||||
private String attributes;
|
||||
|
||||
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.notification.platform;
|
||||
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
/*
|
||||
* Top-level, not nested inside a holder class. Spring Data does not discover nested repository
|
||||
* interfaces unless `considerNestedRepositories` is switched on, so these five were declared,
|
||||
* imported by the bootstrap configuration, and never registered as beans — the enabled context
|
||||
* failed to assemble on five missing constructor arguments. Switching the global flag on would fix
|
||||
* these five and also start discovering every nested interface anywhere on the scan path, including
|
||||
* ones written inside tests.
|
||||
*/
|
||||
|
||||
/** Operator audit records, doubling as the admin idempotency record. */
|
||||
public interface AdminAuditJpaRepository extends JpaRepository<AdminAuditEntity, UUID> {
|
||||
|
||||
/** Previous result for an operation id. */
|
||||
Optional<AdminAuditEntity> findByOperationId(String operationId);
|
||||
|
||||
/**
|
||||
* Claims an operation id, or reports who already holds it.
|
||||
*
|
||||
* <p>{@code ON CONFLICT DO NOTHING} is the claim: exactly one caller inserts the row, and the
|
||||
* update count tells that caller it won. Everything the admin path used to do — read, act, write
|
||||
* — happened between two callers' reads, so both acted.
|
||||
*
|
||||
* @return 1 when this caller claimed it, 0 when somebody else already had
|
||||
*/
|
||||
@org.springframework.data.jpa.repository.Modifying
|
||||
@org.springframework.data.jpa.repository.Query(
|
||||
value =
|
||||
"INSERT INTO notification_admin_audit ("
|
||||
+ " id, operation_id, action, actor_ref, attributes, dry_run, occurred_at,"
|
||||
+ " command_fingerprint, phase, claimed_at)"
|
||||
+ " VALUES (:id, :operationId, :action, :actorRef, '{}'::jsonb, false, :now,"
|
||||
+ " :fingerprint, 'CLAIMED', :now)"
|
||||
+ " ON CONFLICT (operation_id) DO NOTHING",
|
||||
nativeQuery = true)
|
||||
int claimOperation(
|
||||
@org.springframework.data.repository.query.Param("id") java.util.UUID id,
|
||||
@org.springframework.data.repository.query.Param("operationId") String operationId,
|
||||
@org.springframework.data.repository.query.Param("action") String action,
|
||||
@org.springframework.data.repository.query.Param("actorRef") String actorRef,
|
||||
@org.springframework.data.repository.query.Param("fingerprint") String fingerprint,
|
||||
@org.springframework.data.repository.query.Param("now") java.time.Instant now);
|
||||
|
||||
/** The phase and fingerprint of an operation somebody else claimed. */
|
||||
@org.springframework.data.jpa.repository.Query(
|
||||
value =
|
||||
"SELECT phase, command_fingerprint FROM notification_admin_audit"
|
||||
+ " WHERE operation_id = :operationId",
|
||||
nativeQuery = true)
|
||||
java.util.List<Object[]> phaseOf(
|
||||
@org.springframework.data.repository.query.Param("operationId") String operationId);
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.notification.platform;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
/*
|
||||
* Top-level, not nested inside a holder class. Spring Data does not discover nested repository
|
||||
* interfaces unless `considerNestedRepositories` is switched on, so these five were declared,
|
||||
* imported by the bootstrap configuration, and never registered as beans — the enabled context
|
||||
* failed to assemble on five missing constructor arguments. Switching the global flag on would fix
|
||||
* these five and also start discovering every nested interface anywhere on the scan path, including
|
||||
* ones written inside tests.
|
||||
*/
|
||||
|
||||
/** Recorded consent decisions. */
|
||||
public interface ConsentJpaRepository extends JpaRepository<ConsentEntity, UUID> {
|
||||
|
||||
/** Consent history for a recipient. */
|
||||
List<ConsentEntity> findByTenantIdAndRecipientRefOrderByRecordedAtAsc(
|
||||
String tenantId, String recipientRef);
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.notification.platform;
|
||||
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
/*
|
||||
* Top-level, not nested inside a holder class. Spring Data does not discover nested repository
|
||||
* interfaces unless `considerNestedRepositories` is switched on, so these five were declared,
|
||||
* imported by the bootstrap configuration, and never registered as beans — the enabled context
|
||||
* failed to assemble on five missing constructor arguments. Switching the global flag on would fix
|
||||
* these five and also start discovering every nested interface anywhere on the scan path, including
|
||||
* ones written inside tests.
|
||||
*/
|
||||
|
||||
/** Per-window deduplication claims. */
|
||||
public interface DeduplicationClaimJpaRepository
|
||||
extends JpaRepository<DeduplicationClaimEntity, UUID> {
|
||||
|
||||
/** Existing claim for one window. */
|
||||
Optional<DeduplicationClaimEntity>
|
||||
findByTenantIdAndRecipientRefAndCategoryAndDedupKeyAndWindowBucket(
|
||||
String tenantId,
|
||||
String recipientRef,
|
||||
String category,
|
||||
String dedupKey,
|
||||
long windowBucket);
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user