feat: jpa, messaging, notification, mongo, graphql 어댑터터 구현체 추가

This commit is contained in:
DongHyeonka
2026-08-15 13:01:58 +09:00
parent e23ed9486e
commit f56dd3e4eb
174 changed files with 10338 additions and 1274 deletions
@@ -1,78 +0,0 @@
package dev.caskeleton.messaging.admin.runtime;
import java.time.Instant;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
/**
* Remembers which approvals have already been executed.
*
* <p>An approval authorises one execution, not a standing permission. Without this, re-running the
* same approved redrive twice is a single command-history arrow-up away — and the second run
* republishes messages the first one already moved, which on a destination without an inbox is
* indistinguishable from a duplicate storm.
*
* <p>Claiming is atomic and returns the previous claim rather than a boolean, so a duplicate
* attempt can tell the operator <em>when</em> it ran and by which operation id instead of just
* refusing.
*/
public final class AdminOperationIdempotencyStore {
private final Map<String, Claim> claims = new ConcurrentHashMap<>();
/**
* One recorded execution of an approval.
*
* @param approvalTicket the approval that was executed
* @param operationId the operation identity that claimed it
* @param executedAt when it ran
*/
public record Claim(String approvalTicket, String operationId, Instant executedAt) {
public Claim {
Objects.requireNonNull(executedAt, "executedAt must not be null");
if (approvalTicket == null || approvalTicket.isBlank()) {
throw new IllegalArgumentException("approvalTicket must not be blank");
}
if (operationId == null || operationId.isBlank()) {
throw new IllegalArgumentException("operationId must not be blank");
}
}
}
/**
* Claims an approval for execution.
*
* @param approvalTicket the approval to claim
* @param operationId the operation attempting it
* @param now the current instant
* @return empty when the claim succeeded; the existing claim when it was already executed
*/
public Optional<Claim> claim(String approvalTicket, String operationId, Instant now) {
Objects.requireNonNull(now, "now must not be null");
Claim candidate = new Claim(approvalTicket, operationId, now);
Claim existing = claims.putIfAbsent(approvalTicket, candidate);
return Optional.ofNullable(existing);
}
/**
* Returns the recorded execution of an approval.
*
* @param approvalTicket the approval
* @return the claim, when the approval has been executed
*/
public Optional<Claim> find(String approvalTicket) {
return Optional.ofNullable(claims.get(approvalTicket));
}
/**
* Returns how many approvals have been executed.
*
* @return the claim count
*/
public int size() {
return claims.size();
}
}
@@ -1,5 +1,7 @@
package dev.caskeleton.messaging.admin.runtime;
import dev.caskeleton.messaging.admin.AdminOperationJournal;
import dev.caskeleton.messaging.admin.AdminOperationLease;
import dev.caskeleton.messaging.admin.ApprovedRedrivePlan;
import dev.caskeleton.messaging.admin.ApprovedReplayPlan;
import dev.caskeleton.messaging.admin.RedrivePlan;
@@ -10,7 +12,6 @@ import dev.caskeleton.messaging.admin.ReplayRequest;
import dev.caskeleton.messaging.admin.ReplayResult;
import dev.caskeleton.messaging.admin.TopologyManifest;
import dev.caskeleton.messaging.admin.TopologyValidationReport;
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
@@ -30,15 +31,29 @@ import java.util.function.Supplier;
* <li>Only then does anything move.
* </ol>
*
* <p>The idempotency claim comes <em>before</em> the work rather than after it. Claiming afterwards
* leaves a window where a second execution starts while the first is still running, which is
* precisely the double-redrive this store exists to prevent.
* <p>The journal entry is written <em>before</em> the work rather than after it. Writing it
* afterwards leaves a window where a second execution starts while the first is still running,
* which is precisely the double-redrive the journal exists to prevent.
*
* <p>Writing it first used to have a cost the previous store never paid: an operation that died
* halfway had consumed its approval and left no record of how far it got. The journal keeps a
* checkpoint and hands back a lease that says where to resume, so a retry continues the same
* operation instead of either redoing it or requiring a new approval.
*/
public final class DefaultMessagingAdminService implements MessagingAdminService {
/**
* How long a runtime holds an operation before another may take it over.
*
* <p>Long enough that a slow batch does not lose its lease mid-flight, short enough that a dead
* replica does not park an approval for an hour.
*/
private static final Duration LEASE_DURATION = Duration.ofMinutes(5);
private final CompositeTopologyValidator topologyValidator;
private final BrokerTopologyInspector inspector;
private final AdminOperationIdempotencyStore idempotency;
private final AdminOperationJournal journal;
private final String leaseOwner;
private final ReplayService replayService;
private final RedriveService redriveService;
private final Supplier<List<TopologyManifest>> manifests;
@@ -51,7 +66,8 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
*
* @param topologyValidator compares declared topology against the broker
* @param inspector reads the broker's topology version
* @param idempotency records which approvals have been executed
* @param journal records which approvals have been executed, and how far each got
* @param leaseOwner identifies this runtime in the journal's leases
* @param replayService performs replays
* @param redriveService performs redrives
* @param manifests supplies the declared topologies
@@ -62,7 +78,8 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
public DefaultMessagingAdminService(
CompositeTopologyValidator topologyValidator,
BrokerTopologyInspector inspector,
AdminOperationIdempotencyStore idempotency,
AdminOperationJournal journal,
String leaseOwner,
ReplayService replayService,
RedriveService redriveService,
Supplier<List<TopologyManifest>> manifests,
@@ -72,7 +89,11 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
this.topologyValidator =
Objects.requireNonNull(topologyValidator, "topologyValidator required");
this.inspector = Objects.requireNonNull(inspector, "inspector must not be null");
this.idempotency = Objects.requireNonNull(idempotency, "idempotency must not be null");
this.journal = Objects.requireNonNull(journal, "journal must not be null");
if (leaseOwner == null || leaseOwner.isBlank()) {
throw new IllegalArgumentException("leaseOwner must not be blank");
}
this.leaseOwner = leaseOwner;
this.replayService = Objects.requireNonNull(replayService, "replayService must not be null");
this.redriveService = Objects.requireNonNull(redriveService, "redriveService must not be null");
this.manifests = Objects.requireNonNull(manifests, "manifests must not be null");
@@ -104,12 +125,28 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
ReplayRequest request = plan.plan().request();
plan.requireExecutable(now, inspector.topologyVersion());
claimOrRefuse(plan.approval().ticket(), request.replayId().toString(), now);
AdminOperationLease lease =
journal.begin(
plan.approval().ticket(),
plan.plan().digest(),
request.replayId().toString(),
leaseOwner,
LEASE_DURATION,
now);
Instant startedAt = clock.get();
ReplayReport report =
replayService.replay(
request, Optional.of(plan.approval()), plan.approval().approvedBy(), now);
ReplayReport report;
try {
report =
replayService.replay(
request, Optional.of(plan.approval()), plan.approval().approvedBy(), now);
} catch (RuntimeException failure) {
// The operation stays resumable rather than silently consuming the approval: the journal
// entry moves to FAILED at its checkpoint, and a retry takes it over from there.
journal.fail(lease, lease.resumeFrom(), failureCodeOf(failure), clock.get());
throw failure;
}
journal.complete(lease, Math.toIntExact(report.replayed()), clock.get());
return new ReplayResult(
request.replayId(),
@@ -139,12 +176,31 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
RedriveRequest request = plan.plan().request();
plan.requireExecutable(now, inspector.topologyVersion());
claimOrRefuse(plan.approval().ticket(), request.redriveId().toString(), now);
AdminOperationLease lease =
journal.begin(
plan.approval().ticket(),
plan.plan().digest(),
request.redriveId().toString(),
leaseOwner,
LEASE_DURATION,
now);
Instant startedAt = clock.get();
RedriveReport report =
redriveService.redrive(
request, Optional.of(plan.approval()), plan.approval().approvedBy(), now);
RedriveReport report;
try {
report =
redriveService.redrive(
request,
Optional.of(plan.approval()),
plan.approval().approvedBy(),
now,
lease.resumeFrom(),
completed -> journal.checkpoint(lease, completed, LEASE_DURATION, clock.get()));
} catch (RuntimeException failure) {
journal.fail(lease, lease.resumeFrom(), failureCodeOf(failure), clock.get());
throw failure;
}
journal.complete(lease, report.moved() + report.failed(), clock.get());
return new RedriveResult(
request.redriveId(),
@@ -155,17 +211,19 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
report.dryRun());
}
private void claimOrRefuse(String approvalTicket, String operationId, Instant now) {
idempotency
.claim(approvalTicket, operationId, now)
.ifPresent(
existing -> {
throw new MessageAuthorizationException(
"APPROVAL_ALREADY_EXECUTED",
"approval %s was already executed at %s by operation %s; an approval authorises "
.formatted(approvalTicket, existing.executedAt(), existing.operationId())
+ "one execution, not a standing permission");
});
/**
* Reduces a failure to a bounded, sanitised code for the journal.
*
* <p>The journal is read by operators during incidents and its contents outlive the process. A
* raw exception message can carry a destination, a payload fragment, or a credential from a
* driver's own error text, so only the platform's own code or the exception's simple name goes
* in.
*/
private static String failureCodeOf(RuntimeException failure) {
if (failure instanceof dev.caskeleton.messaging.api.error.MessagingException messaging) {
return messaging.failure().code();
}
return failure.getClass().getSimpleName();
}
/** Estimates how many messages a replay would re-deliver. */
@@ -0,0 +1,237 @@
package dev.caskeleton.messaging.admin.runtime;
import dev.caskeleton.messaging.admin.AdminOperationJournal;
import dev.caskeleton.messaging.admin.AdminOperationLease;
import dev.caskeleton.messaging.admin.AdminOperationRecord;
import dev.caskeleton.messaging.admin.AdminOperationState;
import dev.caskeleton.messaging.admin.PlanDigest;
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
/**
* A single-process journal, for tests and for local development.
*
* <p>It reports {@link #isDurable()} as false, and the starter refuses to run a production profile
* on a journal that says so. That declaration is the point of this class existing at all: the
* previous in-memory store was registered as the production default and nothing distinguished it
* from a shared one, so the gap was invisible until two replicas executed the same approval.
*
* <p>The semantics are otherwise the real ones — uniqueness on {@code (ticket, digest)}, lease
* takeover with a monotonic token, resume from checkpoint — so a test that passes here is testing
* the protocol rather than a simplification of it.
*/
public final class InMemoryAdminOperationJournal implements AdminOperationJournal {
private final Map<String, AdminOperationRecord> records = new ConcurrentHashMap<>();
@Override
public AdminOperationLease begin(
String approvalTicket,
PlanDigest planDigest,
String operationId,
String leaseOwner,
Duration leaseDuration,
Instant now) {
Objects.requireNonNull(leaseDuration, "leaseDuration must not be null");
Objects.requireNonNull(now, "now must not be null");
AdminOperationRecord claimed =
records.compute(
key(approvalTicket, planDigest),
(ignored, existing) ->
claim(
existing,
approvalTicket,
planDigest,
operationId,
leaseOwner,
leaseDuration,
now));
return new AdminOperationLease(
approvalTicket,
planDigest,
claimed.operationId(),
claimed.leaseOwner(),
claimed.leaseToken(),
claimed.itemsCompleted());
}
private static AdminOperationRecord claim(
AdminOperationRecord existing,
String approvalTicket,
PlanDigest planDigest,
String operationId,
String leaseOwner,
Duration leaseDuration,
Instant now) {
if (existing == null) {
return new AdminOperationRecord(
approvalTicket,
planDigest,
operationId,
AdminOperationState.STARTED,
0,
leaseOwner,
1L,
now.plus(leaseDuration),
now,
Optional.empty(),
Optional.empty());
}
if (existing.state() == AdminOperationState.COMPLETED) {
throw new MessageAuthorizationException(
"APPROVAL_ALREADY_EXECUTED",
("approval %s was already executed to completion at %s; an approval authorises one"
+ " execution, not a standing permission")
.formatted(approvalTicket, existing.finishedAt().orElse(existing.startedAt())));
}
if (existing.state() == AdminOperationState.STARTED
&& now.isBefore(existing.leaseExpiresAt())) {
throw new MessageAuthorizationException(
"ADMIN_OPERATION_IN_FLIGHT",
("approval %s is being executed by %s until at least %s; two runtimes executing one"
+ " approval is a duplicate storm, not a faster redrive")
.formatted(approvalTicket, existing.leaseOwner(), existing.leaseExpiresAt()));
}
// Either the previous attempt failed, or its lease expired without a completion. Both resume
// from the checkpoint under a new token, which fences the previous holder out.
return new AdminOperationRecord(
approvalTicket,
planDigest,
operationId,
AdminOperationState.STARTED,
existing.itemsCompleted(),
leaseOwner,
existing.leaseToken() + 1,
now.plus(leaseDuration),
existing.startedAt(),
Optional.empty(),
Optional.empty());
}
@Override
public void checkpoint(
AdminOperationLease lease, int itemsCompleted, Duration leaseDuration, Instant now) {
update(
lease,
current ->
new AdminOperationRecord(
current.approvalTicket(),
current.planDigest(),
current.operationId(),
AdminOperationState.STARTED,
Math.max(current.itemsCompleted(), itemsCompleted),
current.leaseOwner(),
current.leaseToken(),
now.plus(leaseDuration),
current.startedAt(),
Optional.empty(),
Optional.empty()));
}
@Override
public void complete(AdminOperationLease lease, int itemsCompleted, Instant now) {
update(
lease,
current ->
new AdminOperationRecord(
current.approvalTicket(),
current.planDigest(),
current.operationId(),
AdminOperationState.COMPLETED,
Math.max(current.itemsCompleted(), itemsCompleted),
current.leaseOwner(),
current.leaseToken(),
current.leaseExpiresAt(),
current.startedAt(),
Optional.of(now),
Optional.empty()));
}
@Override
public void fail(AdminOperationLease lease, int itemsCompleted, String failureCode, Instant now) {
Objects.requireNonNull(failureCode, "failureCode must not be null");
update(
lease,
current ->
new AdminOperationRecord(
current.approvalTicket(),
current.planDigest(),
current.operationId(),
AdminOperationState.FAILED,
Math.max(current.itemsCompleted(), itemsCompleted),
current.leaseOwner(),
current.leaseToken(),
current.leaseExpiresAt(),
current.startedAt(),
Optional.of(now),
Optional.of(failureCode)));
}
@Override
public Optional<AdminOperationRecord> find(String approvalTicket, PlanDigest planDigest) {
return Optional.ofNullable(records.get(key(approvalTicket, planDigest)));
}
@Override
public boolean isDurable() {
return false;
}
/**
* How many operations have been journalled.
*
* @return the record count
*/
public int size() {
return records.size();
}
private void update(
AdminOperationLease lease, java.util.function.UnaryOperator<AdminOperationRecord> mutation) {
Objects.requireNonNull(lease, "lease must not be null");
records.compute(
key(lease.approvalTicket(), lease.planDigest()),
(ignored, current) -> {
if (current == null) {
throw new MessageAuthorizationException(
"ADMIN_OPERATION_NOT_JOURNALLED",
"no journal entry exists for approval " + lease.approvalTicket());
}
if (current.leaseToken() != lease.leaseToken()) {
// The fence. A stalled runtime that wakes up and writes here would otherwise overwrite
// the progress of whichever replica took the operation over.
throw new MessageAuthorizationException(
"ADMIN_OPERATION_LEASE_LOST",
("the lease on approval %s moved from token %d to token %d; this runtime no longer"
+ " owns the operation")
.formatted(lease.approvalTicket(), lease.leaseToken(), current.leaseToken()));
}
return mutation.apply(current);
});
}
private static String key(String approvalTicket, PlanDigest planDigest) {
Objects.requireNonNull(planDigest, "planDigest must not be null");
// Length-prefixed for the same reason the grant's canonical form is: a ticket containing the
// separator must not be able to collide with a different ticket and digest pair.
return String.join(
"", Integer.toString(approvalTicket.length()), ":", approvalTicket, planDigest.value());
}
/**
* Every journalled operation, newest first by start time.
*
* @return the records
*/
public List<AdminOperationRecord> records() {
return records.values().stream()
.sorted(java.util.Comparator.comparing(AdminOperationRecord::startedAt).reversed())
.toList();
}
}
@@ -1,9 +1,9 @@
package dev.caskeleton.messaging.admin.runtime;
import dev.caskeleton.messaging.admin.AdminApproval;
import dev.caskeleton.messaging.admin.DestructiveOperation;
import dev.caskeleton.messaging.admin.DestructiveOperationGuard;
import dev.caskeleton.messaging.admin.RedriveRequest;
import dev.caskeleton.messaging.admin.VerifiedApproval;
import dev.caskeleton.messaging.api.MessageId;
import dev.caskeleton.messaging.api.publish.PublishCompletion;
import dev.caskeleton.messaging.api.publish.PublishResult;
@@ -54,14 +54,46 @@ public final class RedriveService {
* Runs one redrive pass.
*
* @param request what to move
* @param approval the approval, when one was supplied
* @param approval the verified approval, when one was supplied
* @param subject the operator identity
* @param now the current instant
* @return what the pass did
*/
public RedriveReport redrive(
RedriveRequest request, Optional<AdminApproval> approval, String subject, Instant now) {
RedriveRequest request, Optional<VerifiedApproval> approval, String subject, Instant now) {
return redrive(request, approval, subject, now, 0, completed -> {});
}
/**
* Runs one redrive pass, resuming a previous attempt and reporting progress as it goes.
*
* <p>Three things were wrong with running this as a plain loop. A synchronous failure from the
* publisher — a broker that refuses the connection rather than the message — propagated out of
* the loop, so the remaining candidates were never attempted and the audit record was never
* written: the operation left no trace of the items it had already moved. A retry then started
* from the first candidate and republished them. And nothing bounded how many times one message
* could be redriven.
*
* @param request what to move
* @param approval the verified approval, when one was supplied
* @param subject the operator identity
* @param now the current instant
* @param resumeFrom how many candidates a previous attempt durably moved
* @param checkpoint called with the running completed count after each item
* @return what the pass did
*/
public RedriveReport redrive(
RedriveRequest request,
Optional<VerifiedApproval> approval,
String subject,
Instant now,
int resumeFrom,
java.util.function.IntConsumer checkpoint) {
Objects.requireNonNull(request, "request must not be null");
Objects.requireNonNull(checkpoint, "checkpoint must not be null");
if (resumeFrom < 0) {
throw new IllegalArgumentException("resumeFrom must not be negative");
}
guard.authorize(
DestructiveOperation.REDRIVE, request.source(), approval, request.dryRun(), now);
@@ -72,31 +104,57 @@ public final class RedriveService {
List<MessageId> moved = new ArrayList<>();
int failed = 0;
for (MessageId messageId : candidates) {
PublishResult result = publisher.republish(messageId, request.target(), request.redriveId());
if (result.completion() == PublishCompletion.CONFIRMED) {
source.settle(request.source(), messageId);
moved.add(messageId);
} else {
failed++;
int completed = resumeFrom;
try {
// Everything before resumeFrom was moved and settled by the previous attempt. Re-publishing
// it is not a retry, it is a duplicate.
for (MessageId messageId :
candidates.subList(Math.min(resumeFrom, candidates.size()), candidates.size())) {
if (attempt(messageId, request)) {
moved.add(messageId);
} else {
failed++;
}
completed++;
checkpoint.accept(completed);
}
} finally {
// In the finally block on purpose: an operation that dies partway must still leave a record
// of what it moved, because that record is what the resumed attempt and the incident review
// both read.
audit.record(
new dev.caskeleton.messaging.observation.MessagingAuditEvent(
"REDRIVE",
subject,
request.source().value(),
approval.map(VerifiedApproval::ticket).orElse("dry-run"),
now,
java.util.Map.of(
"redriveId", request.redriveId().toString(),
"resumedFrom", Integer.toString(resumeFrom),
"moved", Integer.toString(moved.size()),
"failed", Integer.toString(failed))));
}
audit.record(
new dev.caskeleton.messaging.observation.MessagingAuditEvent(
"REDRIVE",
subject,
request.source().value(),
approval.map(AdminApproval::ticket).orElse("dry-run"),
now,
java.util.Map.of(
"redriveId", request.redriveId().toString(),
"moved", Integer.toString(moved.size()),
"failed", Integer.toString(failed))));
return new RedriveReport(candidates.size(), moved.size(), failed, false);
}
private boolean attempt(MessageId messageId, RedriveRequest request) {
PublishResult result;
try {
result = publisher.republish(messageId, request.target(), request.redriveId());
} catch (RuntimeException failure) {
// One message that cannot be republished is a failed item, not a failed pass. Letting it
// propagate abandoned every candidate behind it.
return false;
}
if (result.completion() != PublishCompletion.CONFIRMED) {
return false;
}
source.settle(request.source(), messageId);
return true;
}
/** Reads and settles messages on a dead letter destination. */
public interface RedriveSource {
@@ -1,9 +1,9 @@
package dev.caskeleton.messaging.admin.runtime;
import dev.caskeleton.messaging.admin.AdminApproval;
import dev.caskeleton.messaging.admin.DestructiveOperation;
import dev.caskeleton.messaging.admin.DestructiveOperationGuard;
import dev.caskeleton.messaging.admin.ReplayRequest;
import dev.caskeleton.messaging.admin.VerifiedApproval;
import dev.caskeleton.messaging.observation.MessagingAuditEvent;
import java.time.Instant;
import java.util.Map;
@@ -45,13 +45,13 @@ public final class ReplayService {
* Runs one replay.
*
* @param request what to replay
* @param approval the approval, when one was supplied
* @param approval the verified approval, when one was supplied
* @param subject the operator identity
* @param now the current instant
* @return what the replay did
*/
public ReplayReport replay(
ReplayRequest request, Optional<AdminApproval> approval, String subject, Instant now) {
ReplayRequest request, Optional<VerifiedApproval> approval, String subject, Instant now) {
Objects.requireNonNull(request, "request must not be null");
Objects.requireNonNull(approval, "approval must not be null");
@@ -74,7 +74,7 @@ public final class ReplayService {
"REPLAY",
subject,
request.destination().value(),
approval.map(AdminApproval::ticket).orElse("isolated"),
approval.map(VerifiedApproval::ticket).orElse("isolated"),
now,
Map.of(
"replayId", request.replayId().toString(),
@@ -0,0 +1,145 @@
package dev.caskeleton.messaging.admin.runtime;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import dev.caskeleton.messaging.admin.AdminOperationLease;
import dev.caskeleton.messaging.admin.AdminOperationState;
import dev.caskeleton.messaging.admin.PlanDigest;
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
import java.time.Duration;
import java.time.Instant;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
/**
* The protocol the journal has to implement, proven against the in-memory one.
*
* <p>What it replaced recorded a single fact — "this approval was claimed" — before any work
* happened, in a map. An operation that died halfway had spent its approval, left no record of what
* it had already moved, and gave a retry no way to continue: either request a new approval and
* republish everything, or leave the redrive half applied.
*/
class AdminOperationJournalTest {
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
private static final Duration LEASE = Duration.ofMinutes(5);
private static final String TICKET = "CHG-1042";
private static final PlanDigest DIGEST = PlanDigest.ofCanonical("REDRIVE|orders.v1.dlq|50");
private static final PlanDigest OTHER_DIGEST = PlanDigest.ofCanonical("REDRIVE|payments.dlq|50");
private final InMemoryAdminOperationJournal journal = new InMemoryAdminOperationJournal();
@Test
@DisplayName("two replicas racing for one approval produce exactly one lease")
void twoReplicasRacingProduceExactlyOneLease() {
journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
assertThatThrownBy(
() -> journal.begin(TICKET, DIGEST, "op-2", "replica-b", LEASE, NOW.plusSeconds(1)))
.as("both replicas executing one approval is a duplicate storm, not a faster redrive")
.isInstanceOf(MessageAuthorizationException.class)
.hasMessageContaining("replica-a");
}
@Test
@DisplayName("a completed operation cannot be executed again")
void aCompletedOperationCannotBeExecutedAgain() {
AdminOperationLease lease = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
journal.complete(lease, 50, NOW.plusSeconds(30));
assertThatThrownBy(
() -> journal.begin(TICKET, DIGEST, "op-2", "replica-a", LEASE, NOW.plusSeconds(60)))
.as("an arrow-up in the shell must not re-run an approved redrive")
.isInstanceOf(MessageAuthorizationException.class)
.hasMessageContaining("already executed");
}
@Test
@DisplayName("a crashed run resumes from its checkpoint rather than starting over")
void aCrashedRunResumesFromItsCheckpoint() {
AdminOperationLease first = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
journal.checkpoint(first, 30, LEASE, NOW.plusSeconds(10));
// No fail() call: this replica died. The lease simply runs out.
AdminOperationLease resumed =
journal.begin(TICKET, DIGEST, "op-1", "replica-b", LEASE, NOW.plus(Duration.ofMinutes(10)));
assertThat(resumed.resumeFrom())
.as("the 30 already moved must not be republished")
.isEqualTo(30);
assertThat(resumed.leaseToken()).isGreaterThan(first.leaseToken());
assertThat(resumed.isResumption()).isTrue();
}
@Test
@DisplayName("a failed run stays resumable without waiting for the lease to expire")
void aFailedRunStaysResumableImmediately() {
AdminOperationLease first = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
journal.checkpoint(first, 12, LEASE, NOW.plusSeconds(5));
journal.fail(first, 12, "MESSAGE_BROKER_UNAVAILABLE", NOW.plusSeconds(6));
AdminOperationLease resumed =
journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW.plusSeconds(7));
assertThat(resumed.resumeFrom()).isEqualTo(12);
assertThat(journal.find(TICKET, DIGEST))
.hasValueSatisfying(
record -> assertThat(record.state()).isEqualTo(AdminOperationState.STARTED));
}
@Test
@DisplayName("a runtime that lost its lease cannot write over the one that took over")
void aRuntimeThatLostItsLeaseCannotWriteOverTheSuccessor() {
AdminOperationLease stale = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
journal.checkpoint(stale, 30, LEASE, NOW.plusSeconds(10));
AdminOperationLease successor =
journal.begin(TICKET, DIGEST, "op-1", "replica-b", LEASE, NOW.plus(Duration.ofMinutes(10)));
journal.checkpoint(successor, 45, LEASE, NOW.plus(Duration.ofMinutes(11)));
assertThatThrownBy(() -> journal.complete(stale, 30, NOW.plus(Duration.ofMinutes(12))))
.as("a stalled replica waking up must not mark the operation done at its own count")
.isInstanceOf(MessageAuthorizationException.class)
.hasMessageContaining("no longer owns");
assertThat(journal.find(TICKET, DIGEST))
.hasValueSatisfying(
record -> {
assertThat(record.itemsCompleted()).isEqualTo(45);
assertThat(record.state()).isEqualTo(AdminOperationState.STARTED);
});
}
@Test
@DisplayName("the same ticket presented for a different plan is a different operation")
void theSameTicketForADifferentPlanIsADifferentOperation() {
journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
// Keyed on the ticket alone this would have been refused as a duplicate execution, which reads
// as protection but is actually the wrong answer: it is a different plan, and it must be
// refused by the approval binding rather than silently treated as the same operation.
assertThatCode(() -> journal.begin(TICKET, OTHER_DIGEST, "op-2", "replica-a", LEASE, NOW))
.doesNotThrowAnyException();
assertThat(journal.size()).isEqualTo(2);
}
@Test
@DisplayName("a checkpoint never moves backwards")
void aCheckpointNeverMovesBackwards() {
AdminOperationLease lease = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
journal.checkpoint(lease, 40, LEASE, NOW.plusSeconds(10));
journal.checkpoint(lease, 5, LEASE, NOW.plusSeconds(20));
assertThat(journal.find(TICKET, DIGEST))
.hasValueSatisfying(record -> assertThat(record.itemsCompleted()).isEqualTo(40));
}
@Test
@DisplayName("the in-memory journal declares itself non-durable")
void theInMemoryJournalDeclaresItselfNonDurable() {
assertThat(journal.isDurable())
.as("the starter refuses a production profile on a journal that says this")
.isFalse();
}
}
@@ -0,0 +1,231 @@
package dev.caskeleton.messaging.admin.runtime;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import dev.caskeleton.messaging.admin.AdminApproval;
import dev.caskeleton.messaging.admin.ApprovalGrant;
import dev.caskeleton.messaging.admin.ApprovedRedrivePlan;
import dev.caskeleton.messaging.admin.DestructiveOperation;
import dev.caskeleton.messaging.admin.HmacApprovalVerifier;
import dev.caskeleton.messaging.admin.RedrivePlan;
import dev.caskeleton.messaging.admin.RedriveRequest;
import dev.caskeleton.messaging.admin.VerifiedApproval;
import dev.caskeleton.messaging.api.destination.DestinationName;
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
import java.util.UUID;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
/**
* Whether "this was approved" is a fact or a claim the caller makes about itself.
*
* <p>It was a claim. {@code AdminApproval} was a public record and {@code ApprovedRedrivePlan} a
* public record over it, so any code that could reach the execute method could write its own
* approval and the platform believed it. Nothing bound an approval to an operation, a destination,
* a topology version, or an impact ceiling either, so a genuine approval for one redrive authorised
* a redrive of anything else inside its window.
*/
class ApprovalForgeryTest {
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
private static final UUID OPERATION_ID = UUID.fromString("0199aaaa-bbbb-7ccc-8ddd-eeeeffff0000");
private static final DestinationName DLQ = new DestinationName("orders.v1.dlq");
private static final DestinationName ORDERS = new DestinationName("orders.v1");
private static final HmacApprovalVerifier ISSUER =
new HmacApprovalVerifier(
"an-issuing-key-of-at-least-32-bytes!".getBytes(StandardCharsets.UTF_8));
private static final HmacApprovalVerifier ANOTHER_ISSUER =
new HmacApprovalVerifier(
"a-different-key-of-at-least-32-bytes".getBytes(StandardCharsets.UTF_8));
@Test
@DisplayName("a verified approval cannot be constructed outside the verifier")
void aVerifiedApprovalCannotBeConstructedOutsideTheVerifier() {
// Written as a reflection probe because the property is about the type's shape, not its
// behaviour: if a public constructor or factory ever appears, every other test here becomes
// decorative.
assertThat(VerifiedApproval.class.getConstructors())
.as("a public constructor would let any caller mint an approval for itself")
.isEmpty();
assertThat(
java.util.Arrays.stream(VerifiedApproval.class.getDeclaredMethods())
.filter(method -> java.lang.reflect.Modifier.isStatic(method.getModifiers()))
.filter(method -> java.lang.reflect.Modifier.isPublic(method.getModifiers()))
.toList())
.as("nor may a public static factory")
.isEmpty();
}
@Test
@DisplayName("an unsigned or altered grant does not verify")
void anAlteredGrantDoesNotVerify() {
RedrivePlan plan = plan(50, 0);
ApprovalGrant signed = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
String signature = ISSUER.sign(signed);
ApprovalGrant raised = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 5_000);
assertThatThrownBy(() -> ISSUER.verify(raised, signature, plan.digest(), NOW))
.as("raising the impact ceiling after the approver signed it")
.isInstanceOf(MessageAuthorizationException.class)
.hasMessageContaining("does not verify");
}
@Test
@DisplayName("a signature from another issuer does not verify")
void aSignatureFromAnotherIssuerDoesNotVerify() {
RedrivePlan plan = plan(50, 0);
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
assertThatThrownBy(
() -> ISSUER.verify(granted, ANOTHER_ISSUER.sign(granted), plan.digest(), NOW))
.isInstanceOf(MessageAuthorizationException.class)
.hasMessageContaining("does not verify");
}
@Test
@DisplayName("an approval for one plan cannot execute another")
void anApprovalForOnePlanCannotExecuteAnother() {
RedrivePlan approvedPlan = plan(50, 0);
RedrivePlan biggerPlan = plan(900, 0);
ApprovalGrant granted = grant(approvedPlan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 900);
String signature = ISSUER.sign(granted);
assertThatThrownBy(() -> ISSUER.verify(granted, signature, biggerPlan.digest(), NOW))
.as("the approver saw 50 candidates; this plan moves 900")
.isInstanceOf(MessageAuthorizationException.class)
.hasMessageContaining("was issued for plan");
}
@Test
@DisplayName("a verified approval for the wrong source cannot be attached to a plan")
void aVerifiedApprovalForTheWrongSourceCannotBeAttached() {
RedrivePlan plan = plan(50, 0);
// A genuine, correctly-signed approval — for a different dead-letter destination, with this
// plan's digest smuggled in. The digest check passes; the source binding is what catches it.
ApprovalGrant granted =
new ApprovalGrant(
approval(),
"operator@example.com",
DestructiveOperation.REDRIVE,
new DestinationName("payments.v1.dlq"),
Optional.of(ORDERS),
"v1",
900,
plan.digest());
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
assertThatThrownBy(() -> new ApprovedRedrivePlan(plan, verified, false))
.isInstanceOf(MessageAuthorizationException.class)
.hasMessageContaining("payments.v1.dlq");
}
@Test
@DisplayName("a plan beyond the approved impact ceiling cannot execute")
void aPlanBeyondTheApprovedImpactCeilingCannotExecute() {
RedrivePlan plan = plan(900, 0);
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 100);
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
assertThatThrownBy(() -> new ApprovedRedrivePlan(plan, verified, false))
.as("the approver agreed to at most 100")
.isInstanceOf(MessageAuthorizationException.class)
.hasMessageContaining("at most 100");
}
@Test
@DisplayName("an approval for a replay cannot authorise a redrive")
void anApprovalForAReplayCannotAuthoriseARedrive() {
RedrivePlan plan = plan(50, 0);
ApprovalGrant granted = grant(plan, DestructiveOperation.REPLAY, DLQ, ORDERS, 900);
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
assertThatThrownBy(() -> new ApprovedRedrivePlan(plan, verified, false))
.isInstanceOf(MessageAuthorizationException.class)
.hasMessageContaining("authorises REPLAY");
}
@Test
@DisplayName("an expired grant does not verify")
void anExpiredGrantDoesNotVerify() {
RedrivePlan plan = plan(50, 0);
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
assertThatThrownBy(
() ->
ISSUER.verify(
granted, ISSUER.sign(granted), plan.digest(), NOW.plus(Duration.ofDays(1))))
.isInstanceOf(MessageAuthorizationException.class)
.hasMessageContaining("not valid at");
}
@Test
@DisplayName("the approver and the operator must be different people")
void theApproverAndTheOperatorMustBeDifferentPeople() {
RedrivePlan plan = plan(50, 0);
assertThatThrownBy(
() ->
new ApprovalGrant(
approval(),
"ops@example.com",
DestructiveOperation.REDRIVE,
DLQ,
Optional.of(ORDERS),
"v1",
50,
plan.digest()))
.as("an operator approving their own destructive operation is the control not existing")
.isInstanceOf(IllegalArgumentException.class);
}
@Test
@DisplayName("a correctly issued approval executes")
void aCorrectlyIssuedApprovalExecutes() {
RedrivePlan plan = plan(50, 0);
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
assertThatCode(
() -> new ApprovedRedrivePlan(plan, verified, false).requireExecutable(NOW, "v1"))
.doesNotThrowAnyException();
}
private static ApprovalGrant grant(
RedrivePlan plan,
DestructiveOperation operation,
DestinationName source,
DestinationName target,
long maxImpact) {
return new ApprovalGrant(
approval(),
"operator@example.com",
operation,
source,
Optional.of(target),
"v1",
maxImpact,
plan.digest());
}
private static AdminApproval approval() {
return new AdminApproval("CHG-1042", "ops@example.com", NOW, NOW.plus(Duration.ofHours(2)));
}
private static RedrivePlan plan(int candidates, int alreadyRedriven) {
return new RedrivePlan(
new RedriveRequest(OPERATION_ID, DLQ, ORDERS, 50, false),
candidates,
alreadyRedriven,
NOW,
"v1");
}
}
@@ -5,14 +5,19 @@ import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import dev.caskeleton.messaging.admin.AdminApproval;
import dev.caskeleton.messaging.admin.ApprovalGrant;
import dev.caskeleton.messaging.admin.ApprovedRedrivePlan;
import dev.caskeleton.messaging.admin.ApprovedReplayPlan;
import dev.caskeleton.messaging.admin.DestructiveOperation;
import dev.caskeleton.messaging.admin.HmacApprovalVerifier;
import dev.caskeleton.messaging.admin.PlanDigest;
import dev.caskeleton.messaging.admin.RedrivePlan;
import dev.caskeleton.messaging.admin.RedriveRequest;
import dev.caskeleton.messaging.admin.RedriveResult;
import dev.caskeleton.messaging.admin.ReplayPlan;
import dev.caskeleton.messaging.admin.ReplayRequest;
import dev.caskeleton.messaging.admin.ReplayResult;
import dev.caskeleton.messaging.admin.VerifiedApproval;
import dev.caskeleton.messaging.api.destination.DestinationName;
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
import java.time.Duration;
@@ -26,10 +31,56 @@ class ApprovedPlanExecutionTest {
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
private static final UUID OPERATION_ID = UUID.fromString("0199aaaa-bbbb-7ccc-8ddd-eeeeffff0000");
private static final byte[] ISSUING_KEY =
"an-issuing-key-of-at-least-32-bytes!".getBytes(java.nio.charset.StandardCharsets.UTF_8);
private static final HmacApprovalVerifier VERIFIER = new HmacApprovalVerifier(ISSUING_KEY);
private static AdminApproval approval() {
return new AdminApproval("CHG-1042", "ops@example.com", NOW, NOW.plus(Duration.ofHours(2)));
}
/** A grant the issuer signed for exactly this plan, verified the way the runtime verifies it. */
static VerifiedApproval verified(
PlanDigest digest,
DestructiveOperation operation,
DestinationName source,
Optional<DestinationName> target,
String topologyVersion,
long maxImpact) {
ApprovalGrant grant =
new ApprovalGrant(
approval(),
"operator@example.com",
operation,
source,
target,
topologyVersion,
maxImpact,
digest);
return VERIFIER.verify(grant, VERIFIER.sign(grant), digest, NOW);
}
private static VerifiedApproval replayApproval(ReplayPlan plan) {
return verified(
plan.digest(),
DestructiveOperation.REPLAY,
plan.request().destination(),
Optional.empty(),
plan.topologyVersion(),
Long.MAX_VALUE);
}
private static VerifiedApproval redriveApproval(RedrivePlan plan) {
return verified(
plan.digest(),
DestructiveOperation.REDRIVE,
plan.request().source(),
Optional.of(plan.request().target()),
plan.topologyVersion(),
Long.MAX_VALUE);
}
private static ReplayRequest replayRequest() {
return new ReplayRequest(
OPERATION_ID,
@@ -57,9 +108,18 @@ class ApprovedPlanExecutionTest {
return new RedrivePlan(redriveRequest(), candidates, alreadyRedriven, NOW, "v1");
}
private static ApprovedReplayPlan approvedReplay(String topologyVersion) {
ReplayPlan plan = replayPlan(topologyVersion);
return new ApprovedReplayPlan(plan, replayApproval(plan));
}
private static ApprovedRedrivePlan approvedRedrive(RedrivePlan plan, boolean loopAcknowledged) {
return new ApprovedRedrivePlan(plan, redriveApproval(plan), loopAcknowledged);
}
@Test
void anExpiredApprovalCannotExecute() {
ApprovedReplayPlan approved = new ApprovedReplayPlan(replayPlan("v1"), approval());
ApprovedReplayPlan approved = approvedReplay("v1");
assertThatThrownBy(() -> approved.requireExecutable(NOW.plus(Duration.ofDays(1)), "v1"))
.isInstanceOf(MessageAuthorizationException.class)
@@ -68,25 +128,26 @@ class ApprovedPlanExecutionTest {
@Test
void aTopologyChangeSinceApprovalInvalidatesThePlan() {
ApprovedReplayPlan approved = new ApprovedReplayPlan(replayPlan("v1"), approval());
ApprovedReplayPlan approved = approvedReplay("v1");
assertThatThrownBy(() -> approved.requireExecutable(NOW, "v2"))
.as("every number in the plan was computed against the old topology")
.isInstanceOf(MessageAuthorizationException.class)
.hasMessageContaining("rebuilt");
// The signed grant carries the topology version too, so the mismatch is now caught against
// what the approver actually signed rather than against a field on the plan the caller
// handed in.
.hasMessageContaining("was signed against topology v1 but the broker is now at v2");
}
@Test
void aValidApprovalOnUnchangedTopologyExecutes() {
assertThatCode(
() -> new ApprovedReplayPlan(replayPlan("v1"), approval()).requireExecutable(NOW, "v1"))
assertThatCode(() -> approvedReplay("v1").requireExecutable(NOW, "v1"))
.doesNotThrowAnyException();
}
@Test
void aRedriveThatWouldLoopNeedsThatAcknowledgedExplicitly() {
ApprovedRedrivePlan approved =
new ApprovedRedrivePlan(redrivePlan(900, 400), approval(), false);
ApprovedRedrivePlan approved = approvedRedrive(redrivePlan(900, 400), false);
assertThatThrownBy(() -> approved.requireExecutable(NOW, "v1"))
.isInstanceOf(MessageAuthorizationException.class)
@@ -95,19 +156,13 @@ class ApprovedPlanExecutionTest {
@Test
void anAcknowledgedLoopMayProceed() {
assertThatCode(
() ->
new ApprovedRedrivePlan(redrivePlan(900, 400), approval(), true)
.requireExecutable(NOW, "v1"))
assertThatCode(() -> approvedRedrive(redrivePlan(900, 400), true).requireExecutable(NOW, "v1"))
.doesNotThrowAnyException();
}
@Test
void aRedriveWithNoPreviouslyRedrivenCandidatesNeedsNoAcknowledgement() {
assertThatCode(
() ->
new ApprovedRedrivePlan(redrivePlan(900, 0), approval(), false)
.requireExecutable(NOW, "v1"))
assertThatCode(() -> approvedRedrive(redrivePlan(900, 0), false).requireExecutable(NOW, "v1"))
.doesNotThrowAnyException();
}
@@ -139,25 +194,6 @@ class ApprovedPlanExecutionTest {
.contains("LIVE consumer group");
}
@Test
void anApprovalAuthorisesOneExecutionNotAStandingPermission() {
AdminOperationIdempotencyStore store = new AdminOperationIdempotencyStore();
assertThat(store.claim("CHG-1042", "op-1", NOW)).isEmpty();
assertThat(store.claim("CHG-1042", "op-2", NOW.plus(Duration.ofMinutes(5))))
.as("an arrow-up in the shell must not re-run an approved redrive")
.hasValueSatisfying(existing -> assertThat(existing.operationId()).isEqualTo("op-1"));
}
@Test
void aDifferentApprovalIsClaimedIndependently() {
AdminOperationIdempotencyStore store = new AdminOperationIdempotencyStore();
store.claim("CHG-1042", "op-1", NOW);
assertThat(store.claim("CHG-1043", "op-2", NOW)).isEmpty();
assertThat(store.size()).isEqualTo(2);
}
@Test
void aReplayResultFlagsAShortfallAgainstTheApprovedEstimate() {
ReplayResult result =
@@ -0,0 +1,212 @@
package dev.caskeleton.messaging.admin.runtime;
import static org.assertj.core.api.Assertions.assertThat;
import dev.caskeleton.messaging.admin.AdminApproval;
import dev.caskeleton.messaging.admin.ApprovalGrant;
import dev.caskeleton.messaging.admin.DestructiveOperation;
import dev.caskeleton.messaging.admin.DestructiveOperationGuard;
import dev.caskeleton.messaging.admin.HmacApprovalVerifier;
import dev.caskeleton.messaging.admin.PlanDigest;
import dev.caskeleton.messaging.admin.RedriveRequest;
import dev.caskeleton.messaging.admin.VerifiedApproval;
import dev.caskeleton.messaging.api.MessageId;
import dev.caskeleton.messaging.api.destination.DestinationName;
import dev.caskeleton.messaging.api.publish.ConfirmationLevel;
import dev.caskeleton.messaging.api.publish.PublishCompletion;
import dev.caskeleton.messaging.api.publish.PublishEvidence;
import dev.caskeleton.messaging.api.publish.PublishResult;
import dev.caskeleton.messaging.api.publish.RoutingOutcome;
import dev.caskeleton.messaging.observation.MessagingAuditEvent;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
/**
* What a redrive does when one message fails, and what it does the second time round.
*
* <p>The loop had no per-item boundary. A publisher that threw — a broker refusing the connection
* rather than the message — propagated straight out, so every candidate behind it was abandoned and
* the audit record was never written. The operation left no trace of what it had already moved, and
* a retry started again from the first candidate and republished it.
*/
class RedriveResumptionTest {
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
private static final DestinationName DLQ = new DestinationName("orders.v1.dlq");
private static final DestinationName ORDERS = new DestinationName("orders.v1");
private static final HmacApprovalVerifier ISSUER =
new HmacApprovalVerifier(
"an-issuing-key-of-at-least-32-bytes!".getBytes(java.nio.charset.StandardCharsets.UTF_8));
private final RecordingSource source = new RecordingSource();
private final RecordingAudit audit = new RecordingAudit();
@Test
@DisplayName("one message that throws does not abandon the ones behind it")
void oneThrowingMessageDoesNotAbandonTheRest() {
List<MessageId> candidates = ids(4);
source.stage(candidates);
RedriveService service =
service(
(messageId, target, redriveId) -> {
if (messageId.equals(candidates.get(1))) {
throw new IllegalStateException("the broker refused the connection");
}
return confirmed();
});
RedriveReport report = service.redrive(request(), approval(), "operator", NOW);
assertThat(report.moved()).isEqualTo(3);
assertThat(report.failed())
.as("the throwing item is a failed item, not a failed pass")
.isEqualTo(1);
assertThat(source.settled)
.containsExactly(candidates.get(0), candidates.get(2), candidates.get(3));
}
@Test
@DisplayName("the audit record is written even when the pass does not finish cleanly")
void theAuditRecordIsWrittenEvenOnFailure() {
source.stage(ids(2));
RedriveService service =
service(
(messageId, target, redriveId) -> {
throw new IllegalStateException("the broker refused the connection");
});
service.redrive(request(), approval(), "operator", NOW);
assertThat(audit.events)
.as("the record of what moved is what the incident review and the resumed run both read")
.singleElement()
.satisfies(event -> assertThat(event.details()).containsEntry("failed", "2"));
}
@Test
@DisplayName("a resumed pass does not republish what the first attempt already moved")
void aResumedPassDoesNotRepublishWhatWasAlreadyMoved() {
List<MessageId> candidates = ids(5);
source.stage(candidates);
List<MessageId> republished = new ArrayList<>();
RedriveService service =
service(
(messageId, target, redriveId) -> {
republished.add(messageId);
return confirmed();
});
RedriveReport report =
service.redrive(request(), approval(), "operator", NOW, 3, completed -> {});
assertThat(republished)
.as("republishing a settled message is a duplicate, not a retry")
.containsExactly(candidates.get(3), candidates.get(4));
assertThat(report.moved()).isEqualTo(2);
}
@Test
@DisplayName("progress is reported after each item so a crash resumes near where it stopped")
void progressIsReportedAfterEachItem() {
source.stage(ids(3));
List<Integer> checkpoints = new ArrayList<>();
RedriveService service = service((messageId, target, redriveId) -> confirmed());
service.redrive(request(), approval(), "operator", NOW, 0, checkpoints::add);
assertThat(checkpoints).containsExactly(1, 2, 3);
}
@Test
@DisplayName("resuming past the end of the batch moves nothing")
void resumingPastTheEndOfTheBatchMovesNothing() {
source.stage(ids(2));
RedriveService service = service((messageId, target, redriveId) -> confirmed());
RedriveReport report =
service.redrive(request(), approval(), "operator", NOW, 10, completed -> {});
assertThat(report.moved()).isZero();
}
private RedriveService service(RedriveService.RedrivePublisher publisher) {
// The guard holds the admin credential and every pass carries a real verified approval: this
// test is about the loop, and a gate that refused would prove nothing about it.
return new RedriveService(new DestructiveOperationGuard(true), source, publisher, audit);
}
private static Optional<VerifiedApproval> approval() {
PlanDigest digest = PlanDigest.ofCanonical("REDRIVE|orders.v1.dlq|orders.v1");
ApprovalGrant grant =
new ApprovalGrant(
new AdminApproval("CHG-1042", "ops@example.com", NOW, NOW.plus(Duration.ofHours(2))),
"operator@example.com",
DestructiveOperation.REDRIVE,
DLQ,
Optional.of(ORDERS),
"v1",
Long.MAX_VALUE,
digest);
return Optional.of(ISSUER.verify(grant, ISSUER.sign(grant), digest, NOW));
}
private static RedriveRequest request() {
return new RedriveRequest(UUID.randomUUID(), DLQ, ORDERS, 50, false);
}
private static PublishResult confirmed() {
return new PublishResult(
PublishCompletion.CONFIRMED,
PublishEvidence.confirmed(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK),
RoutingOutcome.ROUTED,
Optional.empty(),
1,
Duration.ofMillis(2),
Optional.empty());
}
private static List<MessageId> ids(int count) {
List<MessageId> ids = new ArrayList<>();
for (int index = 0; index < count; index++) {
ids.add(MessageId.newId());
}
return List.copyOf(ids);
}
private static final class RecordingSource implements RedriveService.RedriveSource {
private List<MessageId> staged = List.of();
private final List<MessageId> settled = new ArrayList<>();
private void stage(List<MessageId> ids) {
staged = List.copyOf(ids);
}
@Override
public List<MessageId> peek(DestinationName destination, int batchSize) {
return staged;
}
@Override
public void settle(DestinationName destination, MessageId messageId) {
settled.add(messageId);
}
}
private static final class RecordingAudit implements RedriveService.AuditSink {
private final List<MessagingAuditEvent> events = new ArrayList<>();
@Override
public void record(MessagingAuditEvent event) {
events.add(event);
}
}
}