feat: jpa, messaging, notification, mongo, graphql 어댑터터 구현체 추가
This commit is contained in:
-78
@@ -1,78 +0,0 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* Remembers which approvals have already been executed.
|
||||
*
|
||||
* <p>An approval authorises one execution, not a standing permission. Without this, re-running the
|
||||
* same approved redrive twice is a single command-history arrow-up away — and the second run
|
||||
* republishes messages the first one already moved, which on a destination without an inbox is
|
||||
* indistinguishable from a duplicate storm.
|
||||
*
|
||||
* <p>Claiming is atomic and returns the previous claim rather than a boolean, so a duplicate
|
||||
* attempt can tell the operator <em>when</em> it ran and by which operation id instead of just
|
||||
* refusing.
|
||||
*/
|
||||
public final class AdminOperationIdempotencyStore {
|
||||
|
||||
private final Map<String, Claim> claims = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* One recorded execution of an approval.
|
||||
*
|
||||
* @param approvalTicket the approval that was executed
|
||||
* @param operationId the operation identity that claimed it
|
||||
* @param executedAt when it ran
|
||||
*/
|
||||
public record Claim(String approvalTicket, String operationId, Instant executedAt) {
|
||||
|
||||
public Claim {
|
||||
Objects.requireNonNull(executedAt, "executedAt must not be null");
|
||||
if (approvalTicket == null || approvalTicket.isBlank()) {
|
||||
throw new IllegalArgumentException("approvalTicket must not be blank");
|
||||
}
|
||||
if (operationId == null || operationId.isBlank()) {
|
||||
throw new IllegalArgumentException("operationId must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Claims an approval for execution.
|
||||
*
|
||||
* @param approvalTicket the approval to claim
|
||||
* @param operationId the operation attempting it
|
||||
* @param now the current instant
|
||||
* @return empty when the claim succeeded; the existing claim when it was already executed
|
||||
*/
|
||||
public Optional<Claim> claim(String approvalTicket, String operationId, Instant now) {
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
Claim candidate = new Claim(approvalTicket, operationId, now);
|
||||
Claim existing = claims.putIfAbsent(approvalTicket, candidate);
|
||||
return Optional.ofNullable(existing);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the recorded execution of an approval.
|
||||
*
|
||||
* @param approvalTicket the approval
|
||||
* @return the claim, when the approval has been executed
|
||||
*/
|
||||
public Optional<Claim> find(String approvalTicket) {
|
||||
return Optional.ofNullable(claims.get(approvalTicket));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns how many approvals have been executed.
|
||||
*
|
||||
* @return the claim count
|
||||
*/
|
||||
public int size() {
|
||||
return claims.size();
|
||||
}
|
||||
}
|
||||
+85
-27
@@ -1,5 +1,7 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminOperationJournal;
|
||||
import dev.caskeleton.messaging.admin.AdminOperationLease;
|
||||
import dev.caskeleton.messaging.admin.ApprovedRedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.ApprovedReplayPlan;
|
||||
import dev.caskeleton.messaging.admin.RedrivePlan;
|
||||
@@ -10,7 +12,6 @@ import dev.caskeleton.messaging.admin.ReplayRequest;
|
||||
import dev.caskeleton.messaging.admin.ReplayResult;
|
||||
import dev.caskeleton.messaging.admin.TopologyManifest;
|
||||
import dev.caskeleton.messaging.admin.TopologyValidationReport;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
@@ -30,15 +31,29 @@ import java.util.function.Supplier;
|
||||
* <li>Only then does anything move.
|
||||
* </ol>
|
||||
*
|
||||
* <p>The idempotency claim comes <em>before</em> the work rather than after it. Claiming afterwards
|
||||
* leaves a window where a second execution starts while the first is still running, which is
|
||||
* precisely the double-redrive this store exists to prevent.
|
||||
* <p>The journal entry is written <em>before</em> the work rather than after it. Writing it
|
||||
* afterwards leaves a window where a second execution starts while the first is still running,
|
||||
* which is precisely the double-redrive the journal exists to prevent.
|
||||
*
|
||||
* <p>Writing it first used to have a cost the previous store never paid: an operation that died
|
||||
* halfway had consumed its approval and left no record of how far it got. The journal keeps a
|
||||
* checkpoint and hands back a lease that says where to resume, so a retry continues the same
|
||||
* operation instead of either redoing it or requiring a new approval.
|
||||
*/
|
||||
public final class DefaultMessagingAdminService implements MessagingAdminService {
|
||||
|
||||
/**
|
||||
* How long a runtime holds an operation before another may take it over.
|
||||
*
|
||||
* <p>Long enough that a slow batch does not lose its lease mid-flight, short enough that a dead
|
||||
* replica does not park an approval for an hour.
|
||||
*/
|
||||
private static final Duration LEASE_DURATION = Duration.ofMinutes(5);
|
||||
|
||||
private final CompositeTopologyValidator topologyValidator;
|
||||
private final BrokerTopologyInspector inspector;
|
||||
private final AdminOperationIdempotencyStore idempotency;
|
||||
private final AdminOperationJournal journal;
|
||||
private final String leaseOwner;
|
||||
private final ReplayService replayService;
|
||||
private final RedriveService redriveService;
|
||||
private final Supplier<List<TopologyManifest>> manifests;
|
||||
@@ -51,7 +66,8 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
|
||||
*
|
||||
* @param topologyValidator compares declared topology against the broker
|
||||
* @param inspector reads the broker's topology version
|
||||
* @param idempotency records which approvals have been executed
|
||||
* @param journal records which approvals have been executed, and how far each got
|
||||
* @param leaseOwner identifies this runtime in the journal's leases
|
||||
* @param replayService performs replays
|
||||
* @param redriveService performs redrives
|
||||
* @param manifests supplies the declared topologies
|
||||
@@ -62,7 +78,8 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
|
||||
public DefaultMessagingAdminService(
|
||||
CompositeTopologyValidator topologyValidator,
|
||||
BrokerTopologyInspector inspector,
|
||||
AdminOperationIdempotencyStore idempotency,
|
||||
AdminOperationJournal journal,
|
||||
String leaseOwner,
|
||||
ReplayService replayService,
|
||||
RedriveService redriveService,
|
||||
Supplier<List<TopologyManifest>> manifests,
|
||||
@@ -72,7 +89,11 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
|
||||
this.topologyValidator =
|
||||
Objects.requireNonNull(topologyValidator, "topologyValidator required");
|
||||
this.inspector = Objects.requireNonNull(inspector, "inspector must not be null");
|
||||
this.idempotency = Objects.requireNonNull(idempotency, "idempotency must not be null");
|
||||
this.journal = Objects.requireNonNull(journal, "journal must not be null");
|
||||
if (leaseOwner == null || leaseOwner.isBlank()) {
|
||||
throw new IllegalArgumentException("leaseOwner must not be blank");
|
||||
}
|
||||
this.leaseOwner = leaseOwner;
|
||||
this.replayService = Objects.requireNonNull(replayService, "replayService must not be null");
|
||||
this.redriveService = Objects.requireNonNull(redriveService, "redriveService must not be null");
|
||||
this.manifests = Objects.requireNonNull(manifests, "manifests must not be null");
|
||||
@@ -104,12 +125,28 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
|
||||
ReplayRequest request = plan.plan().request();
|
||||
|
||||
plan.requireExecutable(now, inspector.topologyVersion());
|
||||
claimOrRefuse(plan.approval().ticket(), request.replayId().toString(), now);
|
||||
AdminOperationLease lease =
|
||||
journal.begin(
|
||||
plan.approval().ticket(),
|
||||
plan.plan().digest(),
|
||||
request.replayId().toString(),
|
||||
leaseOwner,
|
||||
LEASE_DURATION,
|
||||
now);
|
||||
|
||||
Instant startedAt = clock.get();
|
||||
ReplayReport report =
|
||||
replayService.replay(
|
||||
request, Optional.of(plan.approval()), plan.approval().approvedBy(), now);
|
||||
ReplayReport report;
|
||||
try {
|
||||
report =
|
||||
replayService.replay(
|
||||
request, Optional.of(plan.approval()), plan.approval().approvedBy(), now);
|
||||
} catch (RuntimeException failure) {
|
||||
// The operation stays resumable rather than silently consuming the approval: the journal
|
||||
// entry moves to FAILED at its checkpoint, and a retry takes it over from there.
|
||||
journal.fail(lease, lease.resumeFrom(), failureCodeOf(failure), clock.get());
|
||||
throw failure;
|
||||
}
|
||||
journal.complete(lease, Math.toIntExact(report.replayed()), clock.get());
|
||||
|
||||
return new ReplayResult(
|
||||
request.replayId(),
|
||||
@@ -139,12 +176,31 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
|
||||
RedriveRequest request = plan.plan().request();
|
||||
|
||||
plan.requireExecutable(now, inspector.topologyVersion());
|
||||
claimOrRefuse(plan.approval().ticket(), request.redriveId().toString(), now);
|
||||
AdminOperationLease lease =
|
||||
journal.begin(
|
||||
plan.approval().ticket(),
|
||||
plan.plan().digest(),
|
||||
request.redriveId().toString(),
|
||||
leaseOwner,
|
||||
LEASE_DURATION,
|
||||
now);
|
||||
|
||||
Instant startedAt = clock.get();
|
||||
RedriveReport report =
|
||||
redriveService.redrive(
|
||||
request, Optional.of(plan.approval()), plan.approval().approvedBy(), now);
|
||||
RedriveReport report;
|
||||
try {
|
||||
report =
|
||||
redriveService.redrive(
|
||||
request,
|
||||
Optional.of(plan.approval()),
|
||||
plan.approval().approvedBy(),
|
||||
now,
|
||||
lease.resumeFrom(),
|
||||
completed -> journal.checkpoint(lease, completed, LEASE_DURATION, clock.get()));
|
||||
} catch (RuntimeException failure) {
|
||||
journal.fail(lease, lease.resumeFrom(), failureCodeOf(failure), clock.get());
|
||||
throw failure;
|
||||
}
|
||||
journal.complete(lease, report.moved() + report.failed(), clock.get());
|
||||
|
||||
return new RedriveResult(
|
||||
request.redriveId(),
|
||||
@@ -155,17 +211,19 @@ public final class DefaultMessagingAdminService implements MessagingAdminService
|
||||
report.dryRun());
|
||||
}
|
||||
|
||||
private void claimOrRefuse(String approvalTicket, String operationId, Instant now) {
|
||||
idempotency
|
||||
.claim(approvalTicket, operationId, now)
|
||||
.ifPresent(
|
||||
existing -> {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_ALREADY_EXECUTED",
|
||||
"approval %s was already executed at %s by operation %s; an approval authorises "
|
||||
.formatted(approvalTicket, existing.executedAt(), existing.operationId())
|
||||
+ "one execution, not a standing permission");
|
||||
});
|
||||
/**
|
||||
* Reduces a failure to a bounded, sanitised code for the journal.
|
||||
*
|
||||
* <p>The journal is read by operators during incidents and its contents outlive the process. A
|
||||
* raw exception message can carry a destination, a payload fragment, or a credential from a
|
||||
* driver's own error text, so only the platform's own code or the exception's simple name goes
|
||||
* in.
|
||||
*/
|
||||
private static String failureCodeOf(RuntimeException failure) {
|
||||
if (failure instanceof dev.caskeleton.messaging.api.error.MessagingException messaging) {
|
||||
return messaging.failure().code();
|
||||
}
|
||||
return failure.getClass().getSimpleName();
|
||||
}
|
||||
|
||||
/** Estimates how many messages a replay would re-deliver. */
|
||||
|
||||
+237
@@ -0,0 +1,237 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminOperationJournal;
|
||||
import dev.caskeleton.messaging.admin.AdminOperationLease;
|
||||
import dev.caskeleton.messaging.admin.AdminOperationRecord;
|
||||
import dev.caskeleton.messaging.admin.AdminOperationState;
|
||||
import dev.caskeleton.messaging.admin.PlanDigest;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* A single-process journal, for tests and for local development.
|
||||
*
|
||||
* <p>It reports {@link #isDurable()} as false, and the starter refuses to run a production profile
|
||||
* on a journal that says so. That declaration is the point of this class existing at all: the
|
||||
* previous in-memory store was registered as the production default and nothing distinguished it
|
||||
* from a shared one, so the gap was invisible until two replicas executed the same approval.
|
||||
*
|
||||
* <p>The semantics are otherwise the real ones — uniqueness on {@code (ticket, digest)}, lease
|
||||
* takeover with a monotonic token, resume from checkpoint — so a test that passes here is testing
|
||||
* the protocol rather than a simplification of it.
|
||||
*/
|
||||
public final class InMemoryAdminOperationJournal implements AdminOperationJournal {
|
||||
|
||||
private final Map<String, AdminOperationRecord> records = new ConcurrentHashMap<>();
|
||||
|
||||
@Override
|
||||
public AdminOperationLease begin(
|
||||
String approvalTicket,
|
||||
PlanDigest planDigest,
|
||||
String operationId,
|
||||
String leaseOwner,
|
||||
Duration leaseDuration,
|
||||
Instant now) {
|
||||
Objects.requireNonNull(leaseDuration, "leaseDuration must not be null");
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
AdminOperationRecord claimed =
|
||||
records.compute(
|
||||
key(approvalTicket, planDigest),
|
||||
(ignored, existing) ->
|
||||
claim(
|
||||
existing,
|
||||
approvalTicket,
|
||||
planDigest,
|
||||
operationId,
|
||||
leaseOwner,
|
||||
leaseDuration,
|
||||
now));
|
||||
return new AdminOperationLease(
|
||||
approvalTicket,
|
||||
planDigest,
|
||||
claimed.operationId(),
|
||||
claimed.leaseOwner(),
|
||||
claimed.leaseToken(),
|
||||
claimed.itemsCompleted());
|
||||
}
|
||||
|
||||
private static AdminOperationRecord claim(
|
||||
AdminOperationRecord existing,
|
||||
String approvalTicket,
|
||||
PlanDigest planDigest,
|
||||
String operationId,
|
||||
String leaseOwner,
|
||||
Duration leaseDuration,
|
||||
Instant now) {
|
||||
if (existing == null) {
|
||||
return new AdminOperationRecord(
|
||||
approvalTicket,
|
||||
planDigest,
|
||||
operationId,
|
||||
AdminOperationState.STARTED,
|
||||
0,
|
||||
leaseOwner,
|
||||
1L,
|
||||
now.plus(leaseDuration),
|
||||
now,
|
||||
Optional.empty(),
|
||||
Optional.empty());
|
||||
}
|
||||
if (existing.state() == AdminOperationState.COMPLETED) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_ALREADY_EXECUTED",
|
||||
("approval %s was already executed to completion at %s; an approval authorises one"
|
||||
+ " execution, not a standing permission")
|
||||
.formatted(approvalTicket, existing.finishedAt().orElse(existing.startedAt())));
|
||||
}
|
||||
if (existing.state() == AdminOperationState.STARTED
|
||||
&& now.isBefore(existing.leaseExpiresAt())) {
|
||||
throw new MessageAuthorizationException(
|
||||
"ADMIN_OPERATION_IN_FLIGHT",
|
||||
("approval %s is being executed by %s until at least %s; two runtimes executing one"
|
||||
+ " approval is a duplicate storm, not a faster redrive")
|
||||
.formatted(approvalTicket, existing.leaseOwner(), existing.leaseExpiresAt()));
|
||||
}
|
||||
// Either the previous attempt failed, or its lease expired without a completion. Both resume
|
||||
// from the checkpoint under a new token, which fences the previous holder out.
|
||||
return new AdminOperationRecord(
|
||||
approvalTicket,
|
||||
planDigest,
|
||||
operationId,
|
||||
AdminOperationState.STARTED,
|
||||
existing.itemsCompleted(),
|
||||
leaseOwner,
|
||||
existing.leaseToken() + 1,
|
||||
now.plus(leaseDuration),
|
||||
existing.startedAt(),
|
||||
Optional.empty(),
|
||||
Optional.empty());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkpoint(
|
||||
AdminOperationLease lease, int itemsCompleted, Duration leaseDuration, Instant now) {
|
||||
update(
|
||||
lease,
|
||||
current ->
|
||||
new AdminOperationRecord(
|
||||
current.approvalTicket(),
|
||||
current.planDigest(),
|
||||
current.operationId(),
|
||||
AdminOperationState.STARTED,
|
||||
Math.max(current.itemsCompleted(), itemsCompleted),
|
||||
current.leaseOwner(),
|
||||
current.leaseToken(),
|
||||
now.plus(leaseDuration),
|
||||
current.startedAt(),
|
||||
Optional.empty(),
|
||||
Optional.empty()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void complete(AdminOperationLease lease, int itemsCompleted, Instant now) {
|
||||
update(
|
||||
lease,
|
||||
current ->
|
||||
new AdminOperationRecord(
|
||||
current.approvalTicket(),
|
||||
current.planDigest(),
|
||||
current.operationId(),
|
||||
AdminOperationState.COMPLETED,
|
||||
Math.max(current.itemsCompleted(), itemsCompleted),
|
||||
current.leaseOwner(),
|
||||
current.leaseToken(),
|
||||
current.leaseExpiresAt(),
|
||||
current.startedAt(),
|
||||
Optional.of(now),
|
||||
Optional.empty()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fail(AdminOperationLease lease, int itemsCompleted, String failureCode, Instant now) {
|
||||
Objects.requireNonNull(failureCode, "failureCode must not be null");
|
||||
update(
|
||||
lease,
|
||||
current ->
|
||||
new AdminOperationRecord(
|
||||
current.approvalTicket(),
|
||||
current.planDigest(),
|
||||
current.operationId(),
|
||||
AdminOperationState.FAILED,
|
||||
Math.max(current.itemsCompleted(), itemsCompleted),
|
||||
current.leaseOwner(),
|
||||
current.leaseToken(),
|
||||
current.leaseExpiresAt(),
|
||||
current.startedAt(),
|
||||
Optional.of(now),
|
||||
Optional.of(failureCode)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<AdminOperationRecord> find(String approvalTicket, PlanDigest planDigest) {
|
||||
return Optional.ofNullable(records.get(key(approvalTicket, planDigest)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDurable() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* How many operations have been journalled.
|
||||
*
|
||||
* @return the record count
|
||||
*/
|
||||
public int size() {
|
||||
return records.size();
|
||||
}
|
||||
|
||||
private void update(
|
||||
AdminOperationLease lease, java.util.function.UnaryOperator<AdminOperationRecord> mutation) {
|
||||
Objects.requireNonNull(lease, "lease must not be null");
|
||||
records.compute(
|
||||
key(lease.approvalTicket(), lease.planDigest()),
|
||||
(ignored, current) -> {
|
||||
if (current == null) {
|
||||
throw new MessageAuthorizationException(
|
||||
"ADMIN_OPERATION_NOT_JOURNALLED",
|
||||
"no journal entry exists for approval " + lease.approvalTicket());
|
||||
}
|
||||
if (current.leaseToken() != lease.leaseToken()) {
|
||||
// The fence. A stalled runtime that wakes up and writes here would otherwise overwrite
|
||||
// the progress of whichever replica took the operation over.
|
||||
throw new MessageAuthorizationException(
|
||||
"ADMIN_OPERATION_LEASE_LOST",
|
||||
("the lease on approval %s moved from token %d to token %d; this runtime no longer"
|
||||
+ " owns the operation")
|
||||
.formatted(lease.approvalTicket(), lease.leaseToken(), current.leaseToken()));
|
||||
}
|
||||
return mutation.apply(current);
|
||||
});
|
||||
}
|
||||
|
||||
private static String key(String approvalTicket, PlanDigest planDigest) {
|
||||
Objects.requireNonNull(planDigest, "planDigest must not be null");
|
||||
// Length-prefixed for the same reason the grant's canonical form is: a ticket containing the
|
||||
// separator must not be able to collide with a different ticket and digest pair.
|
||||
return String.join(
|
||||
"", Integer.toString(approvalTicket.length()), ":", approvalTicket, planDigest.value());
|
||||
}
|
||||
|
||||
/**
|
||||
* Every journalled operation, newest first by start time.
|
||||
*
|
||||
* @return the records
|
||||
*/
|
||||
public List<AdminOperationRecord> records() {
|
||||
return records.values().stream()
|
||||
.sorted(java.util.Comparator.comparing(AdminOperationRecord::startedAt).reversed())
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
+80
-22
@@ -1,9 +1,9 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminApproval;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperation;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperationGuard;
|
||||
import dev.caskeleton.messaging.admin.RedriveRequest;
|
||||
import dev.caskeleton.messaging.admin.VerifiedApproval;
|
||||
import dev.caskeleton.messaging.api.MessageId;
|
||||
import dev.caskeleton.messaging.api.publish.PublishCompletion;
|
||||
import dev.caskeleton.messaging.api.publish.PublishResult;
|
||||
@@ -54,14 +54,46 @@ public final class RedriveService {
|
||||
* Runs one redrive pass.
|
||||
*
|
||||
* @param request what to move
|
||||
* @param approval the approval, when one was supplied
|
||||
* @param approval the verified approval, when one was supplied
|
||||
* @param subject the operator identity
|
||||
* @param now the current instant
|
||||
* @return what the pass did
|
||||
*/
|
||||
public RedriveReport redrive(
|
||||
RedriveRequest request, Optional<AdminApproval> approval, String subject, Instant now) {
|
||||
RedriveRequest request, Optional<VerifiedApproval> approval, String subject, Instant now) {
|
||||
return redrive(request, approval, subject, now, 0, completed -> {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one redrive pass, resuming a previous attempt and reporting progress as it goes.
|
||||
*
|
||||
* <p>Three things were wrong with running this as a plain loop. A synchronous failure from the
|
||||
* publisher — a broker that refuses the connection rather than the message — propagated out of
|
||||
* the loop, so the remaining candidates were never attempted and the audit record was never
|
||||
* written: the operation left no trace of the items it had already moved. A retry then started
|
||||
* from the first candidate and republished them. And nothing bounded how many times one message
|
||||
* could be redriven.
|
||||
*
|
||||
* @param request what to move
|
||||
* @param approval the verified approval, when one was supplied
|
||||
* @param subject the operator identity
|
||||
* @param now the current instant
|
||||
* @param resumeFrom how many candidates a previous attempt durably moved
|
||||
* @param checkpoint called with the running completed count after each item
|
||||
* @return what the pass did
|
||||
*/
|
||||
public RedriveReport redrive(
|
||||
RedriveRequest request,
|
||||
Optional<VerifiedApproval> approval,
|
||||
String subject,
|
||||
Instant now,
|
||||
int resumeFrom,
|
||||
java.util.function.IntConsumer checkpoint) {
|
||||
Objects.requireNonNull(request, "request must not be null");
|
||||
Objects.requireNonNull(checkpoint, "checkpoint must not be null");
|
||||
if (resumeFrom < 0) {
|
||||
throw new IllegalArgumentException("resumeFrom must not be negative");
|
||||
}
|
||||
guard.authorize(
|
||||
DestructiveOperation.REDRIVE, request.source(), approval, request.dryRun(), now);
|
||||
|
||||
@@ -72,31 +104,57 @@ public final class RedriveService {
|
||||
|
||||
List<MessageId> moved = new ArrayList<>();
|
||||
int failed = 0;
|
||||
for (MessageId messageId : candidates) {
|
||||
PublishResult result = publisher.republish(messageId, request.target(), request.redriveId());
|
||||
if (result.completion() == PublishCompletion.CONFIRMED) {
|
||||
source.settle(request.source(), messageId);
|
||||
moved.add(messageId);
|
||||
} else {
|
||||
failed++;
|
||||
int completed = resumeFrom;
|
||||
try {
|
||||
// Everything before resumeFrom was moved and settled by the previous attempt. Re-publishing
|
||||
// it is not a retry, it is a duplicate.
|
||||
for (MessageId messageId :
|
||||
candidates.subList(Math.min(resumeFrom, candidates.size()), candidates.size())) {
|
||||
if (attempt(messageId, request)) {
|
||||
moved.add(messageId);
|
||||
} else {
|
||||
failed++;
|
||||
}
|
||||
completed++;
|
||||
checkpoint.accept(completed);
|
||||
}
|
||||
} finally {
|
||||
// In the finally block on purpose: an operation that dies partway must still leave a record
|
||||
// of what it moved, because that record is what the resumed attempt and the incident review
|
||||
// both read.
|
||||
audit.record(
|
||||
new dev.caskeleton.messaging.observation.MessagingAuditEvent(
|
||||
"REDRIVE",
|
||||
subject,
|
||||
request.source().value(),
|
||||
approval.map(VerifiedApproval::ticket).orElse("dry-run"),
|
||||
now,
|
||||
java.util.Map.of(
|
||||
"redriveId", request.redriveId().toString(),
|
||||
"resumedFrom", Integer.toString(resumeFrom),
|
||||
"moved", Integer.toString(moved.size()),
|
||||
"failed", Integer.toString(failed))));
|
||||
}
|
||||
|
||||
audit.record(
|
||||
new dev.caskeleton.messaging.observation.MessagingAuditEvent(
|
||||
"REDRIVE",
|
||||
subject,
|
||||
request.source().value(),
|
||||
approval.map(AdminApproval::ticket).orElse("dry-run"),
|
||||
now,
|
||||
java.util.Map.of(
|
||||
"redriveId", request.redriveId().toString(),
|
||||
"moved", Integer.toString(moved.size()),
|
||||
"failed", Integer.toString(failed))));
|
||||
|
||||
return new RedriveReport(candidates.size(), moved.size(), failed, false);
|
||||
}
|
||||
|
||||
private boolean attempt(MessageId messageId, RedriveRequest request) {
|
||||
PublishResult result;
|
||||
try {
|
||||
result = publisher.republish(messageId, request.target(), request.redriveId());
|
||||
} catch (RuntimeException failure) {
|
||||
// One message that cannot be republished is a failed item, not a failed pass. Letting it
|
||||
// propagate abandoned every candidate behind it.
|
||||
return false;
|
||||
}
|
||||
if (result.completion() != PublishCompletion.CONFIRMED) {
|
||||
return false;
|
||||
}
|
||||
source.settle(request.source(), messageId);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Reads and settles messages on a dead letter destination. */
|
||||
public interface RedriveSource {
|
||||
|
||||
|
||||
+4
-4
@@ -1,9 +1,9 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminApproval;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperation;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperationGuard;
|
||||
import dev.caskeleton.messaging.admin.ReplayRequest;
|
||||
import dev.caskeleton.messaging.admin.VerifiedApproval;
|
||||
import dev.caskeleton.messaging.observation.MessagingAuditEvent;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
@@ -45,13 +45,13 @@ public final class ReplayService {
|
||||
* Runs one replay.
|
||||
*
|
||||
* @param request what to replay
|
||||
* @param approval the approval, when one was supplied
|
||||
* @param approval the verified approval, when one was supplied
|
||||
* @param subject the operator identity
|
||||
* @param now the current instant
|
||||
* @return what the replay did
|
||||
*/
|
||||
public ReplayReport replay(
|
||||
ReplayRequest request, Optional<AdminApproval> approval, String subject, Instant now) {
|
||||
ReplayRequest request, Optional<VerifiedApproval> approval, String subject, Instant now) {
|
||||
Objects.requireNonNull(request, "request must not be null");
|
||||
Objects.requireNonNull(approval, "approval must not be null");
|
||||
|
||||
@@ -74,7 +74,7 @@ public final class ReplayService {
|
||||
"REPLAY",
|
||||
subject,
|
||||
request.destination().value(),
|
||||
approval.map(AdminApproval::ticket).orElse("isolated"),
|
||||
approval.map(VerifiedApproval::ticket).orElse("isolated"),
|
||||
now,
|
||||
Map.of(
|
||||
"replayId", request.replayId().toString(),
|
||||
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminOperationLease;
|
||||
import dev.caskeleton.messaging.admin.AdminOperationState;
|
||||
import dev.caskeleton.messaging.admin.PlanDigest;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* The protocol the journal has to implement, proven against the in-memory one.
|
||||
*
|
||||
* <p>What it replaced recorded a single fact — "this approval was claimed" — before any work
|
||||
* happened, in a map. An operation that died halfway had spent its approval, left no record of what
|
||||
* it had already moved, and gave a retry no way to continue: either request a new approval and
|
||||
* republish everything, or leave the redrive half applied.
|
||||
*/
|
||||
class AdminOperationJournalTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
|
||||
private static final Duration LEASE = Duration.ofMinutes(5);
|
||||
private static final String TICKET = "CHG-1042";
|
||||
private static final PlanDigest DIGEST = PlanDigest.ofCanonical("REDRIVE|orders.v1.dlq|50");
|
||||
private static final PlanDigest OTHER_DIGEST = PlanDigest.ofCanonical("REDRIVE|payments.dlq|50");
|
||||
|
||||
private final InMemoryAdminOperationJournal journal = new InMemoryAdminOperationJournal();
|
||||
|
||||
@Test
|
||||
@DisplayName("two replicas racing for one approval produce exactly one lease")
|
||||
void twoReplicasRacingProduceExactlyOneLease() {
|
||||
journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
|
||||
assertThatThrownBy(
|
||||
() -> journal.begin(TICKET, DIGEST, "op-2", "replica-b", LEASE, NOW.plusSeconds(1)))
|
||||
.as("both replicas executing one approval is a duplicate storm, not a faster redrive")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("replica-a");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a completed operation cannot be executed again")
|
||||
void aCompletedOperationCannotBeExecutedAgain() {
|
||||
AdminOperationLease lease = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
journal.complete(lease, 50, NOW.plusSeconds(30));
|
||||
|
||||
assertThatThrownBy(
|
||||
() -> journal.begin(TICKET, DIGEST, "op-2", "replica-a", LEASE, NOW.plusSeconds(60)))
|
||||
.as("an arrow-up in the shell must not re-run an approved redrive")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("already executed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a crashed run resumes from its checkpoint rather than starting over")
|
||||
void aCrashedRunResumesFromItsCheckpoint() {
|
||||
AdminOperationLease first = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
journal.checkpoint(first, 30, LEASE, NOW.plusSeconds(10));
|
||||
// No fail() call: this replica died. The lease simply runs out.
|
||||
|
||||
AdminOperationLease resumed =
|
||||
journal.begin(TICKET, DIGEST, "op-1", "replica-b", LEASE, NOW.plus(Duration.ofMinutes(10)));
|
||||
|
||||
assertThat(resumed.resumeFrom())
|
||||
.as("the 30 already moved must not be republished")
|
||||
.isEqualTo(30);
|
||||
assertThat(resumed.leaseToken()).isGreaterThan(first.leaseToken());
|
||||
assertThat(resumed.isResumption()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a failed run stays resumable without waiting for the lease to expire")
|
||||
void aFailedRunStaysResumableImmediately() {
|
||||
AdminOperationLease first = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
journal.checkpoint(first, 12, LEASE, NOW.plusSeconds(5));
|
||||
journal.fail(first, 12, "MESSAGE_BROKER_UNAVAILABLE", NOW.plusSeconds(6));
|
||||
|
||||
AdminOperationLease resumed =
|
||||
journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW.plusSeconds(7));
|
||||
|
||||
assertThat(resumed.resumeFrom()).isEqualTo(12);
|
||||
assertThat(journal.find(TICKET, DIGEST))
|
||||
.hasValueSatisfying(
|
||||
record -> assertThat(record.state()).isEqualTo(AdminOperationState.STARTED));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a runtime that lost its lease cannot write over the one that took over")
|
||||
void aRuntimeThatLostItsLeaseCannotWriteOverTheSuccessor() {
|
||||
AdminOperationLease stale = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
journal.checkpoint(stale, 30, LEASE, NOW.plusSeconds(10));
|
||||
AdminOperationLease successor =
|
||||
journal.begin(TICKET, DIGEST, "op-1", "replica-b", LEASE, NOW.plus(Duration.ofMinutes(10)));
|
||||
journal.checkpoint(successor, 45, LEASE, NOW.plus(Duration.ofMinutes(11)));
|
||||
|
||||
assertThatThrownBy(() -> journal.complete(stale, 30, NOW.plus(Duration.ofMinutes(12))))
|
||||
.as("a stalled replica waking up must not mark the operation done at its own count")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("no longer owns");
|
||||
|
||||
assertThat(journal.find(TICKET, DIGEST))
|
||||
.hasValueSatisfying(
|
||||
record -> {
|
||||
assertThat(record.itemsCompleted()).isEqualTo(45);
|
||||
assertThat(record.state()).isEqualTo(AdminOperationState.STARTED);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the same ticket presented for a different plan is a different operation")
|
||||
void theSameTicketForADifferentPlanIsADifferentOperation() {
|
||||
journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
|
||||
// Keyed on the ticket alone this would have been refused as a duplicate execution, which reads
|
||||
// as protection but is actually the wrong answer: it is a different plan, and it must be
|
||||
// refused by the approval binding rather than silently treated as the same operation.
|
||||
assertThatCode(() -> journal.begin(TICKET, OTHER_DIGEST, "op-2", "replica-a", LEASE, NOW))
|
||||
.doesNotThrowAnyException();
|
||||
assertThat(journal.size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a checkpoint never moves backwards")
|
||||
void aCheckpointNeverMovesBackwards() {
|
||||
AdminOperationLease lease = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
journal.checkpoint(lease, 40, LEASE, NOW.plusSeconds(10));
|
||||
journal.checkpoint(lease, 5, LEASE, NOW.plusSeconds(20));
|
||||
|
||||
assertThat(journal.find(TICKET, DIGEST))
|
||||
.hasValueSatisfying(record -> assertThat(record.itemsCompleted()).isEqualTo(40));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the in-memory journal declares itself non-durable")
|
||||
void theInMemoryJournalDeclaresItselfNonDurable() {
|
||||
assertThat(journal.isDurable())
|
||||
.as("the starter refuses a production profile on a journal that says this")
|
||||
.isFalse();
|
||||
}
|
||||
}
|
||||
+231
@@ -0,0 +1,231 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminApproval;
|
||||
import dev.caskeleton.messaging.admin.ApprovalGrant;
|
||||
import dev.caskeleton.messaging.admin.ApprovedRedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperation;
|
||||
import dev.caskeleton.messaging.admin.HmacApprovalVerifier;
|
||||
import dev.caskeleton.messaging.admin.RedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.RedriveRequest;
|
||||
import dev.caskeleton.messaging.admin.VerifiedApproval;
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Whether "this was approved" is a fact or a claim the caller makes about itself.
|
||||
*
|
||||
* <p>It was a claim. {@code AdminApproval} was a public record and {@code ApprovedRedrivePlan} a
|
||||
* public record over it, so any code that could reach the execute method could write its own
|
||||
* approval and the platform believed it. Nothing bound an approval to an operation, a destination,
|
||||
* a topology version, or an impact ceiling either, so a genuine approval for one redrive authorised
|
||||
* a redrive of anything else inside its window.
|
||||
*/
|
||||
class ApprovalForgeryTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
|
||||
private static final UUID OPERATION_ID = UUID.fromString("0199aaaa-bbbb-7ccc-8ddd-eeeeffff0000");
|
||||
private static final DestinationName DLQ = new DestinationName("orders.v1.dlq");
|
||||
private static final DestinationName ORDERS = new DestinationName("orders.v1");
|
||||
|
||||
private static final HmacApprovalVerifier ISSUER =
|
||||
new HmacApprovalVerifier(
|
||||
"an-issuing-key-of-at-least-32-bytes!".getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
private static final HmacApprovalVerifier ANOTHER_ISSUER =
|
||||
new HmacApprovalVerifier(
|
||||
"a-different-key-of-at-least-32-bytes".getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
@Test
|
||||
@DisplayName("a verified approval cannot be constructed outside the verifier")
|
||||
void aVerifiedApprovalCannotBeConstructedOutsideTheVerifier() {
|
||||
// Written as a reflection probe because the property is about the type's shape, not its
|
||||
// behaviour: if a public constructor or factory ever appears, every other test here becomes
|
||||
// decorative.
|
||||
assertThat(VerifiedApproval.class.getConstructors())
|
||||
.as("a public constructor would let any caller mint an approval for itself")
|
||||
.isEmpty();
|
||||
assertThat(
|
||||
java.util.Arrays.stream(VerifiedApproval.class.getDeclaredMethods())
|
||||
.filter(method -> java.lang.reflect.Modifier.isStatic(method.getModifiers()))
|
||||
.filter(method -> java.lang.reflect.Modifier.isPublic(method.getModifiers()))
|
||||
.toList())
|
||||
.as("nor may a public static factory")
|
||||
.isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an unsigned or altered grant does not verify")
|
||||
void anAlteredGrantDoesNotVerify() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
ApprovalGrant signed = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
|
||||
String signature = ISSUER.sign(signed);
|
||||
|
||||
ApprovalGrant raised = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 5_000);
|
||||
|
||||
assertThatThrownBy(() -> ISSUER.verify(raised, signature, plan.digest(), NOW))
|
||||
.as("raising the impact ceiling after the approver signed it")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("does not verify");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a signature from another issuer does not verify")
|
||||
void aSignatureFromAnotherIssuerDoesNotVerify() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
|
||||
|
||||
assertThatThrownBy(
|
||||
() -> ISSUER.verify(granted, ANOTHER_ISSUER.sign(granted), plan.digest(), NOW))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("does not verify");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an approval for one plan cannot execute another")
|
||||
void anApprovalForOnePlanCannotExecuteAnother() {
|
||||
RedrivePlan approvedPlan = plan(50, 0);
|
||||
RedrivePlan biggerPlan = plan(900, 0);
|
||||
ApprovalGrant granted = grant(approvedPlan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 900);
|
||||
String signature = ISSUER.sign(granted);
|
||||
|
||||
assertThatThrownBy(() -> ISSUER.verify(granted, signature, biggerPlan.digest(), NOW))
|
||||
.as("the approver saw 50 candidates; this plan moves 900")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("was issued for plan");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a verified approval for the wrong source cannot be attached to a plan")
|
||||
void aVerifiedApprovalForTheWrongSourceCannotBeAttached() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
// A genuine, correctly-signed approval — for a different dead-letter destination, with this
|
||||
// plan's digest smuggled in. The digest check passes; the source binding is what catches it.
|
||||
ApprovalGrant granted =
|
||||
new ApprovalGrant(
|
||||
approval(),
|
||||
"operator@example.com",
|
||||
DestructiveOperation.REDRIVE,
|
||||
new DestinationName("payments.v1.dlq"),
|
||||
Optional.of(ORDERS),
|
||||
"v1",
|
||||
900,
|
||||
plan.digest());
|
||||
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
|
||||
|
||||
assertThatThrownBy(() -> new ApprovedRedrivePlan(plan, verified, false))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("payments.v1.dlq");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a plan beyond the approved impact ceiling cannot execute")
|
||||
void aPlanBeyondTheApprovedImpactCeilingCannotExecute() {
|
||||
RedrivePlan plan = plan(900, 0);
|
||||
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 100);
|
||||
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
|
||||
|
||||
assertThatThrownBy(() -> new ApprovedRedrivePlan(plan, verified, false))
|
||||
.as("the approver agreed to at most 100")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("at most 100");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an approval for a replay cannot authorise a redrive")
|
||||
void anApprovalForAReplayCannotAuthoriseARedrive() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
ApprovalGrant granted = grant(plan, DestructiveOperation.REPLAY, DLQ, ORDERS, 900);
|
||||
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
|
||||
|
||||
assertThatThrownBy(() -> new ApprovedRedrivePlan(plan, verified, false))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("authorises REPLAY");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an expired grant does not verify")
|
||||
void anExpiredGrantDoesNotVerify() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
ISSUER.verify(
|
||||
granted, ISSUER.sign(granted), plan.digest(), NOW.plus(Duration.ofDays(1))))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("not valid at");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the approver and the operator must be different people")
|
||||
void theApproverAndTheOperatorMustBeDifferentPeople() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
new ApprovalGrant(
|
||||
approval(),
|
||||
"ops@example.com",
|
||||
DestructiveOperation.REDRIVE,
|
||||
DLQ,
|
||||
Optional.of(ORDERS),
|
||||
"v1",
|
||||
50,
|
||||
plan.digest()))
|
||||
.as("an operator approving their own destructive operation is the control not existing")
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a correctly issued approval executes")
|
||||
void aCorrectlyIssuedApprovalExecutes() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
|
||||
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
|
||||
|
||||
assertThatCode(
|
||||
() -> new ApprovedRedrivePlan(plan, verified, false).requireExecutable(NOW, "v1"))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
private static ApprovalGrant grant(
|
||||
RedrivePlan plan,
|
||||
DestructiveOperation operation,
|
||||
DestinationName source,
|
||||
DestinationName target,
|
||||
long maxImpact) {
|
||||
return new ApprovalGrant(
|
||||
approval(),
|
||||
"operator@example.com",
|
||||
operation,
|
||||
source,
|
||||
Optional.of(target),
|
||||
"v1",
|
||||
maxImpact,
|
||||
plan.digest());
|
||||
}
|
||||
|
||||
private static AdminApproval approval() {
|
||||
return new AdminApproval("CHG-1042", "ops@example.com", NOW, NOW.plus(Duration.ofHours(2)));
|
||||
}
|
||||
|
||||
private static RedrivePlan plan(int candidates, int alreadyRedriven) {
|
||||
return new RedrivePlan(
|
||||
new RedriveRequest(OPERATION_ID, DLQ, ORDERS, 50, false),
|
||||
candidates,
|
||||
alreadyRedriven,
|
||||
NOW,
|
||||
"v1");
|
||||
}
|
||||
}
|
||||
+70
-34
@@ -5,14 +5,19 @@ import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminApproval;
|
||||
import dev.caskeleton.messaging.admin.ApprovalGrant;
|
||||
import dev.caskeleton.messaging.admin.ApprovedRedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.ApprovedReplayPlan;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperation;
|
||||
import dev.caskeleton.messaging.admin.HmacApprovalVerifier;
|
||||
import dev.caskeleton.messaging.admin.PlanDigest;
|
||||
import dev.caskeleton.messaging.admin.RedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.RedriveRequest;
|
||||
import dev.caskeleton.messaging.admin.RedriveResult;
|
||||
import dev.caskeleton.messaging.admin.ReplayPlan;
|
||||
import dev.caskeleton.messaging.admin.ReplayRequest;
|
||||
import dev.caskeleton.messaging.admin.ReplayResult;
|
||||
import dev.caskeleton.messaging.admin.VerifiedApproval;
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Duration;
|
||||
@@ -26,10 +31,56 @@ class ApprovedPlanExecutionTest {
|
||||
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
|
||||
private static final UUID OPERATION_ID = UUID.fromString("0199aaaa-bbbb-7ccc-8ddd-eeeeffff0000");
|
||||
|
||||
private static final byte[] ISSUING_KEY =
|
||||
"an-issuing-key-of-at-least-32-bytes!".getBytes(java.nio.charset.StandardCharsets.UTF_8);
|
||||
|
||||
private static final HmacApprovalVerifier VERIFIER = new HmacApprovalVerifier(ISSUING_KEY);
|
||||
|
||||
private static AdminApproval approval() {
|
||||
return new AdminApproval("CHG-1042", "ops@example.com", NOW, NOW.plus(Duration.ofHours(2)));
|
||||
}
|
||||
|
||||
/** A grant the issuer signed for exactly this plan, verified the way the runtime verifies it. */
|
||||
static VerifiedApproval verified(
|
||||
PlanDigest digest,
|
||||
DestructiveOperation operation,
|
||||
DestinationName source,
|
||||
Optional<DestinationName> target,
|
||||
String topologyVersion,
|
||||
long maxImpact) {
|
||||
ApprovalGrant grant =
|
||||
new ApprovalGrant(
|
||||
approval(),
|
||||
"operator@example.com",
|
||||
operation,
|
||||
source,
|
||||
target,
|
||||
topologyVersion,
|
||||
maxImpact,
|
||||
digest);
|
||||
return VERIFIER.verify(grant, VERIFIER.sign(grant), digest, NOW);
|
||||
}
|
||||
|
||||
private static VerifiedApproval replayApproval(ReplayPlan plan) {
|
||||
return verified(
|
||||
plan.digest(),
|
||||
DestructiveOperation.REPLAY,
|
||||
plan.request().destination(),
|
||||
Optional.empty(),
|
||||
plan.topologyVersion(),
|
||||
Long.MAX_VALUE);
|
||||
}
|
||||
|
||||
private static VerifiedApproval redriveApproval(RedrivePlan plan) {
|
||||
return verified(
|
||||
plan.digest(),
|
||||
DestructiveOperation.REDRIVE,
|
||||
plan.request().source(),
|
||||
Optional.of(plan.request().target()),
|
||||
plan.topologyVersion(),
|
||||
Long.MAX_VALUE);
|
||||
}
|
||||
|
||||
private static ReplayRequest replayRequest() {
|
||||
return new ReplayRequest(
|
||||
OPERATION_ID,
|
||||
@@ -57,9 +108,18 @@ class ApprovedPlanExecutionTest {
|
||||
return new RedrivePlan(redriveRequest(), candidates, alreadyRedriven, NOW, "v1");
|
||||
}
|
||||
|
||||
private static ApprovedReplayPlan approvedReplay(String topologyVersion) {
|
||||
ReplayPlan plan = replayPlan(topologyVersion);
|
||||
return new ApprovedReplayPlan(plan, replayApproval(plan));
|
||||
}
|
||||
|
||||
private static ApprovedRedrivePlan approvedRedrive(RedrivePlan plan, boolean loopAcknowledged) {
|
||||
return new ApprovedRedrivePlan(plan, redriveApproval(plan), loopAcknowledged);
|
||||
}
|
||||
|
||||
@Test
|
||||
void anExpiredApprovalCannotExecute() {
|
||||
ApprovedReplayPlan approved = new ApprovedReplayPlan(replayPlan("v1"), approval());
|
||||
ApprovedReplayPlan approved = approvedReplay("v1");
|
||||
|
||||
assertThatThrownBy(() -> approved.requireExecutable(NOW.plus(Duration.ofDays(1)), "v1"))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
@@ -68,25 +128,26 @@ class ApprovedPlanExecutionTest {
|
||||
|
||||
@Test
|
||||
void aTopologyChangeSinceApprovalInvalidatesThePlan() {
|
||||
ApprovedReplayPlan approved = new ApprovedReplayPlan(replayPlan("v1"), approval());
|
||||
ApprovedReplayPlan approved = approvedReplay("v1");
|
||||
|
||||
assertThatThrownBy(() -> approved.requireExecutable(NOW, "v2"))
|
||||
.as("every number in the plan was computed against the old topology")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("rebuilt");
|
||||
// The signed grant carries the topology version too, so the mismatch is now caught against
|
||||
// what the approver actually signed rather than against a field on the plan the caller
|
||||
// handed in.
|
||||
.hasMessageContaining("was signed against topology v1 but the broker is now at v2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aValidApprovalOnUnchangedTopologyExecutes() {
|
||||
assertThatCode(
|
||||
() -> new ApprovedReplayPlan(replayPlan("v1"), approval()).requireExecutable(NOW, "v1"))
|
||||
assertThatCode(() -> approvedReplay("v1").requireExecutable(NOW, "v1"))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRedriveThatWouldLoopNeedsThatAcknowledgedExplicitly() {
|
||||
ApprovedRedrivePlan approved =
|
||||
new ApprovedRedrivePlan(redrivePlan(900, 400), approval(), false);
|
||||
ApprovedRedrivePlan approved = approvedRedrive(redrivePlan(900, 400), false);
|
||||
|
||||
assertThatThrownBy(() -> approved.requireExecutable(NOW, "v1"))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
@@ -95,19 +156,13 @@ class ApprovedPlanExecutionTest {
|
||||
|
||||
@Test
|
||||
void anAcknowledgedLoopMayProceed() {
|
||||
assertThatCode(
|
||||
() ->
|
||||
new ApprovedRedrivePlan(redrivePlan(900, 400), approval(), true)
|
||||
.requireExecutable(NOW, "v1"))
|
||||
assertThatCode(() -> approvedRedrive(redrivePlan(900, 400), true).requireExecutable(NOW, "v1"))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRedriveWithNoPreviouslyRedrivenCandidatesNeedsNoAcknowledgement() {
|
||||
assertThatCode(
|
||||
() ->
|
||||
new ApprovedRedrivePlan(redrivePlan(900, 0), approval(), false)
|
||||
.requireExecutable(NOW, "v1"))
|
||||
assertThatCode(() -> approvedRedrive(redrivePlan(900, 0), false).requireExecutable(NOW, "v1"))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@@ -139,25 +194,6 @@ class ApprovedPlanExecutionTest {
|
||||
.contains("LIVE consumer group");
|
||||
}
|
||||
|
||||
@Test
|
||||
void anApprovalAuthorisesOneExecutionNotAStandingPermission() {
|
||||
AdminOperationIdempotencyStore store = new AdminOperationIdempotencyStore();
|
||||
|
||||
assertThat(store.claim("CHG-1042", "op-1", NOW)).isEmpty();
|
||||
assertThat(store.claim("CHG-1042", "op-2", NOW.plus(Duration.ofMinutes(5))))
|
||||
.as("an arrow-up in the shell must not re-run an approved redrive")
|
||||
.hasValueSatisfying(existing -> assertThat(existing.operationId()).isEqualTo("op-1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aDifferentApprovalIsClaimedIndependently() {
|
||||
AdminOperationIdempotencyStore store = new AdminOperationIdempotencyStore();
|
||||
store.claim("CHG-1042", "op-1", NOW);
|
||||
|
||||
assertThat(store.claim("CHG-1043", "op-2", NOW)).isEmpty();
|
||||
assertThat(store.size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aReplayResultFlagsAShortfallAgainstTheApprovedEstimate() {
|
||||
ReplayResult result =
|
||||
|
||||
+212
@@ -0,0 +1,212 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminApproval;
|
||||
import dev.caskeleton.messaging.admin.ApprovalGrant;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperation;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperationGuard;
|
||||
import dev.caskeleton.messaging.admin.HmacApprovalVerifier;
|
||||
import dev.caskeleton.messaging.admin.PlanDigest;
|
||||
import dev.caskeleton.messaging.admin.RedriveRequest;
|
||||
import dev.caskeleton.messaging.admin.VerifiedApproval;
|
||||
import dev.caskeleton.messaging.api.MessageId;
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.publish.ConfirmationLevel;
|
||||
import dev.caskeleton.messaging.api.publish.PublishCompletion;
|
||||
import dev.caskeleton.messaging.api.publish.PublishEvidence;
|
||||
import dev.caskeleton.messaging.api.publish.PublishResult;
|
||||
import dev.caskeleton.messaging.api.publish.RoutingOutcome;
|
||||
import dev.caskeleton.messaging.observation.MessagingAuditEvent;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* What a redrive does when one message fails, and what it does the second time round.
|
||||
*
|
||||
* <p>The loop had no per-item boundary. A publisher that threw — a broker refusing the connection
|
||||
* rather than the message — propagated straight out, so every candidate behind it was abandoned and
|
||||
* the audit record was never written. The operation left no trace of what it had already moved, and
|
||||
* a retry started again from the first candidate and republished it.
|
||||
*/
|
||||
class RedriveResumptionTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
|
||||
private static final DestinationName DLQ = new DestinationName("orders.v1.dlq");
|
||||
private static final DestinationName ORDERS = new DestinationName("orders.v1");
|
||||
|
||||
private static final HmacApprovalVerifier ISSUER =
|
||||
new HmacApprovalVerifier(
|
||||
"an-issuing-key-of-at-least-32-bytes!".getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
|
||||
private final RecordingSource source = new RecordingSource();
|
||||
private final RecordingAudit audit = new RecordingAudit();
|
||||
|
||||
@Test
|
||||
@DisplayName("one message that throws does not abandon the ones behind it")
|
||||
void oneThrowingMessageDoesNotAbandonTheRest() {
|
||||
List<MessageId> candidates = ids(4);
|
||||
source.stage(candidates);
|
||||
RedriveService service =
|
||||
service(
|
||||
(messageId, target, redriveId) -> {
|
||||
if (messageId.equals(candidates.get(1))) {
|
||||
throw new IllegalStateException("the broker refused the connection");
|
||||
}
|
||||
return confirmed();
|
||||
});
|
||||
|
||||
RedriveReport report = service.redrive(request(), approval(), "operator", NOW);
|
||||
|
||||
assertThat(report.moved()).isEqualTo(3);
|
||||
assertThat(report.failed())
|
||||
.as("the throwing item is a failed item, not a failed pass")
|
||||
.isEqualTo(1);
|
||||
assertThat(source.settled)
|
||||
.containsExactly(candidates.get(0), candidates.get(2), candidates.get(3));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the audit record is written even when the pass does not finish cleanly")
|
||||
void theAuditRecordIsWrittenEvenOnFailure() {
|
||||
source.stage(ids(2));
|
||||
RedriveService service =
|
||||
service(
|
||||
(messageId, target, redriveId) -> {
|
||||
throw new IllegalStateException("the broker refused the connection");
|
||||
});
|
||||
|
||||
service.redrive(request(), approval(), "operator", NOW);
|
||||
|
||||
assertThat(audit.events)
|
||||
.as("the record of what moved is what the incident review and the resumed run both read")
|
||||
.singleElement()
|
||||
.satisfies(event -> assertThat(event.details()).containsEntry("failed", "2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a resumed pass does not republish what the first attempt already moved")
|
||||
void aResumedPassDoesNotRepublishWhatWasAlreadyMoved() {
|
||||
List<MessageId> candidates = ids(5);
|
||||
source.stage(candidates);
|
||||
List<MessageId> republished = new ArrayList<>();
|
||||
RedriveService service =
|
||||
service(
|
||||
(messageId, target, redriveId) -> {
|
||||
republished.add(messageId);
|
||||
return confirmed();
|
||||
});
|
||||
|
||||
RedriveReport report =
|
||||
service.redrive(request(), approval(), "operator", NOW, 3, completed -> {});
|
||||
|
||||
assertThat(republished)
|
||||
.as("republishing a settled message is a duplicate, not a retry")
|
||||
.containsExactly(candidates.get(3), candidates.get(4));
|
||||
assertThat(report.moved()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("progress is reported after each item so a crash resumes near where it stopped")
|
||||
void progressIsReportedAfterEachItem() {
|
||||
source.stage(ids(3));
|
||||
List<Integer> checkpoints = new ArrayList<>();
|
||||
RedriveService service = service((messageId, target, redriveId) -> confirmed());
|
||||
|
||||
service.redrive(request(), approval(), "operator", NOW, 0, checkpoints::add);
|
||||
|
||||
assertThat(checkpoints).containsExactly(1, 2, 3);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("resuming past the end of the batch moves nothing")
|
||||
void resumingPastTheEndOfTheBatchMovesNothing() {
|
||||
source.stage(ids(2));
|
||||
RedriveService service = service((messageId, target, redriveId) -> confirmed());
|
||||
|
||||
RedriveReport report =
|
||||
service.redrive(request(), approval(), "operator", NOW, 10, completed -> {});
|
||||
|
||||
assertThat(report.moved()).isZero();
|
||||
}
|
||||
|
||||
private RedriveService service(RedriveService.RedrivePublisher publisher) {
|
||||
// The guard holds the admin credential and every pass carries a real verified approval: this
|
||||
// test is about the loop, and a gate that refused would prove nothing about it.
|
||||
return new RedriveService(new DestructiveOperationGuard(true), source, publisher, audit);
|
||||
}
|
||||
|
||||
private static Optional<VerifiedApproval> approval() {
|
||||
PlanDigest digest = PlanDigest.ofCanonical("REDRIVE|orders.v1.dlq|orders.v1");
|
||||
ApprovalGrant grant =
|
||||
new ApprovalGrant(
|
||||
new AdminApproval("CHG-1042", "ops@example.com", NOW, NOW.plus(Duration.ofHours(2))),
|
||||
"operator@example.com",
|
||||
DestructiveOperation.REDRIVE,
|
||||
DLQ,
|
||||
Optional.of(ORDERS),
|
||||
"v1",
|
||||
Long.MAX_VALUE,
|
||||
digest);
|
||||
return Optional.of(ISSUER.verify(grant, ISSUER.sign(grant), digest, NOW));
|
||||
}
|
||||
|
||||
private static RedriveRequest request() {
|
||||
return new RedriveRequest(UUID.randomUUID(), DLQ, ORDERS, 50, false);
|
||||
}
|
||||
|
||||
private static PublishResult confirmed() {
|
||||
return new PublishResult(
|
||||
PublishCompletion.CONFIRMED,
|
||||
PublishEvidence.confirmed(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK),
|
||||
RoutingOutcome.ROUTED,
|
||||
Optional.empty(),
|
||||
1,
|
||||
Duration.ofMillis(2),
|
||||
Optional.empty());
|
||||
}
|
||||
|
||||
private static List<MessageId> ids(int count) {
|
||||
List<MessageId> ids = new ArrayList<>();
|
||||
for (int index = 0; index < count; index++) {
|
||||
ids.add(MessageId.newId());
|
||||
}
|
||||
return List.copyOf(ids);
|
||||
}
|
||||
|
||||
private static final class RecordingSource implements RedriveService.RedriveSource {
|
||||
|
||||
private List<MessageId> staged = List.of();
|
||||
private final List<MessageId> settled = new ArrayList<>();
|
||||
|
||||
private void stage(List<MessageId> ids) {
|
||||
staged = List.copyOf(ids);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<MessageId> peek(DestinationName destination, int batchSize) {
|
||||
return staged;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void settle(DestinationName destination, MessageId messageId) {
|
||||
settled.add(messageId);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class RecordingAudit implements RedriveService.AuditSink {
|
||||
|
||||
private final List<MessagingAuditEvent> events = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public void record(MessagingAuditEvent event) {
|
||||
events.add(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user