review-followup Wave 3 라운드 3(P21): messaging 플랫폼 — Kafka 소비(MessageHandler 하나), API 정리
- 결정 10: consume 계약을 `MessageHandler<T>` 하나로, starter-kafka가 consumer 조립(handler bean이 있으면 켜짐, `app.messaging.enabled=false`로 끔). 오프셋은 handler Success(inbox면 트랜잭션 커밋) 뒤 또는 DLQ 쓰기 확인 뒤에만 커밋, auto-commit 금지, `read_committed`, 재시도 소진 → DLQ(없으면 멈춤) — Kafka+PG로 왕복·중복 제거·DLQ 검증 - 미사용 API 타입·Blocking/Reactive/Batch facade·reactor api 노출 삭제, `PublishResult` 정적 factory, `MessagingCapabilities` → EnumSet, API leaf 병합(core-api+schema-api, admin-api+admin-runtime), starter god config 분해, Kafka broker 설정은 starter-kafka로, 등급 용어 정리 - 결함 수정: Retry 응답이 시도 횟수를 세지 않아 partition 정지(P21-new-1), deliveryAttempt 항상 1(P21-new-2) - 통합: app-bootstrap `MessagingPublicSurfaceContractTest` 기대값을 P21 변경에 맞춤, starter-kafka에 `api kafka-clients` 검증: ./gradlew check architectureCheck verifyDependencyLocks :app-bootstrap:verifyEnvKeys :app-bootstrap:integrationTest :sample-portfolio:integrationTest :messaging:messaging-spring-boot-starter:integrationTest :messaging:messaging-reliability-jdbc-postgresql:integrationTest mainCheck --continue 통과(812 task) Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
6397f52dc8
commit
763392c8b2
+45
@@ -0,0 +1,45 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* The approval that authorises a destructive messaging operation.
|
||||
*
|
||||
* <p>Approvals expire. An open-ended approval becomes a standing permission, which is the same
|
||||
* thing as no approval at all — the window is what keeps "we approved a redrive last quarter" from
|
||||
* authorising one today.
|
||||
*
|
||||
* @param ticket the change reference
|
||||
* @param approvedBy the approver's identity
|
||||
* @param approvedAt when the approval was granted
|
||||
* @param validUntil when the approval stops authorising anything
|
||||
*/
|
||||
public record AdminApproval(
|
||||
String ticket, String approvedBy, Instant approvedAt, Instant validUntil) {
|
||||
|
||||
public AdminApproval {
|
||||
Objects.requireNonNull(approvedAt, "approvedAt must not be null");
|
||||
Objects.requireNonNull(validUntil, "validUntil must not be null");
|
||||
if (ticket == null || ticket.isBlank()) {
|
||||
throw new IllegalArgumentException("ticket must not be blank");
|
||||
}
|
||||
if (approvedBy == null || approvedBy.isBlank()) {
|
||||
throw new IllegalArgumentException("approvedBy must not be blank");
|
||||
}
|
||||
if (validUntil.isBefore(approvedAt)) {
|
||||
throw new IllegalArgumentException("an approval cannot expire before it was granted");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether the approval still authorises an operation.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @return true while inside the window
|
||||
*/
|
||||
public boolean isValidAt(Instant now) {
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
return !now.isBefore(approvedAt) && now.isBefore(validUntil);
|
||||
}
|
||||
}
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Where a destructive operation records that it started, how far it got, and how it ended.
|
||||
*
|
||||
* <p>The implementation this replaced was a {@code ConcurrentHashMap} registered by the starter as
|
||||
* the default. Two consequences followed, and both are worse than having no store at all because
|
||||
* the map made the platform look protected. A restart forgot every claim, so the same approval
|
||||
* could be executed again by the same process. And two replicas each had their own map, so both
|
||||
* could claim the same approval at the same moment and each believe it was the only one.
|
||||
*
|
||||
* <p>Implementations must therefore be shared and durable, must enforce uniqueness on {@code
|
||||
* (approvalTicket, planDigest)}, and must hand out leases with a monotonic fencing token so a
|
||||
* process that stalled past its lease cannot write over the replica that took over from it.
|
||||
*/
|
||||
public interface AdminOperationJournal {
|
||||
|
||||
/**
|
||||
* Claims an operation, or resumes the previous attempt at it.
|
||||
*
|
||||
* @param approvalTicket the approval authorising the operation
|
||||
* @param planDigest the exact plan
|
||||
* @param operationId this execution's identity
|
||||
* @param leaseOwner the runtime claiming it
|
||||
* @param leaseDuration how long the lease holds before another runtime may take over
|
||||
* @param now the current instant
|
||||
* @return the lease, carrying the point to resume from
|
||||
* @throws dev.caskeleton.messaging.api.error.MessageAuthorizationException when the operation
|
||||
* already completed, or another runtime holds a live lease on it
|
||||
*/
|
||||
AdminOperationLease begin(
|
||||
String approvalTicket,
|
||||
PlanDigest planDigest,
|
||||
String operationId,
|
||||
String leaseOwner,
|
||||
Duration leaseDuration,
|
||||
Instant now);
|
||||
|
||||
/**
|
||||
* Records durable progress and extends the lease.
|
||||
*
|
||||
* @param lease the lease held
|
||||
* @param itemsCompleted how many items are now durably done
|
||||
* @param leaseDuration how much longer to hold the lease
|
||||
* @param now the current instant
|
||||
* @throws dev.caskeleton.messaging.api.error.MessageAuthorizationException when the lease has
|
||||
* been taken over by a newer token
|
||||
*/
|
||||
void checkpoint(
|
||||
AdminOperationLease lease, int itemsCompleted, Duration leaseDuration, Instant now);
|
||||
|
||||
/**
|
||||
* Marks the operation finished. The approval is spent.
|
||||
*
|
||||
* @param lease the lease held
|
||||
* @param itemsCompleted the final count
|
||||
* @param now the current instant
|
||||
*/
|
||||
void complete(AdminOperationLease lease, int itemsCompleted, Instant now);
|
||||
|
||||
/**
|
||||
* Marks the operation failed at its current checkpoint, leaving it resumable.
|
||||
*
|
||||
* @param lease the lease held
|
||||
* @param itemsCompleted how many items are durably done
|
||||
* @param failureCode a sanitised, bounded reason
|
||||
* @param now the current instant
|
||||
*/
|
||||
void fail(AdminOperationLease lease, int itemsCompleted, String failureCode, Instant now);
|
||||
|
||||
/**
|
||||
* Reads the journal entry for an operation.
|
||||
*
|
||||
* @param approvalTicket the approval
|
||||
* @param planDigest the plan
|
||||
* @return the record, when the operation has been journalled
|
||||
*/
|
||||
Optional<AdminOperationRecord> find(String approvalTicket, PlanDigest planDigest);
|
||||
|
||||
/**
|
||||
* Whether this implementation survives a restart and is shared between replicas.
|
||||
*
|
||||
* <p>Declared rather than inferred so the starter can refuse to run a production profile on an
|
||||
* in-memory journal instead of discovering the gap during an incident.
|
||||
*
|
||||
* @return true when the journal is durable and shared
|
||||
*/
|
||||
boolean isDurable();
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* The right to execute one journalled operation, and the point to resume it from.
|
||||
*
|
||||
* <p>{@code resumeFrom} is the whole reason a lease is handed out rather than a boolean. A retry
|
||||
* after a crash is not a new execution of the approval — it is the same operation continuing, and
|
||||
* treating it as new is what republishes the items the first attempt already moved.
|
||||
*
|
||||
* @param approvalTicket the approval being executed
|
||||
* @param planDigest the plan being executed
|
||||
* @param operationId this execution's identity
|
||||
* @param leaseOwner the runtime holding the lease
|
||||
* @param leaseToken the fencing token; a write from an older token must be refused
|
||||
* @param resumeFrom how many items the previous attempt durably completed
|
||||
*/
|
||||
public record AdminOperationLease(
|
||||
String approvalTicket,
|
||||
PlanDigest planDigest,
|
||||
String operationId,
|
||||
String leaseOwner,
|
||||
long leaseToken,
|
||||
int resumeFrom) {
|
||||
|
||||
/** Validates the lease. */
|
||||
public AdminOperationLease {
|
||||
Objects.requireNonNull(planDigest, "planDigest must not be null");
|
||||
if (approvalTicket == null || approvalTicket.isBlank()) {
|
||||
throw new IllegalArgumentException("approvalTicket must not be blank");
|
||||
}
|
||||
if (operationId == null || operationId.isBlank()) {
|
||||
throw new IllegalArgumentException("operationId must not be blank");
|
||||
}
|
||||
if (leaseOwner == null || leaseOwner.isBlank()) {
|
||||
throw new IllegalArgumentException("leaseOwner must not be blank");
|
||||
}
|
||||
if (leaseToken < 1) {
|
||||
throw new IllegalArgumentException("a lease token starts at one");
|
||||
}
|
||||
if (resumeFrom < 0) {
|
||||
throw new IllegalArgumentException("resumeFrom must not be negative");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether this lease continues an earlier attempt.
|
||||
*
|
||||
* @return true when items were already completed
|
||||
*/
|
||||
public boolean isResumption() {
|
||||
return resumeFrom > 0;
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* One row of the destructive-operation journal.
|
||||
*
|
||||
* <p>Keyed by {@code (approvalTicket, planDigest)} rather than by the ticket alone, because the
|
||||
* ticket alone cannot distinguish "this approval already ran" from "this approval is being reused
|
||||
* for a different plan" — and those need opposite answers.
|
||||
*
|
||||
* @param approvalTicket the approval that authorised the operation
|
||||
* @param planDigest the exact plan that was authorised
|
||||
* @param operationId the caller-side identity of this execution
|
||||
* @param state where the operation stands
|
||||
* @param itemsCompleted how many items are durably done
|
||||
* @param leaseOwner which runtime currently holds the lease
|
||||
* @param leaseToken the fencing token, incremented on every takeover
|
||||
* @param leaseExpiresAt when another runtime may take the lease over
|
||||
* @param startedAt when the operation first began
|
||||
* @param finishedAt when it reached a terminal state
|
||||
* @param failureCode the sanitised reason it failed, when it did
|
||||
*/
|
||||
public record AdminOperationRecord(
|
||||
String approvalTicket,
|
||||
PlanDigest planDigest,
|
||||
String operationId,
|
||||
AdminOperationState state,
|
||||
int itemsCompleted,
|
||||
String leaseOwner,
|
||||
long leaseToken,
|
||||
Instant leaseExpiresAt,
|
||||
Instant startedAt,
|
||||
Optional<Instant> finishedAt,
|
||||
Optional<String> failureCode) {
|
||||
|
||||
/** Validates the record. */
|
||||
public AdminOperationRecord {
|
||||
Objects.requireNonNull(planDigest, "planDigest must not be null");
|
||||
Objects.requireNonNull(state, "state must not be null");
|
||||
Objects.requireNonNull(leaseExpiresAt, "leaseExpiresAt must not be null");
|
||||
Objects.requireNonNull(startedAt, "startedAt must not be null");
|
||||
Objects.requireNonNull(finishedAt, "finishedAt must not be null");
|
||||
Objects.requireNonNull(failureCode, "failureCode must not be null");
|
||||
if (approvalTicket == null || approvalTicket.isBlank()) {
|
||||
throw new IllegalArgumentException("approvalTicket must not be blank");
|
||||
}
|
||||
if (operationId == null || operationId.isBlank()) {
|
||||
throw new IllegalArgumentException("operationId must not be blank");
|
||||
}
|
||||
if (leaseOwner == null || leaseOwner.isBlank()) {
|
||||
throw new IllegalArgumentException("leaseOwner must not be blank");
|
||||
}
|
||||
if (itemsCompleted < 0) {
|
||||
throw new IllegalArgumentException("itemsCompleted must not be negative");
|
||||
}
|
||||
if (leaseToken < 1) {
|
||||
throw new IllegalArgumentException("a lease token starts at one");
|
||||
}
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
/**
|
||||
* Where a journalled destructive operation stands.
|
||||
*
|
||||
* <p>The store this replaces recorded one fact — "this approval was claimed" — and recorded it
|
||||
* before any work happened. A run that died halfway had consumed its approval, left no record of
|
||||
* how far it got, and offered the operator two equally bad choices: request a fresh approval and
|
||||
* redo work that may already have been done, or leave the operation half-applied.
|
||||
*/
|
||||
public enum AdminOperationState {
|
||||
|
||||
/** Claimed and in progress. A lease is held; the checkpoint says how far it got. */
|
||||
STARTED,
|
||||
|
||||
/** Every item finished. The approval is spent and must not run again. */
|
||||
COMPLETED,
|
||||
|
||||
/** The run stopped before finishing. It may be resumed from its checkpoint under a new lease. */
|
||||
FAILED
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Everything an issuer signs when granting one destructive operation.
|
||||
*
|
||||
* <p>{@link AdminApproval} carried a ticket, an approver, and a window. Nothing in it said which
|
||||
* operation, on which destination, against which topology, or up to what impact — so the same
|
||||
* approval authorised every plan presented inside the window, and the audit trail recorded a ticket
|
||||
* that proved nothing about what was executed.
|
||||
*
|
||||
* <p>Every field here is part of the signed canonical form, which is what makes the grant
|
||||
* non-transferable: change the target, the topology version, the impact ceiling, or the plan, and
|
||||
* the signature no longer verifies.
|
||||
*
|
||||
* @param approval the ticket, approver, and validity window
|
||||
* @param requestedBy the operator who will execute it, kept separate from the approver
|
||||
* @param operation the destructive operation being authorised
|
||||
* @param source the destination read from
|
||||
* @param target the destination written to, when the operation has one
|
||||
* @param topologyVersion the topology the impact estimate was computed against
|
||||
* @param maxImpact the largest number of messages the approver agreed to
|
||||
* @param planDigest the fingerprint of the exact plan
|
||||
*/
|
||||
public record ApprovalGrant(
|
||||
AdminApproval approval,
|
||||
String requestedBy,
|
||||
DestructiveOperation operation,
|
||||
DestinationName source,
|
||||
Optional<DestinationName> target,
|
||||
String topologyVersion,
|
||||
long maxImpact,
|
||||
PlanDigest planDigest) {
|
||||
|
||||
/** Validates the grant and enforces subject separation. */
|
||||
public ApprovalGrant {
|
||||
Objects.requireNonNull(approval, "approval must not be null");
|
||||
Objects.requireNonNull(operation, "operation must not be null");
|
||||
Objects.requireNonNull(source, "source must not be null");
|
||||
Objects.requireNonNull(target, "target must not be null");
|
||||
Objects.requireNonNull(topologyVersion, "topologyVersion must not be null");
|
||||
Objects.requireNonNull(planDigest, "planDigest must not be null");
|
||||
if (requestedBy == null || requestedBy.isBlank()) {
|
||||
throw new IllegalArgumentException("requestedBy must not be blank");
|
||||
}
|
||||
if (approval.approvedBy().equals(requestedBy)) {
|
||||
// Four-eyes, expressed where it cannot be skipped: an operator approving their own
|
||||
// destructive operation is the control not existing.
|
||||
throw new IllegalArgumentException(
|
||||
"the approver and the operator must be different people; both are " + requestedBy);
|
||||
}
|
||||
if (maxImpact < 0) {
|
||||
throw new IllegalArgumentException("maxImpact must not be negative");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the grant deterministically for signing and verification.
|
||||
*
|
||||
* <p>Every field is length-prefixed rather than delimited. A delimiter can be smuggled into a
|
||||
* ticket or an identity to make two different grants render identically; a length prefix cannot.
|
||||
*
|
||||
* @return the canonical form
|
||||
*/
|
||||
public String canonicalForm() {
|
||||
StringBuilder canonical = new StringBuilder("v1");
|
||||
appendField(canonical, approval.ticket());
|
||||
appendField(canonical, approval.approvedBy());
|
||||
appendField(canonical, requestedBy);
|
||||
appendField(canonical, operation.name());
|
||||
appendField(canonical, source.value());
|
||||
appendField(canonical, target.map(DestinationName::value).orElse(""));
|
||||
appendField(canonical, topologyVersion);
|
||||
appendField(canonical, Long.toString(maxImpact));
|
||||
appendField(canonical, planDigest.value());
|
||||
appendField(canonical, approval.approvedAt().toString());
|
||||
appendField(canonical, approval.validUntil().toString());
|
||||
return canonical.toString();
|
||||
}
|
||||
|
||||
private static void appendField(StringBuilder canonical, String value) {
|
||||
canonical.append(value.length()).append(':').append(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether the grant still authorises anything.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @return true while inside the approval window
|
||||
*/
|
||||
public boolean isValidAt(Instant now) {
|
||||
return approval.isValidAt(now);
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* The only thing that can turn a signed grant into a {@link VerifiedApproval}.
|
||||
*
|
||||
* <p>An implementation must check, at minimum: the issuer's signature over {@link
|
||||
* ApprovalGrant#canonicalForm()}, that the approval window is open at {@code now}, and that the
|
||||
* digest the caller is about to execute is the digest that was signed. Everything else the grant
|
||||
* binds — operation, source, target, topology version, impact ceiling, approver-versus-operator
|
||||
* separation — is inside the canonical form, so a signature check covers them all at once.
|
||||
*/
|
||||
public interface ApprovalVerifier {
|
||||
|
||||
/**
|
||||
* Verifies a signed grant against the plan actually about to run.
|
||||
*
|
||||
* @param grant what the issuer signed
|
||||
* @param signature the issuer's signature over the grant's canonical form
|
||||
* @param executingDigest the digest of the plan the caller is about to execute
|
||||
* @param now the current instant
|
||||
* @return the verified approval
|
||||
* @throws dev.caskeleton.messaging.api.error.MessageAuthorizationException when anything about
|
||||
* the grant, the signature, the window, or the digest does not hold
|
||||
*/
|
||||
VerifiedApproval verify(
|
||||
ApprovalGrant grant, String signature, PlanDigest executingDigest, Instant now);
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* A redrive plan that a named human has approved.
|
||||
*
|
||||
* <p>Carries {@code loopAcknowledged} separately from the approval itself. Approving a redrive of
|
||||
* 900 parked messages and approving a redrive that will re-fail 400 of them are different
|
||||
* decisions, and the second one needs the approver to have seen the number — so the plan cannot
|
||||
* execute on a loop-risking set unless that was acknowledged explicitly.
|
||||
*
|
||||
* @param plan the plan that was approved
|
||||
* @param approval the approval authorising it
|
||||
* @param loopAcknowledged whether the approver accepted the previously-redriven candidates
|
||||
*/
|
||||
public record ApprovedRedrivePlan(
|
||||
RedrivePlan plan, VerifiedApproval approval, boolean loopAcknowledged) {
|
||||
|
||||
/** Binds the approval to this exact plan at construction. */
|
||||
public ApprovedRedrivePlan {
|
||||
Objects.requireNonNull(plan, "plan must not be null");
|
||||
Objects.requireNonNull(approval, "approval must not be null");
|
||||
if (!approval.grant().planDigest().equals(plan.digest())) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_PLAN_MISMATCH",
|
||||
"approval %s was issued for a different plan than the one it is attached to"
|
||||
.formatted(approval.ticket()));
|
||||
}
|
||||
if (approval.grant().operation() != DestructiveOperation.REDRIVE) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_OPERATION_MISMATCH",
|
||||
"approval %s authorises %s, not a redrive"
|
||||
.formatted(approval.ticket(), approval.grant().operation()));
|
||||
}
|
||||
if (!approval.grant().source().equals(plan.request().source())
|
||||
|| !approval.grant().target().equals(java.util.Optional.of(plan.request().target()))) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_SOURCE_MISMATCH",
|
||||
"approval %s authorises %s to %s, not %s to %s"
|
||||
.formatted(
|
||||
approval.ticket(),
|
||||
approval.grant().source().value(),
|
||||
approval.grant().target().map(DestinationName::value).orElse("(none)"),
|
||||
plan.request().source().value(),
|
||||
plan.request().target().value()));
|
||||
}
|
||||
if (plan.candidates() > approval.grant().maxImpact()) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_IMPACT_EXCEEDED",
|
||||
"the plan moves %d messages but the approver agreed to at most %d"
|
||||
.formatted(plan.candidates(), approval.grant().maxImpact()));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses execution when the approval, the topology, or the loop risk no longer permits it.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @param currentTopologyVersion the topology version at execution time
|
||||
* @throws MessageAuthorizationException when the plan may not execute
|
||||
*/
|
||||
public void requireExecutable(Instant now, String currentTopologyVersion) {
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
Objects.requireNonNull(currentTopologyVersion, "currentTopologyVersion must not be null");
|
||||
|
||||
if (!approval.isValidAt(now)) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_EXPIRED", "approval %s is not valid at %s".formatted(approval.ticket(), now));
|
||||
}
|
||||
if (!approval.grant().topologyVersion().equals(currentTopologyVersion)) {
|
||||
throw new MessageAuthorizationException(
|
||||
"TOPOLOGY_CHANGED_SINCE_APPROVAL",
|
||||
"approval %s was signed against topology %s but the broker is now at %s"
|
||||
.formatted(
|
||||
approval.ticket(), approval.grant().topologyVersion(), currentTopologyVersion));
|
||||
}
|
||||
if (!plan.topologyVersion().equals(currentTopologyVersion)) {
|
||||
throw new MessageAuthorizationException(
|
||||
"TOPOLOGY_CHANGED_SINCE_APPROVAL",
|
||||
"the plan was approved against topology %s but the broker is now at %s"
|
||||
.formatted(plan.topologyVersion(), currentTopologyVersion));
|
||||
}
|
||||
if (plan.risksALoop() && !loopAcknowledged) {
|
||||
throw new MessageAuthorizationException(
|
||||
"REDRIVE_LOOP_NOT_ACKNOWLEDGED",
|
||||
"%d of the %d candidates already failed a previous redrive; re-running them without "
|
||||
.formatted(plan.alreadyRedrivenCandidates(), plan.candidates())
|
||||
+ "fixing the cause produces a loop that looks like progress");
|
||||
}
|
||||
}
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* A replay plan that a named human has approved.
|
||||
*
|
||||
* <p>A distinct type from {@link ReplayPlan} rather than a boolean on it. The execute method takes
|
||||
* this type, so an unapproved plan cannot reach it — the authorisation is enforced by the compiler
|
||||
* instead of by a runtime check somebody can forget to write.
|
||||
*
|
||||
* @param plan the plan that was approved
|
||||
* @param approval the approval authorising it
|
||||
*/
|
||||
public record ApprovedReplayPlan(ReplayPlan plan, VerifiedApproval approval) {
|
||||
|
||||
/** Binds the approval to this exact plan at construction. */
|
||||
public ApprovedReplayPlan {
|
||||
Objects.requireNonNull(plan, "plan must not be null");
|
||||
Objects.requireNonNull(approval, "approval must not be null");
|
||||
if (!approval.grant().planDigest().equals(plan.digest())) {
|
||||
// A genuine, verified approval for a different plan. Checking it here means the mismatch
|
||||
// cannot survive to the execute call under any code path.
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_PLAN_MISMATCH",
|
||||
"approval %s was issued for a different plan than the one it is attached to"
|
||||
.formatted(approval.ticket()));
|
||||
}
|
||||
if (approval.grant().operation() != DestructiveOperation.REPLAY) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_OPERATION_MISMATCH",
|
||||
"approval %s authorises %s, not a replay"
|
||||
.formatted(approval.ticket(), approval.grant().operation()));
|
||||
}
|
||||
if (!approval.grant().source().equals(plan.request().destination())) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_SOURCE_MISMATCH",
|
||||
"approval %s authorises %s, not %s"
|
||||
.formatted(
|
||||
approval.ticket(),
|
||||
approval.grant().source().value(),
|
||||
plan.request().destination().value()));
|
||||
}
|
||||
if (plan.estimatedMessages() > approval.grant().maxImpact()) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_IMPACT_EXCEEDED",
|
||||
"the plan moves about %d messages but the approver agreed to at most %d"
|
||||
.formatted(plan.estimatedMessages(), approval.grant().maxImpact()));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses execution when the approval or the plan no longer applies.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @param currentTopologyVersion the topology version at execution time
|
||||
* @throws MessageAuthorizationException when the approval has expired or the topology moved
|
||||
*/
|
||||
public void requireExecutable(Instant now, String currentTopologyVersion) {
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
Objects.requireNonNull(currentTopologyVersion, "currentTopologyVersion must not be null");
|
||||
|
||||
if (!approval.isValidAt(now)) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_EXPIRED", "approval %s is not valid at %s".formatted(approval.ticket(), now));
|
||||
}
|
||||
if (!approval.grant().topologyVersion().equals(currentTopologyVersion)) {
|
||||
throw new MessageAuthorizationException(
|
||||
"TOPOLOGY_CHANGED_SINCE_APPROVAL",
|
||||
"approval %s was signed against topology %s but the broker is now at %s"
|
||||
.formatted(
|
||||
approval.ticket(), approval.grant().topologyVersion(), currentTopologyVersion));
|
||||
}
|
||||
if (!plan.topologyVersion().equals(currentTopologyVersion)) {
|
||||
// Every number in the plan was computed against the old topology, so the approver agreed to
|
||||
// an impact estimate that no longer describes what would happen.
|
||||
throw new MessageAuthorizationException(
|
||||
"TOPOLOGY_CHANGED_SINCE_APPROVAL",
|
||||
"the plan was approved against topology %s but the broker is now at %s; the estimated "
|
||||
.formatted(plan.topologyVersion(), currentTopologyVersion)
|
||||
+ "impact no longer applies and the plan must be rebuilt");
|
||||
}
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* What the broker actually reports for one destination.
|
||||
*
|
||||
* <p>The counterpart to {@link TopologyManifest}: the manifest is what was declared, this is what
|
||||
* exists. Kept as a separate type rather than reusing the manifest so that a comparison cannot
|
||||
* accidentally compare a manifest with itself and report success.
|
||||
*
|
||||
* @param physicalName the broker-side name
|
||||
* @param partitions the observed partition count
|
||||
* @param replicationFactor the observed replication factor
|
||||
* @param configuration the observed configuration entries
|
||||
* @param exists whether the destination is present at all
|
||||
*/
|
||||
public record DestinationTopology(
|
||||
String physicalName,
|
||||
int partitions,
|
||||
int replicationFactor,
|
||||
Map<String, String> configuration,
|
||||
boolean exists) {
|
||||
|
||||
public DestinationTopology {
|
||||
Objects.requireNonNull(configuration, "configuration must not be null");
|
||||
if (physicalName == null || physicalName.isBlank()) {
|
||||
throw new IllegalArgumentException("physicalName must not be blank");
|
||||
}
|
||||
if (exists && partitions < 1) {
|
||||
throw new IllegalArgumentException("an existing destination has at least one partition");
|
||||
}
|
||||
configuration = Map.copyOf(configuration);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the topology for a destination the broker does not have.
|
||||
*
|
||||
* @param physicalName the broker-side name that was looked up
|
||||
* @return the absent topology
|
||||
*/
|
||||
public static DestinationTopology absent(String physicalName) {
|
||||
return new DestinationTopology(physicalName, 0, 0, Map.of(), false);
|
||||
}
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
/**
|
||||
* The operations that cannot be undone.
|
||||
*
|
||||
* <p>Enumerated so the guard is exhaustive rather than a list of {@code if} statements that a new
|
||||
* operation can quietly avoid.
|
||||
*/
|
||||
public enum DestructiveOperation {
|
||||
|
||||
/** Re-read a destination from an earlier position. */
|
||||
REPLAY,
|
||||
|
||||
/** Move messages from a dead letter destination back to their source. */
|
||||
REDRIVE,
|
||||
|
||||
/** Move a consumer group's committed position. */
|
||||
OFFSET_RESET,
|
||||
|
||||
/** Discard the contents of a destination. */
|
||||
PURGE,
|
||||
|
||||
/** Remove a destination entirely. */
|
||||
DELETE_DESTINATION
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* The single gate every destructive messaging operation passes through.
|
||||
*
|
||||
* <p>Four conditions, all required. The caller must hold the admin credential — an application
|
||||
* runtime does not, by construction. The approval must be present and inside its validity window.
|
||||
* And a dry run is always permitted, because the way to make operators plan before they act is to
|
||||
* make planning free.
|
||||
*
|
||||
* <p>Centralised so that adding a new destructive operation means adding an enum constant, not
|
||||
* remembering to re-implement the checks.
|
||||
*/
|
||||
public final class DestructiveOperationGuard {
|
||||
|
||||
private final boolean adminCredentialPresent;
|
||||
|
||||
/**
|
||||
* Creates a guard for a runtime.
|
||||
*
|
||||
* @param adminCredentialPresent whether this runtime holds the admin credential
|
||||
*/
|
||||
public DestructiveOperationGuard(boolean adminCredentialPresent) {
|
||||
this.adminCredentialPresent = adminCredentialPresent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Authorises one destructive operation.
|
||||
*
|
||||
* @param operation the operation
|
||||
* @param destination the destination affected
|
||||
* @param approval the verified approval, when one was supplied
|
||||
* @param dryRun whether this is a plan-only run
|
||||
* @param now the current instant
|
||||
* @throws MessageAuthorizationException when the operation is not authorised
|
||||
*/
|
||||
public void authorize(
|
||||
DestructiveOperation operation,
|
||||
DestinationName destination,
|
||||
Optional<VerifiedApproval> approval,
|
||||
boolean dryRun,
|
||||
Instant now) {
|
||||
Objects.requireNonNull(operation, "operation must not be null");
|
||||
Objects.requireNonNull(destination, "destination must not be null");
|
||||
Objects.requireNonNull(approval, "approval must not be null");
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
|
||||
if (dryRun) {
|
||||
return;
|
||||
}
|
||||
if (!adminCredentialPresent) {
|
||||
throw new MessageAuthorizationException(
|
||||
"ADMIN_CREDENTIAL_REQUIRED",
|
||||
operation + " on " + destination.value() + " requires the admin credential");
|
||||
}
|
||||
VerifiedApproval granted =
|
||||
approval.orElseThrow(
|
||||
() ->
|
||||
new MessageAuthorizationException(
|
||||
"APPROVAL_REQUIRED",
|
||||
operation + " on " + destination.value() + " requires an approval"));
|
||||
if (!granted.isValidAt(now)) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_EXPIRED", "approval " + granted.ticket() + " is outside its validity window");
|
||||
}
|
||||
if (granted.grant().operation() != operation) {
|
||||
// The approval verified — for something else. A guard that only checks presence and window
|
||||
// lets a verified redrive approval authorise a destination deletion.
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_OPERATION_MISMATCH",
|
||||
"approval "
|
||||
+ granted.ticket()
|
||||
+ " authorises "
|
||||
+ granted.grant().operation()
|
||||
+ ", not "
|
||||
+ operation);
|
||||
}
|
||||
if (!granted.grant().source().equals(destination)) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_SOURCE_MISMATCH",
|
||||
"approval "
|
||||
+ granted.ticket()
|
||||
+ " authorises "
|
||||
+ granted.grant().source().value()
|
||||
+ ", not "
|
||||
+ destination.value());
|
||||
}
|
||||
}
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.Instant;
|
||||
import java.util.HexFormat;
|
||||
import java.util.Objects;
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
/**
|
||||
* Verifies grants with a shared HMAC-SHA256 issuing key.
|
||||
*
|
||||
* <p>A shared secret rather than a public-key signature is a deliberate scope choice: the issuer is
|
||||
* the change-management system operated by the same organisation as the runtime, so key
|
||||
* distribution is already solved and the property that matters — that an operator cannot mint an
|
||||
* approval for themselves — holds as long as the key is not on the operator's machine. A public-key
|
||||
* verifier can be substituted by implementing {@link ApprovalVerifier}; nothing else in the
|
||||
* platform depends on how the signature is produced.
|
||||
*
|
||||
* <p>Comparison is constant-time. A verifier that compares with {@code equals} leaks the signature
|
||||
* one byte at a time to a caller who can retry, which for an offline attacker with unlimited
|
||||
* attempts is the whole secret.
|
||||
*/
|
||||
public final class HmacApprovalVerifier implements ApprovalVerifier {
|
||||
|
||||
private static final String ALGORITHM = "HmacSHA256";
|
||||
|
||||
private final byte[] issuingKey;
|
||||
|
||||
/**
|
||||
* Creates a verifier over the issuing key.
|
||||
*
|
||||
* @param issuingKey the shared secret the approval issuer signs with
|
||||
*/
|
||||
public HmacApprovalVerifier(byte[] issuingKey) {
|
||||
Objects.requireNonNull(issuingKey, "issuingKey must not be null");
|
||||
if (issuingKey.length < 32) {
|
||||
throw new IllegalArgumentException(
|
||||
"the approval issuing key must be at least 256 bits; a shorter key makes the signature"
|
||||
+ " brute-forceable offline");
|
||||
}
|
||||
this.issuingKey = issuingKey.clone();
|
||||
}
|
||||
|
||||
/**
|
||||
* Produces the signature an issuer would attach to a grant.
|
||||
*
|
||||
* <p>Present because the issuer and the verifier must agree on the canonical form exactly, and a
|
||||
* second implementation of that agreement is a second thing to drift. Holding this key is what
|
||||
* makes a caller an issuer — it is not, and must not become, available to the runtime that
|
||||
* executes operations.
|
||||
*
|
||||
* @param grant the grant to sign
|
||||
* @return the hexadecimal signature
|
||||
*/
|
||||
public String sign(ApprovalGrant grant) {
|
||||
Objects.requireNonNull(grant, "grant must not be null");
|
||||
return HexFormat.of().formatHex(mac(grant.canonicalForm()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public VerifiedApproval verify(
|
||||
ApprovalGrant grant, String signature, PlanDigest executingDigest, Instant now) {
|
||||
Objects.requireNonNull(grant, "grant must not be null");
|
||||
Objects.requireNonNull(signature, "signature must not be null");
|
||||
Objects.requireNonNull(executingDigest, "executingDigest must not be null");
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
|
||||
if (!grant.planDigest().equals(executingDigest)) {
|
||||
// The signature would verify: this is a genuine approval, for a different plan. Replaying one
|
||||
// approval across plans is the attack the digest binding exists to stop.
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_PLAN_MISMATCH",
|
||||
"approval %s was issued for plan %s but the plan about to execute is %s"
|
||||
.formatted(
|
||||
grant.approval().ticket(), grant.planDigest().value(), executingDigest.value()));
|
||||
}
|
||||
if (!grant.isValidAt(now)) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_EXPIRED",
|
||||
"approval %s is not valid at %s".formatted(grant.approval().ticket(), now));
|
||||
}
|
||||
byte[] presented;
|
||||
try {
|
||||
presented = HexFormat.of().parseHex(signature);
|
||||
} catch (IllegalArgumentException malformed) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_SIGNATURE_INVALID", "the approval signature is not hexadecimal");
|
||||
}
|
||||
if (!MessageDigest.isEqual(mac(grant.canonicalForm()), presented)) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_SIGNATURE_INVALID",
|
||||
("approval %s does not verify against the issuing key; the grant was altered after it"
|
||||
+ " was signed, or it was never issued")
|
||||
.formatted(grant.approval().ticket()));
|
||||
}
|
||||
return VerifiedApproval.of(grant);
|
||||
}
|
||||
|
||||
private byte[] mac(String canonicalForm) {
|
||||
try {
|
||||
Mac mac = Mac.getInstance(ALGORITHM);
|
||||
mac.init(new SecretKeySpec(issuingKey, ALGORITHM));
|
||||
return mac.doFinal(canonicalForm.getBytes(StandardCharsets.UTF_8));
|
||||
} catch (java.security.GeneralSecurityException failure) {
|
||||
throw new IllegalStateException("HMAC-SHA256 is required of every Java runtime", failure);
|
||||
}
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.HexFormat;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* The fingerprint of exactly what an approver agreed to.
|
||||
*
|
||||
* <p>An approval used to name a ticket and a window and nothing else, so it authorised any plan
|
||||
* presented within that window. The operator who got a redrive of one dead-letter destination
|
||||
* approved could execute a redrive of a different one with the same ticket, and every audit record
|
||||
* would look correct. Binding the signature to this digest is what makes an approval about a plan
|
||||
* rather than about a period of time.
|
||||
*
|
||||
* @param value the lowercase hexadecimal SHA-256 of the plan's canonical form
|
||||
*/
|
||||
public record PlanDigest(String value) {
|
||||
|
||||
/** Validates the digest shape. */
|
||||
public PlanDigest {
|
||||
Objects.requireNonNull(value, "value must not be null");
|
||||
if (!value.matches("[0-9a-f]{64}")) {
|
||||
throw new IllegalArgumentException("a plan digest is 64 lowercase hexadecimal characters");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Digests a plan's canonical form.
|
||||
*
|
||||
* @param canonicalForm the plan rendered deterministically, field by field
|
||||
* @return the digest
|
||||
*/
|
||||
public static PlanDigest ofCanonical(String canonicalForm) {
|
||||
Objects.requireNonNull(canonicalForm, "canonicalForm must not be null");
|
||||
try {
|
||||
byte[] digest =
|
||||
MessageDigest.getInstance("SHA-256")
|
||||
.digest(canonicalForm.getBytes(StandardCharsets.UTF_8));
|
||||
return new PlanDigest(HexFormat.of().formatHex(digest));
|
||||
} catch (NoSuchAlgorithmException impossible) {
|
||||
throw new IllegalStateException("SHA-256 is required of every Java runtime", impossible);
|
||||
}
|
||||
}
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* What a redrive would do, produced before anything is moved.
|
||||
*
|
||||
* <p>{@code alreadyRedrivenCandidates} is the number that matters most. A message with a non-zero
|
||||
* redrive count has been sent back to its source before and failed again; redriving it a second
|
||||
* time without fixing the cause produces a loop that looks like progress in every dashboard.
|
||||
* Surfacing the count at plan time is what lets an operator notice before starting it.
|
||||
*
|
||||
* @param request the request this plan was built from
|
||||
* @param candidates how many messages are eligible
|
||||
* @param alreadyRedrivenCandidates how many of those have been redriven before
|
||||
* @param plannedAt when the plan was produced
|
||||
* @param topologyVersion the topology the estimate was computed against
|
||||
*/
|
||||
public record RedrivePlan(
|
||||
RedriveRequest request,
|
||||
int candidates,
|
||||
int alreadyRedrivenCandidates,
|
||||
Instant plannedAt,
|
||||
String topologyVersion) {
|
||||
|
||||
public RedrivePlan {
|
||||
Objects.requireNonNull(request, "request must not be null");
|
||||
Objects.requireNonNull(plannedAt, "plannedAt must not be null");
|
||||
if (candidates < 0) {
|
||||
throw new IllegalArgumentException("candidates must not be negative");
|
||||
}
|
||||
if (alreadyRedrivenCandidates < 0 || alreadyRedrivenCandidates > candidates) {
|
||||
throw new IllegalArgumentException(
|
||||
"alreadyRedrivenCandidates must be between 0 and the candidate count");
|
||||
}
|
||||
if (topologyVersion == null || topologyVersion.isBlank()) {
|
||||
throw new IllegalArgumentException("topologyVersion must not be blank");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The fingerprint an approver's signature is bound to.
|
||||
*
|
||||
* <p>Source, target, batch size, and both candidate counts are in it, so an approval for a
|
||||
* redrive of one dead-letter destination cannot be presented for a redrive of another.
|
||||
*
|
||||
* @return the digest of this exact plan
|
||||
*/
|
||||
public PlanDigest digest() {
|
||||
return PlanDigest.ofCanonical(
|
||||
String.join(
|
||||
"|",
|
||||
"REDRIVE",
|
||||
request.redriveId().toString(),
|
||||
request.source().value(),
|
||||
request.target().value(),
|
||||
Integer.toString(request.batchSize()),
|
||||
Integer.toString(candidates),
|
||||
Integer.toString(alreadyRedrivenCandidates),
|
||||
topologyVersion));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether this redrive would replay messages that already failed a redrive.
|
||||
*
|
||||
* @return true when any candidate has been redriven before
|
||||
*/
|
||||
public boolean risksALoop() {
|
||||
return alreadyRedrivenCandidates > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a one-line operator-facing summary of the impact.
|
||||
*
|
||||
* @return the sanitized summary
|
||||
*/
|
||||
public String describeImpact() {
|
||||
String loop =
|
||||
risksALoop()
|
||||
? ", %d of which already failed a previous redrive".formatted(alreadyRedrivenCandidates)
|
||||
: "";
|
||||
return "redrive %d messages from %s to %s%s"
|
||||
.formatted(candidates, request.source().value(), request.target().value(), loop);
|
||||
}
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* A request to move messages from a dead letter destination back to their source.
|
||||
*
|
||||
* <p>The redrive id is a separate identifier from the message id, and both travel with the message.
|
||||
* Reusing the message id as the operation id would make "this message was redriven" and "this
|
||||
* message is a different message" indistinguishable, and an operator could not tell a redrive loop
|
||||
* from ordinary traffic.
|
||||
*
|
||||
* @param redriveId the operation identity
|
||||
* @param source the dead letter destination to drain
|
||||
* @param target the destination to publish back to
|
||||
* @param batchSize how many messages to move per pass
|
||||
* @param dryRun whether to plan without moving anything
|
||||
*/
|
||||
public record RedriveRequest(
|
||||
UUID redriveId, DestinationName source, DestinationName target, int batchSize, boolean dryRun) {
|
||||
|
||||
private static final int MAX_BATCH = 100;
|
||||
|
||||
public RedriveRequest {
|
||||
Objects.requireNonNull(redriveId, "redriveId must not be null");
|
||||
Objects.requireNonNull(source, "source must not be null");
|
||||
Objects.requireNonNull(target, "target must not be null");
|
||||
if (source.equals(target)) {
|
||||
throw new IllegalArgumentException("a redrive cannot target its own source");
|
||||
}
|
||||
if (batchSize < 1 || batchSize > MAX_BATCH) {
|
||||
throw new IllegalArgumentException("redrive batch size must be between 1 and " + MAX_BATCH);
|
||||
}
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* What a redrive actually did.
|
||||
*
|
||||
* <p>{@code stillParked} is not simply {@code candidates - moved}. A message stays parked when its
|
||||
* republish did not confirm, and the redrive deliberately leaves it there rather than settling it —
|
||||
* the DLQ-confirm-before-settle rule applies to a redrive exactly as it does to the original
|
||||
* dead-lettering, because a redrive that settles an unconfirmed republish deletes the last copy.
|
||||
*
|
||||
* @param redriveId the operation identity
|
||||
* @param candidates how many messages were eligible
|
||||
* @param moved how many were republished and settled
|
||||
* @param stillParked how many stayed on the dead letter destination
|
||||
* @param elapsed how long the redrive took
|
||||
* @param dryRun whether nothing was actually moved
|
||||
*/
|
||||
public record RedriveResult(
|
||||
UUID redriveId, int candidates, int moved, int stillParked, Duration elapsed, boolean dryRun) {
|
||||
|
||||
public RedriveResult {
|
||||
Objects.requireNonNull(redriveId, "redriveId must not be null");
|
||||
Objects.requireNonNull(elapsed, "elapsed must not be null");
|
||||
if (candidates < 0 || moved < 0 || stillParked < 0) {
|
||||
throw new IllegalArgumentException("redrive counters must not be negative");
|
||||
}
|
||||
if (moved + stillParked > candidates) {
|
||||
throw new IllegalArgumentException(
|
||||
"a redrive cannot account for more messages than it had candidates");
|
||||
}
|
||||
if (dryRun && moved > 0) {
|
||||
throw new IllegalArgumentException("a dry run must not move anything");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether every candidate was accounted for.
|
||||
*
|
||||
* <p>An unaccounted message is a bug, not a partial success: it was neither republished nor left
|
||||
* parked, which means the redrive lost track of it.
|
||||
*
|
||||
* @return true when moved plus still-parked covers every candidate
|
||||
*/
|
||||
public boolean isFullyAccounted() {
|
||||
return moved + stillParked == candidates;
|
||||
}
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* What a replay would do, produced before anything is read.
|
||||
*
|
||||
* <p>The plan exists so the estimate can be reviewed. "Replay from yesterday" is a sentence; "this
|
||||
* will re-deliver 4.2 million messages into the live consumer group" is a decision, and the only
|
||||
* moment an operator can make it is before the replay starts.
|
||||
*
|
||||
* <p>{@code topologyVersion} is captured here and re-checked at execution. A plan approved against
|
||||
* one topology and executed against another is estimating a different thing entirely — a partition
|
||||
* count that changed in between invalidates every number in this record.
|
||||
*
|
||||
* @param request the request this plan was built from
|
||||
* @param estimatedMessages how many messages the window covers
|
||||
* @param plannedAt when the plan was produced
|
||||
* @param topologyVersion the topology the estimate was computed against
|
||||
* @param targetsLiveConsumerGroup whether the replay would feed the live group rather than an
|
||||
* isolated one
|
||||
*/
|
||||
public record ReplayPlan(
|
||||
ReplayRequest request,
|
||||
long estimatedMessages,
|
||||
Instant plannedAt,
|
||||
String topologyVersion,
|
||||
boolean targetsLiveConsumerGroup) {
|
||||
|
||||
public ReplayPlan {
|
||||
Objects.requireNonNull(request, "request must not be null");
|
||||
Objects.requireNonNull(plannedAt, "plannedAt must not be null");
|
||||
if (estimatedMessages < 0) {
|
||||
throw new IllegalArgumentException("estimatedMessages must not be negative");
|
||||
}
|
||||
if (topologyVersion == null || topologyVersion.isBlank()) {
|
||||
throw new IllegalArgumentException("topologyVersion must not be blank");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The fingerprint an approver's signature is bound to.
|
||||
*
|
||||
* <p>Every field that changes what would happen is in it. An approval signed over this digest
|
||||
* cannot be presented for a plan that reads a different destination, targets a different consumer
|
||||
* group, starts from a different position, or estimates a different impact.
|
||||
*
|
||||
* @return the digest of this exact plan
|
||||
*/
|
||||
public PlanDigest digest() {
|
||||
return PlanDigest.ofCanonical(
|
||||
String.join(
|
||||
"|",
|
||||
"REPLAY",
|
||||
request.replayId().toString(),
|
||||
request.destination().value(),
|
||||
Boolean.toString(request.isolatedConsumerGroup()),
|
||||
request.from().toString(),
|
||||
request.to().map(Object::toString).orElse(""),
|
||||
Long.toString(estimatedMessages),
|
||||
topologyVersion,
|
||||
Boolean.toString(targetsLiveConsumerGroup)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a one-line operator-facing summary of the impact.
|
||||
*
|
||||
* @return the sanitized summary
|
||||
*/
|
||||
public String describeImpact() {
|
||||
return "replay %s from %s: about %d messages into %s"
|
||||
.formatted(
|
||||
request.destination().value(),
|
||||
request.from(),
|
||||
estimatedMessages,
|
||||
targetsLiveConsumerGroup ? "the LIVE consumer group" : "an isolated group");
|
||||
}
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* A request to re-read a destination from an earlier position.
|
||||
*
|
||||
* @param replayId the operation identity
|
||||
* @param destination the destination to replay
|
||||
* @param from the replay start point
|
||||
* @param to the replay end point, when bounded
|
||||
* @param isolatedConsumerGroup whether to replay into a throwaway group
|
||||
* @param dryRun whether to plan without reading anything
|
||||
*/
|
||||
public record ReplayRequest(
|
||||
UUID replayId,
|
||||
DestinationName destination,
|
||||
Instant from,
|
||||
Optional<Instant> to,
|
||||
boolean isolatedConsumerGroup,
|
||||
boolean dryRun) {
|
||||
|
||||
public ReplayRequest {
|
||||
Objects.requireNonNull(replayId, "replayId must not be null");
|
||||
Objects.requireNonNull(destination, "destination must not be null");
|
||||
Objects.requireNonNull(from, "from must not be null");
|
||||
Objects.requireNonNull(to, "to must not be null");
|
||||
if (to.filter(end -> end.isBefore(from)).isPresent()) {
|
||||
throw new IllegalArgumentException("replay window ends before it starts");
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* What a replay actually did.
|
||||
*
|
||||
* <p>Reports the delivered count against the plan's estimate. They routinely differ — retention may
|
||||
* have expired part of the window, or the stream may have grown while the plan was being approved —
|
||||
* and the difference is the operator's signal that the replay covered something other than what was
|
||||
* approved.
|
||||
*
|
||||
* @param replayId the operation identity
|
||||
* @param estimatedMessages what the plan predicted
|
||||
* @param deliveredMessages what was actually re-delivered
|
||||
* @param elapsed how long the replay took
|
||||
* @param completed whether the whole window was covered
|
||||
* @param dryRun whether nothing was actually read
|
||||
*/
|
||||
public record ReplayResult(
|
||||
UUID replayId,
|
||||
long estimatedMessages,
|
||||
long deliveredMessages,
|
||||
Duration elapsed,
|
||||
boolean completed,
|
||||
boolean dryRun) {
|
||||
|
||||
public ReplayResult {
|
||||
Objects.requireNonNull(replayId, "replayId must not be null");
|
||||
Objects.requireNonNull(elapsed, "elapsed must not be null");
|
||||
if (estimatedMessages < 0 || deliveredMessages < 0) {
|
||||
throw new IllegalArgumentException("replay counters must not be negative");
|
||||
}
|
||||
if (dryRun && deliveredMessages > 0) {
|
||||
throw new IllegalArgumentException("a dry run must not deliver anything");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether the replay covered materially less than was approved.
|
||||
*
|
||||
* @return true when fewer than 90% of the estimated messages were delivered
|
||||
*/
|
||||
public boolean fellShortOfTheEstimate() {
|
||||
return completed && estimatedMessages > 0 && deliveredMessages * 10 < estimatedMessages * 9;
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* One discrepancy between a declared topology and what the broker actually has.
|
||||
*
|
||||
* <p>Severity is part of the finding because the two kinds behave differently at startup. A {@link
|
||||
* Severity#BLOCKING} issue means the destination cannot deliver its declared guarantee —
|
||||
* replication factor 1 on a destination promising durability is not a warning, it is a promise the
|
||||
* platform cannot keep — so the context refuses to start. A {@link Severity#ADVISORY} issue is a
|
||||
* drift worth reporting that does not break a guarantee.
|
||||
*
|
||||
* @param destination the logical destination
|
||||
* @param attribute the topology attribute that differs
|
||||
* @param declared what the manifest declared
|
||||
* @param actual what the broker reported
|
||||
* @param severity how the platform should react
|
||||
*/
|
||||
public record TopologyIssue(
|
||||
String destination, String attribute, String declared, String actual, Severity severity) {
|
||||
|
||||
/** How the platform reacts to a topology discrepancy. */
|
||||
public enum Severity {
|
||||
/** The destination cannot deliver a declared guarantee; startup must fail. */
|
||||
BLOCKING,
|
||||
/** Drift worth reporting that does not break a guarantee. */
|
||||
ADVISORY
|
||||
}
|
||||
|
||||
public TopologyIssue {
|
||||
Objects.requireNonNull(severity, "severity must not be null");
|
||||
requireText(destination, "destination");
|
||||
requireText(attribute, "attribute");
|
||||
requireText(declared, "declared");
|
||||
requireText(actual, "actual");
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a blocking issue.
|
||||
*
|
||||
* @param destination the logical destination
|
||||
* @param attribute the differing attribute
|
||||
* @param declared the declared value
|
||||
* @param actual the observed value
|
||||
* @return the issue
|
||||
*/
|
||||
public static TopologyIssue blocking(
|
||||
String destination, String attribute, String declared, String actual) {
|
||||
return new TopologyIssue(destination, attribute, declared, actual, Severity.BLOCKING);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an advisory issue.
|
||||
*
|
||||
* @param destination the logical destination
|
||||
* @param attribute the differing attribute
|
||||
* @param declared the declared value
|
||||
* @param actual the observed value
|
||||
* @return the issue
|
||||
*/
|
||||
public static TopologyIssue advisory(
|
||||
String destination, String attribute, String declared, String actual) {
|
||||
return new TopologyIssue(destination, attribute, declared, actual, Severity.ADVISORY);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a one-line operator-facing description.
|
||||
*
|
||||
* @return the sanitized description
|
||||
*/
|
||||
public String describe() {
|
||||
return "%s: %s declared %s but the broker has %s"
|
||||
.formatted(destination, attribute, declared, actual);
|
||||
}
|
||||
|
||||
private static void requireText(String value, String field) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException(field + " must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
|
||||
|
||||
/**
|
||||
* Whether the application may create broker topology, or only check it.
|
||||
*
|
||||
* <p>{@link #VALIDATE_ONLY} in production, always. An application that auto-creates topology will
|
||||
* auto-create it after a configuration typo too, and the topic it makes is indistinguishable from a
|
||||
* real one — same broker, same client, same metrics — while carrying the broker's default partition
|
||||
* count and replication factor instead of the ones the destination needs. The failure surfaces
|
||||
* weeks later as data loss on a partition that was never replicated.
|
||||
*/
|
||||
public enum TopologyManagementMode {
|
||||
|
||||
/** Compare the declared topology against the broker and fail on a mismatch. */
|
||||
VALIDATE_ONLY,
|
||||
|
||||
/** Create missing topology. Permitted outside production only. */
|
||||
CREATE_IF_MISSING;
|
||||
|
||||
/**
|
||||
* Refuses auto-creation on a production runtime.
|
||||
*
|
||||
* @param production whether this runtime is production
|
||||
* @throws MessagingConfigurationException when auto-creation is configured in production
|
||||
*/
|
||||
public void requireSafeFor(boolean production) {
|
||||
if (production && this == CREATE_IF_MISSING) {
|
||||
throw new MessagingConfigurationException(
|
||||
"AUTO_CREATE_IN_PRODUCTION",
|
||||
"topology auto-creation is not permitted in production: a mistyped destination would be "
|
||||
+ "created with the broker's default partition count and replication factor, and "
|
||||
+ "would look exactly like a correctly provisioned one");
|
||||
}
|
||||
}
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* The declared shape of a destination's broker topology.
|
||||
*
|
||||
* <p>Production topology is created by infrastructure code and only <em>validated</em> by the
|
||||
* application. An application that creates topology on startup will happily create it in the wrong
|
||||
* place after a configuration mistake, and the resulting topic looks exactly like a real one.
|
||||
*
|
||||
* @param destination the logical destination
|
||||
* @param physicalName the broker-side name
|
||||
* @param partitions the expected partition count, where the broker has partitions
|
||||
* @param replicationFactor the expected replication factor
|
||||
* @param requiredConfiguration configuration entries that must match exactly
|
||||
*/
|
||||
public record TopologyManifest(
|
||||
String destination,
|
||||
String physicalName,
|
||||
int partitions,
|
||||
int replicationFactor,
|
||||
Map<String, String> requiredConfiguration) {
|
||||
|
||||
public TopologyManifest {
|
||||
Objects.requireNonNull(requiredConfiguration, "requiredConfiguration must not be null");
|
||||
if (destination == null || destination.isBlank()) {
|
||||
throw new IllegalArgumentException("destination must not be blank");
|
||||
}
|
||||
if (physicalName == null || physicalName.isBlank()) {
|
||||
throw new IllegalArgumentException("physicalName must not be blank");
|
||||
}
|
||||
if (partitions < 1) {
|
||||
throw new IllegalArgumentException("partitions must be at least 1");
|
||||
}
|
||||
if (replicationFactor < 1) {
|
||||
throw new IllegalArgumentException("replicationFactor must be at least 1");
|
||||
}
|
||||
requiredConfiguration = Map.copyOf(requiredConfiguration);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares this manifest against what the broker actually reports.
|
||||
*
|
||||
* @param actualPartitions the observed partition count
|
||||
* @param actualReplicationFactor the observed replication factor
|
||||
* @param actualConfiguration the observed configuration
|
||||
* @return the differences, empty when the topology matches
|
||||
*/
|
||||
public List<String> differencesFrom(
|
||||
int actualPartitions, int actualReplicationFactor, Map<String, String> actualConfiguration) {
|
||||
Objects.requireNonNull(actualConfiguration, "actualConfiguration must not be null");
|
||||
List<String> differences = new java.util.ArrayList<>();
|
||||
|
||||
if (actualPartitions != partitions) {
|
||||
differences.add("partitions expected " + partitions + " but found " + actualPartitions);
|
||||
}
|
||||
if (actualReplicationFactor != replicationFactor) {
|
||||
differences.add(
|
||||
"replicationFactor expected "
|
||||
+ replicationFactor
|
||||
+ " but found "
|
||||
+ actualReplicationFactor);
|
||||
}
|
||||
requiredConfiguration.forEach(
|
||||
(key, expected) -> {
|
||||
String actual = actualConfiguration.get(key);
|
||||
if (!expected.equals(actual)) {
|
||||
differences.add(key + " expected " + expected + " but found " + actual);
|
||||
}
|
||||
});
|
||||
return List.copyOf(differences);
|
||||
}
|
||||
}
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* The outcome of comparing every declared topology against the broker.
|
||||
*
|
||||
* <p>Reports both severities together rather than failing on the first blocking issue. An operator
|
||||
* fixing a topology wants the whole list — fixing one attribute, redeploying, and discovering the
|
||||
* next one is how a ten-minute fix becomes an afternoon.
|
||||
*
|
||||
* @param issues every discrepancy found
|
||||
* @param destinationsChecked how many destinations were compared
|
||||
*/
|
||||
public record TopologyValidationReport(List<TopologyIssue> issues, int destinationsChecked) {
|
||||
|
||||
public TopologyValidationReport {
|
||||
Objects.requireNonNull(issues, "issues must not be null");
|
||||
if (destinationsChecked < 0) {
|
||||
throw new IllegalArgumentException("destinationsChecked must not be negative");
|
||||
}
|
||||
issues = List.copyOf(issues);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a report with nothing to fix.
|
||||
*
|
||||
* @param destinationsChecked how many destinations were compared
|
||||
* @return the clean report
|
||||
*/
|
||||
public static TopologyValidationReport clean(int destinationsChecked) {
|
||||
return new TopologyValidationReport(List.of(), destinationsChecked);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the issues that must fail startup.
|
||||
*
|
||||
* @return the blocking issues
|
||||
*/
|
||||
public List<TopologyIssue> blocking() {
|
||||
return issues.stream()
|
||||
.filter(issue -> issue.severity() == TopologyIssue.Severity.BLOCKING)
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the issues worth reporting that do not break a guarantee.
|
||||
*
|
||||
* @return the advisory issues
|
||||
*/
|
||||
public List<TopologyIssue> advisory() {
|
||||
return issues.stream()
|
||||
.filter(issue -> issue.severity() == TopologyIssue.Severity.ADVISORY)
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether the topology may be used.
|
||||
*
|
||||
* @return true when nothing blocking was found
|
||||
*/
|
||||
public boolean isAcceptable() {
|
||||
return blocking().isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
* Fails startup when any blocking issue was found.
|
||||
*
|
||||
* @throws MessagingConfigurationException listing every blocking issue
|
||||
*/
|
||||
public void requireAcceptable() {
|
||||
List<TopologyIssue> blocking = blocking();
|
||||
if (blocking.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
throw new MessagingConfigurationException(
|
||||
"TOPOLOGY_MISMATCH",
|
||||
"the broker topology cannot deliver the declared guarantees: "
|
||||
+ String.join("; ", blocking.stream().map(TopologyIssue::describe).toList()));
|
||||
}
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* A grant that an {@link ApprovalVerifier} has checked, and that nobody else can manufacture.
|
||||
*
|
||||
* <p>The approved-plan types used to hold a plain {@code AdminApproval} record with a public
|
||||
* constructor, so "this plan was approved" was a claim the caller made about itself. Any code that
|
||||
* could reach the execute method could write {@code new AdminApproval("TICKET-1", "someone", now,
|
||||
* later)} and the platform believed it.
|
||||
*
|
||||
* <p>This class has no public constructor and no public factory. The only way to obtain one is to
|
||||
* present a grant and a signature to a verifier, which is exactly the property the type is for:
|
||||
* holding a {@code VerifiedApproval} is itself the evidence that verification happened.
|
||||
*/
|
||||
public final class VerifiedApproval {
|
||||
|
||||
private final ApprovalGrant grant;
|
||||
|
||||
private VerifiedApproval(ApprovalGrant grant) {
|
||||
this.grant = Objects.requireNonNull(grant, "grant must not be null");
|
||||
}
|
||||
|
||||
/**
|
||||
* Mints a verified approval. Package-private on purpose: only a verifier in this package may call
|
||||
* it, so the type cannot be forged from application or adapter code.
|
||||
*/
|
||||
static VerifiedApproval of(ApprovalGrant grant) {
|
||||
return new VerifiedApproval(grant);
|
||||
}
|
||||
|
||||
/**
|
||||
* The grant that was verified.
|
||||
*
|
||||
* @return the grant
|
||||
*/
|
||||
public ApprovalGrant grant() {
|
||||
return grant;
|
||||
}
|
||||
|
||||
/**
|
||||
* The approval ticket, for audit records and journal keys.
|
||||
*
|
||||
* @return the ticket
|
||||
*/
|
||||
public String ticket() {
|
||||
return grant.approval().ticket();
|
||||
}
|
||||
|
||||
/**
|
||||
* Who approved the operation.
|
||||
*
|
||||
* @return the approver identity
|
||||
*/
|
||||
public String approvedBy() {
|
||||
return grant.approval().approvedBy();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether the approval window is still open.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @return true while inside the window
|
||||
*/
|
||||
public boolean isValidAt(Instant now) {
|
||||
return grant.isValidAt(now);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other) {
|
||||
return other instanceof VerifiedApproval verified && grant.equals(verified.grant);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return grant.hashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
// Deliberately not the whole grant: this string reaches logs, and the plan digest plus the
|
||||
// ticket identify the operation without restating who approved what to an audience that has
|
||||
// not been authorised to read it.
|
||||
return "VerifiedApproval[" + ticket() + " " + grant.planDigest().value().substring(0, 12) + "]";
|
||||
}
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.DestinationTopology;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Reads the broker's current topology.
|
||||
*
|
||||
* <p>Read-only by construction. The inspector is what the application's own credential uses, and
|
||||
* that credential holds no destructive grant, so the type exposes no way to create, alter, or
|
||||
* delete — an application cannot reach a destructive operation even by mistake because there is no
|
||||
* method to reach.
|
||||
*/
|
||||
public interface BrokerTopologyInspector {
|
||||
|
||||
/**
|
||||
* Describes one destination.
|
||||
*
|
||||
* @param physicalName the broker-side name
|
||||
* @return the observed topology, absent when the broker has no such destination
|
||||
*/
|
||||
Optional<DestinationTopology> describe(String physicalName);
|
||||
|
||||
/**
|
||||
* Returns an opaque version for the broker's current topology.
|
||||
*
|
||||
* <p>Used to invalidate an approved plan whose impact estimate was computed against an earlier
|
||||
* shape. Any value that changes when the topology changes is sufficient; it is never parsed.
|
||||
*
|
||||
* @return the topology version
|
||||
*/
|
||||
String topologyVersion();
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.DestinationTopology;
|
||||
import dev.caskeleton.messaging.admin.TopologyIssue;
|
||||
import dev.caskeleton.messaging.admin.TopologyManifest;
|
||||
import dev.caskeleton.messaging.admin.TopologyValidationReport;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Validates every declared topology in one pass and reports the whole result.
|
||||
*
|
||||
* <p>Collects all issues rather than stopping at the first blocking one. An operator fixing a
|
||||
* topology mismatch wants the complete list — discovering the next problem only after a redeploy
|
||||
* turns one fix into a sequence of them, and each redeploy is another restart of a production
|
||||
* service.
|
||||
*/
|
||||
public final class CompositeTopologyValidator {
|
||||
|
||||
private final BrokerTopologyInspector inspector;
|
||||
private final TopologyValidator validator = new TopologyValidator();
|
||||
|
||||
/**
|
||||
* Creates a validator over a broker inspector.
|
||||
*
|
||||
* @param inspector reads the broker's current topology
|
||||
*/
|
||||
public CompositeTopologyValidator(BrokerTopologyInspector inspector) {
|
||||
this.inspector = Objects.requireNonNull(inspector, "inspector must not be null");
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares every manifest against the broker.
|
||||
*
|
||||
* @param manifests the declared topologies
|
||||
* @return the complete report
|
||||
*/
|
||||
public TopologyValidationReport validate(List<TopologyManifest> manifests) {
|
||||
Objects.requireNonNull(manifests, "manifests must not be null");
|
||||
|
||||
List<TopologyIssue> issues = new ArrayList<>();
|
||||
for (TopologyManifest manifest : manifests) {
|
||||
DestinationTopology observed =
|
||||
inspector
|
||||
.describe(manifest.physicalName())
|
||||
.orElseGet(() -> DestinationTopology.absent(manifest.physicalName()));
|
||||
issues.addAll(validator.compare(manifest, observed));
|
||||
}
|
||||
return new TopologyValidationReport(issues, manifests.size());
|
||||
}
|
||||
}
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminOperationJournal;
|
||||
import dev.caskeleton.messaging.admin.AdminOperationLease;
|
||||
import dev.caskeleton.messaging.admin.ApprovedRedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.ApprovedReplayPlan;
|
||||
import dev.caskeleton.messaging.admin.RedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.RedriveRequest;
|
||||
import dev.caskeleton.messaging.admin.RedriveResult;
|
||||
import dev.caskeleton.messaging.admin.ReplayPlan;
|
||||
import dev.caskeleton.messaging.admin.ReplayRequest;
|
||||
import dev.caskeleton.messaging.admin.ReplayResult;
|
||||
import dev.caskeleton.messaging.admin.TopologyManifest;
|
||||
import dev.caskeleton.messaging.admin.TopologyValidationReport;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* Wires plan, approval, and execution together for the non-destructive admin operations.
|
||||
*
|
||||
* <p>Execution runs four checks, in this order, and the order is the point.
|
||||
*
|
||||
* <ol>
|
||||
* <li>The approval is still inside its window.
|
||||
* <li>The topology has not changed since the plan was approved.
|
||||
* <li>The approval has not already been executed.
|
||||
* <li>Only then does anything move.
|
||||
* </ol>
|
||||
*
|
||||
* <p>The journal entry is written <em>before</em> the work rather than after it. Writing it
|
||||
* afterwards leaves a window where a second execution starts while the first is still running,
|
||||
* which is precisely the double-redrive the journal exists to prevent.
|
||||
*
|
||||
* <p>Writing it first used to have a cost the previous store never paid: an operation that died
|
||||
* halfway had consumed its approval and left no record of how far it got. The journal keeps a
|
||||
* checkpoint and hands back a lease that says where to resume, so a retry continues the same
|
||||
* operation instead of either redoing it or requiring a new approval.
|
||||
*/
|
||||
public final class DefaultMessagingAdminService implements MessagingAdminService {
|
||||
|
||||
/**
|
||||
* How long a runtime holds an operation before another may take it over.
|
||||
*
|
||||
* <p>Long enough that a slow batch does not lose its lease mid-flight, short enough that a dead
|
||||
* replica does not park an approval for an hour.
|
||||
*/
|
||||
private static final Duration LEASE_DURATION = Duration.ofMinutes(5);
|
||||
|
||||
private final CompositeTopologyValidator topologyValidator;
|
||||
private final BrokerTopologyInspector inspector;
|
||||
private final AdminOperationJournal journal;
|
||||
private final String leaseOwner;
|
||||
private final ReplayService replayService;
|
||||
private final RedriveService redriveService;
|
||||
private final Supplier<List<TopologyManifest>> manifests;
|
||||
private final ReplayEstimator replayEstimator;
|
||||
private final RedriveEstimator redriveEstimator;
|
||||
private final Supplier<Instant> clock;
|
||||
|
||||
/**
|
||||
* Creates the admin service.
|
||||
*
|
||||
* @param topologyValidator compares declared topology against the broker
|
||||
* @param inspector reads the broker's topology version
|
||||
* @param journal records which approvals have been executed, and how far each got
|
||||
* @param leaseOwner identifies this runtime in the journal's leases
|
||||
* @param replayService performs replays
|
||||
* @param redriveService performs redrives
|
||||
* @param manifests supplies the declared topologies
|
||||
* @param replayEstimator estimates a replay's message count
|
||||
* @param redriveEstimator estimates a redrive's candidates
|
||||
* @param clock supplies the current instant
|
||||
*/
|
||||
public DefaultMessagingAdminService(
|
||||
CompositeTopologyValidator topologyValidator,
|
||||
BrokerTopologyInspector inspector,
|
||||
AdminOperationJournal journal,
|
||||
String leaseOwner,
|
||||
ReplayService replayService,
|
||||
RedriveService redriveService,
|
||||
Supplier<List<TopologyManifest>> manifests,
|
||||
ReplayEstimator replayEstimator,
|
||||
RedriveEstimator redriveEstimator,
|
||||
Supplier<Instant> clock) {
|
||||
this.topologyValidator =
|
||||
Objects.requireNonNull(topologyValidator, "topologyValidator required");
|
||||
this.inspector = Objects.requireNonNull(inspector, "inspector must not be null");
|
||||
this.journal = Objects.requireNonNull(journal, "journal must not be null");
|
||||
if (leaseOwner == null || leaseOwner.isBlank()) {
|
||||
throw new IllegalArgumentException("leaseOwner must not be blank");
|
||||
}
|
||||
this.leaseOwner = leaseOwner;
|
||||
this.replayService = Objects.requireNonNull(replayService, "replayService must not be null");
|
||||
this.redriveService = Objects.requireNonNull(redriveService, "redriveService must not be null");
|
||||
this.manifests = Objects.requireNonNull(manifests, "manifests must not be null");
|
||||
this.replayEstimator = Objects.requireNonNull(replayEstimator, "replayEstimator required");
|
||||
this.redriveEstimator = Objects.requireNonNull(redriveEstimator, "redriveEstimator required");
|
||||
this.clock = Objects.requireNonNull(clock, "clock must not be null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public TopologyValidationReport validateTopology() {
|
||||
return topologyValidator.validate(manifests.get());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ReplayPlan planReplay(ReplayRequest request) {
|
||||
Objects.requireNonNull(request, "request must not be null");
|
||||
return new ReplayPlan(
|
||||
request,
|
||||
replayEstimator.estimate(request),
|
||||
clock.get(),
|
||||
inspector.topologyVersion(),
|
||||
!request.isolatedConsumerGroup());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ReplayResult executeReplay(ApprovedReplayPlan plan) {
|
||||
Objects.requireNonNull(plan, "plan must not be null");
|
||||
Instant now = clock.get();
|
||||
ReplayRequest request = plan.plan().request();
|
||||
|
||||
plan.requireExecutable(now, inspector.topologyVersion());
|
||||
AdminOperationLease lease =
|
||||
journal.begin(
|
||||
plan.approval().ticket(),
|
||||
plan.plan().digest(),
|
||||
request.replayId().toString(),
|
||||
leaseOwner,
|
||||
LEASE_DURATION,
|
||||
now);
|
||||
|
||||
Instant startedAt = clock.get();
|
||||
ReplayReport report;
|
||||
try {
|
||||
report =
|
||||
replayService.replay(
|
||||
request, Optional.of(plan.approval()), plan.approval().approvedBy(), now);
|
||||
} catch (RuntimeException failure) {
|
||||
// The operation stays resumable rather than silently consuming the approval: the journal
|
||||
// entry moves to FAILED at its checkpoint, and a retry takes it over from there.
|
||||
journal.fail(lease, lease.resumeFrom(), failureCodeOf(failure), clock.get());
|
||||
throw failure;
|
||||
}
|
||||
journal.complete(lease, Math.toIntExact(report.replayed()), clock.get());
|
||||
|
||||
return new ReplayResult(
|
||||
request.replayId(),
|
||||
plan.plan().estimatedMessages(),
|
||||
report.replayed(),
|
||||
Duration.between(startedAt, clock.get()),
|
||||
true,
|
||||
report.dryRun());
|
||||
}
|
||||
|
||||
@Override
|
||||
public RedrivePlan planRedrive(RedriveRequest request) {
|
||||
Objects.requireNonNull(request, "request must not be null");
|
||||
RedriveEstimate estimate = redriveEstimator.estimate(request);
|
||||
return new RedrivePlan(
|
||||
request,
|
||||
estimate.candidates(),
|
||||
estimate.alreadyRedriven(),
|
||||
clock.get(),
|
||||
inspector.topologyVersion());
|
||||
}
|
||||
|
||||
@Override
|
||||
public RedriveResult executeRedrive(ApprovedRedrivePlan plan) {
|
||||
Objects.requireNonNull(plan, "plan must not be null");
|
||||
Instant now = clock.get();
|
||||
RedriveRequest request = plan.plan().request();
|
||||
|
||||
plan.requireExecutable(now, inspector.topologyVersion());
|
||||
AdminOperationLease lease =
|
||||
journal.begin(
|
||||
plan.approval().ticket(),
|
||||
plan.plan().digest(),
|
||||
request.redriveId().toString(),
|
||||
leaseOwner,
|
||||
LEASE_DURATION,
|
||||
now);
|
||||
|
||||
Instant startedAt = clock.get();
|
||||
RedriveReport report;
|
||||
try {
|
||||
report =
|
||||
redriveService.redrive(
|
||||
request,
|
||||
Optional.of(plan.approval()),
|
||||
plan.approval().approvedBy(),
|
||||
now,
|
||||
lease.resumeFrom(),
|
||||
completed -> journal.checkpoint(lease, completed, LEASE_DURATION, clock.get()));
|
||||
} catch (RuntimeException failure) {
|
||||
journal.fail(lease, lease.resumeFrom(), failureCodeOf(failure), clock.get());
|
||||
throw failure;
|
||||
}
|
||||
journal.complete(lease, report.moved() + report.failed(), clock.get());
|
||||
|
||||
return new RedriveResult(
|
||||
request.redriveId(),
|
||||
report.candidates(),
|
||||
report.moved(),
|
||||
report.failed(),
|
||||
Duration.between(startedAt, clock.get()),
|
||||
report.dryRun());
|
||||
}
|
||||
|
||||
/**
|
||||
* Reduces a failure to a bounded, sanitised code for the journal.
|
||||
*
|
||||
* <p>The journal is read by operators during incidents and its contents outlive the process. A
|
||||
* raw exception message can carry a destination, a payload fragment, or a credential from a
|
||||
* driver's own error text, so only the platform's own code or the exception's simple name goes
|
||||
* in.
|
||||
*/
|
||||
private static String failureCodeOf(RuntimeException failure) {
|
||||
if (failure instanceof dev.caskeleton.messaging.api.error.MessagingException messaging) {
|
||||
return messaging.failure().code();
|
||||
}
|
||||
return failure.getClass().getSimpleName();
|
||||
}
|
||||
|
||||
/** Estimates how many messages a replay would re-deliver. */
|
||||
@FunctionalInterface
|
||||
public interface ReplayEstimator {
|
||||
|
||||
/**
|
||||
* Estimates a replay's message count without reading anything.
|
||||
*
|
||||
* @param request the replay request
|
||||
* @return the estimated count
|
||||
*/
|
||||
long estimate(ReplayRequest request);
|
||||
}
|
||||
|
||||
/** How many messages a redrive would move, and how many already failed one. */
|
||||
record RedriveEstimate(int candidates, int alreadyRedriven) {}
|
||||
|
||||
/** Estimates a redrive's candidates. */
|
||||
@FunctionalInterface
|
||||
public interface RedriveEstimator {
|
||||
|
||||
/**
|
||||
* Estimates a redrive's candidates without moving anything.
|
||||
*
|
||||
* @param request the redrive request
|
||||
* @return the estimate
|
||||
*/
|
||||
RedriveEstimate estimate(RedriveRequest request);
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperation;
|
||||
import dev.caskeleton.messaging.admin.VerifiedApproval;
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* The operations that destroy data an application cannot recreate.
|
||||
*
|
||||
* <p>A separate interface from {@link MessagingAdminService}, and no bean for it is ever registered
|
||||
* in an application runtime. The separation is the control: an application that never receives this
|
||||
* type cannot purge a topic even if every other guard is bypassed, because the method does not
|
||||
* exist on anything it holds.
|
||||
*
|
||||
* <p>Each operation takes an {@link Approved} argument rather than an approval parameter, so the
|
||||
* authorisation cannot be forgotten at a call site — there is no way to call these without one.
|
||||
*/
|
||||
public interface DestructiveMessagingAdmin {
|
||||
|
||||
/** An authorised destructive request. */
|
||||
record Approved(
|
||||
DestructiveOperation operation,
|
||||
DestinationName destination,
|
||||
VerifiedApproval approval,
|
||||
long estimatedMessagesAffected) {
|
||||
|
||||
public Approved {
|
||||
Objects.requireNonNull(operation, "operation must not be null");
|
||||
Objects.requireNonNull(destination, "destination must not be null");
|
||||
Objects.requireNonNull(approval, "approval must not be null");
|
||||
if (estimatedMessagesAffected < 0) {
|
||||
throw new IllegalArgumentException("estimatedMessagesAffected must not be negative");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** What a destructive operation did. */
|
||||
record DestructiveResult(
|
||||
DestructiveOperation operation,
|
||||
DestinationName destination,
|
||||
long messagesAffected,
|
||||
Duration elapsed,
|
||||
Instant executedAt) {
|
||||
|
||||
public DestructiveResult {
|
||||
Objects.requireNonNull(operation, "operation must not be null");
|
||||
Objects.requireNonNull(destination, "destination must not be null");
|
||||
Objects.requireNonNull(elapsed, "elapsed must not be null");
|
||||
Objects.requireNonNull(executedAt, "executedAt must not be null");
|
||||
if (messagesAffected < 0) {
|
||||
throw new IllegalArgumentException("messagesAffected must not be negative");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves a consumer group's committed position.
|
||||
*
|
||||
* @param request the authorised request
|
||||
* @return what the reset did
|
||||
*/
|
||||
DestructiveResult resetOffset(Approved request);
|
||||
|
||||
/**
|
||||
* Discards the messages a destination currently holds.
|
||||
*
|
||||
* @param request the authorised request
|
||||
* @return what the purge did
|
||||
*/
|
||||
DestructiveResult purge(Approved request);
|
||||
|
||||
/**
|
||||
* Removes a destination entirely.
|
||||
*
|
||||
* @param request the authorised request
|
||||
* @return what the deletion did
|
||||
*/
|
||||
DestructiveResult deleteDestination(Approved request);
|
||||
}
|
||||
+237
@@ -0,0 +1,237 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminOperationJournal;
|
||||
import dev.caskeleton.messaging.admin.AdminOperationLease;
|
||||
import dev.caskeleton.messaging.admin.AdminOperationRecord;
|
||||
import dev.caskeleton.messaging.admin.AdminOperationState;
|
||||
import dev.caskeleton.messaging.admin.PlanDigest;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* A single-process journal, for tests and for local development.
|
||||
*
|
||||
* <p>It reports {@link #isDurable()} as false, and the starter refuses to run a production profile
|
||||
* on a journal that says so. That declaration is the point of this class existing at all: the
|
||||
* previous in-memory store was registered as the production default and nothing distinguished it
|
||||
* from a shared one, so the gap was invisible until two replicas executed the same approval.
|
||||
*
|
||||
* <p>The semantics are otherwise the real ones — uniqueness on {@code (ticket, digest)}, lease
|
||||
* takeover with a monotonic token, resume from checkpoint — so a test that passes here is testing
|
||||
* the protocol rather than a simplification of it.
|
||||
*/
|
||||
public final class InMemoryAdminOperationJournal implements AdminOperationJournal {
|
||||
|
||||
private final Map<String, AdminOperationRecord> records = new ConcurrentHashMap<>();
|
||||
|
||||
@Override
|
||||
public AdminOperationLease begin(
|
||||
String approvalTicket,
|
||||
PlanDigest planDigest,
|
||||
String operationId,
|
||||
String leaseOwner,
|
||||
Duration leaseDuration,
|
||||
Instant now) {
|
||||
Objects.requireNonNull(leaseDuration, "leaseDuration must not be null");
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
AdminOperationRecord claimed =
|
||||
records.compute(
|
||||
key(approvalTicket, planDigest),
|
||||
(ignored, existing) ->
|
||||
claim(
|
||||
existing,
|
||||
approvalTicket,
|
||||
planDigest,
|
||||
operationId,
|
||||
leaseOwner,
|
||||
leaseDuration,
|
||||
now));
|
||||
return new AdminOperationLease(
|
||||
approvalTicket,
|
||||
planDigest,
|
||||
claimed.operationId(),
|
||||
claimed.leaseOwner(),
|
||||
claimed.leaseToken(),
|
||||
claimed.itemsCompleted());
|
||||
}
|
||||
|
||||
private static AdminOperationRecord claim(
|
||||
AdminOperationRecord existing,
|
||||
String approvalTicket,
|
||||
PlanDigest planDigest,
|
||||
String operationId,
|
||||
String leaseOwner,
|
||||
Duration leaseDuration,
|
||||
Instant now) {
|
||||
if (existing == null) {
|
||||
return new AdminOperationRecord(
|
||||
approvalTicket,
|
||||
planDigest,
|
||||
operationId,
|
||||
AdminOperationState.STARTED,
|
||||
0,
|
||||
leaseOwner,
|
||||
1L,
|
||||
now.plus(leaseDuration),
|
||||
now,
|
||||
Optional.empty(),
|
||||
Optional.empty());
|
||||
}
|
||||
if (existing.state() == AdminOperationState.COMPLETED) {
|
||||
throw new MessageAuthorizationException(
|
||||
"APPROVAL_ALREADY_EXECUTED",
|
||||
("approval %s was already executed to completion at %s; an approval authorises one"
|
||||
+ " execution, not a standing permission")
|
||||
.formatted(approvalTicket, existing.finishedAt().orElse(existing.startedAt())));
|
||||
}
|
||||
if (existing.state() == AdminOperationState.STARTED
|
||||
&& now.isBefore(existing.leaseExpiresAt())) {
|
||||
throw new MessageAuthorizationException(
|
||||
"ADMIN_OPERATION_IN_FLIGHT",
|
||||
("approval %s is being executed by %s until at least %s; two runtimes executing one"
|
||||
+ " approval is a duplicate storm, not a faster redrive")
|
||||
.formatted(approvalTicket, existing.leaseOwner(), existing.leaseExpiresAt()));
|
||||
}
|
||||
// Either the previous attempt failed, or its lease expired without a completion. Both resume
|
||||
// from the checkpoint under a new token, which fences the previous holder out.
|
||||
return new AdminOperationRecord(
|
||||
approvalTicket,
|
||||
planDigest,
|
||||
operationId,
|
||||
AdminOperationState.STARTED,
|
||||
existing.itemsCompleted(),
|
||||
leaseOwner,
|
||||
existing.leaseToken() + 1,
|
||||
now.plus(leaseDuration),
|
||||
existing.startedAt(),
|
||||
Optional.empty(),
|
||||
Optional.empty());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkpoint(
|
||||
AdminOperationLease lease, int itemsCompleted, Duration leaseDuration, Instant now) {
|
||||
update(
|
||||
lease,
|
||||
current ->
|
||||
new AdminOperationRecord(
|
||||
current.approvalTicket(),
|
||||
current.planDigest(),
|
||||
current.operationId(),
|
||||
AdminOperationState.STARTED,
|
||||
Math.max(current.itemsCompleted(), itemsCompleted),
|
||||
current.leaseOwner(),
|
||||
current.leaseToken(),
|
||||
now.plus(leaseDuration),
|
||||
current.startedAt(),
|
||||
Optional.empty(),
|
||||
Optional.empty()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void complete(AdminOperationLease lease, int itemsCompleted, Instant now) {
|
||||
update(
|
||||
lease,
|
||||
current ->
|
||||
new AdminOperationRecord(
|
||||
current.approvalTicket(),
|
||||
current.planDigest(),
|
||||
current.operationId(),
|
||||
AdminOperationState.COMPLETED,
|
||||
Math.max(current.itemsCompleted(), itemsCompleted),
|
||||
current.leaseOwner(),
|
||||
current.leaseToken(),
|
||||
current.leaseExpiresAt(),
|
||||
current.startedAt(),
|
||||
Optional.of(now),
|
||||
Optional.empty()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fail(AdminOperationLease lease, int itemsCompleted, String failureCode, Instant now) {
|
||||
Objects.requireNonNull(failureCode, "failureCode must not be null");
|
||||
update(
|
||||
lease,
|
||||
current ->
|
||||
new AdminOperationRecord(
|
||||
current.approvalTicket(),
|
||||
current.planDigest(),
|
||||
current.operationId(),
|
||||
AdminOperationState.FAILED,
|
||||
Math.max(current.itemsCompleted(), itemsCompleted),
|
||||
current.leaseOwner(),
|
||||
current.leaseToken(),
|
||||
current.leaseExpiresAt(),
|
||||
current.startedAt(),
|
||||
Optional.of(now),
|
||||
Optional.of(failureCode)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<AdminOperationRecord> find(String approvalTicket, PlanDigest planDigest) {
|
||||
return Optional.ofNullable(records.get(key(approvalTicket, planDigest)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDurable() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* How many operations have been journalled.
|
||||
*
|
||||
* @return the record count
|
||||
*/
|
||||
public int size() {
|
||||
return records.size();
|
||||
}
|
||||
|
||||
private void update(
|
||||
AdminOperationLease lease, java.util.function.UnaryOperator<AdminOperationRecord> mutation) {
|
||||
Objects.requireNonNull(lease, "lease must not be null");
|
||||
records.compute(
|
||||
key(lease.approvalTicket(), lease.planDigest()),
|
||||
(ignored, current) -> {
|
||||
if (current == null) {
|
||||
throw new MessageAuthorizationException(
|
||||
"ADMIN_OPERATION_NOT_JOURNALLED",
|
||||
"no journal entry exists for approval " + lease.approvalTicket());
|
||||
}
|
||||
if (current.leaseToken() != lease.leaseToken()) {
|
||||
// The fence. A stalled runtime that wakes up and writes here would otherwise overwrite
|
||||
// the progress of whichever replica took the operation over.
|
||||
throw new MessageAuthorizationException(
|
||||
"ADMIN_OPERATION_LEASE_LOST",
|
||||
("the lease on approval %s moved from token %d to token %d; this runtime no longer"
|
||||
+ " owns the operation")
|
||||
.formatted(lease.approvalTicket(), lease.leaseToken(), current.leaseToken()));
|
||||
}
|
||||
return mutation.apply(current);
|
||||
});
|
||||
}
|
||||
|
||||
private static String key(String approvalTicket, PlanDigest planDigest) {
|
||||
Objects.requireNonNull(planDigest, "planDigest must not be null");
|
||||
// Length-prefixed for the same reason the grant's canonical form is: a ticket containing the
|
||||
// separator must not be able to collide with a different ticket and digest pair.
|
||||
return String.join(
|
||||
"", Integer.toString(approvalTicket.length()), ":", approvalTicket, planDigest.value());
|
||||
}
|
||||
|
||||
/**
|
||||
* Every journalled operation, newest first by start time.
|
||||
*
|
||||
* @return the records
|
||||
*/
|
||||
public List<AdminOperationRecord> records() {
|
||||
return records.values().stream()
|
||||
.sorted(java.util.Comparator.comparing(AdminOperationRecord::startedAt).reversed())
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.ApprovedRedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.ApprovedReplayPlan;
|
||||
import dev.caskeleton.messaging.admin.RedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.RedriveRequest;
|
||||
import dev.caskeleton.messaging.admin.RedriveResult;
|
||||
import dev.caskeleton.messaging.admin.ReplayPlan;
|
||||
import dev.caskeleton.messaging.admin.ReplayRequest;
|
||||
import dev.caskeleton.messaging.admin.ReplayResult;
|
||||
import dev.caskeleton.messaging.admin.TopologyValidationReport;
|
||||
|
||||
/**
|
||||
* The non-destructive half of the admin plane.
|
||||
*
|
||||
* <p>Every mutating operation is split into plan and execute, and the execute methods take an
|
||||
* {@code Approved*} type. A caller cannot execute something it has not planned, because it has no
|
||||
* way to construct the argument — the plan/approve/execute sequence is enforced by the types rather
|
||||
* than by a runtime check.
|
||||
*
|
||||
* <p>Destructive operations live in {@link DestructiveMessagingAdmin}, a separate interface that an
|
||||
* application runtime never receives a bean for. Splitting them means a compromised handler that
|
||||
* somehow reaches this service still has no method that deletes anything.
|
||||
*/
|
||||
public interface MessagingAdminService {
|
||||
|
||||
/**
|
||||
* Compares every declared topology against the broker.
|
||||
*
|
||||
* @return the discrepancies found
|
||||
*/
|
||||
TopologyValidationReport validateTopology();
|
||||
|
||||
/**
|
||||
* Estimates what a replay would do, without reading anything.
|
||||
*
|
||||
* @param request the replay request
|
||||
* @return the plan, including its impact estimate
|
||||
*/
|
||||
ReplayPlan planReplay(ReplayRequest request);
|
||||
|
||||
/**
|
||||
* Executes an approved replay.
|
||||
*
|
||||
* @param plan the approved plan
|
||||
* @return what the replay actually did
|
||||
*/
|
||||
ReplayResult executeReplay(ApprovedReplayPlan plan);
|
||||
|
||||
/**
|
||||
* Estimates what a redrive would do, without moving anything.
|
||||
*
|
||||
* @param request the redrive request
|
||||
* @return the plan, including how many candidates already failed a redrive
|
||||
*/
|
||||
RedrivePlan planRedrive(RedriveRequest request);
|
||||
|
||||
/**
|
||||
* Executes an approved redrive.
|
||||
*
|
||||
* @param plan the approved plan
|
||||
* @return what the redrive actually did
|
||||
*/
|
||||
RedriveResult executeRedrive(ApprovedRedrivePlan plan);
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
/**
|
||||
* What one redrive pass did.
|
||||
*
|
||||
* @param candidates how many messages were eligible
|
||||
* @param moved how many were republished and settled
|
||||
* @param failed how many did not confirm and stay parked
|
||||
* @param dryRun whether this was a plan-only run
|
||||
*/
|
||||
public record RedriveReport(int candidates, int moved, int failed, boolean dryRun) {
|
||||
|
||||
public RedriveReport {
|
||||
if (candidates < 0 || moved < 0 || failed < 0) {
|
||||
throw new IllegalArgumentException("redrive counters must not be negative");
|
||||
}
|
||||
}
|
||||
}
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperation;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperationGuard;
|
||||
import dev.caskeleton.messaging.admin.RedriveRequest;
|
||||
import dev.caskeleton.messaging.admin.VerifiedApproval;
|
||||
import dev.caskeleton.messaging.api.MessageId;
|
||||
import dev.caskeleton.messaging.api.publish.PublishCompletion;
|
||||
import dev.caskeleton.messaging.api.publish.PublishResult;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Moves messages from a dead letter destination back to their source.
|
||||
*
|
||||
* <p>A redrive is a publish followed by a settlement, in that order, exactly like dead lettering in
|
||||
* reverse. A message whose republish did not confirm stays in the dead letter destination: losing
|
||||
* it on the way back would be the one outcome worse than leaving it parked.
|
||||
*
|
||||
* <p>The redrive id and a redrive counter travel with each message. Without them a message that
|
||||
* fails again is indistinguishable from a new one, and a redrive loop is invisible until the dead
|
||||
* letter destination is full.
|
||||
*/
|
||||
public final class RedriveService {
|
||||
|
||||
private final DestructiveOperationGuard guard;
|
||||
private final RedriveSource source;
|
||||
private final RedrivePublisher publisher;
|
||||
private final AuditSink audit;
|
||||
|
||||
/**
|
||||
* Creates a redrive service.
|
||||
*
|
||||
* @param guard the destructive operation guard
|
||||
* @param source reads and settles dead letter messages
|
||||
* @param publisher republishes to the target destination
|
||||
* @param audit records the operation
|
||||
*/
|
||||
public RedriveService(
|
||||
DestructiveOperationGuard guard,
|
||||
RedriveSource source,
|
||||
RedrivePublisher publisher,
|
||||
AuditSink audit) {
|
||||
this.guard = Objects.requireNonNull(guard, "guard must not be null");
|
||||
this.source = Objects.requireNonNull(source, "source must not be null");
|
||||
this.publisher = Objects.requireNonNull(publisher, "publisher must not be null");
|
||||
this.audit = Objects.requireNonNull(audit, "audit must not be null");
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one redrive pass.
|
||||
*
|
||||
* @param request what to move
|
||||
* @param approval the verified approval, when one was supplied
|
||||
* @param subject the operator identity
|
||||
* @param now the current instant
|
||||
* @return what the pass did
|
||||
*/
|
||||
public RedriveReport redrive(
|
||||
RedriveRequest request, Optional<VerifiedApproval> approval, String subject, Instant now) {
|
||||
return redrive(request, approval, subject, now, 0, completed -> {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one redrive pass, resuming a previous attempt and reporting progress as it goes.
|
||||
*
|
||||
* <p>Three things were wrong with running this as a plain loop. A synchronous failure from the
|
||||
* publisher — a broker that refuses the connection rather than the message — propagated out of
|
||||
* the loop, so the remaining candidates were never attempted and the audit record was never
|
||||
* written: the operation left no trace of the items it had already moved. A retry then started
|
||||
* from the first candidate and republished them. And nothing bounded how many times one message
|
||||
* could be redriven.
|
||||
*
|
||||
* @param request what to move
|
||||
* @param approval the verified approval, when one was supplied
|
||||
* @param subject the operator identity
|
||||
* @param now the current instant
|
||||
* @param resumeFrom how many candidates a previous attempt durably moved
|
||||
* @param checkpoint called with the running completed count after each item
|
||||
* @return what the pass did
|
||||
*/
|
||||
public RedriveReport redrive(
|
||||
RedriveRequest request,
|
||||
Optional<VerifiedApproval> approval,
|
||||
String subject,
|
||||
Instant now,
|
||||
int resumeFrom,
|
||||
java.util.function.IntConsumer checkpoint) {
|
||||
Objects.requireNonNull(request, "request must not be null");
|
||||
Objects.requireNonNull(checkpoint, "checkpoint must not be null");
|
||||
if (resumeFrom < 0) {
|
||||
throw new IllegalArgumentException("resumeFrom must not be negative");
|
||||
}
|
||||
guard.authorize(
|
||||
DestructiveOperation.REDRIVE, request.source(), approval, request.dryRun(), now);
|
||||
|
||||
List<MessageId> candidates = source.peek(request.source(), request.batchSize());
|
||||
if (request.dryRun()) {
|
||||
return new RedriveReport(candidates.size(), 0, 0, true);
|
||||
}
|
||||
|
||||
List<MessageId> moved = new ArrayList<>();
|
||||
int failed = 0;
|
||||
int completed = resumeFrom;
|
||||
try {
|
||||
// Everything before resumeFrom was moved and settled by the previous attempt. Re-publishing
|
||||
// it is not a retry, it is a duplicate.
|
||||
for (MessageId messageId :
|
||||
candidates.subList(Math.min(resumeFrom, candidates.size()), candidates.size())) {
|
||||
if (attempt(messageId, request)) {
|
||||
moved.add(messageId);
|
||||
} else {
|
||||
failed++;
|
||||
}
|
||||
completed++;
|
||||
checkpoint.accept(completed);
|
||||
}
|
||||
} finally {
|
||||
// In the finally block on purpose: an operation that dies partway must still leave a record
|
||||
// of what it moved, because that record is what the resumed attempt and the incident review
|
||||
// both read.
|
||||
audit.record(
|
||||
new dev.caskeleton.messaging.observation.MessagingAuditEvent(
|
||||
"REDRIVE",
|
||||
subject,
|
||||
request.source().value(),
|
||||
approval.map(VerifiedApproval::ticket).orElse("dry-run"),
|
||||
now,
|
||||
java.util.Map.of(
|
||||
"redriveId", request.redriveId().toString(),
|
||||
"resumedFrom", Integer.toString(resumeFrom),
|
||||
"moved", Integer.toString(moved.size()),
|
||||
"failed", Integer.toString(failed))));
|
||||
}
|
||||
|
||||
return new RedriveReport(candidates.size(), moved.size(), failed, false);
|
||||
}
|
||||
|
||||
private boolean attempt(MessageId messageId, RedriveRequest request) {
|
||||
PublishResult result;
|
||||
try {
|
||||
result = publisher.republish(messageId, request.target(), request.redriveId());
|
||||
} catch (RuntimeException failure) {
|
||||
// One message that cannot be republished is a failed item, not a failed pass. Letting it
|
||||
// propagate abandoned every candidate behind it.
|
||||
return false;
|
||||
}
|
||||
if (result.completion() != PublishCompletion.CONFIRMED) {
|
||||
return false;
|
||||
}
|
||||
source.settle(request.source(), messageId);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Reads and settles messages on a dead letter destination. */
|
||||
public interface RedriveSource {
|
||||
|
||||
/**
|
||||
* Returns the next candidates without settling them.
|
||||
*
|
||||
* @param destination the dead letter destination
|
||||
* @param batchSize how many to return
|
||||
* @return the candidate identities
|
||||
*/
|
||||
List<MessageId> peek(
|
||||
dev.caskeleton.messaging.api.destination.DestinationName destination, int batchSize);
|
||||
|
||||
/**
|
||||
* Settles a message that has been successfully republished.
|
||||
*
|
||||
* @param destination the dead letter destination
|
||||
* @param messageId the message identity
|
||||
*/
|
||||
void settle(
|
||||
dev.caskeleton.messaging.api.destination.DestinationName destination, MessageId messageId);
|
||||
}
|
||||
|
||||
/** Republishes a dead lettered message to its target destination. */
|
||||
@FunctionalInterface
|
||||
public interface RedrivePublisher {
|
||||
|
||||
/**
|
||||
* Republishes one message under its original identity.
|
||||
*
|
||||
* @param messageId the message identity
|
||||
* @param target the destination to publish back to
|
||||
* @param redriveId the operation identity stamped on the message
|
||||
* @return the publish outcome
|
||||
*/
|
||||
PublishResult republish(
|
||||
MessageId messageId,
|
||||
dev.caskeleton.messaging.api.destination.DestinationName target,
|
||||
java.util.UUID redriveId);
|
||||
}
|
||||
|
||||
/** Records privileged operations. */
|
||||
@FunctionalInterface
|
||||
public interface AuditSink {
|
||||
|
||||
/**
|
||||
* Records one audit event.
|
||||
*
|
||||
* @param event the event
|
||||
*/
|
||||
void record(dev.caskeleton.messaging.observation.MessagingAuditEvent event);
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* What one replay did.
|
||||
*
|
||||
* @param replayId the operation identity
|
||||
* @param replayed how many messages were re-read
|
||||
* @param dryRun whether this was a plan-only run
|
||||
*/
|
||||
public record ReplayReport(UUID replayId, long replayed, boolean dryRun) {
|
||||
|
||||
public ReplayReport {
|
||||
Objects.requireNonNull(replayId, "replayId must not be null");
|
||||
if (replayed < 0) {
|
||||
throw new IllegalArgumentException("replayed must not be negative");
|
||||
}
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperation;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperationGuard;
|
||||
import dev.caskeleton.messaging.admin.ReplayRequest;
|
||||
import dev.caskeleton.messaging.admin.VerifiedApproval;
|
||||
import dev.caskeleton.messaging.observation.MessagingAuditEvent;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Re-reads a destination from an earlier position.
|
||||
*
|
||||
* <p>The request carries an {@code isolatedConsumerGroup} intent flag, but a caller-supplied
|
||||
* boolean is not proof that the executor actually created or used an isolated broker group.
|
||||
* Therefore every non-dry-run replay goes through the destructive guard until the runtime can
|
||||
* attest isolation itself. Dry-run remains approval-free because it changes no broker state.
|
||||
*/
|
||||
public final class ReplayService {
|
||||
|
||||
private final DestructiveOperationGuard guard;
|
||||
private final ReplayExecutor executor;
|
||||
private final RedriveService.AuditSink audit;
|
||||
|
||||
/**
|
||||
* Creates a replay service.
|
||||
*
|
||||
* @param guard the destructive operation guard
|
||||
* @param executor performs the replay
|
||||
* @param audit records the operation
|
||||
*/
|
||||
public ReplayService(
|
||||
DestructiveOperationGuard guard, ReplayExecutor executor, RedriveService.AuditSink audit) {
|
||||
this.guard = Objects.requireNonNull(guard, "guard must not be null");
|
||||
this.executor = Objects.requireNonNull(executor, "executor must not be null");
|
||||
this.audit = Objects.requireNonNull(audit, "audit must not be null");
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one replay.
|
||||
*
|
||||
* @param request what to replay
|
||||
* @param approval the verified approval, when one was supplied
|
||||
* @param subject the operator identity
|
||||
* @param now the current instant
|
||||
* @return what the replay did
|
||||
*/
|
||||
public ReplayReport replay(
|
||||
ReplayRequest request, Optional<VerifiedApproval> approval, String subject, Instant now) {
|
||||
Objects.requireNonNull(request, "request must not be null");
|
||||
Objects.requireNonNull(approval, "approval must not be null");
|
||||
|
||||
guard.authorize(
|
||||
DestructiveOperation.REPLAY, request.destination(), approval, request.dryRun(), now);
|
||||
|
||||
if (request.dryRun()) {
|
||||
return new ReplayReport(request.replayId(), 0, true);
|
||||
}
|
||||
|
||||
long replayed = executor.replay(request);
|
||||
|
||||
audit.record(
|
||||
new MessagingAuditEvent(
|
||||
"REPLAY",
|
||||
subject,
|
||||
request.destination().value(),
|
||||
approval.map(VerifiedApproval::ticket).orElse("dry-run"),
|
||||
now,
|
||||
Map.of(
|
||||
"replayId", request.replayId().toString(),
|
||||
"isolated", Boolean.toString(request.isolatedConsumerGroup()),
|
||||
"replayed", Long.toString(replayed))));
|
||||
|
||||
return new ReplayReport(request.replayId(), replayed, false);
|
||||
}
|
||||
|
||||
/** Performs the replay against a broker. */
|
||||
@FunctionalInterface
|
||||
public interface ReplayExecutor {
|
||||
|
||||
/**
|
||||
* Replays a destination and returns how many messages were re-read.
|
||||
*
|
||||
* @param request the replay request
|
||||
* @return the replayed message count
|
||||
*/
|
||||
long replay(ReplayRequest request);
|
||||
}
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.TopologyManifest;
|
||||
import dev.caskeleton.messaging.api.error.MessageTopologyException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Validates declared topology against what the broker actually has.
|
||||
*
|
||||
* <p>Validate-only, and it fails startup rather than logging. A partition count that silently
|
||||
* differs from the manifest changes the ordering guarantee the destination advertises, and a
|
||||
* missing {@code min.insync.replicas} changes what {@code acks=all} actually means — both are the
|
||||
* kind of drift that is invisible until the incident.
|
||||
*/
|
||||
public final class TopologyValidationRuntime {
|
||||
|
||||
private final TopologyReader reader;
|
||||
|
||||
/**
|
||||
* Creates a validator over a broker reader.
|
||||
*
|
||||
* @param reader reads the observed topology
|
||||
*/
|
||||
public TopologyValidationRuntime(TopologyReader reader) {
|
||||
this.reader = Objects.requireNonNull(reader, "reader must not be null");
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates every manifest, reporting all differences at once.
|
||||
*
|
||||
* @param manifests the declared topology
|
||||
* @throws MessageTopologyException when the broker does not match
|
||||
*/
|
||||
public void validate(List<TopologyManifest> manifests) {
|
||||
Objects.requireNonNull(manifests, "manifests must not be null");
|
||||
List<String> problems = new ArrayList<>();
|
||||
|
||||
for (TopologyManifest manifest : manifests) {
|
||||
ObservedTopology observed = reader.read(manifest.physicalName());
|
||||
if (observed == null) {
|
||||
problems.add(manifest.physicalName() + " does not exist");
|
||||
continue;
|
||||
}
|
||||
manifest
|
||||
.differencesFrom(
|
||||
observed.partitions(), observed.replicationFactor(), observed.configuration())
|
||||
.forEach(difference -> problems.add(manifest.physicalName() + ": " + difference));
|
||||
}
|
||||
|
||||
if (!problems.isEmpty()) {
|
||||
throw new MessageTopologyException(
|
||||
"TOPOLOGY_MISMATCH",
|
||||
"broker topology does not match the manifest: " + String.join("; ", problems));
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads the observed topology for a physical destination. */
|
||||
@FunctionalInterface
|
||||
public interface TopologyReader {
|
||||
|
||||
/**
|
||||
* Reads one destination's topology.
|
||||
*
|
||||
* @param physicalName the broker-side name
|
||||
* @return the observed topology, or null when it does not exist
|
||||
*/
|
||||
ObservedTopology read(String physicalName);
|
||||
}
|
||||
|
||||
/**
|
||||
* What the broker reports for a destination.
|
||||
*
|
||||
* @param partitions the observed partition count
|
||||
* @param replicationFactor the observed replication factor
|
||||
* @param configuration the observed configuration
|
||||
*/
|
||||
public record ObservedTopology(
|
||||
int partitions, int replicationFactor, Map<String, String> configuration) {
|
||||
|
||||
public ObservedTopology {
|
||||
Objects.requireNonNull(configuration, "configuration must not be null");
|
||||
configuration = Map.copyOf(configuration);
|
||||
}
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import dev.caskeleton.messaging.admin.DestinationTopology;
|
||||
import dev.caskeleton.messaging.admin.TopologyIssue;
|
||||
import dev.caskeleton.messaging.admin.TopologyManifest;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Compares one declared topology against what the broker reports.
|
||||
*
|
||||
* <p>Which discrepancies block is a judgement encoded here rather than left to configuration.
|
||||
* Replication factor and absence are blocking because a destination that is missing or unreplicated
|
||||
* cannot deliver the durability its profile promises. A partition count that is <em>higher</em>
|
||||
* than declared is advisory rather than blocking: extra partitions do not break durability, and
|
||||
* someone scaling a topic up deliberately should not be met with a refusal to start.
|
||||
*
|
||||
* <p>A partition count that is <em>lower</em> is blocking, because it silently reduces the
|
||||
* concurrency the destination was sized for and, on a keyed topic, changes which key lands where.
|
||||
*/
|
||||
public final class TopologyValidator {
|
||||
|
||||
/**
|
||||
* Compares a manifest against observed topology.
|
||||
*
|
||||
* @param manifest what was declared
|
||||
* @param observed what the broker reports
|
||||
* @return the discrepancies found, empty when they agree
|
||||
*/
|
||||
public List<TopologyIssue> compare(TopologyManifest manifest, DestinationTopology observed) {
|
||||
Objects.requireNonNull(manifest, "manifest must not be null");
|
||||
Objects.requireNonNull(observed, "observed must not be null");
|
||||
|
||||
List<TopologyIssue> issues = new ArrayList<>();
|
||||
String destination = manifest.destination();
|
||||
|
||||
if (!observed.exists()) {
|
||||
issues.add(
|
||||
TopologyIssue.blocking(destination, "existence", manifest.physicalName(), "absent"));
|
||||
return List.copyOf(issues);
|
||||
}
|
||||
|
||||
if (!manifest.physicalName().equals(observed.physicalName())) {
|
||||
issues.add(
|
||||
TopologyIssue.blocking(
|
||||
destination, "physicalName", manifest.physicalName(), observed.physicalName()));
|
||||
}
|
||||
|
||||
if (observed.partitions() < manifest.partitions()) {
|
||||
issues.add(
|
||||
TopologyIssue.blocking(
|
||||
destination,
|
||||
"partitions",
|
||||
Integer.toString(manifest.partitions()),
|
||||
Integer.toString(observed.partitions())));
|
||||
} else if (observed.partitions() > manifest.partitions()) {
|
||||
// Scaling a topic up is a legitimate operation; refusing to start would punish it.
|
||||
issues.add(
|
||||
TopologyIssue.advisory(
|
||||
destination,
|
||||
"partitions",
|
||||
Integer.toString(manifest.partitions()),
|
||||
Integer.toString(observed.partitions())));
|
||||
}
|
||||
|
||||
if (observed.replicationFactor() < manifest.replicationFactor()) {
|
||||
issues.add(
|
||||
TopologyIssue.blocking(
|
||||
destination,
|
||||
"replicationFactor",
|
||||
Integer.toString(manifest.replicationFactor()),
|
||||
Integer.toString(observed.replicationFactor())));
|
||||
}
|
||||
|
||||
for (Map.Entry<String, String> required : manifest.requiredConfiguration().entrySet()) {
|
||||
String actual = observed.configuration().get(required.getKey());
|
||||
if (!required.getValue().equals(actual)) {
|
||||
issues.add(
|
||||
TopologyIssue.blocking(
|
||||
destination,
|
||||
required.getKey(),
|
||||
required.getValue(),
|
||||
actual == null ? "unset" : actual));
|
||||
}
|
||||
}
|
||||
|
||||
return List.copyOf(issues);
|
||||
}
|
||||
}
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
package dev.caskeleton.messaging.admin;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class DestructiveOperationGuardTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z");
|
||||
private static final DestinationName ORDERS = new DestinationName("order-events");
|
||||
|
||||
private static final HmacApprovalVerifier ISSUER =
|
||||
new HmacApprovalVerifier(
|
||||
"an-issuing-key-of-at-least-32-bytes!".getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
|
||||
private static final VerifiedApproval VALID =
|
||||
verified(DestructiveOperation.REPLAY, ORDERS, NOW.minusSeconds(60), NOW.plusSeconds(3600));
|
||||
|
||||
/**
|
||||
* A grant the issuer signed, verified the way the runtime verifies it.
|
||||
*
|
||||
* <p>The guard used to take a plain {@code AdminApproval} record, which any caller could
|
||||
* construct. Every case below now has to go through a real signature to get an approval object at
|
||||
* all, which is the property the type change was made for.
|
||||
*/
|
||||
private static VerifiedApproval verified(
|
||||
DestructiveOperation operation, DestinationName source, Instant from, Instant until) {
|
||||
PlanDigest digest = PlanDigest.ofCanonical(operation + "|" + source.value());
|
||||
ApprovalGrant grant =
|
||||
new ApprovalGrant(
|
||||
new AdminApproval("CHG-1001", "approver", from, until),
|
||||
"operator",
|
||||
operation,
|
||||
source,
|
||||
Optional.empty(),
|
||||
"v1",
|
||||
Long.MAX_VALUE,
|
||||
digest);
|
||||
return ISSUER.verify(grant, ISSUER.sign(grant), digest, from);
|
||||
}
|
||||
|
||||
@Test
|
||||
void anApplicationRuntimeCannotRedrive() {
|
||||
DestructiveOperationGuard guard = new DestructiveOperationGuard(false);
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
guard.authorize(
|
||||
DestructiveOperation.REDRIVE, ORDERS, Optional.of(VALID), false, NOW))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("admin credential");
|
||||
}
|
||||
|
||||
@Test
|
||||
void anAdminRuntimeStillNeedsAnApproval() {
|
||||
DestructiveOperationGuard guard = new DestructiveOperationGuard(true);
|
||||
|
||||
assertThatThrownBy(
|
||||
() -> guard.authorize(DestructiveOperation.PURGE, ORDERS, Optional.empty(), false, NOW))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("approval");
|
||||
}
|
||||
|
||||
@Test
|
||||
void anExpiredApprovalDoesNotAuthorise() {
|
||||
DestructiveOperationGuard guard = new DestructiveOperationGuard(true);
|
||||
VerifiedApproval expired =
|
||||
verified(
|
||||
DestructiveOperation.OFFSET_RESET,
|
||||
ORDERS,
|
||||
NOW.minusSeconds(7200),
|
||||
NOW.minusSeconds(60));
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
guard.authorize(
|
||||
DestructiveOperation.OFFSET_RESET, ORDERS, Optional.of(expired), false, NOW))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("validity window");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aDryRunIsAlwaysPermitted() {
|
||||
DestructiveOperationGuard guard = new DestructiveOperationGuard(false);
|
||||
|
||||
assertThatCode(
|
||||
() ->
|
||||
guard.authorize(
|
||||
DestructiveOperation.DELETE_DESTINATION, ORDERS, Optional.empty(), true, NOW))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void anApprovedAdminOperationIsAuthorised() {
|
||||
DestructiveOperationGuard guard = new DestructiveOperationGuard(true);
|
||||
|
||||
assertThatCode(
|
||||
() ->
|
||||
guard.authorize(
|
||||
DestructiveOperation.REPLAY, ORDERS, Optional.of(VALID), false, NOW))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRedriveCannotTargetItsOwnSource() {
|
||||
assertThatThrownBy(() -> new RedriveRequest(UUID.randomUUID(), ORDERS, ORDERS, 100, false))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRedriveBatchIsBoundedSoOneOperationCannotFloodTheSource() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
new RedriveRequest(
|
||||
UUID.randomUUID(), new DestinationName("order-events-dlq"), ORDERS, 101, false))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aTopologyManifestReportsEveryDifference() {
|
||||
TopologyManifest manifest =
|
||||
new TopologyManifest(
|
||||
"order-events", "order.events.v1", 6, 3, Map.of("min.insync.replicas", "2"));
|
||||
|
||||
assertThat(manifest.differencesFrom(3, 3, Map.of("min.insync.replicas", "1")))
|
||||
.hasSize(2)
|
||||
.anySatisfy(difference -> assertThat(difference).contains("partitions"))
|
||||
.anySatisfy(difference -> assertThat(difference).contains("min.insync.replicas"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aMatchingTopologyReportsNoDifferences() {
|
||||
TopologyManifest manifest =
|
||||
new TopologyManifest(
|
||||
"order-events", "order.events.v1", 6, 3, Map.of("min.insync.replicas", "2"));
|
||||
|
||||
assertThat(manifest.differencesFrom(6, 3, Map.of("min.insync.replicas", "2"))).isEmpty();
|
||||
}
|
||||
}
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminOperationLease;
|
||||
import dev.caskeleton.messaging.admin.AdminOperationState;
|
||||
import dev.caskeleton.messaging.admin.PlanDigest;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* The protocol the journal has to implement, proven against the in-memory one.
|
||||
*
|
||||
* <p>What it replaced recorded a single fact — "this approval was claimed" — before any work
|
||||
* happened, in a map. An operation that died halfway had spent its approval, left no record of what
|
||||
* it had already moved, and gave a retry no way to continue: either request a new approval and
|
||||
* republish everything, or leave the redrive half applied.
|
||||
*/
|
||||
class AdminOperationJournalTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
|
||||
private static final Duration LEASE = Duration.ofMinutes(5);
|
||||
private static final String TICKET = "CHG-1042";
|
||||
private static final PlanDigest DIGEST = PlanDigest.ofCanonical("REDRIVE|orders.v1.dlq|50");
|
||||
private static final PlanDigest OTHER_DIGEST = PlanDigest.ofCanonical("REDRIVE|payments.dlq|50");
|
||||
|
||||
private final InMemoryAdminOperationJournal journal = new InMemoryAdminOperationJournal();
|
||||
|
||||
@Test
|
||||
@DisplayName("two replicas racing for one approval produce exactly one lease")
|
||||
void twoReplicasRacingProduceExactlyOneLease() {
|
||||
journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
|
||||
assertThatThrownBy(
|
||||
() -> journal.begin(TICKET, DIGEST, "op-2", "replica-b", LEASE, NOW.plusSeconds(1)))
|
||||
.as("both replicas executing one approval is a duplicate storm, not a faster redrive")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("replica-a");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a completed operation cannot be executed again")
|
||||
void aCompletedOperationCannotBeExecutedAgain() {
|
||||
AdminOperationLease lease = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
journal.complete(lease, 50, NOW.plusSeconds(30));
|
||||
|
||||
assertThatThrownBy(
|
||||
() -> journal.begin(TICKET, DIGEST, "op-2", "replica-a", LEASE, NOW.plusSeconds(60)))
|
||||
.as("an arrow-up in the shell must not re-run an approved redrive")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("already executed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a crashed run resumes from its checkpoint rather than starting over")
|
||||
void aCrashedRunResumesFromItsCheckpoint() {
|
||||
AdminOperationLease first = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
journal.checkpoint(first, 30, LEASE, NOW.plusSeconds(10));
|
||||
// No fail() call: this replica died. The lease simply runs out.
|
||||
|
||||
AdminOperationLease resumed =
|
||||
journal.begin(TICKET, DIGEST, "op-1", "replica-b", LEASE, NOW.plus(Duration.ofMinutes(10)));
|
||||
|
||||
assertThat(resumed.resumeFrom())
|
||||
.as("the 30 already moved must not be republished")
|
||||
.isEqualTo(30);
|
||||
assertThat(resumed.leaseToken()).isGreaterThan(first.leaseToken());
|
||||
assertThat(resumed.isResumption()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a failed run stays resumable without waiting for the lease to expire")
|
||||
void aFailedRunStaysResumableImmediately() {
|
||||
AdminOperationLease first = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
journal.checkpoint(first, 12, LEASE, NOW.plusSeconds(5));
|
||||
journal.fail(first, 12, "MESSAGE_BROKER_UNAVAILABLE", NOW.plusSeconds(6));
|
||||
|
||||
AdminOperationLease resumed =
|
||||
journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW.plusSeconds(7));
|
||||
|
||||
assertThat(resumed.resumeFrom()).isEqualTo(12);
|
||||
assertThat(journal.find(TICKET, DIGEST))
|
||||
.hasValueSatisfying(
|
||||
record -> assertThat(record.state()).isEqualTo(AdminOperationState.STARTED));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a runtime that lost its lease cannot write over the one that took over")
|
||||
void aRuntimeThatLostItsLeaseCannotWriteOverTheSuccessor() {
|
||||
AdminOperationLease stale = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
journal.checkpoint(stale, 30, LEASE, NOW.plusSeconds(10));
|
||||
AdminOperationLease successor =
|
||||
journal.begin(TICKET, DIGEST, "op-1", "replica-b", LEASE, NOW.plus(Duration.ofMinutes(10)));
|
||||
journal.checkpoint(successor, 45, LEASE, NOW.plus(Duration.ofMinutes(11)));
|
||||
|
||||
assertThatThrownBy(() -> journal.complete(stale, 30, NOW.plus(Duration.ofMinutes(12))))
|
||||
.as("a stalled replica waking up must not mark the operation done at its own count")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("no longer owns");
|
||||
|
||||
assertThat(journal.find(TICKET, DIGEST))
|
||||
.hasValueSatisfying(
|
||||
record -> {
|
||||
assertThat(record.itemsCompleted()).isEqualTo(45);
|
||||
assertThat(record.state()).isEqualTo(AdminOperationState.STARTED);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the same ticket presented for a different plan is a different operation")
|
||||
void theSameTicketForADifferentPlanIsADifferentOperation() {
|
||||
journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
|
||||
// Keyed on the ticket alone this would have been refused as a duplicate execution, which reads
|
||||
// as protection but is actually the wrong answer: it is a different plan, and it must be
|
||||
// refused by the approval binding rather than silently treated as the same operation.
|
||||
assertThatCode(() -> journal.begin(TICKET, OTHER_DIGEST, "op-2", "replica-a", LEASE, NOW))
|
||||
.doesNotThrowAnyException();
|
||||
assertThat(journal.size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a checkpoint never moves backwards")
|
||||
void aCheckpointNeverMovesBackwards() {
|
||||
AdminOperationLease lease = journal.begin(TICKET, DIGEST, "op-1", "replica-a", LEASE, NOW);
|
||||
journal.checkpoint(lease, 40, LEASE, NOW.plusSeconds(10));
|
||||
journal.checkpoint(lease, 5, LEASE, NOW.plusSeconds(20));
|
||||
|
||||
assertThat(journal.find(TICKET, DIGEST))
|
||||
.hasValueSatisfying(record -> assertThat(record.itemsCompleted()).isEqualTo(40));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the in-memory journal declares itself non-durable")
|
||||
void theInMemoryJournalDeclaresItselfNonDurable() {
|
||||
assertThat(journal.isDurable())
|
||||
.as("the starter refuses a production profile on a journal that says this")
|
||||
.isFalse();
|
||||
}
|
||||
}
|
||||
+243
@@ -0,0 +1,243 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminApproval;
|
||||
import dev.caskeleton.messaging.admin.ApprovalGrant;
|
||||
import dev.caskeleton.messaging.admin.ApprovedRedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperation;
|
||||
import dev.caskeleton.messaging.admin.HmacApprovalVerifier;
|
||||
import dev.caskeleton.messaging.admin.RedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.RedriveRequest;
|
||||
import dev.caskeleton.messaging.admin.VerifiedApproval;
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Whether "this was approved" is a fact or a claim the caller makes about itself.
|
||||
*
|
||||
* <p>It was a claim. {@code AdminApproval} was a public record and {@code ApprovedRedrivePlan} a
|
||||
* public record over it, so any code that could reach the execute method could write its own
|
||||
* approval and the platform believed it. Nothing bound an approval to an operation, a destination,
|
||||
* a topology version, or an impact ceiling either, so a genuine approval for one redrive authorised
|
||||
* a redrive of anything else inside its window.
|
||||
*/
|
||||
class ApprovalForgeryTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
|
||||
private static final UUID OPERATION_ID = UUID.fromString("0199aaaa-bbbb-7ccc-8ddd-eeeeffff0000");
|
||||
private static final DestinationName DLQ = new DestinationName("orders.v1.dlq");
|
||||
private static final DestinationName ORDERS = new DestinationName("orders.v1");
|
||||
|
||||
private static final HmacApprovalVerifier ISSUER =
|
||||
new HmacApprovalVerifier(
|
||||
"an-issuing-key-of-at-least-32-bytes!".getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
private static final HmacApprovalVerifier ANOTHER_ISSUER =
|
||||
new HmacApprovalVerifier(
|
||||
"a-different-key-of-at-least-32-bytes".getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
@Test
|
||||
@DisplayName("a verified approval cannot be constructed outside the verifier")
|
||||
void aVerifiedApprovalCannotBeConstructedOutsideTheVerifier() {
|
||||
// Written as a reflection probe because the property is about the type's shape, not its
|
||||
// behaviour: if a public constructor or factory ever appears, every other test here becomes
|
||||
// decorative.
|
||||
assertThat(VerifiedApproval.class.getConstructors())
|
||||
.as("a public constructor would let any caller mint an approval for itself")
|
||||
.isEmpty();
|
||||
assertThat(
|
||||
java.util.Arrays.stream(VerifiedApproval.class.getDeclaredMethods())
|
||||
.filter(method -> java.lang.reflect.Modifier.isStatic(method.getModifiers()))
|
||||
.filter(method -> java.lang.reflect.Modifier.isPublic(method.getModifiers()))
|
||||
.toList())
|
||||
.as("nor may a public static factory")
|
||||
.isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an unsigned or altered grant does not verify")
|
||||
void anAlteredGrantDoesNotVerify() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
ApprovalGrant signed = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
|
||||
String signature = ISSUER.sign(signed);
|
||||
|
||||
ApprovalGrant raised = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 5_000);
|
||||
|
||||
assertThatThrownBy(() -> ISSUER.verify(raised, signature, plan.digest(), NOW))
|
||||
.as("raising the impact ceiling after the approver signed it")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("does not verify");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a signature from another issuer does not verify")
|
||||
void aSignatureFromAnotherIssuerDoesNotVerify() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
|
||||
|
||||
assertThatThrownBy(
|
||||
() -> ISSUER.verify(granted, ANOTHER_ISSUER.sign(granted), plan.digest(), NOW))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("does not verify");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an approval for one plan cannot execute another")
|
||||
void anApprovalForOnePlanCannotExecuteAnother() {
|
||||
RedrivePlan approvedPlan = plan(50, 0);
|
||||
RedrivePlan biggerPlan = plan(900, 0);
|
||||
ApprovalGrant granted = grant(approvedPlan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 900);
|
||||
String signature = ISSUER.sign(granted);
|
||||
|
||||
assertThatThrownBy(() -> ISSUER.verify(granted, signature, biggerPlan.digest(), NOW))
|
||||
.as("the approver saw 50 candidates; this plan moves 900")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("was issued for plan");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a verified approval for the wrong source cannot be attached to a plan")
|
||||
void aVerifiedApprovalForTheWrongSourceCannotBeAttached() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
// A genuine, correctly-signed approval — for a different dead-letter destination, with this
|
||||
// plan's digest smuggled in. The digest check passes; the source binding is what catches it.
|
||||
ApprovalGrant granted =
|
||||
new ApprovalGrant(
|
||||
approval(),
|
||||
"operator@example.com",
|
||||
DestructiveOperation.REDRIVE,
|
||||
new DestinationName("payments.v1.dlq"),
|
||||
Optional.of(ORDERS),
|
||||
"v1",
|
||||
900,
|
||||
plan.digest());
|
||||
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
|
||||
|
||||
assertThatThrownBy(() -> new ApprovedRedrivePlan(plan, verified, false))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("payments.v1.dlq");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a plan beyond the approved impact ceiling cannot execute")
|
||||
void aPlanBeyondTheApprovedImpactCeilingCannotExecute() {
|
||||
RedrivePlan plan = plan(900, 0);
|
||||
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 100);
|
||||
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
|
||||
|
||||
assertThatThrownBy(() -> new ApprovedRedrivePlan(plan, verified, false))
|
||||
.as("the approver agreed to at most 100")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("at most 100");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an approval for a replay cannot authorise a redrive")
|
||||
void anApprovalForAReplayCannotAuthoriseARedrive() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
ApprovalGrant granted = grant(plan, DestructiveOperation.REPLAY, DLQ, ORDERS, 900);
|
||||
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
|
||||
|
||||
assertThatThrownBy(() -> new ApprovedRedrivePlan(plan, verified, false))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("authorises REPLAY");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an expired grant does not verify")
|
||||
void anExpiredGrantDoesNotVerify() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
ISSUER.verify(
|
||||
granted, ISSUER.sign(granted), plan.digest(), NOW.plus(Duration.ofDays(1))))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("not valid at");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the approver and the operator must be different people")
|
||||
void theApproverAndTheOperatorMustBeDifferentPeople() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
new ApprovalGrant(
|
||||
approval(),
|
||||
"ops@example.com",
|
||||
DestructiveOperation.REDRIVE,
|
||||
DLQ,
|
||||
Optional.of(ORDERS),
|
||||
"v1",
|
||||
50,
|
||||
plan.digest()))
|
||||
.as("an operator approving their own destructive operation is the control not existing")
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("destructive admin plans can only carry a verified approval")
|
||||
void destructiveAdminUsesTheUnforgeableApprovalType() {
|
||||
assertThat(
|
||||
java.util.Arrays.stream(DestructiveMessagingAdmin.Approved.class.getRecordComponents())
|
||||
.filter(component -> component.getName().equals("approval"))
|
||||
.findFirst()
|
||||
.orElseThrow()
|
||||
.getType())
|
||||
.isEqualTo(VerifiedApproval.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a correctly issued approval executes")
|
||||
void aCorrectlyIssuedApprovalExecutes() {
|
||||
RedrivePlan plan = plan(50, 0);
|
||||
ApprovalGrant granted = grant(plan, DestructiveOperation.REDRIVE, DLQ, ORDERS, 50);
|
||||
VerifiedApproval verified = ISSUER.verify(granted, ISSUER.sign(granted), plan.digest(), NOW);
|
||||
|
||||
assertThatCode(
|
||||
() -> new ApprovedRedrivePlan(plan, verified, false).requireExecutable(NOW, "v1"))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
private static ApprovalGrant grant(
|
||||
RedrivePlan plan,
|
||||
DestructiveOperation operation,
|
||||
DestinationName source,
|
||||
DestinationName target,
|
||||
long maxImpact) {
|
||||
return new ApprovalGrant(
|
||||
approval(),
|
||||
"operator@example.com",
|
||||
operation,
|
||||
source,
|
||||
Optional.of(target),
|
||||
"v1",
|
||||
maxImpact,
|
||||
plan.digest());
|
||||
}
|
||||
|
||||
private static AdminApproval approval() {
|
||||
return new AdminApproval("CHG-1042", "ops@example.com", NOW, NOW.plus(Duration.ofHours(2)));
|
||||
}
|
||||
|
||||
private static RedrivePlan plan(int candidates, int alreadyRedriven) {
|
||||
return new RedrivePlan(
|
||||
new RedriveRequest(OPERATION_ID, DLQ, ORDERS, 50, false),
|
||||
candidates,
|
||||
alreadyRedriven,
|
||||
NOW,
|
||||
"v1");
|
||||
}
|
||||
}
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminApproval;
|
||||
import dev.caskeleton.messaging.admin.ApprovalGrant;
|
||||
import dev.caskeleton.messaging.admin.ApprovedRedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.ApprovedReplayPlan;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperation;
|
||||
import dev.caskeleton.messaging.admin.HmacApprovalVerifier;
|
||||
import dev.caskeleton.messaging.admin.PlanDigest;
|
||||
import dev.caskeleton.messaging.admin.RedrivePlan;
|
||||
import dev.caskeleton.messaging.admin.RedriveRequest;
|
||||
import dev.caskeleton.messaging.admin.RedriveResult;
|
||||
import dev.caskeleton.messaging.admin.ReplayPlan;
|
||||
import dev.caskeleton.messaging.admin.ReplayRequest;
|
||||
import dev.caskeleton.messaging.admin.ReplayResult;
|
||||
import dev.caskeleton.messaging.admin.VerifiedApproval;
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class ApprovedPlanExecutionTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
|
||||
private static final UUID OPERATION_ID = UUID.fromString("0199aaaa-bbbb-7ccc-8ddd-eeeeffff0000");
|
||||
|
||||
private static final byte[] ISSUING_KEY =
|
||||
"an-issuing-key-of-at-least-32-bytes!".getBytes(java.nio.charset.StandardCharsets.UTF_8);
|
||||
|
||||
private static final HmacApprovalVerifier VERIFIER = new HmacApprovalVerifier(ISSUING_KEY);
|
||||
|
||||
private static AdminApproval approval() {
|
||||
return new AdminApproval("CHG-1042", "ops@example.com", NOW, NOW.plus(Duration.ofHours(2)));
|
||||
}
|
||||
|
||||
/** A grant the issuer signed for exactly this plan, verified the way the runtime verifies it. */
|
||||
static VerifiedApproval verified(
|
||||
PlanDigest digest,
|
||||
DestructiveOperation operation,
|
||||
DestinationName source,
|
||||
Optional<DestinationName> target,
|
||||
String topologyVersion,
|
||||
long maxImpact) {
|
||||
ApprovalGrant grant =
|
||||
new ApprovalGrant(
|
||||
approval(),
|
||||
"operator@example.com",
|
||||
operation,
|
||||
source,
|
||||
target,
|
||||
topologyVersion,
|
||||
maxImpact,
|
||||
digest);
|
||||
return VERIFIER.verify(grant, VERIFIER.sign(grant), digest, NOW);
|
||||
}
|
||||
|
||||
private static VerifiedApproval replayApproval(ReplayPlan plan) {
|
||||
return verified(
|
||||
plan.digest(),
|
||||
DestructiveOperation.REPLAY,
|
||||
plan.request().destination(),
|
||||
Optional.empty(),
|
||||
plan.topologyVersion(),
|
||||
Long.MAX_VALUE);
|
||||
}
|
||||
|
||||
private static VerifiedApproval redriveApproval(RedrivePlan plan) {
|
||||
return verified(
|
||||
plan.digest(),
|
||||
DestructiveOperation.REDRIVE,
|
||||
plan.request().source(),
|
||||
Optional.of(plan.request().target()),
|
||||
plan.topologyVersion(),
|
||||
Long.MAX_VALUE);
|
||||
}
|
||||
|
||||
private static ReplayRequest replayRequest() {
|
||||
return new ReplayRequest(
|
||||
OPERATION_ID,
|
||||
new DestinationName("orders.v1"),
|
||||
NOW.minus(Duration.ofDays(1)),
|
||||
Optional.empty(),
|
||||
true,
|
||||
false);
|
||||
}
|
||||
|
||||
private static ReplayPlan replayPlan(String topologyVersion) {
|
||||
return new ReplayPlan(replayRequest(), 4_200_000, NOW, topologyVersion, false);
|
||||
}
|
||||
|
||||
private static RedriveRequest redriveRequest() {
|
||||
return new RedriveRequest(
|
||||
OPERATION_ID,
|
||||
new DestinationName("orders.v1.dlq"),
|
||||
new DestinationName("orders.v1"),
|
||||
50,
|
||||
false);
|
||||
}
|
||||
|
||||
private static RedrivePlan redrivePlan(int candidates, int alreadyRedriven) {
|
||||
return new RedrivePlan(redriveRequest(), candidates, alreadyRedriven, NOW, "v1");
|
||||
}
|
||||
|
||||
private static ApprovedReplayPlan approvedReplay(String topologyVersion) {
|
||||
ReplayPlan plan = replayPlan(topologyVersion);
|
||||
return new ApprovedReplayPlan(plan, replayApproval(plan));
|
||||
}
|
||||
|
||||
private static ApprovedRedrivePlan approvedRedrive(RedrivePlan plan, boolean loopAcknowledged) {
|
||||
return new ApprovedRedrivePlan(plan, redriveApproval(plan), loopAcknowledged);
|
||||
}
|
||||
|
||||
@Test
|
||||
void anExpiredApprovalCannotExecute() {
|
||||
ApprovedReplayPlan approved = approvedReplay("v1");
|
||||
|
||||
assertThatThrownBy(() -> approved.requireExecutable(NOW.plus(Duration.ofDays(1)), "v1"))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("CHG-1042");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aTopologyChangeSinceApprovalInvalidatesThePlan() {
|
||||
ApprovedReplayPlan approved = approvedReplay("v1");
|
||||
|
||||
assertThatThrownBy(() -> approved.requireExecutable(NOW, "v2"))
|
||||
.as("every number in the plan was computed against the old topology")
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
// The signed grant carries the topology version too, so the mismatch is now caught against
|
||||
// what the approver actually signed rather than against a field on the plan the caller
|
||||
// handed in.
|
||||
.hasMessageContaining("was signed against topology v1 but the broker is now at v2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aValidApprovalOnUnchangedTopologyExecutes() {
|
||||
assertThatCode(() -> approvedReplay("v1").requireExecutable(NOW, "v1"))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRedriveThatWouldLoopNeedsThatAcknowledgedExplicitly() {
|
||||
ApprovedRedrivePlan approved = approvedRedrive(redrivePlan(900, 400), false);
|
||||
|
||||
assertThatThrownBy(() -> approved.requireExecutable(NOW, "v1"))
|
||||
.isInstanceOf(MessageAuthorizationException.class)
|
||||
.hasMessageContaining("looks like progress");
|
||||
}
|
||||
|
||||
@Test
|
||||
void anAcknowledgedLoopMayProceed() {
|
||||
assertThatCode(() -> approvedRedrive(redrivePlan(900, 400), true).requireExecutable(NOW, "v1"))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRedriveWithNoPreviouslyRedrivenCandidatesNeedsNoAcknowledgement() {
|
||||
assertThatCode(() -> approvedRedrive(redrivePlan(900, 0), false).requireExecutable(NOW, "v1"))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void thePlanDescribesItsImpactBeforeAnythingRuns() {
|
||||
assertThat(replayPlan("v1").describeImpact()).contains("4200000").contains("isolated group");
|
||||
assertThat(redrivePlan(900, 400).describeImpact())
|
||||
.contains("already failed a previous redrive");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aReplayIntoTheLiveGroupSaysSoInCapitals() {
|
||||
ReplayPlan live =
|
||||
new ReplayPlan(
|
||||
new ReplayRequest(
|
||||
OPERATION_ID,
|
||||
new DestinationName("orders.v1"),
|
||||
NOW.minus(Duration.ofDays(1)),
|
||||
Optional.empty(),
|
||||
false,
|
||||
false),
|
||||
4_200_000,
|
||||
NOW,
|
||||
"v1",
|
||||
true);
|
||||
|
||||
assertThat(live.describeImpact())
|
||||
.as("re-delivering millions of messages into the live group is the decision to flag")
|
||||
.contains("LIVE consumer group");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aReplayResultFlagsAShortfallAgainstTheApprovedEstimate() {
|
||||
ReplayResult result =
|
||||
new ReplayResult(OPERATION_ID, 1_000_000, 100_000, Duration.ofMinutes(3), true, false);
|
||||
|
||||
assertThat(result.fellShortOfTheEstimate())
|
||||
.as("retention may have expired part of the approved window")
|
||||
.isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRedriveResultMustAccountForEveryCandidate() {
|
||||
RedriveResult accounted =
|
||||
new RedriveResult(OPERATION_ID, 100, 80, 20, Duration.ofSeconds(4), false);
|
||||
RedriveResult unaccounted =
|
||||
new RedriveResult(OPERATION_ID, 100, 80, 10, Duration.ofSeconds(4), false);
|
||||
|
||||
assertThat(accounted.isFullyAccounted()).isTrue();
|
||||
assertThat(unaccounted.isFullyAccounted())
|
||||
.as("a message that was neither moved nor left parked has been lost track of")
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aDryRunCannotClaimToHaveMovedAnything() {
|
||||
assertThatThrownBy(
|
||||
() -> new RedriveResult(OPERATION_ID, 100, 5, 0, Duration.ofSeconds(1), true))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
}
|
||||
+212
@@ -0,0 +1,212 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.messaging.admin.AdminApproval;
|
||||
import dev.caskeleton.messaging.admin.ApprovalGrant;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperation;
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperationGuard;
|
||||
import dev.caskeleton.messaging.admin.HmacApprovalVerifier;
|
||||
import dev.caskeleton.messaging.admin.PlanDigest;
|
||||
import dev.caskeleton.messaging.admin.RedriveRequest;
|
||||
import dev.caskeleton.messaging.admin.VerifiedApproval;
|
||||
import dev.caskeleton.messaging.api.MessageId;
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.publish.ConfirmationLevel;
|
||||
import dev.caskeleton.messaging.api.publish.PublishCompletion;
|
||||
import dev.caskeleton.messaging.api.publish.PublishEvidence;
|
||||
import dev.caskeleton.messaging.api.publish.PublishResult;
|
||||
import dev.caskeleton.messaging.api.publish.RoutingOutcome;
|
||||
import dev.caskeleton.messaging.observation.MessagingAuditEvent;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* What a redrive does when one message fails, and what it does the second time round.
|
||||
*
|
||||
* <p>The loop had no per-item boundary. A publisher that threw — a broker refusing the connection
|
||||
* rather than the message — propagated straight out, so every candidate behind it was abandoned and
|
||||
* the audit record was never written. The operation left no trace of what it had already moved, and
|
||||
* a retry started again from the first candidate and republished it.
|
||||
*/
|
||||
class RedriveResumptionTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z");
|
||||
private static final DestinationName DLQ = new DestinationName("orders.v1.dlq");
|
||||
private static final DestinationName ORDERS = new DestinationName("orders.v1");
|
||||
|
||||
private static final HmacApprovalVerifier ISSUER =
|
||||
new HmacApprovalVerifier(
|
||||
"an-issuing-key-of-at-least-32-bytes!".getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
|
||||
private final RecordingSource source = new RecordingSource();
|
||||
private final RecordingAudit audit = new RecordingAudit();
|
||||
|
||||
@Test
|
||||
@DisplayName("one message that throws does not abandon the ones behind it")
|
||||
void oneThrowingMessageDoesNotAbandonTheRest() {
|
||||
List<MessageId> candidates = ids(4);
|
||||
source.stage(candidates);
|
||||
RedriveService service =
|
||||
service(
|
||||
(messageId, target, redriveId) -> {
|
||||
if (messageId.equals(candidates.get(1))) {
|
||||
throw new IllegalStateException("the broker refused the connection");
|
||||
}
|
||||
return confirmed();
|
||||
});
|
||||
|
||||
RedriveReport report = service.redrive(request(), approval(), "operator", NOW);
|
||||
|
||||
assertThat(report.moved()).isEqualTo(3);
|
||||
assertThat(report.failed())
|
||||
.as("the throwing item is a failed item, not a failed pass")
|
||||
.isEqualTo(1);
|
||||
assertThat(source.settled)
|
||||
.containsExactly(candidates.get(0), candidates.get(2), candidates.get(3));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the audit record is written even when the pass does not finish cleanly")
|
||||
void theAuditRecordIsWrittenEvenOnFailure() {
|
||||
source.stage(ids(2));
|
||||
RedriveService service =
|
||||
service(
|
||||
(messageId, target, redriveId) -> {
|
||||
throw new IllegalStateException("the broker refused the connection");
|
||||
});
|
||||
|
||||
service.redrive(request(), approval(), "operator", NOW);
|
||||
|
||||
assertThat(audit.events)
|
||||
.as("the record of what moved is what the incident review and the resumed run both read")
|
||||
.singleElement()
|
||||
.satisfies(event -> assertThat(event.details()).containsEntry("failed", "2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a resumed pass does not republish what the first attempt already moved")
|
||||
void aResumedPassDoesNotRepublishWhatWasAlreadyMoved() {
|
||||
List<MessageId> candidates = ids(5);
|
||||
source.stage(candidates);
|
||||
List<MessageId> republished = new ArrayList<>();
|
||||
RedriveService service =
|
||||
service(
|
||||
(messageId, target, redriveId) -> {
|
||||
republished.add(messageId);
|
||||
return confirmed();
|
||||
});
|
||||
|
||||
RedriveReport report =
|
||||
service.redrive(request(), approval(), "operator", NOW, 3, completed -> {});
|
||||
|
||||
assertThat(republished)
|
||||
.as("republishing a settled message is a duplicate, not a retry")
|
||||
.containsExactly(candidates.get(3), candidates.get(4));
|
||||
assertThat(report.moved()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("progress is reported after each item so a crash resumes near where it stopped")
|
||||
void progressIsReportedAfterEachItem() {
|
||||
source.stage(ids(3));
|
||||
List<Integer> checkpoints = new ArrayList<>();
|
||||
RedriveService service = service((messageId, target, redriveId) -> confirmed());
|
||||
|
||||
service.redrive(request(), approval(), "operator", NOW, 0, checkpoints::add);
|
||||
|
||||
assertThat(checkpoints).containsExactly(1, 2, 3);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("resuming past the end of the batch moves nothing")
|
||||
void resumingPastTheEndOfTheBatchMovesNothing() {
|
||||
source.stage(ids(2));
|
||||
RedriveService service = service((messageId, target, redriveId) -> confirmed());
|
||||
|
||||
RedriveReport report =
|
||||
service.redrive(request(), approval(), "operator", NOW, 10, completed -> {});
|
||||
|
||||
assertThat(report.moved()).isZero();
|
||||
}
|
||||
|
||||
private RedriveService service(RedriveService.RedrivePublisher publisher) {
|
||||
// The guard holds the admin credential and every pass carries a real verified approval: this
|
||||
// test is about the loop, and a gate that refused would prove nothing about it.
|
||||
return new RedriveService(new DestructiveOperationGuard(true), source, publisher, audit);
|
||||
}
|
||||
|
||||
private static Optional<VerifiedApproval> approval() {
|
||||
PlanDigest digest = PlanDigest.ofCanonical("REDRIVE|orders.v1.dlq|orders.v1");
|
||||
ApprovalGrant grant =
|
||||
new ApprovalGrant(
|
||||
new AdminApproval("CHG-1042", "ops@example.com", NOW, NOW.plus(Duration.ofHours(2))),
|
||||
"operator@example.com",
|
||||
DestructiveOperation.REDRIVE,
|
||||
DLQ,
|
||||
Optional.of(ORDERS),
|
||||
"v1",
|
||||
Long.MAX_VALUE,
|
||||
digest);
|
||||
return Optional.of(ISSUER.verify(grant, ISSUER.sign(grant), digest, NOW));
|
||||
}
|
||||
|
||||
private static RedriveRequest request() {
|
||||
return new RedriveRequest(UUID.randomUUID(), DLQ, ORDERS, 50, false);
|
||||
}
|
||||
|
||||
private static PublishResult confirmed() {
|
||||
return new PublishResult(
|
||||
PublishCompletion.CONFIRMED,
|
||||
PublishEvidence.confirmed(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK),
|
||||
RoutingOutcome.ROUTED,
|
||||
Optional.empty(),
|
||||
1,
|
||||
Duration.ofMillis(2),
|
||||
Optional.empty());
|
||||
}
|
||||
|
||||
private static List<MessageId> ids(int count) {
|
||||
List<MessageId> ids = new ArrayList<>();
|
||||
for (int index = 0; index < count; index++) {
|
||||
ids.add(MessageId.newId());
|
||||
}
|
||||
return List.copyOf(ids);
|
||||
}
|
||||
|
||||
private static final class RecordingSource implements RedriveService.RedriveSource {
|
||||
|
||||
private List<MessageId> staged = List.of();
|
||||
private final List<MessageId> settled = new ArrayList<>();
|
||||
|
||||
private void stage(List<MessageId> ids) {
|
||||
staged = List.copyOf(ids);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<MessageId> peek(DestinationName destination, int batchSize) {
|
||||
return staged;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void settle(DestinationName destination, MessageId messageId) {
|
||||
settled.add(messageId);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class RecordingAudit implements RedriveService.AuditSink {
|
||||
|
||||
private final List<MessagingAuditEvent> events = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public void record(MessagingAuditEvent event) {
|
||||
events.add(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.admin.DestructiveOperationGuard;
|
||||
import dev.caskeleton.messaging.admin.ReplayRequest;
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.error.MessageAuthorizationException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class ReplayServiceTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-09-21T00:00:00Z");
|
||||
private static final DestinationName DESTINATION = new DestinationName("orders.v1");
|
||||
|
||||
@Test
|
||||
void callerClaimedIsolationDoesNotBypassApprovalForARealReplay() {
|
||||
AtomicInteger executions = new AtomicInteger();
|
||||
ReplayService service =
|
||||
new ReplayService(
|
||||
new DestructiveOperationGuard(true),
|
||||
request -> {
|
||||
executions.incrementAndGet();
|
||||
return 1;
|
||||
},
|
||||
event -> {});
|
||||
|
||||
ReplayRequest request =
|
||||
new ReplayRequest(
|
||||
UUID.fromString("0199aaaa-bbbb-7ccc-8ddd-eeeeffff1000"),
|
||||
DESTINATION,
|
||||
NOW.minus(Duration.ofHours(1)),
|
||||
Optional.empty(),
|
||||
true,
|
||||
false);
|
||||
|
||||
assertThatThrownBy(() -> service.replay(request, Optional.empty(), "operator", NOW))
|
||||
.isInstanceOfSatisfying(
|
||||
MessageAuthorizationException.class,
|
||||
failure -> assertThat(failure.failure().code()).isEqualTo("APPROVAL_REQUIRED"));
|
||||
assertThat(executions).hasValue(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dryRunRemainsApprovalFreeEvenWhenIsolationIsOnlyAnIntentFlag() {
|
||||
AtomicInteger executions = new AtomicInteger();
|
||||
ReplayService service =
|
||||
new ReplayService(
|
||||
new DestructiveOperationGuard(true),
|
||||
request -> {
|
||||
executions.incrementAndGet();
|
||||
return 1;
|
||||
},
|
||||
event -> {});
|
||||
|
||||
ReplayRequest request =
|
||||
new ReplayRequest(
|
||||
UUID.fromString("0199aaaa-bbbb-7ccc-8ddd-eeeeffff1001"),
|
||||
DESTINATION,
|
||||
NOW.minus(Duration.ofHours(1)),
|
||||
Optional.empty(),
|
||||
true,
|
||||
true);
|
||||
|
||||
ReplayReport report = service.replay(request, Optional.empty(), "operator", NOW);
|
||||
|
||||
assertThat(report.dryRun()).isTrue();
|
||||
assertThat(report.replayed()).isZero();
|
||||
assertThat(executions).hasValue(0);
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.admin.TopologyManifest;
|
||||
import dev.caskeleton.messaging.api.error.MessageTopologyException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class TopologyValidationRuntimeTest {
|
||||
|
||||
private static final TopologyManifest ORDERS =
|
||||
new TopologyManifest(
|
||||
"order-events", "order.events.v1", 6, 3, Map.of("min.insync.replicas", "2"));
|
||||
|
||||
@Test
|
||||
void aMatchingTopologyValidates() {
|
||||
TopologyValidationRuntime runtime =
|
||||
new TopologyValidationRuntime(
|
||||
name ->
|
||||
new TopologyValidationRuntime.ObservedTopology(
|
||||
6, 3, Map.of("min.insync.replicas", "2")));
|
||||
|
||||
assertThatCode(() -> runtime.validate(List.of(ORDERS))).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aMissingDestinationFailsStartup() {
|
||||
TopologyValidationRuntime runtime = new TopologyValidationRuntime(name -> null);
|
||||
|
||||
assertThatThrownBy(() -> runtime.validate(List.of(ORDERS)))
|
||||
.isInstanceOf(MessageTopologyException.class)
|
||||
.hasMessageContaining("does not exist");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aWeakenedReplicationSettingFailsStartup() {
|
||||
TopologyValidationRuntime runtime =
|
||||
new TopologyValidationRuntime(
|
||||
name ->
|
||||
new TopologyValidationRuntime.ObservedTopology(
|
||||
6, 3, Map.of("min.insync.replicas", "1")));
|
||||
|
||||
assertThatThrownBy(() -> runtime.validate(List.of(ORDERS)))
|
||||
.isInstanceOf(MessageTopologyException.class)
|
||||
.hasMessageContaining("min.insync.replicas");
|
||||
}
|
||||
|
||||
@Test
|
||||
void everyDifferenceIsReportedAtOnce() {
|
||||
TopologyValidationRuntime runtime =
|
||||
new TopologyValidationRuntime(
|
||||
name ->
|
||||
new TopologyValidationRuntime.ObservedTopology(
|
||||
3, 1, Map.of("min.insync.replicas", "1")));
|
||||
|
||||
assertThatThrownBy(() -> runtime.validate(List.of(ORDERS)))
|
||||
.isInstanceOf(MessageTopologyException.class)
|
||||
.hasMessageContaining("partitions")
|
||||
.hasMessageContaining("replicationFactor")
|
||||
.hasMessageContaining("min.insync.replicas");
|
||||
}
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
package dev.caskeleton.messaging.admin.runtime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.admin.DestinationTopology;
|
||||
import dev.caskeleton.messaging.admin.TopologyIssue;
|
||||
import dev.caskeleton.messaging.admin.TopologyManagementMode;
|
||||
import dev.caskeleton.messaging.admin.TopologyManifest;
|
||||
import dev.caskeleton.messaging.admin.TopologyValidationReport;
|
||||
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class TopologyValidatorTest {
|
||||
|
||||
private final TopologyValidator validator = new TopologyValidator();
|
||||
|
||||
private static TopologyManifest manifest() {
|
||||
return new TopologyManifest(
|
||||
"orders.v1", "orders-v1", 12, 3, Map.of("min.insync.replicas", "2"));
|
||||
}
|
||||
|
||||
private static DestinationTopology observed(
|
||||
int partitions, int replication, Map<String, String> config) {
|
||||
return new DestinationTopology("orders-v1", partitions, replication, config, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
void anAgreeingTopologyProducesNoIssues() {
|
||||
assertThat(validator.compare(manifest(), observed(12, 3, Map.of("min.insync.replicas", "2"))))
|
||||
.isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void anAbsentDestinationBlocksStartup() {
|
||||
List<TopologyIssue> issues =
|
||||
validator.compare(manifest(), DestinationTopology.absent("orders-v1"));
|
||||
|
||||
assertThat(issues)
|
||||
.singleElement()
|
||||
.satisfies(
|
||||
issue -> {
|
||||
assertThat(issue.attribute()).isEqualTo("existence");
|
||||
assertThat(issue.severity()).isEqualTo(TopologyIssue.Severity.BLOCKING);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void tooFewPartitionsBlocksBecauseKeysWouldLandDifferently() {
|
||||
List<TopologyIssue> issues =
|
||||
validator.compare(manifest(), observed(6, 3, Map.of("min.insync.replicas", "2")));
|
||||
|
||||
assertThat(issues)
|
||||
.singleElement()
|
||||
.satisfies(
|
||||
issue -> assertThat(issue.severity()).isEqualTo(TopologyIssue.Severity.BLOCKING));
|
||||
}
|
||||
|
||||
@Test
|
||||
void extraPartitionsAreAdvisoryBecauseScalingUpIsLegitimate() {
|
||||
List<TopologyIssue> issues =
|
||||
validator.compare(manifest(), observed(24, 3, Map.of("min.insync.replicas", "2")));
|
||||
|
||||
assertThat(issues)
|
||||
.singleElement()
|
||||
.satisfies(
|
||||
issue -> assertThat(issue.severity()).isEqualTo(TopologyIssue.Severity.ADVISORY));
|
||||
}
|
||||
|
||||
@Test
|
||||
void tooLittleReplicationBlocksBecauseDurabilityIsPromised() {
|
||||
List<TopologyIssue> issues =
|
||||
validator.compare(manifest(), observed(12, 1, Map.of("min.insync.replicas", "2")));
|
||||
|
||||
assertThat(issues)
|
||||
.singleElement()
|
||||
.satisfies(
|
||||
issue -> {
|
||||
assertThat(issue.attribute()).isEqualTo("replicationFactor");
|
||||
assertThat(issue.severity()).isEqualTo(TopologyIssue.Severity.BLOCKING);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void aMissingRequiredConfigurationEntryBlocks() {
|
||||
List<TopologyIssue> issues = validator.compare(manifest(), observed(12, 3, Map.of()));
|
||||
|
||||
assertThat(issues)
|
||||
.singleElement()
|
||||
.satisfies(issue -> assertThat(issue.actual()).isEqualTo("unset"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void everyIssueIsCollectedRatherThanFailingOnTheFirst() {
|
||||
List<TopologyIssue> issues = validator.compare(manifest(), observed(6, 1, Map.of()));
|
||||
|
||||
assertThat(issues)
|
||||
.as("discovering the next problem only after a redeploy turns one fix into several")
|
||||
.hasSize(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aReportRefusesStartupWhenAnythingBlocks() {
|
||||
TopologyValidationReport report =
|
||||
new TopologyValidationReport(
|
||||
validator.compare(manifest(), observed(12, 1, Map.of("min.insync.replicas", "2"))), 1);
|
||||
|
||||
assertThat(report.isAcceptable()).isFalse();
|
||||
assertThatThrownBy(report::requireAcceptable)
|
||||
.isInstanceOf(MessagingConfigurationException.class)
|
||||
.hasMessageContaining("replicationFactor");
|
||||
}
|
||||
|
||||
@Test
|
||||
void anAdvisoryOnlyReportStillStarts() {
|
||||
TopologyValidationReport report =
|
||||
new TopologyValidationReport(
|
||||
validator.compare(manifest(), observed(24, 3, Map.of("min.insync.replicas", "2"))), 1);
|
||||
|
||||
assertThat(report.isAcceptable()).isTrue();
|
||||
assertThat(report.advisory()).hasSize(1);
|
||||
assertThatCode(report::requireAcceptable).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void theCompositeValidatorChecksEveryManifest() {
|
||||
CompositeTopologyValidator composite =
|
||||
new CompositeTopologyValidator(
|
||||
new BrokerTopologyInspector() {
|
||||
@Override
|
||||
public Optional<DestinationTopology> describe(String physicalName) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String topologyVersion() {
|
||||
return "v1";
|
||||
}
|
||||
});
|
||||
|
||||
TopologyValidationReport report =
|
||||
composite.validate(
|
||||
List.of(
|
||||
manifest(), new TopologyManifest("payments.v1", "payments-v1", 3, 3, Map.of())));
|
||||
|
||||
assertThat(report.destinationsChecked()).isEqualTo(2);
|
||||
assertThat(report.blocking()).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void autoCreationIsRefusedInProduction() {
|
||||
assertThatThrownBy(() -> TopologyManagementMode.CREATE_IF_MISSING.requireSafeFor(true))
|
||||
.as("a mistyped destination would be created and look exactly like a real one")
|
||||
.isInstanceOf(MessagingConfigurationException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void autoCreationIsAllowedOutsideProduction() {
|
||||
assertThatCode(() -> TopologyManagementMode.CREATE_IF_MISSING.requireSafeFor(false))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateOnlyIsAlwaysSafe() {
|
||||
assertThatCode(() -> TopologyManagementMode.VALIDATE_ONLY.requireSafeFor(true))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user