Light, fluffy, and always free - The AWS Local Emulator alternative
fix: add Scheduler retry backoff and DLQ handling (#3687)
Niels Kootstra682989288ad3
diff --git a/docs/services/scheduler.md b/docs/services/scheduler.mdindex 2a5e14a1c8..da21742be2 100644--- a/docs/services/scheduler.md+++ b/docs/services/scheduler.md@@ -38,6 +38,29 @@ window are skipped. The dispatcher ticks everySupported target types: SQS, Lambda, SNS, EventBridge `PutEvents`.+### Retries and dead-letter queues++A failed invocation is retried after a deterministic capped exponential delay+until the target's `RetryPolicy` is exhausted. The delay starts at 60 seconds+and is capped at 86400 seconds. Missing fields use the AWS defaults:+`MaximumEventAgeInSeconds=86400` and `MaximumRetryAttempts=185`. Retries run+when due, without undocumented AWS jitter. Each occurrence retries+independently, so a failing occurrence does not delay later `rate` or `cron`+fires. Pending retries are in memory and are dropped when the process restarts,+or when a schedule is disabled, updated, or deleted. Targets Floci cannot+deliver to count as failed invocations. Lambda targets are invoked+asynchronously, so function errors are not retried by the scheduler.++When an occurrence is exhausted and `DeadLetterConfig.Arn` names a standard SQS+queue, Floci sends the fully materialized target request as a JSON body to that+queue with the message attributes+`ERROR_CODE`, `ERROR_MESSAGE`, `EXECUTION_ID`, `EXHAUSTED_RETRY_CONDITION`,+`IS_PAYLOAD_TRUNCATED`, `RETRY_ATTEMPTS`, `SCHEDULED_TIME`, `SCHEDULE_ARN`, and+`TARGET_ARN`. FIFO queues and non-SQS ARNs are logged and skipped. An `at`+schedule with `ActionAfterCompletion=DELETE` is deleted once its occurrence is+delivered or exhausted. For an SQS target, the body contains `MessageBody` and+`QueueUrl`. Non-JSON target input is encoded as a JSON string field.+## Configuration| Variable | Default | Description |@@ -48,7 +71,6 @@ Supported target types: SQS, Lambda, SNS, EventBridge `PutEvents`.## Not Yet Supported-- `RetryPolicy` and `DeadLetterConfig` on failed invocations (stored but not honored)- `FlexibleTimeWindow` jitter (fires deterministically at the scheduled time)- `NextToken`-based pagination for List operationsdiff --git a/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcher.java b/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcher.javaindex 7f6734670c..f9c2f54a07 100644--- a/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcher.java+++ b/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcher.java@@ -2,9 +2,15 @@ package io.github.hectorvent.floci.services.scheduler;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.services.scheduler.SchedulerExpressionParser.Kind;+import io.github.hectorvent.floci.services.scheduler.model.DeadLetterConfig;+import io.github.hectorvent.floci.services.scheduler.model.RetryPolicy;import io.github.hectorvent.floci.services.scheduler.model.Schedule;+import io.github.hectorvent.floci.services.scheduler.model.Target;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;import io.quarkus.runtime.ShutdownEvent;import io.quarkus.runtime.StartupEvent;import jakarta.enterprise.context.ApplicationScoped;@@ -13,7 +19,14 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.time.Instant;+import java.time.temporal.ChronoUnit;+import java.util.HashMap;+import java.util.LinkedHashMap;import java.util.List;+import java.util.Map;+import java.util.Objects;+import java.util.Set;+import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.Executors;import java.util.concurrent.ScheduledExecutorService;@@ -33,11 +46,16 @@ import java.util.concurrent.TimeUnit;* with optional {@code ScheduleExpressionTimezone}.</li>* <li>Gating: {@code State=DISABLED}, {@code StartDate}/{@code EndDate}.</li>* <li>Completion: {@code ActionAfterCompletion=DELETE} removes one-time- * {@code at(...)} schedules once fired.</li>+ * {@code at(...)} schedules once their occurrence is delivered or exhausted.</li>* <li>Targets: whatever {@link ScheduleInvoker} can deliver to (SQS, Lambda,- * SNS, EventBridge).</li>- * <li>Failures: logged; {@code RetryPolicy} / {@code DeadLetterConfig} are- * stored but not yet honored.</li>+ * SNS, EventBridge, ECS). Unsupported targets count as failed invocations.</li>+ * <li>Failures: each occurrence (schedule ARN + scheduled time) is retried on+ * later ticks until its {@code RetryPolicy} is exhausted, using the AWS+ * defaults (86400 seconds, 185 retries) for missing fields. Exhausted+ * occurrences are sent to the standard SQS queue named by+ * {@code DeadLetterConfig} with the AWS dead-letter message attributes.+ * Pending retries do not delay later occurrences and are dropped when the+ * schedule is disabled, updated, or deleted.</li>* </ul>*/@ApplicationScoped@@ -45,20 +63,32 @@ public class ScheduleDispatcher implements Resettable {private static final Logger LOG = Logger.getLogger(ScheduleDispatcher.class);+ static final int DEFAULT_MAXIMUM_EVENT_AGE_SECONDS = 86400;+ static final int DEFAULT_MAXIMUM_RETRY_ATTEMPTS = 185;+ private static final String DEFAULT_ERROR_CODE = "AWS.Scheduler.InternalServerError";+ private static final String EXHAUSTED_BY_ATTEMPTS = "MaximumRetryAttempts";+ private static final String EXHAUSTED_BY_AGE = "MaximumEventAgeInSeconds";+private final SchedulerService schedulerService;private final ScheduleInvoker invoker;+ private final SqsService sqsService;+ private final String baseUrl;private final long tickIntervalSeconds;private final boolean enabled;private final ScheduledExecutorService executor;private final ConcurrentHashMap<String, Instant> lastFireByArn = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, Boolean> firedOnceByArn = new ConcurrentHashMap<>();+ private final ConcurrentHashMap<Occurrence, Delivery> pendingRetries = new ConcurrentHashMap<>();@Injectpublic ScheduleDispatcher(SchedulerService schedulerService,ScheduleInvoker invoker,+ SqsService sqsService,EmulatorConfig config) {this.schedulerService = schedulerService;this.invoker = invoker;+ this.sqsService = sqsService;+ this.baseUrl = config.baseUrl();this.tickIntervalSeconds = config.services().scheduler().tickIntervalSeconds();this.enabled = config.services().scheduler().enabled()&& config.services().scheduler().invocationEnabled();@@ -85,6 +115,7 @@ public class ScheduleDispatcher implements Resettable {public void clear() {lastFireByArn.clear();firedOnceByArn.clear();+ pendingRetries.clear();}void tickSafely() {@@ -97,6 +128,9 @@ public class ScheduleDispatcher implements Resettable {void tick(Instant now) {List<Schedule> schedules = schedulerService.listAllSchedules();+ // Retry only occurrences that failed on earlier ticks, and retry them after evaluation so a+ // schedule deleted on completion is not evaluated again from this tick's stale listing.+ Set<Occurrence> dueRetries = Set.copyOf(pendingRetries.keySet());for (Schedule schedule : schedules) {try {evaluate(schedule, now);@@ -104,6 +138,42 @@ public class ScheduleDispatcher implements Resettable {LOG.warnv("Failed to evaluate schedule {0}: {1}", schedule.getArn(), e.getMessage());}}+ retryPending(dueRetries, schedules, now);+ }++ private void retryPending(Set<Occurrence> dueRetries, List<Schedule> schedules, Instant now) {+ if (dueRetries.isEmpty()) {+ return;+ }+ Map<String, Schedule> schedulesByArn = new HashMap<>();+ for (Schedule schedule : schedules) {+ schedulesByArn.put(schedule.getArn(), schedule);+ }+ for (Occurrence occurrence : dueRetries) {+ Delivery delivery = pendingRetries.get(occurrence);+ if (delivery == null) {+ continue;+ }+ Schedule schedule = schedulesByArn.get(occurrence.scheduleArn());+ if (isStale(schedule, delivery)) {+ pendingRetries.remove(occurrence);+ LOG.debugv("Dropped pending retry for {0}: schedule deleted, disabled, or updated",+ occurrence.scheduleArn());+ continue;+ }+ if (delivery.nextAttemptAt() != null && now.isBefore(delivery.nextAttemptAt())) {+ continue;+ }+ try {+ if (hasExpired(schedule, occurrence, now)) {+ exhaust(schedule, occurrence, delivery, EXHAUSTED_BY_AGE);+ } else {+ attempt(schedule, occurrence, delivery.nextRetry(), now);+ }+ } catch (Exception e) {+ LOG.warnv("Failed to retry schedule {0}: {1}", occurrence.scheduleArn(), e.getMessage());+ }+ }}private void evaluate(Schedule schedule, Instant now) {@@ -134,19 +204,11 @@ public class ScheduleDispatcher implements Resettable {return;}- fire(schedule);+ // Record the fire before delivering so a failing occurrence never holds back the next one.recordFire(schedule, now);-- if (kind == Kind.AT && isDeleteAfterCompletion(schedule)) {- try {- schedulerService.deleteScheduleForAccount(- schedule.getAccountId(), schedule.getName(), schedule.getGroupName(), regionOf(schedule));- lastFireByArn.remove(schedule.getArn());- firedOnceByArn.remove(schedule.getArn());- } catch (Exception e) {- LOG.warnv("Post-completion delete failed for {0}: {1}", schedule.getArn(), e.getMessage());- }- }+ String requestBody = invoker.materializeRequest(schedule.getTarget(), regionOf(schedule));+ attempt(schedule, new Occurrence(schedule.getArn(), nextFire),+ Delivery.first(kind, schedule, requestBody), now);}private Instant computeNextFire(Schedule schedule, Kind kind, Instant now) {@@ -177,14 +239,120 @@ public class ScheduleDispatcher implements Resettable {};}- private void fire(Schedule schedule) {- String region = regionOf(schedule);+ private void attempt(Schedule schedule, Occurrence occurrence, Delivery delivery, Instant now) {try {- invoker.invoke(schedule.getTarget(), region);- LOG.infov("Fired schedule {0} in group {1}", schedule.getName(), schedule.getGroupName());+ invoker.invoke(schedule.getTarget(), regionOf(schedule));} catch (Exception e) {LOG.warnv("Schedule {0} invocation failed: {1}", schedule.getArn(), e.getMessage());+ Delivery failed = delivery.failedWith(e, now);+ if (failed.retryAttempts() >= maximumRetryAttempts(schedule)) {+ exhaust(schedule, occurrence, failed, EXHAUSTED_BY_ATTEMPTS);+ } else if (hasExpired(schedule, occurrence, now)) {+ exhaust(schedule, occurrence, failed, EXHAUSTED_BY_AGE);+ } else {+ pendingRetries.put(occurrence, failed);+ }+ return;}+ LOG.infov("Fired schedule {0} in group {1}", schedule.getName(), schedule.getGroupName());+ resolve(schedule, occurrence, delivery);+ }++ private void exhaust(Schedule schedule, Occurrence occurrence, Delivery delivery, String condition) {+ LOG.warnv("Schedule {0} occurrence {1} exhausted by {2} after {3} retries",+ schedule.getArn(), occurrence.scheduledAt(), condition, delivery.retryAttempts());+ sendToDeadLetter(schedule, occurrence, delivery, condition);+ resolve(schedule, occurrence, delivery);+ }++ private void resolve(Schedule schedule, Occurrence occurrence, Delivery delivery) {+ pendingRetries.remove(occurrence);+ if (delivery.kind() != Kind.AT || !isDeleteAfterCompletion(schedule)) {+ return;+ }+ try {+ schedulerService.deleteScheduleForAccount(+ schedule.getAccountId(), schedule.getName(), schedule.getGroupName(), regionOf(schedule));+ lastFireByArn.remove(schedule.getArn());+ firedOnceByArn.remove(schedule.getArn());+ } catch (Exception e) {+ LOG.warnv("Post-completion delete failed for {0}: {1}", schedule.getArn(), e.getMessage());+ }+ }++ private void sendToDeadLetter(Schedule schedule, Occurrence occurrence, Delivery delivery, String condition) {+ Target target = schedule.getTarget();+ DeadLetterConfig deadLetterConfig = target.getDeadLetterConfig();+ if (deadLetterConfig == null || deadLetterConfig.getArn() == null+ || deadLetterConfig.getArn().isBlank()) {+ return;+ }+ String queueArn = deadLetterConfig.getArn();+ if (!isStandardSqsQueueArn(queueArn)) {+ LOG.warnv("Skipping dead-letter delivery for schedule {0}: {1} is not a standard SQS queue ARN",+ schedule.getArn(), queueArn);+ return;+ }++ Map<String, MessageAttributeValue> attributes = new LinkedHashMap<>();+ attributes.put("ERROR_CODE", stringAttribute(delivery.errorCode()));+ attributes.put("ERROR_MESSAGE", stringAttribute(delivery.errorMessage()));+ attributes.put("EXECUTION_ID", stringAttribute(delivery.executionId()));+ attributes.put("EXHAUSTED_RETRY_CONDITION", stringAttribute(condition));+ attributes.put("IS_PAYLOAD_TRUNCATED", stringAttribute("false"));+ attributes.put("RETRY_ATTEMPTS", stringAttribute(String.valueOf(delivery.retryAttempts())));+ attributes.put("SCHEDULED_TIME", stringAttribute(+ occurrence.scheduledAt().truncatedTo(ChronoUnit.SECONDS).toString()));+ attributes.put("SCHEDULE_ARN", stringAttribute(schedule.getArn()));+ attributes.put("TARGET_ARN", stringAttribute(target.getArn()));++ String body = delivery.requestBody() != null ? delivery.requestBody() : "{}";+ try {+ sqsService.sendMessage(AwsArnUtils.arnToQueueUrl(queueArn, baseUrl), body, 0, null, null,+ attributes, AwsArnUtils.regionOrDefault(queueArn, regionOf(schedule)));+ } catch (Exception e) {+ LOG.warnv("Dead-letter delivery failed for schedule {0}: {1}", schedule.getArn(), e.getMessage());+ }+ }++ private static boolean isStale(Schedule current, Delivery delivery) {+ return current == null+ || current.getTarget() == null+ || !"ENABLED".equalsIgnoreCase(current.getState())+ || !Objects.equals(current.getLastModificationDate(), delivery.lastModificationDate());+ }++ private static boolean hasExpired(Schedule schedule, Occurrence occurrence, Instant now) {+ return now.isAfter(occurrence.scheduledAt().plusSeconds(maximumEventAgeSeconds(schedule)));+ }++ private static int maximumEventAgeSeconds(Schedule schedule) {+ RetryPolicy retryPolicy = schedule.getTarget().getRetryPolicy();+ return retryPolicy != null && retryPolicy.getMaximumEventAgeInSeconds() != null+ ? retryPolicy.getMaximumEventAgeInSeconds()+ : DEFAULT_MAXIMUM_EVENT_AGE_SECONDS;+ }++ private static int maximumRetryAttempts(Schedule schedule) {+ RetryPolicy retryPolicy = schedule.getTarget().getRetryPolicy();+ return retryPolicy != null && retryPolicy.getMaximumRetryAttempts() != null+ ? retryPolicy.getMaximumRetryAttempts()+ : DEFAULT_MAXIMUM_RETRY_ATTEMPTS;+ }++ private static boolean isStandardSqsQueueArn(String arn) {+ try {+ AwsArnUtils.Arn parsed = AwsArnUtils.parse(arn);+ return "sqs".equals(parsed.service())+ && !parsed.resource().isBlank()+ && !parsed.resource().endsWith(".fifo");+ } catch (IllegalArgumentException e) {+ return false;+ }+ }++ private static MessageAttributeValue stringAttribute(String value) {+ return new MessageAttributeValue(value, "String");}private void recordFire(Schedule schedule, Instant now) {@@ -199,4 +367,35 @@ public class ScheduleDispatcher implements Resettable {private static String regionOf(Schedule schedule) {return AwsArnUtils.regionOrDefault(schedule.getArn(), "us-east-1");}++ private record Occurrence(String scheduleArn, Instant scheduledAt) {+ }++ private record Delivery(Kind kind, Instant lastModificationDate, String executionId,+ int retryAttempts, String errorCode, String errorMessage,+ String requestBody, Instant nextAttemptAt) {++ static Delivery first(Kind kind, Schedule schedule, String requestBody) {+ String executionId = UUID.randomUUID().toString().replace("-", "").substring(0, 16);+ return new Delivery(kind, schedule.getLastModificationDate(), executionId, 0, null, null,+ requestBody, null);+ }++ Delivery nextRetry() {+ return new Delivery(kind, lastModificationDate, executionId, retryAttempts + 1, errorCode, errorMessage,+ requestBody, null);+ }++ Delivery failedWith(Exception e, Instant now) {+ String code = e instanceof AwsException aws && aws.getErrorCode() != null+ ? aws.getErrorCode()+ : DEFAULT_ERROR_CODE;+ String message = e.getMessage() != null && !e.getMessage().isBlank()+ ? e.getMessage()+ : e.getClass().getSimpleName();+ long delaySeconds = Math.min(86400L, 60L << Math.min(retryAttempts, 11));+ return new Delivery(kind, lastModificationDate, executionId, retryAttempts, code, message,+ requestBody, now.plusSeconds(delaySeconds));+ }+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvoker.java b/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvoker.javaindex bf1962578a..10a8619ddf 100644--- a/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvoker.java+++ b/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvoker.java@@ -23,6 +23,7 @@ import org.jboss.logging.Logger;import java.util.Base64;import java.util.HashMap;+import java.util.LinkedHashMap;import java.util.List;import java.util.Map;@@ -64,12 +65,13 @@ public class ScheduleInvoker {this.baseUrl = config.baseUrl();}- public void invoke(Target target, String region) {+ public String invoke(Target target, String region) {if (target == null || target.getArn() == null) {- return;+ return "{}";}String arn = target.getArn();String payload = target.getInput() != null ? target.getInput() : "{}";+ String requestBody = materializeRequest(target, region);// Universal targets (arn:aws:scheduler:::aws-sdk:<service>:<action>) carry the// real resource identifiers inside Input, not in the target ARN. Detect and@@ -78,7 +80,7 @@ public class ScheduleInvoker {int sdkIdx = arn.indexOf(":aws-sdk:");if (sdkIdx >= 0) {invokeUniversalTarget(arn.substring(sdkIdx + ":aws-sdk:".length()), payload, region);- return;+ return requestBody;}String targetRegion = extractRegion(arn, region);@@ -101,7 +103,49 @@ public class ScheduleInvoker {deliverToEventBridge(target, payload, targetRegion);LOG.debugv("Scheduler delivered to EventBridge: {0}", arn);} else {- LOG.warnv("Scheduler: unsupported target ARN type: {0}", arn);+ throw new UnsupportedOperationException("Scheduler: unsupported target ARN type: " + arn);+ }+ return requestBody;+ }++ /** Returns the JSON request sent to the target service, for invocation diagnostics and DLQs. */+ public String materializeRequest(Target target, String region) {+ if (target == null || target.getArn() == null) {+ return "{}";+ }+ String arn = target.getArn();+ String payload = target.getInput() != null ? target.getInput() : "{}";+ int sdkIdx = arn.indexOf(":aws-sdk:");+ if (sdkIdx >= 0) {+ return validJsonOrString(payload);+ }+ if (arn.contains(":sqs:")) {+ Map<String, Object> request = new LinkedHashMap<>();+ request.put("MessageBody", payload);+ request.put("QueueUrl", AwsArnUtils.arnToQueueUrl(arn, baseUrl));+ if (target.getSqsParameters() != null+ && target.getSqsParameters().getMessageGroupId() != null) {+ request.put("MessageGroupId", target.getSqsParameters().getMessageGroupId());+ }+ return writeJson(request);+ }+ return validJsonOrString(payload);+ }++ private String validJsonOrString(String payload) {+ try {+ objectMapper.readTree(payload);+ return payload;+ } catch (Exception e) {+ return writeJson(Map.of("Input", payload));+ }+ }++ private String writeJson(Object value) {+ try {+ return objectMapper.writeValueAsString(value);+ } catch (Exception e) {+ return "{}";}}@@ -152,16 +196,16 @@ public class ScheduleInvoker {/*** Dispatches an EventBridge Scheduler universal target ({@code aws-sdk:<service>:<action>}),* reading the call parameters from the target's {@code Input} payload. Supports the- * common {@code sns:publish} and {@code sqs:sendMessage} actions; other actions are- * logged as unsupported.+ * common {@code sns:publish} and {@code sqs:sendMessage} actions; other actions fail+ * as unsupported.*/private void invokeUniversalTarget(String serviceAction, String input, String region) {JsonNode params;try {params = objectMapper.readTree(input == null || input.isBlank() ? "{}" : input);} catch (Exception e) {- LOG.warnv("Scheduler: universal target {0} has unparseable Input: {1}", serviceAction, e.getMessage());- return;+ throw new AwsException("InvalidParameterValue",+ "Universal target Input is not valid JSON", 400);}switch (serviceAction) {case "sns:publish" -> {@@ -188,7 +232,8 @@ public class ScheduleInvoker {messageAttributes, region);LOG.debugv("Scheduler delivered to SQS (universal target): {0}", queueUrl);}- default -> LOG.warnv("Scheduler: unsupported universal target action: {0}", serviceAction);+ default -> throw new UnsupportedOperationException(+ "Scheduler: unsupported universal target action: " + serviceAction);}}diff --git a/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcherTest.java b/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcherTest.javaindex b57f72c0c6..b7e22bc30e 100644--- a/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcherTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcherTest.java@@ -1,13 +1,20 @@package io.github.hectorvent.floci.services.scheduler;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.scheduler.model.Schedule;+import io.github.hectorvent.floci.services.scheduler.model.DeadLetterConfig;+import io.github.hectorvent.floci.services.scheduler.model.RetryPolicy;import io.github.hectorvent.floci.services.scheduler.model.Target;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;import java.time.Instant;import java.util.List;+import java.util.Map;import static org.junit.jupiter.api.Assertions.*;import static org.mockito.ArgumentMatchers.*;@@ -17,15 +24,21 @@ class ScheduleDispatcherTest {private static final String ARN_PREFIX = "arn:aws:scheduler:eu-central-1:000000000000:schedule/default/";private static final String SQS_TARGET_ARN = "arn:aws:sqs:eu-central-1:000000000000:test-queue";+ private static final String DLQ_TARGET_ARN = "arn:aws:sqs:eu-central-1:000000000000:dead-letter";+ private static final String DLQ_QUEUE_URL = "http://localhost:4566/000000000000/dead-letter";+ private static final String TARGET_INPUT = "{\"hello\":\"world\"}";+ private static final String TARGET_REQUEST = "{\"MessageBody\":\"{\\\"hello\\\":\\\"world\\\"}\",\"QueueUrl\":\"http://localhost:4566/000000000000/test-queue\"}";private SchedulerService schedulerService;private ScheduleInvoker invoker;+ private SqsService sqsService;private ScheduleDispatcher dispatcher;@BeforeEachvoid setUp() {schedulerService = mock(SchedulerService.class);invoker = mock(ScheduleInvoker.class);+ sqsService = mock(SqsService.class);EmulatorConfig.SchedulerServiceConfig schedulerCfg = mock(EmulatorConfig.SchedulerServiceConfig.class);when(schedulerCfg.enabled()).thenReturn(true);@@ -35,8 +48,9 @@ class ScheduleDispatcherTest {when(servicesCfg.scheduler()).thenReturn(schedulerCfg);EmulatorConfig config = mock(EmulatorConfig.class);when(config.services()).thenReturn(servicesCfg);+ when(config.baseUrl()).thenReturn("http://localhost:4566");- dispatcher = new ScheduleDispatcher(schedulerService, invoker, config);+ dispatcher = new ScheduleDispatcher(schedulerService, invoker, sqsService, config);}private Schedule newSchedule(String name, String expression, String state) {@@ -49,12 +63,44 @@ class ScheduleDispatcherTest {Target target = new Target();target.setArn(SQS_TARGET_ARN);target.setRoleArn("arn:aws:iam::000000000000:role/test");- target.setInput("{\"hello\":\"world\"}");+ target.setInput(TARGET_INPUT);s.setTarget(target);s.setCreationDate(Instant.parse("2026-04-21T09:00:00Z"));return s;}+ private Schedule failingSchedule(String name, String expression, RetryPolicy retryPolicy) {+ Schedule s = newSchedule(name, expression, "ENABLED");+ s.getTarget().setRetryPolicy(retryPolicy);+ s.getTarget().setDeadLetterConfig(new DeadLetterConfig(DLQ_TARGET_ARN));+ when(schedulerService.listAllSchedules()).thenReturn(List.of(s));+ doThrow(new RuntimeException("target unavailable")).when(invoker)+ .invoke(eq(s.getTarget()), anyString());+ when(invoker.materializeRequest(eq(s.getTarget()), anyString())).thenReturn(TARGET_REQUEST);+ return s;+ }++ private void tickEveryTenSeconds(String start, int ticks) {+ Instant first = Instant.parse(start);+ for (int i = 0; i < ticks; i++) {+ dispatcher.tick(first.plusSeconds(10L * i));+ }+ }++ private List<Map<String, MessageAttributeValue>> deadLetters(int count) {+ ArgumentCaptor<Map<String, MessageAttributeValue>> attributes = ArgumentCaptor.captor();+ verify(sqsService, times(count)).sendMessage(eq(DLQ_QUEUE_URL), eq(TARGET_REQUEST), eq(0),+ isNull(), isNull(), attributes.capture(), eq("eu-central-1"));+ return attributes.getAllValues();+ }++ private static String attribute(Map<String, MessageAttributeValue> attributes, String name) {+ MessageAttributeValue value = attributes.get(name);+ assertNotNull(value, name);+ assertEquals("String", value.getDataType(), name);+ return value.getStringValue();+ }+@Testvoid firesAtScheduleWhenDue() {Schedule s = newSchedule("at1", "at(2026-04-21T09:17:54)", "ENABLED");@@ -177,4 +223,223 @@ class ScheduleDispatcherTest {verify(invoker, never()).invoke(any(), anyString());}++ @Test+ void successfulDeliveryIsNotRetriedOrDeadLettered() {+ Schedule s = newSchedule("delivered", "at(2026-04-21T09:17:54)", "ENABLED");+ s.getTarget().setDeadLetterConfig(new DeadLetterConfig(DLQ_TARGET_ARN));+ when(schedulerService.listAllSchedules()).thenReturn(List.of(s));++ tickEveryTenSeconds("2026-04-21T09:18:00Z", 3);++ verify(invoker, times(1)).invoke(s.getTarget(), "eu-central-1");+ verifyNoInteractions(sqsService);+ }++ @Test+ void disabledScheduleIsNeitherInvokedNorDeadLettered() {+ Schedule s = failingSchedule("disabled", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 0));+ s.setState("DISABLED");++ tickEveryTenSeconds("2026-04-21T09:18:00Z", 3);++ verify(invoker, never()).invoke(any(), anyString());+ verifyNoInteractions(sqsService);+ }++ @Test+ void retriesFailedInvocationUntilMaximumRetryAttempts() {+ Schedule s = newSchedule("retry", "at(2026-04-21T09:17:54)", "ENABLED");+ s.getTarget().setRetryPolicy(new RetryPolicy(3600, 1));+ when(schedulerService.listAllSchedules()).thenReturn(List.of(s));+ doThrow(new RuntimeException("target unavailable")).when(invoker)+ .invoke(eq(s.getTarget()), anyString());++ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));+ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));+ dispatcher.tick(Instant.parse("2026-04-21T09:21:00Z"));++ verify(invoker, times(2)).invoke(eq(s.getTarget()), anyString());+ }++ @Test+ void doesNotRetryUntilTheCappedBackoffIsDue() {+ Schedule s = failingSchedule("backoff", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 2));++ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));+ dispatcher.tick(Instant.parse("2026-04-21T09:18:59Z"));+ verify(invoker, times(1)).invoke(eq(s.getTarget()), anyString());++ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));+ verify(invoker, times(2)).invoke(eq(s.getTarget()), anyString());++ dispatcher.tick(Instant.parse("2026-04-21T09:20:59Z"));+ verify(invoker, times(2)).invoke(eq(s.getTarget()), anyString());+ dispatcher.tick(Instant.parse("2026-04-21T09:21:00Z"));+ verify(invoker, times(3)).invoke(eq(s.getTarget()), anyString());+ }++ @Test+ void appliesAwsDefaultRetryPolicyWhenTargetHasNone() {+ Schedule s = failingSchedule("no-policy", "at(2026-04-21T09:17:54)", null);++ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));+ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));++ verify(invoker, times(2)).invoke(eq(s.getTarget()), anyString());+ verifyNoInteractions(sqsService);+ }++ @Test+ void missingMaximumEventAgeDefaultsToOneDay() {+ Schedule s = failingSchedule("default-age", "at(2026-04-21T09:17:54)", new RetryPolicy(null, 1));++ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));+ dispatcher.tick(Instant.parse("2026-04-21T11:18:00Z"));++ verify(invoker, times(2)).invoke(eq(s.getTarget()), anyString());+ Map<String, MessageAttributeValue> attributes = deadLetters(1).get(0);+ assertEquals("MaximumRetryAttempts", attribute(attributes, "EXHAUSTED_RETRY_CONDITION"));+ assertEquals("1", attribute(attributes, "RETRY_ATTEMPTS"));+ }++ @Test+ void retriesStopAtMaximumEventAgeAfterFailedAttempts() {+ Schedule s = failingSchedule("max-age", "at(2026-04-21T09:17:54)", new RetryPolicy(60, null));++ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));+ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));+ dispatcher.tick(Instant.parse("2026-04-21T09:21:00Z"));++ verify(invoker, times(1)).invoke(eq(s.getTarget()), anyString());+ Map<String, MessageAttributeValue> attributes = deadLetters(1).get(0);+ assertEquals("MaximumEventAgeInSeconds", attribute(attributes, "EXHAUSTED_RETRY_CONDITION"));+ assertEquals("0", attribute(attributes, "RETRY_ATTEMPTS"));+ }++ @Test+ void sendsExhaustedOccurrenceToDeadLetterQueueWithSchedulerAttributes() {+ Schedule s = failingSchedule("dead-letter", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 2));+ doThrow(new AwsException("AWS.SimpleQueueService.NonExistentQueue",+ "The specified queue does not exist.", 400)).when(invoker).invoke(eq(s.getTarget()), anyString());++ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));+ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));+ dispatcher.tick(Instant.parse("2026-04-21T09:21:00Z"));++ verify(invoker, times(3)).invoke(eq(s.getTarget()), anyString());+ Map<String, MessageAttributeValue> attributes = deadLetters(1).get(0);+ assertEquals(9, attributes.size());+ assertEquals("AWS.SimpleQueueService.NonExistentQueue", attribute(attributes, "ERROR_CODE"));+ assertEquals("The specified queue does not exist.", attribute(attributes, "ERROR_MESSAGE"));+ assertFalse(attribute(attributes, "EXECUTION_ID").isBlank());+ assertEquals("MaximumRetryAttempts", attribute(attributes, "EXHAUSTED_RETRY_CONDITION"));+ assertEquals("false", attribute(attributes, "IS_PAYLOAD_TRUNCATED"));+ assertEquals("2", attribute(attributes, "RETRY_ATTEMPTS"));+ assertEquals("2026-04-21T09:17:54Z", attribute(attributes, "SCHEDULED_TIME"));+ assertEquals(s.getArn(), attribute(attributes, "SCHEDULE_ARN"));+ assertEquals(SQS_TARGET_ARN, attribute(attributes, "TARGET_ARN"));+ verify(invoker, never()).invoke(argThat(target -> DLQ_TARGET_ARN.equals(target.getArn())), anyString());+ }++ @Test+ void skipsDeadLetterArnsThatAreNotStandardSqsQueues() {+ Schedule fifo = newSchedule("fifo-dlq", "at(2026-04-21T09:17:54)", "ENABLED");+ fifo.getTarget().setRetryPolicy(new RetryPolicy(3600, 0));+ fifo.getTarget().setDeadLetterConfig(new DeadLetterConfig(DLQ_TARGET_ARN + ".fifo"));+ Schedule topic = newSchedule("sns-dlq", "at(2026-04-21T09:17:54)", "ENABLED");+ topic.getTarget().setRetryPolicy(new RetryPolicy(3600, 0));+ topic.getTarget().setDeadLetterConfig(new DeadLetterConfig("arn:aws:sns:eu-central-1:000000000000:dead-letter"));+ when(schedulerService.listAllSchedules()).thenReturn(List.of(fifo, topic));+ doThrow(new RuntimeException("target unavailable")).when(invoker).invoke(any(), anyString());++ tickEveryTenSeconds("2026-04-21T09:18:00Z", 2);++ verify(invoker, times(2)).invoke(any(), anyString());+ verifyNoInteractions(sqsService);+ }++ @Test+ void failingRateOccurrenceDoesNotBlockNextOccurrence() {+ Schedule s = failingSchedule("rate-retry", "rate(5 minutes)", new RetryPolicy(3600, 1));++ dispatcher.tick(Instant.parse("2026-04-21T09:05:00Z"));+ dispatcher.tick(Instant.parse("2026-04-21T09:10:00Z"));+ dispatcher.tick(Instant.parse("2026-04-21T09:12:00Z"));++ verify(invoker, times(4)).invoke(eq(s.getTarget()), anyString());+ List<Map<String, MessageAttributeValue>> deadLetters = deadLetters(2);+ assertEquals("2026-04-21T09:05:00Z", attribute(deadLetters.get(0), "SCHEDULED_TIME"));+ assertEquals("2026-04-21T09:10:00Z", attribute(deadLetters.get(1), "SCHEDULED_TIME"));+ assertNotEquals(attribute(deadLetters.get(0), "EXECUTION_ID"), attribute(deadLetters.get(1), "EXECUTION_ID"));+ }++ @Test+ void deletesAtScheduleOnlyAfterRetriedDeliverySucceeds() {+ Schedule s = failingSchedule("at-delete", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 1));+ s.setActionAfterCompletion("DELETE");+ s.setAccountId("000000000000");+ when(invoker.invoke(eq(s.getTarget()), anyString()))+ .thenThrow(new RuntimeException("target unavailable"))+ .thenReturn(TARGET_REQUEST);++ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));+ verify(schedulerService, never()).deleteScheduleForAccount(anyString(), anyString(), anyString(), anyString());++ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));+ verify(schedulerService, times(1)).deleteScheduleForAccount("000000000000", "at-delete", "default", "eu-central-1");+ verifyNoInteractions(sqsService);+ }++ @Test+ void deletesAtScheduleAfterOccurrenceIsExhausted() {+ Schedule s = failingSchedule("at-exhausted", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 1));+ s.setActionAfterCompletion("DELETE");+ s.setAccountId("000000000000");++ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));+ verify(schedulerService, never()).deleteScheduleForAccount(anyString(), anyString(), anyString(), anyString());++ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));+ verify(schedulerService, times(1)).deleteScheduleForAccount("000000000000", "at-exhausted", "default", "eu-central-1");+ deadLetters(1);+ }++ @Test+ void disablingScheduleMidRetryDropsPendingRetry() {+ Schedule s = failingSchedule("disable-mid-retry", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 3));++ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));+ s.setState("DISABLED");+ dispatcher.tick(Instant.parse("2026-04-21T09:18:10Z"));+ s.setState("ENABLED");+ dispatcher.tick(Instant.parse("2026-04-21T09:18:20Z"));++ verify(invoker, times(1)).invoke(eq(s.getTarget()), anyString());+ verifyNoInteractions(sqsService);+ }++ @Test+ void deletingScheduleMidRetryDropsPendingRetry() {+ Schedule s = failingSchedule("delete-mid-retry", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 3));+ when(schedulerService.listAllSchedules()).thenReturn(List.of(s), List.of(), List.of(s));++ tickEveryTenSeconds("2026-04-21T09:18:00Z", 3);++ verify(invoker, times(1)).invoke(eq(s.getTarget()), anyString());+ verifyNoInteractions(sqsService);+ }++ @Test+ void updatingScheduleMidRetryDropsPendingRetry() {+ Schedule s = failingSchedule("update-mid-retry", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 3));+ s.setLastModificationDate(Instant.parse("2026-04-21T09:00:00Z"));++ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));+ s.setLastModificationDate(Instant.parse("2026-04-21T09:18:05Z"));+ dispatcher.tick(Instant.parse("2026-04-21T09:18:10Z"));++ verify(invoker, times(1)).invoke(eq(s.getTarget()), anyString());+ verifyNoInteractions(sqsService);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvokerTest.java b/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvokerTest.javaindex 9e0b3a1169..67b1617541 100644--- a/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvokerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvokerTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.scheduler;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.ecs.EcsService;import io.github.hectorvent.floci.services.ecs.model.ContainerOverride;import io.github.hectorvent.floci.services.ecs.model.LaunchType;@@ -28,8 +29,8 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.mockito.ArgumentMatchers.any;-import static org.mockito.ArgumentMatchers.anyInt;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;import static org.mockito.ArgumentMatchers.isNull;@@ -37,6 +38,7 @@ import static org.mockito.ArgumentMatchers.argThat;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;import static org.mockito.Mockito.when;class ScheduleInvokerTest {@@ -88,6 +90,16 @@ class ScheduleInvokerTest {isNull(), isNull(), eq("us-east-1"));}+ @Test+ void materializesSqsRequestForDeadLetterBody() {+ Target target = new Target();+ target.setArn("arn:aws:sqs:us-east-1:000000000000:queue");+ target.setInput("payload");++ assertEquals("{\"MessageBody\":\"payload\",\"QueueUrl\":\"http://localhost:4566/000000000000/queue\"}",+ invoker.materializeRequest(target, "us-east-1"));+ }+@Testvoid universalSnsPublishForwardsBinaryMessageAttributes() {Target target = new Target();@@ -365,14 +377,35 @@ class ScheduleInvokerTest {}@Test- void unsupportedUniversalActionDoesNotThrowOrDispatch() {+ void unsupportedUniversalActionFailsWithoutDispatch() {Target target = new Target();target.setArn("arn:aws:scheduler:::aws-sdk:dynamodb:putItem");target.setInput("{}");- invoker.invoke(target, "us-east-1");+ assertThrows(UnsupportedOperationException.class, () -> invoker.invoke(target, "us-east-1"));++ verifyNoInteractions(sqsService, lambdaService, snsService, eventBridgeService, ecsService);+ }++ @Test+ void malformedUniversalInputFailsWithoutDispatch() {+ Target target = new Target();+ target.setArn("arn:aws:scheduler:::aws-sdk:sqs:sendMessage");+ target.setInput("{not-json");++ assertThrows(AwsException.class, () -> invoker.invoke(target, "us-east-1"));++ verifyNoInteractions(sqsService, lambdaService, snsService, eventBridgeService, ecsService);+ }++ @Test+ void unsupportedTargetArnFailsWithoutDispatch() {+ Target target = new Target();+ target.setArn("arn:aws:dynamodb:us-east-1:000000000000:table/orders");+ target.setInput("{}");++ assertThrows(UnsupportedOperationException.class, () -> invoker.invoke(target, "us-east-1"));- verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString(), any(), anyString());- verify(snsService, never()).publish(anyString(), any(), anyString(), anyString(), anyString());+ verifyNoInteractions(sqsService, lambdaService, snsService, eventBridgeService, ecsService);}}