Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix: add Scheduler retry backoff and DLQ handling (#3687)

Niels Kootstra682989288ad3
diff --git a/docs/services/scheduler.md b/docs/services/scheduler.md
index 2a5e14a1c8..da21742be2 100644
--- a/docs/services/scheduler.md
+++ b/docs/services/scheduler.md
@@ -38,6 +38,29 @@ window are skipped. The dispatcher ticks every
Supported target types: SQS, Lambda, SNS, EventBridge `PutEvents`.
+### Retries and dead-letter queues
+
+A failed invocation is retried after a deterministic capped exponential delay
+until the target's `RetryPolicy` is exhausted. The delay starts at 60 seconds
+and is capped at 86400 seconds. Missing fields use the AWS defaults:
+`MaximumEventAgeInSeconds=86400` and `MaximumRetryAttempts=185`. Retries run
+when due, without undocumented AWS jitter. Each occurrence retries
+independently, so a failing occurrence does not delay later `rate` or `cron`
+fires. Pending retries are in memory and are dropped when the process restarts,
+or when a schedule is disabled, updated, or deleted. Targets Floci cannot
+deliver to count as failed invocations. Lambda targets are invoked
+asynchronously, so function errors are not retried by the scheduler.
+
+When an occurrence is exhausted and `DeadLetterConfig.Arn` names a standard SQS
+queue, Floci sends the fully materialized target request as a JSON body to that
+queue with the message attributes
+`ERROR_CODE`, `ERROR_MESSAGE`, `EXECUTION_ID`, `EXHAUSTED_RETRY_CONDITION`,
+`IS_PAYLOAD_TRUNCATED`, `RETRY_ATTEMPTS`, `SCHEDULED_TIME`, `SCHEDULE_ARN`, and
+`TARGET_ARN`. FIFO queues and non-SQS ARNs are logged and skipped. An `at`
+schedule with `ActionAfterCompletion=DELETE` is deleted once its occurrence is
+delivered or exhausted. For an SQS target, the body contains `MessageBody` and
+`QueueUrl`. Non-JSON target input is encoded as a JSON string field.
+
## Configuration
| Variable | Default | Description |
@@ -48,7 +71,6 @@ Supported target types: SQS, Lambda, SNS, EventBridge `PutEvents`.
## Not Yet Supported
-- `RetryPolicy` and `DeadLetterConfig` on failed invocations (stored but not honored)
- `FlexibleTimeWindow` jitter (fires deterministically at the scheduled time)
- `NextToken`-based pagination for List operations
diff --git a/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcher.java b/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcher.java
index 7f6734670c..f9c2f54a07 100644
--- a/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcher.java
+++ b/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcher.java
@@ -2,9 +2,15 @@ package io.github.hectorvent.floci.services.scheduler;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.services.scheduler.SchedulerExpressionParser.Kind;
+import io.github.hectorvent.floci.services.scheduler.model.DeadLetterConfig;
+import io.github.hectorvent.floci.services.scheduler.model.RetryPolicy;
import io.github.hectorvent.floci.services.scheduler.model.Schedule;
+import io.github.hectorvent.floci.services.scheduler.model.Target;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;
import io.quarkus.runtime.ShutdownEvent;
import io.quarkus.runtime.StartupEvent;
import jakarta.enterprise.context.ApplicationScoped;
@@ -13,7 +19,14 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -33,11 +46,16 @@ import java.util.concurrent.TimeUnit;
* with optional {@code ScheduleExpressionTimezone}.</li>
* <li>Gating: {@code State=DISABLED}, {@code StartDate}/{@code EndDate}.</li>
* <li>Completion: {@code ActionAfterCompletion=DELETE} removes one-time
- * {@code at(...)} schedules once fired.</li>
+ * {@code at(...)} schedules once their occurrence is delivered or exhausted.</li>
* <li>Targets: whatever {@link ScheduleInvoker} can deliver to (SQS, Lambda,
- * SNS, EventBridge).</li>
- * <li>Failures: logged; {@code RetryPolicy} / {@code DeadLetterConfig} are
- * stored but not yet honored.</li>
+ * SNS, EventBridge, ECS). Unsupported targets count as failed invocations.</li>
+ * <li>Failures: each occurrence (schedule ARN + scheduled time) is retried on
+ * later ticks until its {@code RetryPolicy} is exhausted, using the AWS
+ * defaults (86400 seconds, 185 retries) for missing fields. Exhausted
+ * occurrences are sent to the standard SQS queue named by
+ * {@code DeadLetterConfig} with the AWS dead-letter message attributes.
+ * Pending retries do not delay later occurrences and are dropped when the
+ * schedule is disabled, updated, or deleted.</li>
* </ul>
*/
@ApplicationScoped
@@ -45,20 +63,32 @@ public class ScheduleDispatcher implements Resettable {
private static final Logger LOG = Logger.getLogger(ScheduleDispatcher.class);
+ static final int DEFAULT_MAXIMUM_EVENT_AGE_SECONDS = 86400;
+ static final int DEFAULT_MAXIMUM_RETRY_ATTEMPTS = 185;
+ private static final String DEFAULT_ERROR_CODE = "AWS.Scheduler.InternalServerError";
+ private static final String EXHAUSTED_BY_ATTEMPTS = "MaximumRetryAttempts";
+ private static final String EXHAUSTED_BY_AGE = "MaximumEventAgeInSeconds";
+
private final SchedulerService schedulerService;
private final ScheduleInvoker invoker;
+ private final SqsService sqsService;
+ private final String baseUrl;
private final long tickIntervalSeconds;
private final boolean enabled;
private final ScheduledExecutorService executor;
private final ConcurrentHashMap<String, Instant> lastFireByArn = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Boolean> firedOnceByArn = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<Occurrence, Delivery> pendingRetries = new ConcurrentHashMap<>();
@Inject
public ScheduleDispatcher(SchedulerService schedulerService,
ScheduleInvoker invoker,
+ SqsService sqsService,
EmulatorConfig config) {
this.schedulerService = schedulerService;
this.invoker = invoker;
+ this.sqsService = sqsService;
+ this.baseUrl = config.baseUrl();
this.tickIntervalSeconds = config.services().scheduler().tickIntervalSeconds();
this.enabled = config.services().scheduler().enabled()
&& config.services().scheduler().invocationEnabled();
@@ -85,6 +115,7 @@ public class ScheduleDispatcher implements Resettable {
public void clear() {
lastFireByArn.clear();
firedOnceByArn.clear();
+ pendingRetries.clear();
}
void tickSafely() {
@@ -97,6 +128,9 @@ public class ScheduleDispatcher implements Resettable {
void tick(Instant now) {
List<Schedule> schedules = schedulerService.listAllSchedules();
+ // Retry only occurrences that failed on earlier ticks, and retry them after evaluation so a
+ // schedule deleted on completion is not evaluated again from this tick's stale listing.
+ Set<Occurrence> dueRetries = Set.copyOf(pendingRetries.keySet());
for (Schedule schedule : schedules) {
try {
evaluate(schedule, now);
@@ -104,6 +138,42 @@ public class ScheduleDispatcher implements Resettable {
LOG.warnv("Failed to evaluate schedule {0}: {1}", schedule.getArn(), e.getMessage());
}
}
+ retryPending(dueRetries, schedules, now);
+ }
+
+ private void retryPending(Set<Occurrence> dueRetries, List<Schedule> schedules, Instant now) {
+ if (dueRetries.isEmpty()) {
+ return;
+ }
+ Map<String, Schedule> schedulesByArn = new HashMap<>();
+ for (Schedule schedule : schedules) {
+ schedulesByArn.put(schedule.getArn(), schedule);
+ }
+ for (Occurrence occurrence : dueRetries) {
+ Delivery delivery = pendingRetries.get(occurrence);
+ if (delivery == null) {
+ continue;
+ }
+ Schedule schedule = schedulesByArn.get(occurrence.scheduleArn());
+ if (isStale(schedule, delivery)) {
+ pendingRetries.remove(occurrence);
+ LOG.debugv("Dropped pending retry for {0}: schedule deleted, disabled, or updated",
+ occurrence.scheduleArn());
+ continue;
+ }
+ if (delivery.nextAttemptAt() != null && now.isBefore(delivery.nextAttemptAt())) {
+ continue;
+ }
+ try {
+ if (hasExpired(schedule, occurrence, now)) {
+ exhaust(schedule, occurrence, delivery, EXHAUSTED_BY_AGE);
+ } else {
+ attempt(schedule, occurrence, delivery.nextRetry(), now);
+ }
+ } catch (Exception e) {
+ LOG.warnv("Failed to retry schedule {0}: {1}", occurrence.scheduleArn(), e.getMessage());
+ }
+ }
}
private void evaluate(Schedule schedule, Instant now) {
@@ -134,19 +204,11 @@ public class ScheduleDispatcher implements Resettable {
return;
}
- fire(schedule);
+ // Record the fire before delivering so a failing occurrence never holds back the next one.
recordFire(schedule, now);
-
- if (kind == Kind.AT && isDeleteAfterCompletion(schedule)) {
- try {
- schedulerService.deleteScheduleForAccount(
- schedule.getAccountId(), schedule.getName(), schedule.getGroupName(), regionOf(schedule));
- lastFireByArn.remove(schedule.getArn());
- firedOnceByArn.remove(schedule.getArn());
- } catch (Exception e) {
- LOG.warnv("Post-completion delete failed for {0}: {1}", schedule.getArn(), e.getMessage());
- }
- }
+ String requestBody = invoker.materializeRequest(schedule.getTarget(), regionOf(schedule));
+ attempt(schedule, new Occurrence(schedule.getArn(), nextFire),
+ Delivery.first(kind, schedule, requestBody), now);
}
private Instant computeNextFire(Schedule schedule, Kind kind, Instant now) {
@@ -177,14 +239,120 @@ public class ScheduleDispatcher implements Resettable {
};
}
- private void fire(Schedule schedule) {
- String region = regionOf(schedule);
+ private void attempt(Schedule schedule, Occurrence occurrence, Delivery delivery, Instant now) {
try {
- invoker.invoke(schedule.getTarget(), region);
- LOG.infov("Fired schedule {0} in group {1}", schedule.getName(), schedule.getGroupName());
+ invoker.invoke(schedule.getTarget(), regionOf(schedule));
} catch (Exception e) {
LOG.warnv("Schedule {0} invocation failed: {1}", schedule.getArn(), e.getMessage());
+ Delivery failed = delivery.failedWith(e, now);
+ if (failed.retryAttempts() >= maximumRetryAttempts(schedule)) {
+ exhaust(schedule, occurrence, failed, EXHAUSTED_BY_ATTEMPTS);
+ } else if (hasExpired(schedule, occurrence, now)) {
+ exhaust(schedule, occurrence, failed, EXHAUSTED_BY_AGE);
+ } else {
+ pendingRetries.put(occurrence, failed);
+ }
+ return;
}
+ LOG.infov("Fired schedule {0} in group {1}", schedule.getName(), schedule.getGroupName());
+ resolve(schedule, occurrence, delivery);
+ }
+
+ private void exhaust(Schedule schedule, Occurrence occurrence, Delivery delivery, String condition) {
+ LOG.warnv("Schedule {0} occurrence {1} exhausted by {2} after {3} retries",
+ schedule.getArn(), occurrence.scheduledAt(), condition, delivery.retryAttempts());
+ sendToDeadLetter(schedule, occurrence, delivery, condition);
+ resolve(schedule, occurrence, delivery);
+ }
+
+ private void resolve(Schedule schedule, Occurrence occurrence, Delivery delivery) {
+ pendingRetries.remove(occurrence);
+ if (delivery.kind() != Kind.AT || !isDeleteAfterCompletion(schedule)) {
+ return;
+ }
+ try {
+ schedulerService.deleteScheduleForAccount(
+ schedule.getAccountId(), schedule.getName(), schedule.getGroupName(), regionOf(schedule));
+ lastFireByArn.remove(schedule.getArn());
+ firedOnceByArn.remove(schedule.getArn());
+ } catch (Exception e) {
+ LOG.warnv("Post-completion delete failed for {0}: {1}", schedule.getArn(), e.getMessage());
+ }
+ }
+
+ private void sendToDeadLetter(Schedule schedule, Occurrence occurrence, Delivery delivery, String condition) {
+ Target target = schedule.getTarget();
+ DeadLetterConfig deadLetterConfig = target.getDeadLetterConfig();
+ if (deadLetterConfig == null || deadLetterConfig.getArn() == null
+ || deadLetterConfig.getArn().isBlank()) {
+ return;
+ }
+ String queueArn = deadLetterConfig.getArn();
+ if (!isStandardSqsQueueArn(queueArn)) {
+ LOG.warnv("Skipping dead-letter delivery for schedule {0}: {1} is not a standard SQS queue ARN",
+ schedule.getArn(), queueArn);
+ return;
+ }
+
+ Map<String, MessageAttributeValue> attributes = new LinkedHashMap<>();
+ attributes.put("ERROR_CODE", stringAttribute(delivery.errorCode()));
+ attributes.put("ERROR_MESSAGE", stringAttribute(delivery.errorMessage()));
+ attributes.put("EXECUTION_ID", stringAttribute(delivery.executionId()));
+ attributes.put("EXHAUSTED_RETRY_CONDITION", stringAttribute(condition));
+ attributes.put("IS_PAYLOAD_TRUNCATED", stringAttribute("false"));
+ attributes.put("RETRY_ATTEMPTS", stringAttribute(String.valueOf(delivery.retryAttempts())));
+ attributes.put("SCHEDULED_TIME", stringAttribute(
+ occurrence.scheduledAt().truncatedTo(ChronoUnit.SECONDS).toString()));
+ attributes.put("SCHEDULE_ARN", stringAttribute(schedule.getArn()));
+ attributes.put("TARGET_ARN", stringAttribute(target.getArn()));
+
+ String body = delivery.requestBody() != null ? delivery.requestBody() : "{}";
+ try {
+ sqsService.sendMessage(AwsArnUtils.arnToQueueUrl(queueArn, baseUrl), body, 0, null, null,
+ attributes, AwsArnUtils.regionOrDefault(queueArn, regionOf(schedule)));
+ } catch (Exception e) {
+ LOG.warnv("Dead-letter delivery failed for schedule {0}: {1}", schedule.getArn(), e.getMessage());
+ }
+ }
+
+ private static boolean isStale(Schedule current, Delivery delivery) {
+ return current == null
+ || current.getTarget() == null
+ || !"ENABLED".equalsIgnoreCase(current.getState())
+ || !Objects.equals(current.getLastModificationDate(), delivery.lastModificationDate());
+ }
+
+ private static boolean hasExpired(Schedule schedule, Occurrence occurrence, Instant now) {
+ return now.isAfter(occurrence.scheduledAt().plusSeconds(maximumEventAgeSeconds(schedule)));
+ }
+
+ private static int maximumEventAgeSeconds(Schedule schedule) {
+ RetryPolicy retryPolicy = schedule.getTarget().getRetryPolicy();
+ return retryPolicy != null && retryPolicy.getMaximumEventAgeInSeconds() != null
+ ? retryPolicy.getMaximumEventAgeInSeconds()
+ : DEFAULT_MAXIMUM_EVENT_AGE_SECONDS;
+ }
+
+ private static int maximumRetryAttempts(Schedule schedule) {
+ RetryPolicy retryPolicy = schedule.getTarget().getRetryPolicy();
+ return retryPolicy != null && retryPolicy.getMaximumRetryAttempts() != null
+ ? retryPolicy.getMaximumRetryAttempts()
+ : DEFAULT_MAXIMUM_RETRY_ATTEMPTS;
+ }
+
+ private static boolean isStandardSqsQueueArn(String arn) {
+ try {
+ AwsArnUtils.Arn parsed = AwsArnUtils.parse(arn);
+ return "sqs".equals(parsed.service())
+ && !parsed.resource().isBlank()
+ && !parsed.resource().endsWith(".fifo");
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ }
+
+ private static MessageAttributeValue stringAttribute(String value) {
+ return new MessageAttributeValue(value, "String");
}
private void recordFire(Schedule schedule, Instant now) {
@@ -199,4 +367,35 @@ public class ScheduleDispatcher implements Resettable {
private static String regionOf(Schedule schedule) {
return AwsArnUtils.regionOrDefault(schedule.getArn(), "us-east-1");
}
+
+ private record Occurrence(String scheduleArn, Instant scheduledAt) {
+ }
+
+ private record Delivery(Kind kind, Instant lastModificationDate, String executionId,
+ int retryAttempts, String errorCode, String errorMessage,
+ String requestBody, Instant nextAttemptAt) {
+
+ static Delivery first(Kind kind, Schedule schedule, String requestBody) {
+ String executionId = UUID.randomUUID().toString().replace("-", "").substring(0, 16);
+ return new Delivery(kind, schedule.getLastModificationDate(), executionId, 0, null, null,
+ requestBody, null);
+ }
+
+ Delivery nextRetry() {
+ return new Delivery(kind, lastModificationDate, executionId, retryAttempts + 1, errorCode, errorMessage,
+ requestBody, null);
+ }
+
+ Delivery failedWith(Exception e, Instant now) {
+ String code = e instanceof AwsException aws && aws.getErrorCode() != null
+ ? aws.getErrorCode()
+ : DEFAULT_ERROR_CODE;
+ String message = e.getMessage() != null && !e.getMessage().isBlank()
+ ? e.getMessage()
+ : e.getClass().getSimpleName();
+ long delaySeconds = Math.min(86400L, 60L << Math.min(retryAttempts, 11));
+ return new Delivery(kind, lastModificationDate, executionId, retryAttempts, code, message,
+ requestBody, now.plusSeconds(delaySeconds));
+ }
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvoker.java b/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvoker.java
index bf1962578a..10a8619ddf 100644
--- a/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvoker.java
+++ b/src/main/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvoker.java
@@ -23,6 +23,7 @@ import org.jboss.logging.Logger;
import java.util.Base64;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -64,12 +65,13 @@ public class ScheduleInvoker {
this.baseUrl = config.baseUrl();
}
- public void invoke(Target target, String region) {
+ public String invoke(Target target, String region) {
if (target == null || target.getArn() == null) {
- return;
+ return "{}";
}
String arn = target.getArn();
String payload = target.getInput() != null ? target.getInput() : "{}";
+ String requestBody = materializeRequest(target, region);
// Universal targets (arn:aws:scheduler:::aws-sdk:<service>:<action>) carry the
// real resource identifiers inside Input, not in the target ARN. Detect and
@@ -78,7 +80,7 @@ public class ScheduleInvoker {
int sdkIdx = arn.indexOf(":aws-sdk:");
if (sdkIdx >= 0) {
invokeUniversalTarget(arn.substring(sdkIdx + ":aws-sdk:".length()), payload, region);
- return;
+ return requestBody;
}
String targetRegion = extractRegion(arn, region);
@@ -101,7 +103,49 @@ public class ScheduleInvoker {
deliverToEventBridge(target, payload, targetRegion);
LOG.debugv("Scheduler delivered to EventBridge: {0}", arn);
} else {
- LOG.warnv("Scheduler: unsupported target ARN type: {0}", arn);
+ throw new UnsupportedOperationException("Scheduler: unsupported target ARN type: " + arn);
+ }
+ return requestBody;
+ }
+
+ /** Returns the JSON request sent to the target service, for invocation diagnostics and DLQs. */
+ public String materializeRequest(Target target, String region) {
+ if (target == null || target.getArn() == null) {
+ return "{}";
+ }
+ String arn = target.getArn();
+ String payload = target.getInput() != null ? target.getInput() : "{}";
+ int sdkIdx = arn.indexOf(":aws-sdk:");
+ if (sdkIdx >= 0) {
+ return validJsonOrString(payload);
+ }
+ if (arn.contains(":sqs:")) {
+ Map<String, Object> request = new LinkedHashMap<>();
+ request.put("MessageBody", payload);
+ request.put("QueueUrl", AwsArnUtils.arnToQueueUrl(arn, baseUrl));
+ if (target.getSqsParameters() != null
+ && target.getSqsParameters().getMessageGroupId() != null) {
+ request.put("MessageGroupId", target.getSqsParameters().getMessageGroupId());
+ }
+ return writeJson(request);
+ }
+ return validJsonOrString(payload);
+ }
+
+ private String validJsonOrString(String payload) {
+ try {
+ objectMapper.readTree(payload);
+ return payload;
+ } catch (Exception e) {
+ return writeJson(Map.of("Input", payload));
+ }
+ }
+
+ private String writeJson(Object value) {
+ try {
+ return objectMapper.writeValueAsString(value);
+ } catch (Exception e) {
+ return "{}";
}
}
@@ -152,16 +196,16 @@ public class ScheduleInvoker {
/**
* Dispatches an EventBridge Scheduler universal target ({@code aws-sdk:<service>:<action>}),
* reading the call parameters from the target's {@code Input} payload. Supports the
- * common {@code sns:publish} and {@code sqs:sendMessage} actions; other actions are
- * logged as unsupported.
+ * common {@code sns:publish} and {@code sqs:sendMessage} actions; other actions fail
+ * as unsupported.
*/
private void invokeUniversalTarget(String serviceAction, String input, String region) {
JsonNode params;
try {
params = objectMapper.readTree(input == null || input.isBlank() ? "{}" : input);
} catch (Exception e) {
- LOG.warnv("Scheduler: universal target {0} has unparseable Input: {1}", serviceAction, e.getMessage());
- return;
+ throw new AwsException("InvalidParameterValue",
+ "Universal target Input is not valid JSON", 400);
}
switch (serviceAction) {
case "sns:publish" -> {
@@ -188,7 +232,8 @@ public class ScheduleInvoker {
messageAttributes, region);
LOG.debugv("Scheduler delivered to SQS (universal target): {0}", queueUrl);
}
- default -> LOG.warnv("Scheduler: unsupported universal target action: {0}", serviceAction);
+ default -> throw new UnsupportedOperationException(
+ "Scheduler: unsupported universal target action: " + serviceAction);
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcherTest.java b/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcherTest.java
index b57f72c0c6..b7e22bc30e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcherTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleDispatcherTest.java
@@ -1,13 +1,20 @@
package io.github.hectorvent.floci.services.scheduler;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.scheduler.model.Schedule;
+import io.github.hectorvent.floci.services.scheduler.model.DeadLetterConfig;
+import io.github.hectorvent.floci.services.scheduler.model.RetryPolicy;
import io.github.hectorvent.floci.services.scheduler.model.Target;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
import java.time.Instant;
import java.util.List;
+import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.*;
@@ -17,15 +24,21 @@ class ScheduleDispatcherTest {
private static final String ARN_PREFIX = "arn:aws:scheduler:eu-central-1:000000000000:schedule/default/";
private static final String SQS_TARGET_ARN = "arn:aws:sqs:eu-central-1:000000000000:test-queue";
+ private static final String DLQ_TARGET_ARN = "arn:aws:sqs:eu-central-1:000000000000:dead-letter";
+ private static final String DLQ_QUEUE_URL = "http://localhost:4566/000000000000/dead-letter";
+ private static final String TARGET_INPUT = "{\"hello\":\"world\"}";
+ private static final String TARGET_REQUEST = "{\"MessageBody\":\"{\\\"hello\\\":\\\"world\\\"}\",\"QueueUrl\":\"http://localhost:4566/000000000000/test-queue\"}";
private SchedulerService schedulerService;
private ScheduleInvoker invoker;
+ private SqsService sqsService;
private ScheduleDispatcher dispatcher;
@BeforeEach
void setUp() {
schedulerService = mock(SchedulerService.class);
invoker = mock(ScheduleInvoker.class);
+ sqsService = mock(SqsService.class);
EmulatorConfig.SchedulerServiceConfig schedulerCfg = mock(EmulatorConfig.SchedulerServiceConfig.class);
when(schedulerCfg.enabled()).thenReturn(true);
@@ -35,8 +48,9 @@ class ScheduleDispatcherTest {
when(servicesCfg.scheduler()).thenReturn(schedulerCfg);
EmulatorConfig config = mock(EmulatorConfig.class);
when(config.services()).thenReturn(servicesCfg);
+ when(config.baseUrl()).thenReturn("http://localhost:4566");
- dispatcher = new ScheduleDispatcher(schedulerService, invoker, config);
+ dispatcher = new ScheduleDispatcher(schedulerService, invoker, sqsService, config);
}
private Schedule newSchedule(String name, String expression, String state) {
@@ -49,12 +63,44 @@ class ScheduleDispatcherTest {
Target target = new Target();
target.setArn(SQS_TARGET_ARN);
target.setRoleArn("arn:aws:iam::000000000000:role/test");
- target.setInput("{\"hello\":\"world\"}");
+ target.setInput(TARGET_INPUT);
s.setTarget(target);
s.setCreationDate(Instant.parse("2026-04-21T09:00:00Z"));
return s;
}
+ private Schedule failingSchedule(String name, String expression, RetryPolicy retryPolicy) {
+ Schedule s = newSchedule(name, expression, "ENABLED");
+ s.getTarget().setRetryPolicy(retryPolicy);
+ s.getTarget().setDeadLetterConfig(new DeadLetterConfig(DLQ_TARGET_ARN));
+ when(schedulerService.listAllSchedules()).thenReturn(List.of(s));
+ doThrow(new RuntimeException("target unavailable")).when(invoker)
+ .invoke(eq(s.getTarget()), anyString());
+ when(invoker.materializeRequest(eq(s.getTarget()), anyString())).thenReturn(TARGET_REQUEST);
+ return s;
+ }
+
+ private void tickEveryTenSeconds(String start, int ticks) {
+ Instant first = Instant.parse(start);
+ for (int i = 0; i < ticks; i++) {
+ dispatcher.tick(first.plusSeconds(10L * i));
+ }
+ }
+
+ private List<Map<String, MessageAttributeValue>> deadLetters(int count) {
+ ArgumentCaptor<Map<String, MessageAttributeValue>> attributes = ArgumentCaptor.captor();
+ verify(sqsService, times(count)).sendMessage(eq(DLQ_QUEUE_URL), eq(TARGET_REQUEST), eq(0),
+ isNull(), isNull(), attributes.capture(), eq("eu-central-1"));
+ return attributes.getAllValues();
+ }
+
+ private static String attribute(Map<String, MessageAttributeValue> attributes, String name) {
+ MessageAttributeValue value = attributes.get(name);
+ assertNotNull(value, name);
+ assertEquals("String", value.getDataType(), name);
+ return value.getStringValue();
+ }
+
@Test
void firesAtScheduleWhenDue() {
Schedule s = newSchedule("at1", "at(2026-04-21T09:17:54)", "ENABLED");
@@ -177,4 +223,223 @@ class ScheduleDispatcherTest {
verify(invoker, never()).invoke(any(), anyString());
}
+
+ @Test
+ void successfulDeliveryIsNotRetriedOrDeadLettered() {
+ Schedule s = newSchedule("delivered", "at(2026-04-21T09:17:54)", "ENABLED");
+ s.getTarget().setDeadLetterConfig(new DeadLetterConfig(DLQ_TARGET_ARN));
+ when(schedulerService.listAllSchedules()).thenReturn(List.of(s));
+
+ tickEveryTenSeconds("2026-04-21T09:18:00Z", 3);
+
+ verify(invoker, times(1)).invoke(s.getTarget(), "eu-central-1");
+ verifyNoInteractions(sqsService);
+ }
+
+ @Test
+ void disabledScheduleIsNeitherInvokedNorDeadLettered() {
+ Schedule s = failingSchedule("disabled", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 0));
+ s.setState("DISABLED");
+
+ tickEveryTenSeconds("2026-04-21T09:18:00Z", 3);
+
+ verify(invoker, never()).invoke(any(), anyString());
+ verifyNoInteractions(sqsService);
+ }
+
+ @Test
+ void retriesFailedInvocationUntilMaximumRetryAttempts() {
+ Schedule s = newSchedule("retry", "at(2026-04-21T09:17:54)", "ENABLED");
+ s.getTarget().setRetryPolicy(new RetryPolicy(3600, 1));
+ when(schedulerService.listAllSchedules()).thenReturn(List.of(s));
+ doThrow(new RuntimeException("target unavailable")).when(invoker)
+ .invoke(eq(s.getTarget()), anyString());
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T09:21:00Z"));
+
+ verify(invoker, times(2)).invoke(eq(s.getTarget()), anyString());
+ }
+
+ @Test
+ void doesNotRetryUntilTheCappedBackoffIsDue() {
+ Schedule s = failingSchedule("backoff", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 2));
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:59Z"));
+ verify(invoker, times(1)).invoke(eq(s.getTarget()), anyString());
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));
+ verify(invoker, times(2)).invoke(eq(s.getTarget()), anyString());
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:20:59Z"));
+ verify(invoker, times(2)).invoke(eq(s.getTarget()), anyString());
+ dispatcher.tick(Instant.parse("2026-04-21T09:21:00Z"));
+ verify(invoker, times(3)).invoke(eq(s.getTarget()), anyString());
+ }
+
+ @Test
+ void appliesAwsDefaultRetryPolicyWhenTargetHasNone() {
+ Schedule s = failingSchedule("no-policy", "at(2026-04-21T09:17:54)", null);
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));
+
+ verify(invoker, times(2)).invoke(eq(s.getTarget()), anyString());
+ verifyNoInteractions(sqsService);
+ }
+
+ @Test
+ void missingMaximumEventAgeDefaultsToOneDay() {
+ Schedule s = failingSchedule("default-age", "at(2026-04-21T09:17:54)", new RetryPolicy(null, 1));
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T11:18:00Z"));
+
+ verify(invoker, times(2)).invoke(eq(s.getTarget()), anyString());
+ Map<String, MessageAttributeValue> attributes = deadLetters(1).get(0);
+ assertEquals("MaximumRetryAttempts", attribute(attributes, "EXHAUSTED_RETRY_CONDITION"));
+ assertEquals("1", attribute(attributes, "RETRY_ATTEMPTS"));
+ }
+
+ @Test
+ void retriesStopAtMaximumEventAgeAfterFailedAttempts() {
+ Schedule s = failingSchedule("max-age", "at(2026-04-21T09:17:54)", new RetryPolicy(60, null));
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T09:21:00Z"));
+
+ verify(invoker, times(1)).invoke(eq(s.getTarget()), anyString());
+ Map<String, MessageAttributeValue> attributes = deadLetters(1).get(0);
+ assertEquals("MaximumEventAgeInSeconds", attribute(attributes, "EXHAUSTED_RETRY_CONDITION"));
+ assertEquals("0", attribute(attributes, "RETRY_ATTEMPTS"));
+ }
+
+ @Test
+ void sendsExhaustedOccurrenceToDeadLetterQueueWithSchedulerAttributes() {
+ Schedule s = failingSchedule("dead-letter", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 2));
+ doThrow(new AwsException("AWS.SimpleQueueService.NonExistentQueue",
+ "The specified queue does not exist.", 400)).when(invoker).invoke(eq(s.getTarget()), anyString());
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T09:21:00Z"));
+
+ verify(invoker, times(3)).invoke(eq(s.getTarget()), anyString());
+ Map<String, MessageAttributeValue> attributes = deadLetters(1).get(0);
+ assertEquals(9, attributes.size());
+ assertEquals("AWS.SimpleQueueService.NonExistentQueue", attribute(attributes, "ERROR_CODE"));
+ assertEquals("The specified queue does not exist.", attribute(attributes, "ERROR_MESSAGE"));
+ assertFalse(attribute(attributes, "EXECUTION_ID").isBlank());
+ assertEquals("MaximumRetryAttempts", attribute(attributes, "EXHAUSTED_RETRY_CONDITION"));
+ assertEquals("false", attribute(attributes, "IS_PAYLOAD_TRUNCATED"));
+ assertEquals("2", attribute(attributes, "RETRY_ATTEMPTS"));
+ assertEquals("2026-04-21T09:17:54Z", attribute(attributes, "SCHEDULED_TIME"));
+ assertEquals(s.getArn(), attribute(attributes, "SCHEDULE_ARN"));
+ assertEquals(SQS_TARGET_ARN, attribute(attributes, "TARGET_ARN"));
+ verify(invoker, never()).invoke(argThat(target -> DLQ_TARGET_ARN.equals(target.getArn())), anyString());
+ }
+
+ @Test
+ void skipsDeadLetterArnsThatAreNotStandardSqsQueues() {
+ Schedule fifo = newSchedule("fifo-dlq", "at(2026-04-21T09:17:54)", "ENABLED");
+ fifo.getTarget().setRetryPolicy(new RetryPolicy(3600, 0));
+ fifo.getTarget().setDeadLetterConfig(new DeadLetterConfig(DLQ_TARGET_ARN + ".fifo"));
+ Schedule topic = newSchedule("sns-dlq", "at(2026-04-21T09:17:54)", "ENABLED");
+ topic.getTarget().setRetryPolicy(new RetryPolicy(3600, 0));
+ topic.getTarget().setDeadLetterConfig(new DeadLetterConfig("arn:aws:sns:eu-central-1:000000000000:dead-letter"));
+ when(schedulerService.listAllSchedules()).thenReturn(List.of(fifo, topic));
+ doThrow(new RuntimeException("target unavailable")).when(invoker).invoke(any(), anyString());
+
+ tickEveryTenSeconds("2026-04-21T09:18:00Z", 2);
+
+ verify(invoker, times(2)).invoke(any(), anyString());
+ verifyNoInteractions(sqsService);
+ }
+
+ @Test
+ void failingRateOccurrenceDoesNotBlockNextOccurrence() {
+ Schedule s = failingSchedule("rate-retry", "rate(5 minutes)", new RetryPolicy(3600, 1));
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:05:00Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T09:10:00Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T09:12:00Z"));
+
+ verify(invoker, times(4)).invoke(eq(s.getTarget()), anyString());
+ List<Map<String, MessageAttributeValue>> deadLetters = deadLetters(2);
+ assertEquals("2026-04-21T09:05:00Z", attribute(deadLetters.get(0), "SCHEDULED_TIME"));
+ assertEquals("2026-04-21T09:10:00Z", attribute(deadLetters.get(1), "SCHEDULED_TIME"));
+ assertNotEquals(attribute(deadLetters.get(0), "EXECUTION_ID"), attribute(deadLetters.get(1), "EXECUTION_ID"));
+ }
+
+ @Test
+ void deletesAtScheduleOnlyAfterRetriedDeliverySucceeds() {
+ Schedule s = failingSchedule("at-delete", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 1));
+ s.setActionAfterCompletion("DELETE");
+ s.setAccountId("000000000000");
+ when(invoker.invoke(eq(s.getTarget()), anyString()))
+ .thenThrow(new RuntimeException("target unavailable"))
+ .thenReturn(TARGET_REQUEST);
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));
+ verify(schedulerService, never()).deleteScheduleForAccount(anyString(), anyString(), anyString(), anyString());
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));
+ verify(schedulerService, times(1)).deleteScheduleForAccount("000000000000", "at-delete", "default", "eu-central-1");
+ verifyNoInteractions(sqsService);
+ }
+
+ @Test
+ void deletesAtScheduleAfterOccurrenceIsExhausted() {
+ Schedule s = failingSchedule("at-exhausted", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 1));
+ s.setActionAfterCompletion("DELETE");
+ s.setAccountId("000000000000");
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));
+ verify(schedulerService, never()).deleteScheduleForAccount(anyString(), anyString(), anyString(), anyString());
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:19:00Z"));
+ verify(schedulerService, times(1)).deleteScheduleForAccount("000000000000", "at-exhausted", "default", "eu-central-1");
+ deadLetters(1);
+ }
+
+ @Test
+ void disablingScheduleMidRetryDropsPendingRetry() {
+ Schedule s = failingSchedule("disable-mid-retry", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 3));
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));
+ s.setState("DISABLED");
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:10Z"));
+ s.setState("ENABLED");
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:20Z"));
+
+ verify(invoker, times(1)).invoke(eq(s.getTarget()), anyString());
+ verifyNoInteractions(sqsService);
+ }
+
+ @Test
+ void deletingScheduleMidRetryDropsPendingRetry() {
+ Schedule s = failingSchedule("delete-mid-retry", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 3));
+ when(schedulerService.listAllSchedules()).thenReturn(List.of(s), List.of(), List.of(s));
+
+ tickEveryTenSeconds("2026-04-21T09:18:00Z", 3);
+
+ verify(invoker, times(1)).invoke(eq(s.getTarget()), anyString());
+ verifyNoInteractions(sqsService);
+ }
+
+ @Test
+ void updatingScheduleMidRetryDropsPendingRetry() {
+ Schedule s = failingSchedule("update-mid-retry", "at(2026-04-21T09:17:54)", new RetryPolicy(3600, 3));
+ s.setLastModificationDate(Instant.parse("2026-04-21T09:00:00Z"));
+
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:00Z"));
+ s.setLastModificationDate(Instant.parse("2026-04-21T09:18:05Z"));
+ dispatcher.tick(Instant.parse("2026-04-21T09:18:10Z"));
+
+ verify(invoker, times(1)).invoke(eq(s.getTarget()), anyString());
+ verifyNoInteractions(sqsService);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvokerTest.java b/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvokerTest.java
index 9e0b3a1169..67b1617541 100644
--- a/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvokerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/scheduler/ScheduleInvokerTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.scheduler;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.ecs.EcsService;
import io.github.hectorvent.floci.services.ecs.model.ContainerOverride;
import io.github.hectorvent.floci.services.ecs.model.LaunchType;
@@ -28,8 +29,8 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
@@ -37,6 +38,7 @@ import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
class ScheduleInvokerTest {
@@ -88,6 +90,16 @@ class ScheduleInvokerTest {
isNull(), isNull(), eq("us-east-1"));
}
+ @Test
+ void materializesSqsRequestForDeadLetterBody() {
+ Target target = new Target();
+ target.setArn("arn:aws:sqs:us-east-1:000000000000:queue");
+ target.setInput("payload");
+
+ assertEquals("{\"MessageBody\":\"payload\",\"QueueUrl\":\"http://localhost:4566/000000000000/queue\"}",
+ invoker.materializeRequest(target, "us-east-1"));
+ }
+
@Test
void universalSnsPublishForwardsBinaryMessageAttributes() {
Target target = new Target();
@@ -365,14 +377,35 @@ class ScheduleInvokerTest {
}
@Test
- void unsupportedUniversalActionDoesNotThrowOrDispatch() {
+ void unsupportedUniversalActionFailsWithoutDispatch() {
Target target = new Target();
target.setArn("arn:aws:scheduler:::aws-sdk:dynamodb:putItem");
target.setInput("{}");
- invoker.invoke(target, "us-east-1");
+ assertThrows(UnsupportedOperationException.class, () -> invoker.invoke(target, "us-east-1"));
+
+ verifyNoInteractions(sqsService, lambdaService, snsService, eventBridgeService, ecsService);
+ }
+
+ @Test
+ void malformedUniversalInputFailsWithoutDispatch() {
+ Target target = new Target();
+ target.setArn("arn:aws:scheduler:::aws-sdk:sqs:sendMessage");
+ target.setInput("{not-json");
+
+ assertThrows(AwsException.class, () -> invoker.invoke(target, "us-east-1"));
+
+ verifyNoInteractions(sqsService, lambdaService, snsService, eventBridgeService, ecsService);
+ }
+
+ @Test
+ void unsupportedTargetArnFailsWithoutDispatch() {
+ Target target = new Target();
+ target.setArn("arn:aws:dynamodb:us-east-1:000000000000:table/orders");
+ target.setInput("{}");
+
+ assertThrows(UnsupportedOperationException.class, () -> invoker.invoke(target, "us-east-1"));
- verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString(), any(), anyString());
- verify(snsService, never()).publish(anyString(), any(), anyString(), anyString(), anyString());
+ verifyNoInteractions(sqsService, lambdaService, snsService, eventBridgeService, ecsService);
}
}