Light, fluffy, and always free - The AWS Local Emulator alternative
feat(sqs): publish AWS/SQS queue metrics to CloudWatch (#6280)
Mario Toffiaba353021e150
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsQueueMetricsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsQueueMetricsTest.javanew file mode 100644index 0000000000..c6ea3ea946--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsQueueMetricsTest.java@@ -0,0 +1,80 @@+package com.floci.test;++import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.services.cloudwatch.CloudWatchClient;+import software.amazon.awssdk.services.cloudwatch.model.Datapoint;+import software.amazon.awssdk.services.cloudwatch.model.Dimension;+import software.amazon.awssdk.services.cloudwatch.model.DimensionFilter;+import software.amazon.awssdk.services.cloudwatch.model.Metric;+import software.amazon.awssdk.services.cloudwatch.model.MetricDataResult;+import software.amazon.awssdk.services.cloudwatch.model.StandardUnit;+import software.amazon.awssdk.services.cloudwatch.model.Statistic;+import software.amazon.awssdk.services.sqs.SqsClient;++import java.time.Duration;+import java.time.Instant;+import java.util.List;++import static org.assertj.core.api.Assertions.assertThat;++@DisplayName("SQS - AWS/SQS CloudWatch metrics")+class SqsQueueMetricsTest {++ private static final String NAMESPACE = "AWS/SQS";+ private static final List<String> STANDARD_QUEUE_METRICS = List.of("ApproximateNumberOfMessagesVisible",+ "ApproximateNumberOfMessagesNotVisible", "ApproximateNumberOfMessagesDelayed",+ "ApproximateAgeOfOldestMessage", "NumberOfMessagesSent", "NumberOfMessagesReceived",+ "NumberOfMessagesDeleted", "NumberOfEmptyReceives", "SentMessageSize");++ @Test+ @DisplayName("A queue's depth and sends reach AWS/SQS without PutMetricData")+ void queueMetricsArePublishedUnderAwsSqs() throws InterruptedException {+ try (SqsClient sqs = TestFixtures.sqsClient(); CloudWatchClient cloudWatch = TestFixtures.cloudWatchClient()) {+ String queueName = TestFixtures.uniqueName("sdk-sqs-metrics");+ String queueUrl = sqs.createQueue(request -> request.queueName(queueName)).queueUrl();+ try {+ sqs.sendMessage(request -> request.queueUrl(queueUrl).messageBody("held"));+ Dimension queue = Dimension.builder().name("QueueName").value(queueName).build();+ Instant start = Instant.now().minus(Duration.ofMinutes(5));++ // SQS publishes once a minute. On AWS a newly active queue can take up to 15 minutes,+ // and ListMetrics can trail the statistics by as long.+ TestFixtures.await(() -> visibleMaximum(cloudWatch, queue, start).contains(1.0)+ && sent(cloudWatch, queue, start).stream().mapToDouble(Datapoint::sum).sum() == 1.0+ && listed(cloudWatch, queueName).containsAll(STANDARD_QUEUE_METRICS),+ queueName, "the AWS/SQS metrics of one sent message",+ Duration.ofMinutes(TestFixtures.isRealAws() ? 20 : 3).toNanos(),+ TestFixtures.isRealAws() ? 30_000 : 1_000);++ assertThat(sent(cloudWatch, queue, start)).extracting(Datapoint::unit)+ .containsOnly(StandardUnit.COUNT);+ } finally {+ sqs.deleteQueue(request -> request.queueUrl(queueUrl));+ }+ }+ }++ private static List<Double> visibleMaximum(CloudWatchClient cloudWatch, Dimension queue, Instant start) {+ List<MetricDataResult> results = cloudWatch.getMetricData(request -> request+ .startTime(start).endTime(Instant.now().plus(Duration.ofMinutes(1)))+ .metricDataQueries(query -> query.id("visible").metricStat(stat -> stat+ .metric(metric -> metric.namespace(NAMESPACE)+ .metricName("ApproximateNumberOfMessagesVisible").dimensions(queue))+ .period(60).stat("Maximum")))).metricDataResults();+ return results.isEmpty() ? List.of() : results.get(0).values();+ }++ private static List<Datapoint> sent(CloudWatchClient cloudWatch, Dimension queue, Instant start) {+ return cloudWatch.getMetricStatistics(request -> request.namespace(NAMESPACE)+ .metricName("NumberOfMessagesSent").dimensions(queue).startTime(start)+ .endTime(Instant.now().plus(Duration.ofMinutes(1))).period(60).statistics(Statistic.SUM))+ .datapoints();+ }++ private static List<String> listed(CloudWatchClient cloudWatch, String queueName) {+ return cloudWatch.listMetrics(request -> request.namespace(NAMESPACE)+ .dimensions(DimensionFilter.builder().name("QueueName").value(queueName).build()))+ .metrics().stream().map(Metric::metricName).toList();+ }+}diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.mdindex e34fe0afc6..3ae76f1082 100644--- a/docs/configuration/advanced/application-yml.md+++ b/docs/configuration/advanced/application-yml.md@@ -59,8 +59,8 @@ floci:# mount-user: "1001:1001" # PosixUser: run mounting containers as uid[:gid]# mount-group-add: 2000 # supplementary gid added to mounting containerswal:- # Also the cadence at which journaled stores under persistent mode (CloudWatch Logs events,- # the S3 object index) fold their .wal file into the store's JSON file.+ # Also the cadence at which journaled stores under persistent mode (CloudWatch Logs events and+ # metric datapoints, the S3 object index) fold their .wal file into the store's JSON file.compaction-interval-ms: 30000services:ssm:@@ -136,6 +136,7 @@ floci:max-message-size: 1048576 # Bytes (1 MiB, the AWS maximum)clear-fifo-deduplication-cache-on-purge: false # When true, PurgeQueue clears SQS FIFO dedup and SNS FIFO topic dedup for topics subscribed to that queuereceipt-handle-secret: local-emulator-secret # HMAC secret that signs receipt handles+ cloudwatch-metrics-enabled: true # Publish AWS/SQS queue metrics to CloudWatch once a minuteendpoint-strategy: floci # QueueUrl shape: floci is <base-url>/<account>/<name>, aws is https://sqs.<region>.<dns-suffix>/<account>/<name>s3:@@ -349,6 +350,7 @@ All keys in this table are declared on `EmulatorConfig` and accept environment v| `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | When `true`, `PurgeQueue` clears the FIFO 5-minute deduplication cache for the target queue and matching SNS FIFO topic dedup entries || `FLOCI_SERVICES_SQS_RECEIPT_HANDLE_SECRET` | `local-emulator-secret` | HMAC secret that signs receipt handles, so a handle that was edited or built by hand is rejected || `FLOCI_SERVICES_SQS_ENDPOINT_STRATEGY` | `floci` | Shape of the returned `QueueUrl`. `floci` gives `<FLOCI_BASE_URL>/<account>/<name>` and `aws` gives `https://sqs.<region>.<dns-suffix>/<account>/<name>`. Both forms are accepted as input |+| `FLOCI_SERVICES_SQS_CLOUDWATCH_METRICS_ENABLED` | `true` | Publish the `AWS/SQS` metrics of active queues to CloudWatch once a minute || `FLOCI_SERVICES_S3_DEFAULT_PRESIGN_EXPIRY_SECONDS` | `3600` | Pre-signed URL expiry || `FLOCI_SERVICES_DOCKER_NETWORK` | *(unset)* | Shared Docker network for Lambda, RDS, ElastiCache containers || `FLOCI_SERVICES_RDS_DATA_ENABLED` | `true` | Enable the RDS Data API service |diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.mdindex c56aba835b..db8581f745 100644--- a/docs/configuration/environment-variables.md+++ b/docs/configuration/environment-variables.md@@ -84,7 +84,7 @@ See [TLS / HTTPS](./tls.md) for SDK configuration examples and WebSocket (`wss:/| `FLOCI_STORAGE_PERSISTENT_PATH` | `./data` | Container-side directory for persistent and hybrid storage || `FLOCI_STORAGE_HOST_PERSISTENT_PATH` | `./data` | Host-side path for Docker volume bind-mounts (RDS, OpenSearch, MSK, ECR data). When unset, Floci uses named Docker volumes || `FLOCI_STORAGE_PRUNE_VOLUMES_ON_DELETE` | `false` | Remove named Docker volumes immediately when the resource is deleted |-| `FLOCI_STORAGE_WAL_COMPACTION_INTERVAL_MS` | `30000` | How often (ms) the WAL compaction runs. Applies to `wal` mode and to the stores that are journaled under `persistent` mode (CloudWatch Logs events, the S3 object index) |+| `FLOCI_STORAGE_WAL_COMPACTION_INTERVAL_MS` | `30000` | How often (ms) the WAL compaction runs. Applies to `wal` mode and to the stores that are journaled under `persistent` mode (CloudWatch Logs events and metric datapoints, the S3 object index) |### Per-service storage overrides@@ -208,6 +208,7 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s| `FLOCI_SERVICES_SQS_MAX_MESSAGE_SIZE` | `1048576` | Maximum message body size in bytes (1 MiB) || `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | Reset the deduplication cache when a FIFO queue is purged || `FLOCI_SERVICES_SQS_RECEIPT_HANDLE_SECRET` | `local-emulator-secret` | HMAC secret that signs receipt handles |+| `FLOCI_SERVICES_SQS_CLOUDWATCH_METRICS_ENABLED` | `true` | Publish the `AWS/SQS` metrics of active queues to CloudWatch once a minute || `FLOCI_SERVICES_SQS_ENDPOINT_STRATEGY` | `floci` | Shape of the returned `QueueUrl`. `floci` gives `<FLOCI_BASE_URL>/<account>/<name>` and `aws` gives the AWS form `https://sqs.<region>.<dns-suffix>/<account>/<name>`. Both forms are accepted as input |### SNSdiff --git a/docs/configuration/storage.md b/docs/configuration/storage.mdindex 4e279055de..4ca58899cb 100644--- a/docs/configuration/storage.md+++ b/docs/configuration/storage.md@@ -51,11 +51,12 @@ current after every call but makes the cost of one write grow with the size of tAppend-heavy stores are journaled instead: a change is appended to a `.wal` file next to thestore, and the store's JSON file is rewritten from memory on the `FLOCI_STORAGE_WAL_COMPACTION_INTERVAL_MS`cadence and at shutdown, and only when something changed. Today this applies to CloudWatch Logs-events (`cwlogs-events.json` with `cwlogs-events.wal`) and the S3 object index-(`s3-objects.json` with `s3-objects.wal`), which holds the metadata of every object in every-bucket. After a clean shutdown the JSON file holds every entry; while Floci runs it can be up to+events (`cwlogs-events.json` with `cwlogs-events.wal`), CloudWatch metric datapoints+(`cwmetrics.json` with `cwmetrics.wal`), which services such as SQS publish every minute, and the+S3 object index (`s3-objects.json` with `s3-objects.wal`), which holds the metadata of every+object in every bucket. After a clean shutdown the JSON file holds every entry; while Floci runs it can be up toone compaction interval behind, and the journal is replayed on the next start. An existing-`cwlogs-events.json` or `s3-objects.json` from an older version is picked up as the first snapshot+`cwlogs-events.json`, `cwmetrics.json` or `s3-objects.json` from an older version is picked up as the first snapshotwithout any migration.## Per-Service Overridediff --git a/docs/services/cloudwatch.md b/docs/services/cloudwatch.mdindex 231577a8ef..0b69258a71 100644--- a/docs/services/cloudwatch.md+++ b/docs/services/cloudwatch.md@@ -399,6 +399,9 @@ HTTP 404 `UnknownOperationException`. IAM policies use the `cloudwatch` action pMetric streams are stored as definitions with their `running` or `stopped` state. Floci neverdelivers metrics to the Firehose delivery stream a metric stream names.+SQS publishes its queue metrics under `AWS/SQS` once a minute; see+[SQS CloudWatch Metrics](sqs.md#cloudwatch-metrics).+### SNS alarm actionsState transitions publish to same-account standard SNS topics in `AlarmActions`, `OKActions`, ordiff --git a/docs/services/sqs.md b/docs/services/sqs.mdindex 239cf7c295..9ba7be1536 100644--- a/docs/services/sqs.md+++ b/docs/services/sqs.md@@ -106,6 +106,28 @@ A message older than the queue's `MessageRetentionPeriod` is no longer returned`CreateQueue` and `SetQueueAttributes` reject a `MessageRetentionPeriod` outside 60 to 1209600 seconds with `InvalidAttributeValue`.+## CloudWatch Metrics++Floci publishes the `AWS/SQS` metrics of every active queue to CloudWatch once a minute, on the single `QueueName` dimension, as SQS does. Each publication runs just after a minute boundary and is stamped with the minute that ended, so a call counts in the minute it was made. Alarms, `GetMetricData`, `GetMetricStatistics` and `ListMetrics` see them with no `PutMetricData`. A queue is active while it holds a message, and for six hours after `CreateQueue`, `SendMessage`, `ReceiveMessage`, `DeleteMessage`, `ChangeMessageVisibility`, `PurgeQueue`, `GetQueueAttributes` or `SetQueueAttributes` on it, batch forms included. Publication stops when `FLOCI_SERVICES_SQS_CLOUDWATCH_METRICS_ENABLED` is `false` or the CloudWatch Metrics service is disabled.++| Metric | Unit | Samples each minute |+|---|---|---|+| `ApproximateNumberOfMessagesVisible` | `Count` | One |+| `ApproximateNumberOfMessagesNotVisible` | `Count` | One |+| `ApproximateNumberOfMessagesDelayed` | `Count` | One |+| `ApproximateAgeOfOldestMessage` | `Seconds` | One, `0` for an empty queue |+| `NumberOfMessagesSent` | `Count` | One per sent message, plus a zero |+| `NumberOfMessagesReceived` | `Count` | One per `ReceiveMessage` call that returned messages, valued at their number, plus a zero |+| `NumberOfEmptyReceives` | `Count` | One per `ReceiveMessage` call that returned none, plus a zero |+| `NumberOfMessagesDeleted` | `Count` | One per delete with a valid receipt handle, repeats included, plus a zero |+| `SentMessageSize` | `Bytes` | One per sent message, none in a minute without sends |+| `ApproximateNumberOfGroupsWithInflightMessages` | `Count` | FIFO only: one |+| `NumberOfDeduplicatedSentMessages` | `Count` | FIFO only: one per send, `1` when it was deduplicated, plus a zero |++These match what AWS publishes. A deduplicated FIFO send still counts in `NumberOfMessagesSent`, and a message a redrive policy moves to a dead-letter queue does not. `ApproximateAgeOfOldestMessage` counts a message moved to a FIFO dead-letter queue or by `StartMessageMoveTask` from the move.++Known differences: a `SendMessageBatch` or `DeleteMessageBatch` call adds one sample per entry where AWS adds one per call, so `Sum` matches while `SampleCount`, `Average`, `Minimum` and `Maximum` can differ. A message in a standard dead-letter queue keeps its original send time for its age, where AWS counts from the move, and a standard-queue message received three times or more still counts toward the age, where AWS leaves it out until it is deleted. Floci keeps these datapoints for the 15 days AWS keeps one-minute data and then drops them, where AWS goes on serving them at 5 minute and 1 hour resolution. The fair-queue metrics (`ApproximateNumberOfNoisyGroups` and the `InQuietGroups` variants) are not published, and the delay of up to 15 minutes AWS has before a newly active queue reports is not reproduced.+## Configuration| Variable | Default | Description |@@ -116,6 +138,7 @@ A message older than the queue's `MessageRetentionPeriod` is no longer returned| `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | When `true`, `PurgeQueue` also clears the FIFO deduplication cache for the queue and any SNS FIFO topics subscribed to it || `FLOCI_SERVICES_SQS_RECEIPT_HANDLE_SECRET` | `local-emulator-secret` | HMAC secret that signs receipt handles, so a handle that was edited or built by hand is rejected || `FLOCI_SERVICES_SQS_ENDPOINT_STRATEGY` | `floci` | Shape of the returned `QueueUrl`, see [Queue URL Format](#queue-url-format) |+| `FLOCI_SERVICES_SQS_CLOUDWATCH_METRICS_ENABLED` | `true` | Publish the `AWS/SQS` metrics of active queues to CloudWatch once a minute, see [CloudWatch Metrics](#cloudwatch-metrics) |## Examplesdiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex c04df49898..8d1f377d10 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1384,6 +1384,13 @@ public interface EmulatorConfig {@WithDefault("local-emulator-secret")String receiptHandleSecret();+ /**+ * Publish the AWS/SQS CloudWatch metrics of every active queue once a minute, as SQS does.+ * Env: FLOCI_SERVICES_SQS_CLOUDWATCH_METRICS_ENABLED+ */+ @WithDefault("true")+ boolean cloudwatchMetricsEnabled();+/*** Shape of the QueueUrl returned by CreateQueue, GetQueueUrl, ListQueues and* ListDeadLetterSourceQueues.diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/CloudWatchMetricsService.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/CloudWatchMetricsService.javaindex b8a846922f..27bfbb1b5b 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/CloudWatchMetricsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/CloudWatchMetricsService.java@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.core.storage.WriteProfile;import io.github.hectorvent.floci.services.cloudwatch.metrics.model.Dimension;import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricDatum;@@ -20,6 +21,7 @@ import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.UUID;+import java.util.function.Predicate;import java.util.stream.Collectors;@ApplicationScoped@@ -39,8 +41,10 @@ public class CloudWatchMetricsService {@Injectpublic CloudWatchMetricsService(StorageFactory storageFactory, RegionResolver regionResolver,AlarmSnsNotifier alarmSnsNotifier) {+ // Services publish into this store every minute, SQS for every active queue, so under+ // persistent mode it is journaled rather than rewritten in full on every datum.this.metricStore = storageFactory.create("cloudwatchmetrics", "cwmetrics.json",- new TypeReference<Map<String, MetricDatum>>() {});+ new TypeReference<Map<String, MetricDatum>>() {}, WriteProfile.APPEND_HEAVY);this.alarmStore = storageFactory.create("cloudwatchmetrics", "cwalarms.json",new TypeReference<Map<String, MetricAlarm>>() {});this.regionResolver = regionResolver;@@ -130,6 +134,22 @@ public class CloudWatchMetricsService {}}+ /**+ * Drops the {@code namespace} datapoints stamped before {@code cutoff}, in every account. A service+ * that writes its own metrics each minute, such as SQS, calls it so that history stays bounded.+ */+ public void deleteMetricDataBefore(String namespace, Instant cutoff) {+ long cutoffSeconds = cutoff.getEpochSecond();+ Predicate<MetricDatum> expired = d -> namespace.equals(d.getNamespace()) && d.getTimestamp() < cutoffSeconds;+ if (metricStore instanceof AccountAwareStorageBackend<MetricDatum> aware) {+ aware.deleteAllAccountsMatching(expired);+ return;+ }+ for (String key : List.copyOf(metricStore.keys())) {+ metricStore.get(key).filter(expired).ifPresent(ignored -> metricStore.delete(key));+ }+ }+public record MetricIdentity(String namespace, String metricName, List<Dimension> dimensions) {}public List<MetricIdentity> listMetrics(String namespace, String metricName,diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.javaindex d1c599cf93..e829a3a5c0 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java@@ -4,8 +4,10 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.services.sqs.model.Message;+import java.time.Duration;import java.time.Instant;import java.util.ArrayList;+import java.util.HashSet;import java.util.List;import java.util.Set;import java.util.concurrent.locks.ReentrantLock;@@ -24,6 +26,7 @@ class GuardedMessageQueue {private final StorageBackend<String, List<Message>> messageStore;private final String storageKey;private final String receiptHandleSecret;+ private final SqsQueueActivity activity = new SqsQueueActivity();private volatile boolean closed;@FunctionalInterface@@ -300,6 +303,42 @@ class GuardedMessageQueue {}}+ record Gauges(MessageCounts counts, long ageOfOldestMessageSeconds, long groupsWithInflightMessages) {+ boolean isEmpty() {+ return counts.visible() + counts.inFlight() + counts.delayed() == 0;+ }+ }++ /**+ * The queue contents AWS/SQS reports each minute. The age is that of the oldest message by the+ * time its retention started, so a message moved to a FIFO dead-letter queue or by a move task+ * counts from the move.+ */+ Gauges gauges(Instant now) {+ try (Guard _ = hold()) {+ Instant oldest = null;+ Set<String> groupsInFlight = new HashSet<>();+ for (Message m : messages) {+ if (!m.isVisible() && m.getReceiptHandle() != null && m.getMessageGroupId() != null) {+ groupsInFlight.add(m.getMessageGroupId());+ }+ // ponytail: AWS also leaves out a standard-queue message received 3+ times in this queue;+ // that needs a per-queue receive count, since a redriven message keeps its total.+ Instant enqueued = m.getRetentionStartTimestamp() != null+ ? m.getRetentionStartTimestamp() : m.getSentTimestamp();+ if (enqueued != null && (oldest == null || enqueued.isBefore(oldest))) {+ oldest = enqueued;+ }+ }+ long age = oldest == null ? 0 : Math.max(0, Duration.between(oldest, now).toSeconds());+ return new Gauges(messageCounts(), age, groupsInFlight.size());+ }+ }++ SqsQueueActivity activity() {+ return activity;+ }+List<Message> peekAll() {try (Guard _ = hold()) {return new ArrayList<>(messages);diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsMetricsPublisher.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsMetricsPublisher.javanew file mode 100644index 0000000000..644d75d3ee--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsMetricsPublisher.java@@ -0,0 +1,159 @@+package io.github.hectorvent.floci.services.sqs;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.Dimension;+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricDatum;+import io.quarkus.runtime.ShutdownEvent;+import io.quarkus.runtime.StartupEvent;+import jakarta.annotation.Priority;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.event.Observes;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.time.Duration;+import java.time.Instant;+import java.time.temporal.ChronoUnit;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.Executors;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.TimeUnit;++/**+ * Sends the AWS/SQS metrics of every active queue to CloudWatch once a minute, shaped as SQS sends+ * them: one sample of each depth gauge, and for each counter one sample per call in the minute plus+ * the zero sample AWS adds to every minute. SentMessageSize has one sample per message and no zero+ * sample. A tick runs just after each minute boundary and stamps its samples with the minute that+ * ended, so a call lands in the minute it was made, as on AWS. Once an hour it drops the AWS/SQS+ * datapoints older than the 15 days AWS keeps one-minute data, so the generated history stays bounded.+ */+@ApplicationScoped+public class SqsMetricsPublisher {++ static final String NAMESPACE = "AWS/SQS";+ private static final Logger LOG = Logger.getLogger(SqsMetricsPublisher.class);+ private static final long MINUTE_MILLIS = 60_000;+ private static final long TICK_OFFSET_MILLIS = 1_000;+ private static final Duration RETENTION = Duration.ofDays(15);+ private static final String COUNT = "Count";++ private final SqsService sqsService;+ private final CloudWatchMetricsService metricsService;+ private final boolean enabled;+ private final ScheduledExecutorService scheduler;++ @Inject+ public SqsMetricsPublisher(SqsService sqsService, CloudWatchMetricsService metricsService, EmulatorConfig config) {+ this.sqsService = sqsService;+ this.metricsService = metricsService;+ this.enabled = config.services().sqs().enabled() && config.services().sqs().cloudwatchMetricsEnabled()+ && config.services().cloudwatchmetrics().enabled();+ this.scheduler = Executors.newSingleThreadScheduledExecutor(r -> {+ Thread t = new Thread(r, "sqs-metrics-publisher");+ t.setDaemon(true);+ return t;+ });+ }++ void onStart(@Observes StartupEvent ignored) {+ if (!enabled) {+ return;+ }+ long untilNextMinute = MINUTE_MILLIS - System.currentTimeMillis() % MINUTE_MILLIS;+ scheduler.scheduleAtFixedRate(() -> publish(Instant.now()), untilNextMinute + TICK_OFFSET_MILLIS,+ MINUTE_MILLIS, TimeUnit.MILLISECONDS);+ }++ /** Before EmulatorLifecycle's default-priority storage shutdown, letting a running tick finish its writes. */+ void onStop(@Observes @Priority(1000) ShutdownEvent ignored) {+ scheduler.shutdown();+ try {+ if (!scheduler.awaitTermination(5, TimeUnit.SECONDS)) {+ LOG.warn("The AWS/SQS metrics publisher did not finish its tick within 5 seconds");+ }+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ LOG.warnv(e, "Interrupted while stopping the AWS/SQS metrics publisher");+ }+ }++ /**+ * Nothing may escape: the scheduler stops for good after a tick throws. One queue's failed write+ * drops only that queue's samples for the minute.+ */+ void publish(Instant now) {+ List<SqsService.QueueMetrics> queues;+ try {+ queues = sqsService.collectQueueMetrics(now);+ } catch (RuntimeException e) {+ LOG.warnv(e, "Collecting AWS/SQS queue metrics failed; this minute's samples are dropped");+ return;+ }+ long minute = endedMinute(now);+ for (SqsService.QueueMetrics queue : queues) {+ try {+ metricsService.putMetricDataForAccount(queue.accountId(), NAMESPACE, datums(queue, minute),+ queue.region());+ } catch (RuntimeException e) {+ LOG.warnv(e, "Publishing AWS/SQS metrics for queue {0} of account {1} in {2} failed; its samples "+ + "for this minute are dropped", queue.queueName(), queue.accountId(), queue.region());+ }+ }+ if (minute % 3600 == 0) {+ try {+ metricsService.deleteMetricDataBefore(NAMESPACE, Instant.ofEpochSecond(minute).minus(RETENTION));+ } catch (RuntimeException e) {+ LOG.warnv(e, "Dropping AWS/SQS datapoints older than {0} failed; the next hour tries again", RETENTION);+ }+ }+ }++ /** The minute a tick's samples belong to. Reading it half a minute back absorbs any drift of the+ * fixed-rate schedule away from the minute boundary. */+ private static long endedMinute(Instant tick) {+ return tick.minusSeconds(30).truncatedTo(ChronoUnit.MINUTES).getEpochSecond();+ }++ private static List<MetricDatum> datums(SqsService.QueueMetrics queue, long timestamp) {+ List<Dimension> dimensions = List.of(new Dimension("QueueName", queue.queueName()));+ GuardedMessageQueue.MessageCounts counts = queue.gauges().counts();+ SqsQueueActivity.Snapshot activity = queue.activity();+ List<MetricDatum> datums = new ArrayList<>(List.of(+ MetricDatum.of("ApproximateNumberOfMessagesVisible", COUNT, counts.visible(), dimensions, timestamp),+ MetricDatum.of("ApproximateNumberOfMessagesNotVisible", COUNT, counts.inFlight(), dimensions,+ timestamp),+ MetricDatum.of("ApproximateNumberOfMessagesDelayed", COUNT, counts.delayed(), dimensions, timestamp),+ MetricDatum.of("ApproximateAgeOfOldestMessage", "Seconds",+ queue.gauges().ageOfOldestMessageSeconds(), dimensions, timestamp),+ counter("NumberOfMessagesSent", activity.sent(), dimensions, timestamp),+ counter("NumberOfMessagesReceived", activity.received(), dimensions, timestamp),+ counter("NumberOfMessagesDeleted", activity.deleted(), dimensions, timestamp),+ counter("NumberOfEmptyReceives", activity.emptyReceives(), dimensions, timestamp)));+ if (activity.sentSize().count() > 0) {+ datums.add(samples("SentMessageSize", "Bytes", activity.sentSize(), dimensions, timestamp));+ }+ if (queue.fifo()) {+ datums.add(MetricDatum.of("ApproximateNumberOfGroupsWithInflightMessages", COUNT,+ queue.gauges().groupsWithInflightMessages(), dimensions, timestamp));+ datums.add(counter("NumberOfDeduplicatedSentMessages", activity.deduplicated(), dimensions, timestamp));+ }+ return datums;+ }++ private static MetricDatum counter(String metricName, SqsQueueActivity.Samples calls,+ List<Dimension> dimensions, long timestamp) {+ return samples(metricName, COUNT, calls.add(0), dimensions, timestamp);+ }++ private static MetricDatum samples(String metricName, String unit, SqsQueueActivity.Samples samples,+ List<Dimension> dimensions, long timestamp) {+ MetricDatum datum = MetricDatum.of(metricName, unit, samples.sum(), dimensions, timestamp);+ datum.setSampleCount(samples.count());+ datum.setSum(samples.sum());+ datum.setMinimum(samples.minimum());+ datum.setMaximum(samples.maximum());+ return datum;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueueActivity.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueueActivity.javanew file mode 100644index 0000000000..337eefac77--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueueActivity.java@@ -0,0 +1,73 @@+package io.github.hectorvent.floci.services.sqs;++import java.time.Instant;++/**+ * What a queue's API calls did since the last AWS/SQS metrics tick, and when the queue was last+ * acted on, which decides whether CloudWatch still considers it active.+ */+final class SqsQueueActivity {++ /** The samples of one metric within one tick. */+ record Samples(long count, double sum, double minimum, double maximum) {++ static final Samples NONE = new Samples(0, 0, 0, 0);++ Samples add(double value) {+ if (count == 0) {+ return new Samples(1, value, value, value);+ }+ return new Samples(count + 1, sum + value, Math.min(minimum, value), Math.max(maximum, value));+ }+ }++ record Snapshot(Samples sent, Samples sentSize, Samples deduplicated, Samples received,+ Samples emptyReceives, Samples deleted, Instant lastActivity) {+ }++ private Samples sent = Samples.NONE;+ private Samples sentSize = Samples.NONE;+ private Samples deduplicated = Samples.NONE;+ private Samples received = Samples.NONE;+ private Samples emptyReceives = Samples.NONE;+ private Samples deleted = Samples.NONE;+ private Instant lastActivity;++ synchronized void accessed() {+ lastActivity = Instant.now();+ }++ /** A FIFO send that was deduplicated still counts as sent, as on AWS. */+ synchronized void sent(int size, boolean wasDeduplicated) {+ sent = sent.add(1);+ sentSize = sentSize.add(size);+ deduplicated = deduplicated.add(wasDeduplicated ? 1 : 0);+ lastActivity = Instant.now();+ }++ synchronized void received(int messages) {+ if (messages == 0) {+ emptyReceives = emptyReceives.add(1);+ } else {+ received = received.add(messages);+ }+ lastActivity = Instant.now();+ }++ synchronized void deleted() {+ deleted = deleted.add(1);+ lastActivity = Instant.now();+ }++ /** Hands over the samples gathered since the previous call and starts the next tick empty. */+ synchronized Snapshot drain() {+ Snapshot snapshot = new Snapshot(sent, sentSize, deduplicated, received, emptyReceives, deleted, lastActivity);+ sent = Samples.NONE;+ sentSize = Samples.NONE;+ deduplicated = Samples.NONE;+ received = Samples.NONE;+ emptyReceives = Samples.NONE;+ deleted = Samples.NONE;+ return snapshot;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.javaindex e373fb381a..91a0e1dd3c 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java@@ -80,6 +80,8 @@ public class SqsService implements Resettable, ResourceProvider {private static final int DEFAULT_MESSAGE_RETENTION_PERIOD_SECONDS = 345600;private static final int MIN_MESSAGE_RETENTION_PERIOD_SECONDS = 60;private static final int MAX_MESSAGE_RETENTION_PERIOD_SECONDS = 1209600;+ /** CloudWatch keeps reporting a queue for six hours after its last action, or while it holds a message. */+ private static final Duration METRICS_ACTIVE_WINDOW = Duration.ofHours(6);private final StorageBackend<String, Queue> queueStore;private final StorageBackend<String, List<Message>> messageStore;@@ -773,6 +775,7 @@ public class SqsService implements Resettable, ResourceProvider {}}}+ getOrCreateQueue(storageKey).activity().accessed();return withCurrentQueueUrl(existing, region);}@@ -799,8 +802,10 @@ public class SqsService implements Resettable, ResourceProvider {}queueStore.put(storageKey, queue);- messagesByQueue.put(storageKey,- new GuardedMessageQueue(new ArrayList<>(), messageStore, storageKey, receiptHandleSecret));+ GuardedMessageQueue messages = new GuardedMessageQueue(new ArrayList<>(), messageStore, storageKey,+ receiptHandleSecret);+ messages.activity().accessed();+ messagesByQueue.put(storageKey, messages);LOG.infov("Created {0} queue: {1} in region {2}", queue.isFifo() ? "FIFO" : "standard", queueName, region);return queue;}@@ -904,7 +909,10 @@ public class SqsService implements Resettable, ResourceProvider {attrs.put("CreatedTimestamp", String.valueOf(queue.getCreatedTimestamp().getEpochSecond()));attrs.put("LastModifiedTimestamp", String.valueOf(queue.getLastModifiedTimestamp().getEpochSecond()));- GuardedMessageQueue.MessageCounts counts = unexpiredMessages(storageKey, queue).messageCounts();+ GuardedMessageQueue messages = unexpiredMessages(storageKey, queue);+ // AWS: a GetQueueAttributes call activates the queue, so CloudWatch reports it again.+ messages.activity().accessed();+ GuardedMessageQueue.MessageCounts counts = messages.messageCounts();attrs.put("ApproximateNumberOfMessages", String.valueOf(counts.visible()));attrs.put("ApproximateNumberOfMessagesNotVisible", String.valueOf(counts.inFlight()));attrs.put("ApproximateNumberOfMessagesDelayed", String.valueOf(counts.delayed()));@@ -1119,6 +1127,7 @@ public class SqsService implements Resettable, ResourceProvider {response.getMessageAttributes().putAll(messageAttributes);response.updateMd5OfMessageAttributes();}+ getOrCreateQueue(storageKey).activity().sent(totalSize, true);return response;}@@ -1138,7 +1147,9 @@ public class SqsService implements Resettable, ResourceProvider {message.updateMd5OfMessageAttributes();}- getOrCreateQueue(storageKey).addMessage(message);+ GuardedMessageQueue messages = getOrCreateQueue(storageKey);+ messages.addMessage(message);+ messages.activity().sent(totalSize, false);deduplicationIdentityCache.computeIfAbsent(storageKey, k -> new ConcurrentHashMap<>()).putIfAbsent(dedupCacheKey,new DeduplicationIdentity(message.getMessageId(), message.getSequenceNumber()));@@ -1166,7 +1177,9 @@ public class SqsService implements Resettable, ResourceProvider {message.updateMd5OfMessageAttributes();}- getOrCreateQueue(storageKey).addMessage(message);+ GuardedMessageQueue messages = getOrCreateQueue(storageKey);+ messages.addMessage(message);+ messages.activity().sent(totalSize, false);notifyReceivers(storageKey);LOG.debugv("Sent message {0} to queue {1}", message.getMessageId(), queueUrl);LOG.tracev("Sent message {0} to queue {1} body={2} attributes={3}",@@ -1405,11 +1418,11 @@ public class SqsService implements Resettable, ResourceProvider {m.getMessageId(), queueUrl, m.getBody(), m.getMessageAttributes());}}- return result;+ return recordReceive(polledQueue, result);}long elapsed = System.currentTimeMillis() - start;if (elapsed >= maxWait) {- return result;+ return recordReceive(polledQueue, result);}try {synchronized (lock) {@@ -1417,11 +1430,17 @@ public class SqsService implements Resettable, ResourceProvider {}} catch (InterruptedException e) {Thread.currentThread().interrupt();- return result;+ return recordReceive(polledQueue, result);}}}+ /** AWS takes one sample per ReceiveMessage call, however long it polled. */+ private static List<Message> recordReceive(GuardedMessageQueue polledQueue, List<Message> received) {+ polledQueue.activity().received(received.size());+ return received;+ }+private int resolveWaitTimeSeconds(Queue queue, Integer requestedWaitTimeSeconds) {if (requestedWaitTimeSeconds != null) {return requestedWaitTimeSeconds;@@ -1533,9 +1552,11 @@ public class SqsService implements Resettable, ResourceProvider {"The specified queue does not exist.", 400));ReceiptHandle handle = checkReceiptHandle(storageKey, receiptHandle, inBatch);- GuardedMessageQueue.Removal removal =- getOrCreateQueue(storageKey).removeByReceiptHandle(handle, queue.isFifo());+ GuardedMessageQueue messages = getOrCreateQueue(storageKey);+ GuardedMessageQueue.Removal removal = messages.removeByReceiptHandle(handle, queue.isFifo());rejectUnusableHandle(removal.result(), receiptHandle, inBatch);+ // AWS counts every delete with a valid handle, including a repeat for a message already gone.+ messages.activity().deleted();// AWS reports success for a message that is already gone.if (removal.result() == GuardedMessageQueue.HandleResult.MESSAGE_GONE) {LOG.debugv("No message for receipt handle {0}", receiptHandle);@@ -1561,8 +1582,9 @@ public class SqsService implements Resettable, ResourceProvider {"The specified queue does not exist.", 400));ReceiptHandle handle = checkReceiptHandle(storageKey, receiptHandle, inBatch);- GuardedMessageQueue.HandleResult result =- getOrCreateQueue(storageKey).changeVisibility(handle, visibilityTimeout, queue.isFifo());+ GuardedMessageQueue messages = getOrCreateQueue(storageKey);+ messages.activity().accessed();+ GuardedMessageQueue.HandleResult result = messages.changeVisibility(handle, visibilityTimeout, queue.isFifo());rejectUnusableHandle(result, receiptHandle, inBatch);if (result == GuardedMessageQueue.HandleResult.MESSAGE_GONE) {throw invalidReceiptHandle(receiptHandle,@@ -1609,7 +1631,9 @@ public class SqsService implements Resettable, ResourceProvider {public void purgeQueue(String queueUrl, String region) {String storageKey = regionKey(region, queueUrl);ensureQueueExists(storageKey);- getOrCreateQueue(storageKey).purge();+ GuardedMessageQueue messages = getOrCreateQueue(storageKey);+ messages.purge();+ messages.activity().accessed();if (clearFifoDeduplicationCacheOnPurge) {deduplicationCache.remove(storageKey);deduplicationIdentityCache.remove(storageKey);@@ -1646,6 +1670,7 @@ public class SqsService implements Resettable, ResourceProvider {}queue.setLastModifiedTimestamp(Instant.now());queueStore.put(storageKey, queue);+ getOrCreateQueue(storageKey).activity().accessed();LOG.infov("Updated attributes for queue: {0}", queueUrl);}@@ -2208,6 +2233,40 @@ public class SqsService implements Resettable, ResourceProvider {}}+ /** One active queue's AWS/SQS metrics for one publication. */+ record QueueMetrics(String accountId, String region, String queueName, boolean fifo,+ GuardedMessageQueue.Gauges gauges, SqsQueueActivity.Snapshot activity) {+ }++ /**+ * The AWS/SQS metrics of every queue CloudWatch considers active: one that holds a message or+ * was acted on in the last six hours. Each queue's activity since the previous call is handed+ * over, so every call starts the next sample window.+ */+ List<QueueMetrics> collectQueueMetrics(Instant now) {+ Instant activeSince = now.minus(METRICS_ACTIVE_WINDOW);+ List<QueueMetrics> metrics = new ArrayList<>();+ messagesByQueue.forEach((storageKey, messages) -> {+ int separator = storageKey.indexOf("::");+ String queueUrl = baseUrl + storageKey.substring(separator + 2);+ Queue queue = getQueueByUrl(storageKey, queueUrl).orElse(null);+ // A queue deleted and created again meanwhile has a new message queue; this one is gone.+ if (queue == null || messagesByQueue.get(storageKey) != messages) {+ return;+ }+ messages.removeExpired(retentionCutoff(queue));+ GuardedMessageQueue.Gauges gauges = messages.gauges(now);+ SqsQueueActivity.Snapshot activity = messages.activity().drain();+ Instant lastActivity = activity.lastActivity();+ if (gauges.isEmpty() && (lastActivity == null || lastActivity.isBefore(activeSince))) {+ return;+ }+ metrics.add(new QueueMetrics(accountFromQueueUrl(queueUrl), storageKey.substring(0, separator),+ queue.getQueueName(), queue.isFifo(), gauges, activity));+ });+ return metrics;+ }+private void removeExpiredMessages(String storageKey) {GuardedMessageQueue messages = messagesByQueue.get(storageKey);if (messages == null) {diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex f5e93af0de..b214ab6dab 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -201,8 +201,8 @@ floci:# mount-user: # run mounting containers as "uid[:gid]" (access point PosixUser)# mount-group-add: # supplementary gid added to mounting containerswal:- # Also the cadence at which journaled stores under persistent mode (CloudWatch Logs events,- # the S3 object index) fold their .wal file into the store's JSON file.+ # Also the cadence at which journaled stores under persistent mode (CloudWatch Logs events and+ # metric datapoints, the S3 object index) fold their .wal file into the store's JSON file.compaction-interval-ms: 30000services:ssm:@@ -395,6 +395,7 @@ floci:max-message-size: 1048576clear-fifo-deduplication-cache-on-purge: falsereceipt-handle-secret: local-emulator-secret+ cloudwatch-metrics-enabled: true # FLOCI_SERVICES_SQS_CLOUDWATCH_METRICS_ENABLED - publish AWS/SQS queue metrics to CloudWatch once a minute# QueueUrl shape. floci is <base-url>/<account>/<name>,# aws is https://sqs.<region>.<partition dns suffix>/<account>/<name>endpoint-strategy: flocidiff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/PublicationTestSupport.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/PublicationTestSupport.javaindex cf40e29e3c..8c249ff003 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/PublicationTestSupport.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/PublicationTestSupport.java@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.core.storage.WriteProfile;import io.github.hectorvent.floci.services.cloudwatch.logs.model.MetricFilter;import io.github.hectorvent.floci.services.cloudwatch.logs.model.MetricTransformation;import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;@@ -56,7 +57,8 @@ class PublicationTestSupport {when(resolver.getAccountId()).thenAnswer(call -> account.get());metricStore = new AccountAwareStorageBackend<>(sink, contexts, ACCOUNT);filters = new AccountAwareStorageBackend<>(new InMemoryStorage<>(), contexts, ACCOUNT);- doReturn(metricStore).when(factory).create(eq("cloudwatchmetrics"), eq("cwmetrics.json"), any());+ doReturn(metricStore).when(factory).create(eq("cloudwatchmetrics"), eq("cwmetrics.json"), any(),+ eq(WriteProfile.APPEND_HEAVY));doReturn(AccountAwareStorageBackend.inMemory(ACCOUNT)).when(factory).create(eq("cloudwatchmetrics"), eq("cwalarms.json"), any());metrics = new CloudWatchMetricsService(factory, resolver);diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceContainerInsightsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceContainerInsightsTest.javaindex f79cf28d5b..0e6089a435 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceContainerInsightsTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceContainerInsightsTest.java@@ -1,11 +1,7 @@package io.github.hectorvent.floci.services.ecs;-import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.RegionResolver;-import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;-import io.github.hectorvent.floci.core.storage.StorageBackend;-import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService.Datapoint;import io.github.hectorvent.floci.services.cloudwatch.metrics.model.Dimension;@@ -14,6 +10,7 @@ import io.github.hectorvent.floci.services.ecs.exec.EcsExecSessionRegistry;import io.github.hectorvent.floci.services.ecs.model.ClusterSetting;import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.LaunchType;+import io.github.hectorvent.floci.testing.InMemoryStorageFactory;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;@@ -22,9 +19,7 @@ import org.junit.jupiter.params.provider.ValueSource;import java.time.Clock;import java.time.Instant;import java.time.ZoneOffset;-import java.util.HashMap;import java.util.List;-import java.util.Map;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -58,7 +53,7 @@ class EcsServiceContainerInsightsTest {@BeforeEachvoid setUp() {- InMemoryStorageFactory storage = new InMemoryStorageFactory();+ InMemoryStorageFactory storage = new InMemoryStorageFactory("000000000000");RegionResolver regionResolver = new RegionResolver(REGION, "000000000000");metrics = spy(new CloudWatchMetricsService(storage, regionResolver));EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);@@ -176,21 +171,4 @@ class EcsServiceContainerInsightsTest {List.of(new Dimension("ServiceName", "web"), new Dimension("ClusterName", "insights")),MINUTE.minusSeconds(600), MINUTE.plusSeconds(600), 60, List.of("Maximum"), null, REGION);}-- private static final class InMemoryStorageFactory extends StorageFactory {- private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();-- private InMemoryStorageFactory() {- super(null, null);- }-- @Override- @SuppressWarnings("unchecked")- public <V> AccountAwareStorageBackend<V> create(String serviceName,- String fileName,- TypeReference<Map<String, V>> typeReference) {- return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,- ignored -> AccountAwareStorageBackend.inMemory("000000000000"));- }- }}diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsMetricsPublisherTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsMetricsPublisherTest.javanew file mode 100644index 0000000000..8187a646d3--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsMetricsPublisherTest.java@@ -0,0 +1,304 @@+package io.github.hectorvent.floci.services.sqs;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;+import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService.Datapoint;+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.Dimension;+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricDatum;+import io.github.hectorvent.floci.services.sqs.model.Message;+import io.github.hectorvent.floci.services.sqs.model.Queue;+import io.github.hectorvent.floci.testing.InMemoryStorageFactory;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.mockito.Answers;++import java.time.Duration;+import java.time.Instant;+import java.time.temporal.ChronoUnit;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.*;++/**+ * The AWS/SQS metrics as real SQS sends them, measured on AWS: per active queue and minute one+ * sample of each depth gauge, and for each counter one sample per API call, valued at the+ * messages that call handled, plus the zero sample AWS adds to every minute.+ */+class SqsMetricsPublisherTest {++ private static final String REGION = "eu-west-1";+ private static final String ACCOUNT = "111111111111";+ private static final String BASE_URL = "http://localhost:4566";++ private SqsService sqs;+ private CloudWatchMetricsService cloudWatch;+ private SqsMetricsPublisher publisher;++ @BeforeEach+ void setUp() {+ RegionResolver resolver = new RegionResolver(REGION, ACCOUNT);+ sqs = SqsServiceFactory.createInMemory(BASE_URL, resolver);+ cloudWatch = new CloudWatchMetricsService(new InMemoryStorageFactory(ACCOUNT), resolver);+ publisher = new SqsMetricsPublisher(sqs, cloudWatch, mock(EmulatorConfig.class, Answers.RETURNS_DEEP_STUBS));+ }++ @Test+ void activeQueueReportsItsDepthAsGauges() {+ String url = sqs.createQueue("depth", null, REGION).getQueueUrl();+ sqs.sendMessage(url, "visible", null, REGION);+ sqs.sendMessage(url, "in-flight", null, REGION);+ sqs.sendMessage(url, "delayed", 600, REGION);+ sqs.receiveMessage(url, 1, 600, 0, REGION);+ Instant now = Instant.now().plusSeconds(100);++ publisher.publish(now);++ assertSamples(datapoint("depth", "ApproximateNumberOfMessagesVisible", now), 1, 1, 1, 1, "Count");+ assertSamples(datapoint("depth", "ApproximateNumberOfMessagesNotVisible", now), 1, 1, 1, 1, "Count");+ assertSamples(datapoint("depth", "ApproximateNumberOfMessagesDelayed", now), 1, 1, 1, 1, "Count");+ Datapoint age = datapoint("depth", "ApproximateAgeOfOldestMessage", now);+ assertEquals("Seconds", age.unit());+ assertTrue(age.maximum() >= 99 && age.maximum() <= 101, "age " + age.maximum());+ }++ @Test+ void eachCallIsOneSampleValuedAtTheMessagesItHandled() {+ String url = sqs.createQueue("calls", null, REGION).getQueueUrl();+ sqs.sendMessage(url, "one", null, REGION);+ sqs.sendMessage(url, "two", null, REGION);+ List<Message> received = sqs.receiveMessage(url, 2, 600, 0, REGION);+ assertEquals(2, received.size());+ sqs.receiveMessage(url, 1, 600, 0, REGION);+ sqs.deleteMessage(url, received.getFirst().getReceiptHandle(), REGION);+ sqs.deleteMessage(url, received.getFirst().getReceiptHandle(), REGION);+ Instant now = Instant.now();++ publisher.publish(now);++ assertSamples(datapoint("calls", "NumberOfMessagesSent", now), 3, 2, 0, 1, "Count");+ assertSamples(datapoint("calls", "NumberOfMessagesReceived", now), 2, 2, 0, 2, "Count");+ assertSamples(datapoint("calls", "NumberOfEmptyReceives", now), 2, 1, 0, 1, "Count");+ assertSamples(datapoint("calls", "NumberOfMessagesDeleted", now), 3, 2, 0, 1, "Count");+ }++ @Test+ void aQuietMinuteGetsOneZeroSamplePerCounterAndNoMessageSize() {+ sqs.createQueue("quiet", null, REGION);+ Instant now = Instant.now();++ publisher.publish(now);++ for (String counter : List.of("NumberOfMessagesSent", "NumberOfMessagesReceived",+ "NumberOfMessagesDeleted", "NumberOfEmptyReceives")) {+ assertSamples(datapoint("quiet", counter, now), 1, 0, 0, 0, "Count");+ }+ assertSamples(datapoint("quiet", "ApproximateNumberOfMessagesVisible", now), 1, 0, 0, 0, "Count");+ assertSamples(datapoint("quiet", "ApproximateAgeOfOldestMessage", now), 1, 0, 0, 0, "Seconds");+ assertNull(datapoint("quiet", "SentMessageSize", now));+ assertNull(datapoint("quiet", "NumberOfDeduplicatedSentMessages", now));+ assertNull(datapoint("quiet", "ApproximateNumberOfGroupsWithInflightMessages", now));+ }++ @Test+ void sentMessageSizeIsOneSamplePerMessage() {+ String url = sqs.createQueue("sizes", null, REGION).getQueueUrl();+ sqs.sendMessage(url, "aa", null, REGION);+ sqs.sendMessage(url, "cccccc", null, REGION);+ Instant now = Instant.now();++ publisher.publish(now);++ assertSamples(datapoint("sizes", "SentMessageSize", now), 2, 8, 2, 6, "Bytes");+ }++ @Test+ void countersStartOverAfterEachPublication() {+ String url = sqs.createQueue("drained", null, REGION).getQueueUrl();+ sqs.sendMessage(url, "once", null, REGION);+ Instant first = Instant.now();+ publisher.publish(first);+ Instant second = first.plusSeconds(60);++ publisher.publish(second);++ assertSamples(datapoint("drained", "NumberOfMessagesSent", second), 1, 0, 0, 0, "Count");+ assertNull(datapoint("drained", "SentMessageSize", second));+ assertSamples(datapoint("drained", "ApproximateNumberOfMessagesVisible", second), 1, 1, 1, 1, "Count");+ }++ @Test+ void fifoQueueAlsoReportsDeduplicatedSendsAndGroupsWithInflightMessages() {+ String url = sqs.createQueue("dedup.fifo", Map.of("FifoQueue", "true"), REGION).getQueueUrl();+ sqs.sendMessage(url, "one", null, "g", "X", REGION);+ sqs.sendMessage(url, "one", null, "g", "X", REGION);+ sqs.sendMessage(url, "two", null, "h", "Y", REGION);+ sqs.receiveMessage(url, 1, 600, 0, REGION);+ Instant now = Instant.now();++ publisher.publish(now);++ assertSamples(datapoint("dedup.fifo", "NumberOfMessagesSent", now), 4, 3, 0, 1, "Count");+ assertSamples(datapoint("dedup.fifo", "NumberOfDeduplicatedSentMessages", now), 4, 1, 0, 1, "Count");+ assertSamples(datapoint("dedup.fifo", "SentMessageSize", now), 3, 9, 3, 3, "Bytes");+ assertSamples(datapoint("dedup.fifo", "ApproximateNumberOfGroupsWithInflightMessages", now),+ 1, 1, 1, 1, "Count");+ }++ @Test+ void anEmptyQueueStopsReportingSixHoursAfterItsLastAction() {+ sqs.createQueue("idle", null, REGION);+ String busy = sqs.createQueue("holding", null, REGION).getQueueUrl();+ sqs.sendMessage(busy, "kept", null, REGION);+ Instant later = Instant.now().plus(Duration.ofHours(6)).plusSeconds(60);++ publisher.publish(later);++ assertNull(datapoint("idle", "ApproximateNumberOfMessagesVisible", later));+ assertSamples(datapoint("holding", "ApproximateNumberOfMessagesVisible", later), 1, 1, 1, 1, "Count");+ }++ @Test+ void aRestoredEmptyQueueIsReportedOnlyOnceAnActionReachesIt() {+ InMemoryStorage<String, Queue> queues = new InMemoryStorage<>();+ InMemoryStorage<String, List<Message>> messages = new InMemoryStorage<>();+ SqsService before = new SqsService(queues, messages, new InMemoryStorage<>(), 30, 1048576, BASE_URL,+ new RegionResolver(REGION, ACCOUNT));+ String url = before.createQueue("restored", null, REGION).getQueueUrl();+ before.sendMessage(url, "handled", null, REGION);+ before.deleteMessage(url, before.receiveMessage(url, 1, 600, 0, REGION).getFirst().getReceiptHandle(),+ REGION);+ SqsService restarted = new SqsService(queues, messages, new InMemoryStorage<>(), 30, 1048576, BASE_URL,+ new RegionResolver(REGION, ACCOUNT));+ SqsMetricsPublisher restartedPublisher = new SqsMetricsPublisher(restarted, cloudWatch,+ mock(EmulatorConfig.class, Answers.RETURNS_DEEP_STUBS));+ Instant first = Instant.now();+ restartedPublisher.publish(first);+ assertNull(datapoint("restored", "ApproximateNumberOfMessagesVisible", first));++ restarted.getQueueAttributes(url, List.of("All"), REGION);+ Instant second = first.plusSeconds(60);+ restartedPublisher.publish(second);++ assertSamples(datapoint("restored", "ApproximateNumberOfMessagesVisible", second), 1, 0, 0, 0, "Count");+ }++ @Test+ void aMessageRedrivenToAStandardDeadLetterQueueKeepsItsAge() {+ String dlq = sqs.createQueue("redriven-dlq", null, REGION).getQueueUrl();+ String dlqArn = sqs.getQueueAttributes(dlq, List.of("QueueArn"), REGION).get("QueueArn");+ String source = sqs.createQueue("redriven-source", Map.of("RedrivePolicy",+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\",\"maxReceiveCount\":\"2\"}"), REGION).getQueueUrl();+ sqs.sendMessage(source, "pill", null, REGION);+ assertEquals(1, sqs.receiveMessage(source, 1, 0, 0, REGION).size());+ assertEquals(1, sqs.receiveMessage(source, 1, 0, 0, REGION).size());+ assertTrue(sqs.receiveMessage(source, 1, 0, 0, REGION).isEmpty());+ Instant now = Instant.now().plusSeconds(100);++ publisher.publish(now);++ assertSamples(datapoint("redriven-dlq", "ApproximateNumberOfMessagesVisible", now), 1, 1, 1, 1, "Count");+ assertTrue(datapoint("redriven-dlq", "ApproximateAgeOfOldestMessage", now).maximum() >= 99);+ }++ @Test+ void aDeletedQueueIsNoLongerReported() {+ String url = sqs.createQueue("gone", null, REGION).getQueueUrl();+ sqs.deleteQueue(url, REGION);+ Instant now = Instant.now();++ publisher.publish(now);++ assertNull(datapoint("gone", "ApproximateNumberOfMessagesVisible", now));+ }++ @Test+ void aTickStampsItsSamplesWithTheMinuteThatJustEnded() {+ sqs.createQueue("stamped", null, REGION);++ publisher.publish(Instant.parse("2026-10-22T12:01:01Z"));++ Instant ended = Instant.parse("2026-10-22T12:00:00Z");+ List<Datapoint> points = cloudWatch.getMetricStatistics("AWS/SQS", "NumberOfMessagesSent",+ List.of(new Dimension("QueueName", "stamped")), ended, ended, 60, List.of("Sum"), null, REGION);+ assertEquals(1, points.size());+ assertEquals(ended, points.getFirst().timestamp());+ }++ @Test+ void theHourlyTickDropsAwsSqsDatapointsOlderThanFifteenDays() {+ Instant tick = Instant.parse("2026-10-22T13:01:01Z");+ Instant hour = Instant.parse("2026-10-22T13:00:00Z");+ Instant expired = hour.minus(Duration.ofDays(16));+ Instant kept = hour.minus(Duration.ofDays(14));+ List<Dimension> queue = List.of(new Dimension("QueueName", "old"));+ cloudWatch.putMetricDataForAccount(ACCOUNT, "AWS/SQS", List.of(+ MetricDatum.of("NumberOfMessagesSent", "Count", 1, queue, expired.getEpochSecond()),+ MetricDatum.of("NumberOfMessagesSent", "Count", 1, queue, kept.getEpochSecond())), REGION);+ cloudWatch.putMetricDataForAccount(ACCOUNT, "Custom", List.of(+ MetricDatum.of("NumberOfMessagesSent", "Count", 1, queue, expired.getEpochSecond())), REGION);++ publisher.publish(tick);++ assertTrue(points("AWS/SQS", queue, expired).isEmpty());+ assertEquals(1, points("AWS/SQS", queue, kept).size());+ assertEquals(1, points("Custom", queue, expired).size());+ }++ private List<Datapoint> points(String namespace, List<Dimension> dimensions, Instant at) {+ return cloudWatch.getMetricStatistics(namespace, "NumberOfMessagesSent", dimensions, at, at, 60,+ List.of("Sum"), null, REGION);+ }++ @Test+ void aQueueWhosePublicationFailsDoesNotStopTheOthers() {+ sqs.createQueue("first", null, REGION);+ sqs.createQueue("second", null, REGION);+ CloudWatchMetricsService failingOnce = spy(cloudWatch);+ doThrow(new IllegalStateException("store failed")).doCallRealMethod()+ .when(failingOnce).putMetricDataForAccount(any(), any(), any(), any());+ Instant now = Instant.now();++ new SqsMetricsPublisher(sqs, failingOnce, mock(EmulatorConfig.class, Answers.RETURNS_DEEP_STUBS))+ .publish(now);++ long published = List.of("first", "second").stream()+ .filter(queue -> datapoint(queue, "ApproximateNumberOfMessagesVisible", now) != null)+ .count();+ assertEquals(1, published);+ }++ @Test+ void aFailingPublicationDoesNotEscapeTheScheduledTask() {+ SqsService failing = mock(SqsService.class);+ when(failing.collectQueueMetrics(any())).thenThrow(new IllegalStateException("collect failed"));+ SqsMetricsPublisher failingPublisher = new SqsMetricsPublisher(failing, cloudWatch,+ mock(EmulatorConfig.class, Answers.RETURNS_DEEP_STUBS));++ assertDoesNotThrow(() -> failingPublisher.publish(Instant.now()));+ }++ /** The datapoint a tick at {@code tick} published, under the minute that had just ended. */+ private Datapoint datapoint(String queueName, String metricName, Instant tick) {+ Instant minute = tick.minusSeconds(30).truncatedTo(ChronoUnit.MINUTES);+ List<Datapoint> points = cloudWatch.getMetricStatistics("AWS/SQS", metricName,+ List.of(new Dimension("QueueName", queueName)), minute, minute, 60, List.of("Sum"), null, REGION);+ assertTrue(points.size() <= 1, metricName + " has " + points.size() + " datapoints");+ return points.isEmpty() ? null : points.getFirst();+ }++ private static void assertSamples(Datapoint point, double sampleCount, double sum, double minimum,+ double maximum, String unit) {+ assertNotNull(point);+ assertAll(+ () -> assertEquals(sampleCount, point.sampleCount(), "SampleCount"),+ () -> assertEquals(sum, point.sum(), "Sum"),+ () -> assertEquals(minimum, point.minimum(), "Minimum"),+ () -> assertEquals(maximum, point.maximum(), "Maximum"),+ () -> assertEquals(unit, point.unit(), "Unit"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsQueueMetricsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsQueueMetricsIntegrationTest.javanew file mode 100644index 0000000000..36f78e7ffa--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsQueueMetricsIntegrationTest.java@@ -0,0 +1,92 @@+package io.github.hectorvent.floci.services.sqs;++import io.github.hectorvent.floci.services.cloudwatch.metrics.AlarmEvaluator;+import io.github.hectorvent.floci.services.cloudwatch.metrics.AlarmEvaluatorTestAccess;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.time.Instant;++import static io.restassured.RestAssured.given;+import static org.awaitility.Awaitility.await;+import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.contains;+import static org.hamcrest.Matchers.equalTo;++/**+ * The acceptance of AWS/SQS queue metrics: an alarm on a queue's ApproximateNumberOfMessagesVisible+ * goes to ALARM after one message is sent and back to OK after a purge, with no PutMetricData.+ *+ * <p>The alarm has the 5 minute period of a typical queue-depth alarm. Each publication is the+ * publisher's tick, run 10 and 5 minutes before the queue was created, so the evaluator sees two+ * complete periods at once whatever minute the test starts in. The test configuration turns the+ * scheduled tick off, so these two are the only publications for the queue.+ */+@QuarkusTest+class SqsQueueMetricsIntegrationTest {++ private static final String JSON = "application/x-amz-json-1.0";+ private static final String QUEUE = "metrics-alarm-queue";+ private static final String ALARM = "metrics-alarm-queue-depth";++ @Inject+ SqsMetricsPublisher publisher;++ @Inject+ AlarmEvaluator alarmEvaluator;++ @BeforeAll+ static void configure() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void queueDepthAlarmFiresOnASentMessageAndClearsAfterAPurge() {+ Instant created = Instant.now();+ String queueUrl = call("AmazonSQS.CreateQueue", "{\"QueueName\":\"" + QUEUE + "\"}")+ .jsonPath().getString("QueueUrl");+ call("GraniteServiceVersion20100801.PutMetricAlarm", """+ {"AlarmName":"%s","Namespace":"AWS/SQS","MetricName":"ApproximateNumberOfMessagesVisible",+ "Dimensions":[{"Name":"QueueName","Value":"%s"}],"Statistic":"Maximum","Period":300,+ "EvaluationPeriods":1,"Threshold":1,"ComparisonOperator":"GreaterThanOrEqualToThreshold"}+ """.formatted(ALARM, QUEUE));++ call("AmazonSQS.SendMessage", "{\"QueueUrl\":\"" + queueUrl + "\",\"MessageBody\":\"held\"}");+ publisher.publish(created.minusSeconds(600));+ awaitAlarmState("ALARM");++ call("AmazonSQS.PurgeQueue", "{\"QueueUrl\":\"" + queueUrl + "\"}");+ publisher.publish(created.minusSeconds(300));+ awaitAlarmState("OK");++ Response data = call("GraniteServiceVersion20100801.GetMetricData", """+ {"StartTime":%d,"EndTime":%d,"ScanBy":"TimestampAscending","MetricDataQueries":[{"Id":"visible",+ "MetricStat":{"Metric":{"Namespace":"AWS/SQS","MetricName":"ApproximateNumberOfMessagesVisible",+ "Dimensions":[{"Name":"QueueName","Value":"%s"}]},"Period":300,"Stat":"Maximum"}}]}+ """.formatted(created.minusSeconds(1200).getEpochSecond(), created.getEpochSecond() / 300 * 300 - 1,+ QUEUE));+ assertThat(data.jsonPath().getList("MetricDataResults[0].Values", Double.class), contains(1.0, 0.0));+ }++ /** The scheduled evaluator runs on its own thread and may finish a pass that read the data+ * before the last publication; the next pass corrects it. */+ private void awaitAlarmState(String expected) {+ await().atMost(Duration.ofSeconds(15)).pollInterval(Duration.ofMillis(200)).untilAsserted(() -> {+ AlarmEvaluatorTestAccess.evaluateNow(alarmEvaluator);+ call("GraniteServiceVersion20100801.DescribeAlarms", "{\"AlarmNames\":[\"" + ALARM + "\"]}")+ .then().body("MetricAlarms[0].StateValue", equalTo(expected));+ });+ }++ private static Response call(String target, String body) {+ return given().contentType(JSON).header("X-Amz-Target", target).body(body)+ .when().post("/")+ .then().statusCode(200)+ .extract().response();+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 72c376bbf9..905120dd0a 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -140,6 +140,7 @@ floci:default-visibility-timeout: 30max-message-size: 1048576endpoint-strategy: floci+ cloudwatch-metrics-enabled: false # tests publish themselves, so no tick stamps another test's queue behind its backs3:enabled: trueenforce-auth: false