Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(ses): add v2 export jobs (#4327)

Kenshin Okinaka08c6ce0768f7
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesExportJobTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesExportJobTest.java
new file mode 100644
index 0000000000..1873ff0e30
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesExportJobTest.java
@@ -0,0 +1,233 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import software.amazon.awssdk.services.sesv2.SesV2Client;
+import software.amazon.awssdk.services.sesv2.model.BadRequestException;
+import software.amazon.awssdk.services.sesv2.model.Body;
+import software.amazon.awssdk.services.sesv2.model.CancelExportJobRequest;
+import software.amazon.awssdk.services.sesv2.model.Content;
+import software.amazon.awssdk.services.sesv2.model.CreateEmailIdentityRequest;
+import software.amazon.awssdk.services.sesv2.model.CreateExportJobRequest;
+import software.amazon.awssdk.services.sesv2.model.DataFormat;
+import software.amazon.awssdk.services.sesv2.model.DeleteEmailIdentityRequest;
+import software.amazon.awssdk.services.sesv2.model.Destination;
+import software.amazon.awssdk.services.sesv2.model.EmailContent;
+import software.amazon.awssdk.services.sesv2.model.ExportDataSource;
+import software.amazon.awssdk.services.sesv2.model.ExportDestination;
+import software.amazon.awssdk.services.sesv2.model.ExportSourceType;
+import software.amazon.awssdk.services.sesv2.model.GetAccountRequest;
+import software.amazon.awssdk.services.sesv2.model.GetAccountResponse;
+import software.amazon.awssdk.services.sesv2.model.GetExportJobRequest;
+import software.amazon.awssdk.services.sesv2.model.GetExportJobResponse;
+import software.amazon.awssdk.services.sesv2.model.ListExportJobsRequest;
+import software.amazon.awssdk.services.sesv2.model.ListExportJobsResponse;
+import software.amazon.awssdk.services.sesv2.model.Message;
+import software.amazon.awssdk.services.sesv2.model.MessageInsightsDataSource;
+import software.amazon.awssdk.services.sesv2.model.PutAccountVdmAttributesRequest;
+import software.amazon.awssdk.services.sesv2.model.SendEmailRequest;
+import software.amazon.awssdk.services.sesv2.model.VdmAttributes;
+
+import java.time.Duration;
+import java.time.Instant;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * SDK compatibility test for the SES v2 export jobs against a live Floci instance: the Virtual
+ * Deliverability Manager gate, which answers BadRequestException here rather than the
+ * NotFoundException the other two gated operations use, the refusal of a caller-chosen S3Url, a
+ * job followed to completion with its presigned URL and its echoed data source, and the two
+ * server-side checks the SDK cannot catch on the client.
+ */
+@DisplayName("SES v2 Export Jobs")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class SesExportJobTest {
+
+ private static final String SENDER = "[email protected]";
+
+ private static SesV2Client sesV2;
+ private static VdmAttributes originalVdm;
+ private static String completedJobId;
+
+ @BeforeAll
+ static void setup() {
+ sesV2 = TestFixtures.sesV2Client();
+ GetAccountResponse account = sesV2.getAccount(GetAccountRequest.builder().build());
+ originalVdm = account.vdmAttributes();
+ quietly(() -> sesV2.deleteEmailIdentity(
+ DeleteEmailIdentityRequest.builder().emailIdentity(SENDER).build()));
+ setVdmEnabled(false);
+ sesV2.createEmailIdentity(CreateEmailIdentityRequest.builder().emailIdentity(SENDER).build());
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (sesV2 == null) {
+ return;
+ }
+ quietly(() -> sesV2.deleteEmailIdentity(
+ DeleteEmailIdentityRequest.builder().emailIdentity(SENDER).build()));
+ quietly(SesExportJobTest::restoreVdm);
+ sesV2.close();
+ }
+
+ @Test
+ @Order(1)
+ @DisplayName("the VDM gate is a BadRequest here, not the NotFound the other gated operations use")
+ void vdmGateIsABadRequest() {
+ assertThatThrownBy(() -> sesV2.createExportJob(insightsExport(null)))
+ .isInstanceOf(BadRequestException.class)
+ .hasMessageContaining("Virtual Deliverability Manager");
+ }
+
+ @Test
+ @Order(2)
+ @DisplayName("a caller-chosen destination is refused")
+ void customDestinationIsRefused() {
+ setVdmEnabled(true);
+
+ assertThatThrownBy(() -> sesV2.createExportJob(insightsExport("s3://mine/prefix/")))
+ .isInstanceOf(BadRequestException.class)
+ .hasMessageContaining("Providing a custom S3 URL is not supported");
+ }
+
+ @Test
+ @Order(3)
+ @DisplayName("a job runs to completion and carries a presigned URL and its own data source")
+ void jobCompletesAndCarriesItsDestination() {
+ send();
+ completedJobId = sesV2.createExportJob(insightsExport(null)).jobId();
+ assertThat(completedJobId).isNotBlank();
+
+ GetExportJobResponse job = awaitTerminal(completedJobId);
+
+ assertThat(job.jobStatusAsString()).isEqualTo("COMPLETED");
+ assertThat(job.exportSourceType()).isEqualTo(ExportSourceType.MESSAGE_INSIGHTS);
+ assertThat(job.exportDestination().dataFormat()).isEqualTo(DataFormat.CSV);
+ assertThat(job.exportDestination().s3Url()).isNotBlank();
+ // The union round-trips through the SDK, which is what an integration test asserting raw
+ // JSON cannot show.
+ assertThat(job.exportDataSource().messageInsightsDataSource()).isNotNull();
+ assertThat(job.exportDataSource().metricsDataSource()).isNull();
+ assertThat(job.createdTimestamp()).isNotNull();
+ assertThat(job.completedTimestamp()).isNotNull();
+ }
+
+ @Test
+ @Order(4)
+ @DisplayName("the list reports the job and paginates")
+ void listReportsTheJob() {
+ ListExportJobsResponse all = sesV2.listExportJobs(ListExportJobsRequest.builder()
+ .exportSourceType(ExportSourceType.MESSAGE_INSIGHTS).build());
+ assertThat(all.exportJobs()).isNotEmpty();
+ assertThat(all.exportJobs()).allSatisfy(summary ->
+ assertThat(summary.exportSourceType()).isEqualTo(ExportSourceType.MESSAGE_INSIGHTS));
+
+ ListExportJobsResponse firstPage = sesV2.listExportJobs(
+ ListExportJobsRequest.builder().pageSize(1).build());
+ assertThat(firstPage.exportJobs()).hasSize(1);
+ }
+
+ @Test
+ @Order(5)
+ @DisplayName("a finished job cannot be cancelled")
+ void finishedJobCannotBeCancelled() {
+ assertThatThrownBy(() -> sesV2.cancelExportJob(
+ CancelExportJobRequest.builder().jobId(completedJobId).build()))
+ .isInstanceOf(BadRequestException.class)
+ .hasMessageContaining("Cannot cancel an export job that has a <COMPLETED> status.");
+ }
+
+ @Test
+ @Order(6)
+ @DisplayName("an unknown job id is a BadRequest rather than a NotFound")
+ void unknownJobIsABadRequest() {
+ assertThatThrownBy(() -> sesV2.getExportJob(
+ GetExportJobRequest.builder().jobId("no-such-job").build()))
+ .isInstanceOf(BadRequestException.class)
+ .hasMessageContaining("Invalid JobId");
+ }
+
+ private static CreateExportJobRequest insightsExport(String s3Url) {
+ ExportDestination.Builder destination = ExportDestination.builder().dataFormat(DataFormat.CSV);
+ if (s3Url != null) {
+ destination.s3Url(s3Url);
+ }
+ return CreateExportJobRequest.builder()
+ .exportDataSource(ExportDataSource.builder()
+ .messageInsightsDataSource(MessageInsightsDataSource.builder()
+ .startDate(Instant.now().minus(Duration.ofDays(1)))
+ .endDate(Instant.now().plus(Duration.ofDays(1)))
+ .build())
+ .build())
+ .exportDestination(destination.build())
+ .build();
+ }
+
+ private static GetExportJobResponse awaitTerminal(String jobId) {
+ for (int attempt = 0; attempt < 100; attempt++) {
+ GetExportJobResponse job = sesV2.getExportJob(
+ GetExportJobRequest.builder().jobId(jobId).build());
+ String status = job.jobStatusAsString();
+ if (!"PROCESSING".equals(status) && !"CREATED".equals(status)) {
+ return job;
+ }
+ try {
+ Thread.sleep(50);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError("interrupted while waiting for the job", e);
+ }
+ }
+ throw new AssertionError("export job " + jobId + " never reached a terminal state");
+ }
+
+ private static void send() {
+ sesV2.sendEmail(SendEmailRequest.builder()
+ .fromEmailAddress(SENDER)
+ .destination(Destination.builder().toAddresses("[email protected]").build())
+ .content(EmailContent.builder()
+ .simple(Message.builder()
+ .subject(Content.builder().data("compat-export").build())
+ .body(Body.builder()
+ .text(Content.builder().data("hi").build())
+ .build())
+ .build())
+ .build())
+ .build());
+ }
+
+ private static void setVdmEnabled(boolean enabled) {
+ sesV2.putAccountVdmAttributes(PutAccountVdmAttributesRequest.builder()
+ .vdmAttributes(VdmAttributes.builder()
+ .vdmEnabled(enabled ? "ENABLED" : "DISABLED")
+ .build())
+ .build());
+ }
+
+ private static void restoreVdm() {
+ if (originalVdm == null || originalVdm.vdmEnabled() == null) {
+ setVdmEnabled(false);
+ return;
+ }
+ sesV2.putAccountVdmAttributes(PutAccountVdmAttributesRequest.builder()
+ .vdmAttributes(originalVdm)
+ .build());
+ }
+
+ private static void quietly(Runnable action) {
+ try {
+ action.run();
+ } catch (RuntimeException expected) {
+ // Best effort teardown: the identity or the VDM state may already be gone.
+ }
+ }
+}
diff --git a/docs/services/ses.md b/docs/services/ses.md
index 807f99f83d..63ec1c4fff 100644
--- a/docs/services/ses.md
+++ b/docs/services/ses.md
@@ -379,6 +379,10 @@ Alongside the classic Query API, Floci implements a subset of the SES v2 REST JS
| `POST` | `/v2/email/import-jobs/list` | `ListImportJobs` |
| `GET` | `/v2/email/insights/{MessageId}` | `GetMessageInsights` |
| `POST` | `/v2/email/metrics/batch` | `BatchGetMetricData` |
+| `POST` | `/v2/email/export-jobs` | `CreateExportJob` |
+| `GET` | `/v2/email/export-jobs/{JobId}` | `GetExportJob` |
+| `POST` | `/v2/email/list-export-jobs` | `ListExportJobs` |
+| `PUT` | `/v2/email/export-jobs/{JobId}/cancel` | `CancelExportJob` |
Floci models no leased dedicated IPs: `GetDedicatedIps` is empty and IP-targeted operations return `NotFoundException`, as real AWS does for an account with no leased IPs, with required request members validated first (`BadRequestException`). `PutDedicatedIpPoolScalingAttributes` rejects downgrading a `MANAGED` pool to `STANDARD`, and `PutAccountDedicatedIpWarmupAttributes` stores the flag behind `GetAccount.DedicatedIpAutoWarmupEnabled` (default `true`).
@@ -422,6 +426,8 @@ Suppressed recipients are filtered out of the SMTP relay step (non-suppressed re
`BatchGetMetricData` reads those same stored timelines, so it and `GetMessageInsights` cannot disagree, and it shares the VDM gate. Counts are per recipient, in whole UTC days, and sparse: a day with no data is absent, not zero. `OPEN`, `CLICK` and their delivery variants stay empty because they need a real mailbox, and `TRANSIENT_BOUNCE` because every bounce Floci derives is permanent. Dimensions filter the send: `EMAIL_IDENTITY` on the address or its domain, `CONFIGURATION_SET` on the set it resolved to, `TENANT_NAME` on the tenant it named, and `ISP` only ever on `UNKNOWN_ISP`. Each value is validated with AWS's own rule and message: the configuration set, the tenant and the ISP take ASCII letters, digits, `_` and `-`, and an identity is a bare name or `local@domain`. Where AWS's validation order could not be observed, Floci ranks the member constraints below a duplicate `Id` and the interval cap below the start-before-end check, an absent date fails with a null-constraint message, and a query with two invalid dimension values reports the first in `EMAIL_IDENTITY`, `ISP`, `CONFIGURATION_SET`, `TENANT_NAME` order.
+Export jobs render those same timelines as CSV or JSON. Floci writes the file into its own S3 emulation, under a bucket named `floci-<account>-<region>-ses-export-files`, so the presigned URL on a finished job resolves against Floci. The metrics export emits one row per dimension combination Floci has seen, where real SES emits its whole ISP catalogue with zeros, and a `RATE` is the metric's volume over the send volume for that combination. `LastEngagementEvent` filters select nothing, since Floci derives no opens or clicks. An export finishes in milliseconds, so `CancelExportJob` rarely finds a running job to catch. With S3 authentication enforcement on, the `S3Url` of a finished job answers 403: the shared presigner signs with Floci's own HMAC, which S3 then verifies as SigV4.
+
Import jobs read their source from Floci's own S3 emulation, so the object has to be uploaded there rather than to real S3; the CSV and newline-delimited JSON record formats are AWS's own ([suppression list](https://docs.aws.amazon.com/ses/latest/dg/sending-email-suppression-list.html#sending-email-suppression-list-manual-add-bulk), [contact list](https://docs.aws.amazon.com/ses/latest/dg/sending-email-list-management.html#configuring-list-management-bulk-import)). A job still running when the emulator restarts is marked `FAILED` on the next start. Not emulated: `FailureInfo.FailedRecordsS3Url` (Floci writes no failed-record file), the per-file record limits, the same-region bucket rule, and `ListImportJobs` pagination, which returns every job in one page.
`SendEmail` honors `ListManagementOptions` (`ContactListName`, optional `TopicName`). When present, each recipient is matched against the named contact list and suppressed as a `Bounce` when opted out — reusing the same relay-exclusion and Bounce-event path as suppression-list filtering, matching AWS ("SES will issue a bounce event for a message that is sent to an unsubscribed contact"). A contact is opted out when `UnsubscribeAll` is set; with a `TopicName`, an explicit `OPT_OUT` preference for that topic (or, absent an explicit preference, the topic's `DefaultSubscriptionStatus` being `OPT_OUT`) suppresses; without a `TopicName`, only `UnsubscribeAll` contacts are suppressed. A recipient that is not yet a contact is created on the list automatically (as on AWS) and then evaluated. Referencing a contact list that does not exist fails the send with `NotFoundException`. The topic-default fallback at send time is an intentional deviation — it is not documented by AWS and mirrors the effective-status model AWS uses for `ListContacts`.
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
index a51f50f03c..f1da8511e7 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
@@ -38,6 +38,7 @@ import io.github.hectorvent.floci.services.ses.SesConfigurationSetController;
import io.github.hectorvent.floci.services.ses.SesContactController;
import io.github.hectorvent.floci.services.ses.SesCvetController;
import io.github.hectorvent.floci.services.ses.SesDedicatedIpController;
+import io.github.hectorvent.floci.services.ses.SesExportJobController;
import io.github.hectorvent.floci.services.ses.SesIdentityController;
import io.github.hectorvent.floci.services.ses.SesImportJobController;
import io.github.hectorvent.floci.services.ses.SesInsightsController;
@@ -307,7 +308,8 @@ public class ResolvedServiceCatalog {
Set.of(), Set.of("email", "ses", "sesv2"), Set.of(),
Set.of(SesAccountController.class, SesConfigurationSetController.class,
SesContactController.class, SesCvetController.class,
- SesDedicatedIpController.class, SesIdentityController.class,
+ SesDedicatedIpController.class, SesExportJobController.class,
+ SesIdentityController.class,
SesInsightsController.class, SesMetricsController.class,
SesSendController.class,
SesSuppressionController.class, SesTagController.class,
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesAccountService.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesAccountService.java
index 220d08e575..acd302f72c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ses/SesAccountService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesAccountService.java
@@ -87,10 +87,19 @@ public class SesAccountService {
/** The probe-confirmed gate the VDM-only operations check before reading the request. */
public void requireVdmEnabled(String region) {
+ requireVdmEnabled(region, "NotFoundException", 404);
+ }
+
+ /**
+ * The same gate with the caller's error shape. Message insights and metric data answer
+ * {@code NotFoundException} 404, while export jobs answer {@code BadRequestException} 400 with
+ * the identical sentence, all probe-confirmed.
+ */
+ public void requireVdmEnabled(String region, String errorCode, int status) {
Optional<AccountVdmAttributes> vdm = findAccountVdmAttributes(region);
if (vdm.isEmpty() || !vdm.get().vdmEnabled()) {
- throw new AwsException("NotFoundException",
- "To use this feature you must enable Virtual Deliverability Manager", 404);
+ throw new AwsException(errorCode,
+ "To use this feature you must enable Virtual Deliverability Manager", status);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesExportFilters.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportFilters.java
new file mode 100644
index 0000000000..cd883f9b90
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportFilters.java
@@ -0,0 +1,157 @@
+package io.github.hectorvent.floci.services.ses;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.github.hectorvent.floci.core.common.AwsException;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.function.Function;
+
+import static io.github.hectorvent.floci.services.ses.SesV2Json.stringArrayOrAbsent;
+
+/**
+ * The {@code Include} / {@code Exclude} filters and {@code MaxResults} of a message-insights
+ * export. A row has to match every member present in {@code Include} and no member present in
+ * {@code Exclude}; within one member the values are alternatives.
+ *
+ * <p>{@code TenantName} matches the tenant the send named, which is carried beside the row's
+ * columns because the exported file has no tenant column.
+ *
+ * <p>{@code LastEngagementEvent} filters on opens and clicks, which Floci never derives, so an
+ * include on it selects nothing and an exclude on it removes nothing. That is stated in
+ * {@code docs/services/ses.md} rather than treated as an error, because the request is valid.
+ */
+final class SesExportFilters {
+
+ private static final int NO_LIMIT = -1;
+
+ private SesExportFilters() {}
+
+ static List<SesExportPayloads.InsightsRow> apply(List<SesExportPayloads.InsightsRow> rows,
+ JsonNode insights) {
+ JsonNode include = insights.path("Include");
+ JsonNode exclude = insights.path("Exclude");
+ List<SesExportPayloads.InsightsRow> kept = new ArrayList<>();
+ for (SesExportPayloads.InsightsRow row : rows) {
+ if (include.isObject() && !matches(row, include)) {
+ continue;
+ }
+ if (exclude.isObject() && hasConstraint(exclude) && matches(row, exclude)) {
+ continue;
+ }
+ kept.add(row);
+ }
+ // The service validated the range at create time; this stays defensive so a stored job
+ // written by an older build cannot slice with a negative bound.
+ JsonNode maxResults = insights.path("MaxResults");
+ if (maxResults.isInt() && maxResults.intValue() > 0 && kept.size() > maxResults.intValue()) {
+ return kept.subList(0, maxResults.intValue());
+ }
+ return kept;
+ }
+
+ /**
+ * The filter members, each paired with the value it reads off a row, so validating a request
+ * and filtering its rows can never drift apart.
+ */
+ private enum Member {
+ FROM_EMAIL_ADDRESS("FromEmailAddress", 5, row -> row.columns().get("fromaddress")),
+ DESTINATION("Destination", 5, row -> row.columns().get("destination")),
+ SUBJECT("Subject", 1, row -> row.columns().get("subject")),
+ ISP("Isp", 5, row -> row.columns().get("isp")),
+ LAST_DELIVERY_EVENT("LastDeliveryEvent", 5,
+ row -> row.columns().get("last_delivery_event")),
+ LAST_ENGAGEMENT_EVENT("LastEngagementEvent", 2,
+ row -> row.columns().get("last_engagement_event")),
+ // The tenant filter arrived with the 2026-09 tenant launch and is absent from the pinned
+ // botocore model, so its list has no length to enforce yet.
+ TENANT_NAME("TenantName", NO_LIMIT, SesExportPayloads.InsightsRow::tenantName);
+
+ private final String wireName;
+ private final int maxValues;
+ private final Function<SesExportPayloads.InsightsRow, String> value;
+
+ Member(String wireName, int maxValues,
+ Function<SesExportPayloads.InsightsRow, String> value) {
+ this.wireName = wireName;
+ this.maxValues = maxValues;
+ this.value = value;
+ }
+ }
+
+ /**
+ * Rejects a filter block that is not an object and a member that is not a list of strings,
+ * rather than reading it as no filter at all and quietly returning rows the caller excluded,
+ * then holds each member to the length the model gives it.
+ */
+ static void validate(JsonNode insights) {
+ List<String> violations = new ArrayList<>();
+ requireFilterBlock(insights.path("Include"), "include", violations);
+ requireFilterBlock(insights.path("Exclude"), "exclude", violations);
+ if (!violations.isEmpty()) {
+ String header = violations.size() == 1
+ ? "1 validation error detected: "
+ : violations.size() + " validation errors detected: ";
+ throw new AwsException("BadRequestException",
+ header + String.join("; ", violations), 400);
+ }
+ }
+
+ private static void requireFilterBlock(JsonNode filters, String path, List<String> violations) {
+ if (filters.isMissingNode() || filters.isNull()) {
+ return;
+ }
+ if (!filters.isObject()) {
+ throw new AwsException("SerializationException", null, 400);
+ }
+ for (Member member : Member.values()) {
+ List<String> values = stringArrayOrAbsent(filters, member.wireName);
+ if (values != null && member.maxValues != NO_LIMIT && values.size() > member.maxValues) {
+ violations.add("Value at 'exportDataSource.messageInsightsDataSource." + path + "."
+ + member.wireName.substring(0, 1).toLowerCase(Locale.ROOT)
+ + member.wireName.substring(1)
+ + "' failed to satisfy constraint: Member must have length less than or "
+ + "equal to " + member.maxValues);
+ }
+ }
+ }
+
+ /**
+ * A member with no values constrains nothing, so a block carrying none matches every row. That
+ * reading is harmless for {@code Include} and empties the file for {@code Exclude}, which is
+ * why the exclude side asks first whether there is anything to exclude on.
+ */
+ private static boolean hasConstraint(JsonNode filters) {
+ for (Member member : Member.values()) {
+ JsonNode values = filters.path(member.wireName);
+ if (values.isArray() && !values.isEmpty()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean matches(SesExportPayloads.InsightsRow row, JsonNode filters) {
+ for (Member member : Member.values()) {
+ if (!matchesMember(filters, member.wireName, member.value.apply(row))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean matchesMember(JsonNode filters, String member, String actual) {
+ JsonNode values = filters.path(member);
+ if (!values.isArray() || values.isEmpty()) {
+ return true;
+ }
+ Set<String> wanted = new LinkedHashSet<>();
+ for (JsonNode value : values) {
+ wanted.add(value.asText());
+ }
+ return actual != null && wanted.contains(actual);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobController.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobController.java
new file mode 100644
index 0000000000..1c09e70582
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobController.java
@@ -0,0 +1,171 @@
+package io.github.hectorvent.floci.services.ses;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.PaginatedResult;
+import io.github.hectorvent.floci.core.common.Pagination;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.services.ses.model.ExportJob;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.HttpHeaders;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+
+import java.time.Instant;
+import java.util.Optional;
+
+import static io.github.hectorvent.floci.services.ses.SesV2Json.intMemberOrAbsent;
+import static io.github.hectorvent.floci.services.ses.SesV2Json.readOptionBody;
+import static io.github.hectorvent.floci.services.ses.SesV2Json.stringMemberOrAbsent;
+
+/**
+ * SES V2 export jobs ({@code /v2/email/export-jobs}, {@code /v2/email/list-export-jobs}).
+ *
+ * <p>Probe-confirmed against real SES (2026-09-23): a request that names its own {@code S3Url} is
+ * refused before the Virtual Deliverability Manager gate is reached, which is why the gate is
+ * passed into the service rather than checked first as it is for message insights and metric data.
+ * The gate itself answers {@code BadRequestException} 400 here, where those two answer
+ * {@code NotFoundException} 404 with the same sentence.
+ */
+@Path("/v2/email")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public class SesExportJobController {
+
+ private static final int DEFAULT_PAGE_SIZE = 100;
+ /**
+ * A page no emulator will ever fill, so a larger {@code PageSize} can be clamped instead of
+ * refused. The paginator adds this to the page offset as an int, so {@code Integer.MAX_VALUE}
+ * would overflow that sum and report a further page that does not exist.
+ */
+ private static final int MAX_PAGE_SIZE = 1_000_000;
+
+ private final SesExportJobService exportJobService;
+ private final SesAccountService accountService;
+ private final RegionResolver regionResolver;
+ private final ObjectMapper objectMapper;
+
+ @Inject
+ public SesExportJobController(SesExportJobService exportJobService,
+ SesAccountService accountService,
+ RegionResolver regionResolver, ObjectMapper objectMapper) {
+ this.exportJobService = exportJobService;
+ this.accountService = accountService;
+ this.regionResolver = regionResolver;
+ this.objectMapper = objectMapper;
+ }
+
+ @POST
+ @Path("/export-jobs")
+ public Response createExportJob(@Context HttpHeaders headers, String body) {
+ String region = regionResolver.resolveRegion(headers);
+ JsonNode request = readOptionBody(objectMapper, body);
+ JsonNode destination = request.path("ExportDestination");
+ JsonNode dataSource = request.path("ExportDataSource");
+ ExportJob job = exportJobService.createExportJob(region, regionResolver.getAccountId(),
+ dataSource, destination,
+ () -> accountService.requireVdmEnabled(region, "BadRequestException", 400));
+ ObjectNode result = objectMapper.createObjectNode();
+ result.put("JobId", job.getJobId());
+ return Response.ok(result).build();
+ }
+
+ @GET
+ @Path("/export-jobs/{jobId}")
+ public Response getExportJob(@Context HttpHeaders headers, @PathParam("jobId") String jobId) {
+ String region = regionResolver.resolveRegion(headers);
+ ExportJob job = exportJobService.getExportJob(region, jobId);
+ ObjectNode result = objectMapper.createObjectNode();
+ result.put("JobId", job.getJobId());
+ result.put("ExportSourceType", job.getExportSourceType());
+ result.put("JobStatus", job.getJobStatus());
+
+ ObjectNode destination = result.putObject("ExportDestination");
+ destination.put("DataFormat", job.getDataFormat());
+ exportJobService.presignedUrl(job).ifPresent(url -> destination.put("S3Url", url));
+ result.set("ExportDataSource", readOptionBody(objectMapper, job.getDataSource()));
+
+ putTimestamp(result, "CreatedTimestamp", job.getCreatedTimestamp());
+ putTimestamp(result, "CompletedTimestamp", job.getCompletedTimestamp());
+ if (job.getProcessedRecordsCount() != null) {
+ // AWS returns only ProcessedRecordsCount, never ExportedRecordsCount, so neither does
+ // Floci even though the model carries both.
+ result.putObject("Statistics")
+ .put("ProcessedRecordsCount", job.getProcessedRecordsCount());
+ }
+ if (job.getErrorMessage() != null) {
+ result.putObject("FailureInfo").put("ErrorMessage", job.getErrorMessage());
+ }
+ return Response.ok(result).build();
+ }
+
+ @POST
+ @Path("/list-export-jobs")
+ public Response listExportJobs(@Context HttpHeaders headers, String body) {
+ String region = regionResolver.resolveRegion(headers);
+ JsonNode request = readOptionBody(objectMapper, body);
+ String sourceType = stringMemberOrAbsent(request, "ExportSourceType");
+ String jobStatus = stringMemberOrAbsent(request, "JobStatus");
+ String nextToken = stringMemberOrAbsent(request, "NextToken");
+ // PageSize carries no bound in the model, so a large one is clamped rather than refused;
+ // only a value below 1 is an error, and the default applies when it is absent.
+ Integer pageSize = intMemberOrAbsent(request, "PageSize");
+ if (pageSize != null && pageSize > MAX_PAGE_SIZE) {
+ pageSize = MAX_PAGE_SIZE;
+ }
+
+ PaginatedResult<ExportJob> page = Pagination.paginate(
+ exportJobService.listExportJobs(region, sourceType, jobStatus),
+ SesExportJobController::cursor, pageSize, nextToken, DEFAULT_PAGE_SIZE,
+ MAX_PAGE_SIZE, "BadRequestException");
+
+ ObjectNode result = objectMapper.createObjectNode();
+ ArrayNode jobs = result.putArray("ExportJobs");
+ for (ExportJob job : page.items()) {
+ ObjectNode node = jobs.addObject();
+ node.put("JobId", job.getJobId());
+ node.put("ExportSourceType", job.getExportSourceType());
+ node.put("JobStatus", job.getJobStatus());
+ putTimestamp(node, "CreatedTimestamp", job.getCreatedTimestamp());
+ putTimestamp(node, "CompletedTimestamp", job.getCompletedTimestamp());
+ }
+ if (page.nextToken() != null) {
+ result.put("NextToken", page.nextToken());
+ }
+ return Response.ok(result).build();
+ }
+
+ @PUT
+ @Path("/export-jobs/{jobId}/cancel")
+ public Response cancelExportJob(@Context HttpHeaders headers, @PathParam("jobId") String jobId) {
+ String region = regionResolver.resolveRegion(headers);
+ exportJobService.cancelExportJob(region, jobId);
+ return Response.ok(objectMapper.createObjectNode()).build();
+ }
+
+ /**
+ * The shared paginator orders by the cursor ascending, and AWS returns the newest job first, so
+ * the cursor counts time backwards. The job id breaks ties between jobs created in the same
+ * millisecond.
+ */
+ private static String cursor(ExportJob job) {
+ Instant created = job.getCreatedTimestamp();
+ long descending = Long.MAX_VALUE - (created == null ? 0L : created.toEpochMilli());
+ return String.format("%019d#%s", descending, job.getJobId());
+ }
+
+ private static void putTimestamp(ObjectNode node, String field, Instant value) {
+ Optional.ofNullable(value)
+ .ifPresent(instant -> node.put(field, instant.toEpochMilli() / 1000.0));
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobService.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobService.java
new file mode 100644
index 0000000000..522a4de558
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobService.java
@@ -0,0 +1,607 @@
+package io.github.hectorvent.floci.services.ses;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.core.common.Resettable;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.s3.PreSignedUrlGenerator;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.ses.model.ExportJob;
+import io.github.hectorvent.floci.services.ses.model.SentEmail;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static io.github.hectorvent.floci.services.ses.SesV2Json.intMemberOrAbsent;
+import static io.github.hectorvent.floci.services.ses.SesV2Json.stringMemberOrAbsent;
+
+/**
+ * SES V2 export jobs. A job reads the timelines already stored at send time, renders them through
+ * {@link SesExportPayloads} and writes one object, which {@code GetExportJob} hands back as a
+ * presigned URL.
+ *
+ * <p>Probe-confirmed against real SES (2026-09-23, us-east-1): the caller cannot choose the
+ * destination, a request carrying {@code S3Url} is refused before anything else is checked, and
+ * exactly one data source must be present. The metrics source insists on whole UTC days where the
+ * message-insights source does not. A job is already {@code PROCESSING} on the first read, so
+ * {@code CREATED} is never observed from outside.
+ *
+ * <p>Deviations, all documented in {@code docs/services/ses.md}: real SES writes into a bucket it
+ * owns, while Floci writes into its own S3 emulation so the presigned URL resolves; the metrics
+ * export lists only the dimension values Floci has seen rather than AWS's fixed ISP catalogue; and
+ * the concurrent-job ceiling is the published quota of 20, since the probe hit AWS's own
+ * limit before it could measure one.
+ */
+@ApplicationScoped
+public class SesExportJobService implements Resettable {
+
+ private static final Logger LOG = Logger.getLogger(SesExportJobService.class);
+
+ // The published quota, the same figure SES gives import jobs:
+ // https://docs.aws.amazon.com/ses/latest/dg/quotas.html
+ static final int MAX_CONCURRENT_JOBS = 20;
+ static final int MAX_RESULTS = 10_000;
+ static final long RESET_DRAIN_TIMEOUT_MILLIS = 5_000L;
+ private static final long RESET_ADMISSION_WAIT_MILLIS = 30_000L;
+ static final String RESTART_FAILURE_MESSAGE = "The emulator restarted before the export finished";
+ static final String EXPORT_BUCKET_SUFFIX = "-ses-export-files";
+
+ private static final Set<String> DATA_FORMATS = Set.of(ExportJob.FORMAT_CSV, ExportJob.FORMAT_JSON);
+ private static final Set<String> SOURCE_TYPES =
+ Set.of(ExportJob.SOURCE_METRICS, ExportJob.SOURCE_MESSAGE_INSIGHTS);
+ private static final Set<String> JOB_STATUSES = Set.of(ExportJob.STATUS_CREATED,
+ ExportJob.STATUS_PROCESSING, ExportJob.STATUS_COMPLETED, ExportJob.STATUS_FAILED,
+ ExportJob.STATUS_CANCELLED);
+ private static final DateTimeFormatter OBJECT_STAMP =
+ DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss", Locale.ROOT).withZone(ZoneOffset.UTC);
+ private static final int PRESIGNED_URL_EXPIRY_SECONDS = 3600;
+
+ private final StorageBackend<String, ExportJob> exportJobStore;
+ private final SesSentEmailService sentEmailService;
+ private final S3Service s3Service;
+ private final PreSignedUrlGenerator preSignedUrlGenerator;
+ private final ObjectMapper objectMapper;
+ private final Clock clock;
+ private final Executor workers;
+ private final String baseUrl;
+ private final Object createLock = new Object();
+ // Reset support, following SesImportJobService: beforeReset bumps the generation so a running
+ // worker stops before it writes its object, then waits for the workers already inside one. The
+ // wait is bounded, so a reset still returns when a worker is wedged.
+ private final AtomicInteger generation = new AtomicInteger();
+ private boolean resetting;
+ private final Object workerMonitor = new Object();
+ private int activeWorkers;
+
+ @Inject
+ public SesExportJobService(StorageFactory storageFactory, SesSentEmailService sentEmailService,
+ S3Service s3Service, PreSignedUrlGenerator preSignedUrlGenerator,
+ ObjectMapper objectMapper, Clock clock, EmulatorConfig config) {
+ this(storageFactory.create("ses", "ses-export-jobs.json",
+ new TypeReference<Map<String, ExportJob>>() {}),
+ sentEmailService, s3Service, preSignedUrlGenerator, objectMapper, clock,
+ task -> Thread.ofVirtual().start(task), config.effectiveBaseUrl());
+ }
+
+ SesExportJobService(StorageBackend<String, ExportJob> exportJobStore,
+ SesSentEmailService sentEmailService, S3Service s3Service,
+ PreSignedUrlGenerator preSignedUrlGenerator, ObjectMapper objectMapper,
+ Clock clock, Executor workers, String baseUrl) {
+ this.exportJobStore = exportJobStore;
+ this.sentEmailService = sentEmailService;
+ this.s3Service = s3Service;
+ this.preSignedUrlGenerator = preSignedUrlGenerator;
+ this.objectMapper = objectMapper;
+ this.clock = clock;
+ this.workers = workers;
+ this.baseUrl = baseUrl;
+ recoverInterruptedJobs();
+ }
+
+ /**
+ * The request's already-parsed {@code ExportDataSource} and {@code ExportDestination}. The
+ * controller keeps the JSON shape checks; the service owns the order the service-level rules
+ * fire in.
+ */
+ public ExportJob createExportJob(String region, String accountId, JsonNode exportDataSource,
+ JsonNode exportDestination, Runnable vdmGate) {
+ JsonNode metrics = objectMemberOrAbsent(exportDataSource, "MetricsDataSource");
+ JsonNode insights = objectMemberOrAbsent(exportDataSource, "MessageInsightsDataSource");
+ if (metrics.isObject() == insights.isObject()) {
+ throw new AwsException("BadRequestException", "One data source must be provided", 400);
+ }
+ if (!exportDestination.isObject()) {
+ throw notNull("exportDestination");
+ }
+ String dataFormat = stringMemberOrAbsent(exportDestination, "DataFormat");
+ if (dataFormat == null) {
+ throw notNull("exportDestination.dataFormat");
+ }
+ if (!DATA_FORMATS.contains(dataFormat)) {
+ throw new AwsException("BadRequestException",
+ "1 validation error detected: Value at 'exportDestination.dataFormat' failed to "
+ + "satisfy constraint: Member must satisfy enum value set: [CSV, JSON]", 400);
+ }
+ // Probe-confirmed order: the data source and the destination's own constraints are checked
+ // first, then the refusal of a caller-chosen destination, then the VDM gate.
+ if (exportDestination.hasNonNull("S3Url")) {
+ throw new AwsException("BadRequestException",
+ "Providing a custom S3 URL is not supported", 400);
+ }
+ vdmGate.run();
+ if (metrics.isObject()) {
+ SesExportMetrics.validate(metrics);
+ } else {
+ SesExportFilters.validate(insights);
+ requireMaxResults(insights);
+ }
+
+ JsonNode source = metrics.isObject() ? metrics : insights;
+ Instant start = requireDate(source, "StartDate");
+ Instant end = requireDate(source, "EndDate");
+ if (!start.isBefore(end)) {
+ throw new AwsException("BadRequestException", "Invalid date range", 400);
+ }
+ if (metrics.isObject() && (!isMidnight(start) || !isMidnight(end))) {
+ throw new AwsException("BadRequestException",
+ "To get daily aggregated data you must not specify partial-day timestamps. "
+ + "Please make your interval go from midnight to midnight UTC.", 400);
+ }
+
+ ExportJob job = new ExportJob();
+ job.setJobId(UUID.randomUUID().toString());
+ job.setRegion(region);
+ job.setAccountId(accountId);
+ job.setExportSourceType(metrics.isObject()
+ ? ExportJob.SOURCE_METRICS : ExportJob.SOURCE_MESSAGE_INSIGHTS);
+ job.setDataFormat(dataFormat);
+ job.setDataSource(exportDataSource.toString());
+ job.setJobStatus(ExportJob.STATUS_PROCESSING);
+ job.setCreatedTimestamp(Instant.now(clock));
+ return start(job, accountId);
+ }
+
+ private ExportJob start(ExportJob job, String accountId) {
+ synchronized (createLock) {
+ awaitResetFinished();
+ long active = exportJobStore.scan(k -> k.startsWith(keyPrefix(job.getRegion()))).stream()
+ .filter(candidate -> !candidate.isTerminal())
+ .count();
+ if (active >= MAX_CONCURRENT_JOBS) {
+ throw new AwsException("LimitExceededException",
+ "There are too many <" + job.getExportSourceType() + "> export jobs running"
+ + " - either cancel one of the jobs or wait for it to complete"
+ + " before resubmitting.", 400);
+ }
+ exportJobStore.put(key(job.getRegion(), job.getJobId()), job);
+ // Persist and dispatch under the lock beforeReset takes, so a job is either admitted
+ // before a reset and then abandoned by its stale generation, or admitted after it.
+ int startedIn = generation.get();
+ // The worker runs on a fresh thread, where account-aware storage would otherwise fall
+ // back to the default account and never find this job again.
+ workers.execute(() -> RequestScopes.runAs(accountId,
+ () -> run(job.getRegion(), job.getJobId(), startedIn)));
+ }
+ return job;
+ }
+
+ // Holds createLock. A create arriving while a reset drains would capture the already-bumped
+ // generation, so its worker would write an object after the wipe. It waits for afterReset.
+ private void awaitResetFinished() {
+ long deadline = System.currentTimeMillis() + RESET_ADMISSION_WAIT_MILLIS;
+ while (resetting) {
+ long remaining = deadline - System.currentTimeMillis();
+ if (remaining <= 0) {
+ throw resettingNow();
+ }
+ try {
+ createLock.wait(remaining);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw resettingNow();
+ }
+ }
+ }
+
+ private static AwsException resettingNow() {
+ return new AwsException("TooManyRequestsException",
+ "Export jobs are not accepted while the emulator is resetting.", 429);
+ }
+
+ private void run(String region, String jobId, int startedIn) {
+ synchronized (workerMonitor) {
+ activeWorkers++;
+ }
+ try {
+ runJob(region, jobId, startedIn);
+ } finally {
+ synchronized (workerMonitor) {
+ activeWorkers--;
+ workerMonitor.notifyAll();
+ }
+ }
+ }
+
+ private void runJob(String region, String jobId, int startedIn) {
+ try {
+ Optional<ExportJob> current = exportJobStore.get(key(region, jobId));
+ if (current.isEmpty() || current.get().isTerminal()) {
+ // Cancelled between the create and the worker starting.
+ return;
+ }
+ ExportJob job = current.get();
+ long processed = processedRecords(job);
+ byte[] payload = render(job);
+ if (generation.get() != startedIn) {
+ // A reset wiped the stores while this job was rendering. Writing the object now
+ // would put a bucket back that the reset removed.
+ LOG.infov("SES export job {0} abandoned by an emulator reset", jobId);
+ return;
+ }
+ String bucket = ensureExportBucket(job);
+ String objectKey = objectKey(job);
+ s3Service.putObject(bucket, objectKey, payload, contentType(job.getDataFormat()),
+ Map.of());
+ if (!finish(region, jobId, ExportJob.STATUS_COMPLETED, objectKey, null, processed)) {
+ // Cancelled while the object was being written. The job will never point at it,
+ // so the object goes too rather than sitting in the bucket unreachable.
+ s3Service.deleteObject(bucket, objectKey);
+ }
+ } catch (RuntimeException e) {
+ LOG.warnv("SES export job {0} failed: {1}", jobId, e.getMessage());
+ finish(region, jobId, ExportJob.STATUS_FAILED, null, e.getMessage(), null);
+ }
+ }
+
+ /** False when the job already reached a terminal state, which means a cancel won the race. */
+ private boolean finish(String region, String jobId, String status, String objectKey,
+ String error, Long processedRecords) {
+ synchronized (createLock) {
+ Optional<ExportJob> current = exportJobStore.get(key(region, jobId));
+ if (current.isEmpty() || current.get().isTerminal()) {
+ return false;
+ }
+ ExportJob job = current.get();
+ job.setJobStatus(status);
+ job.setObjectKey(objectKey);
+ job.setErrorMessage(error);
+ job.setProcessedRecordsCount(processedRecords);
+ job.setCompletedTimestamp(Instant.now(clock));
+ exportJobStore.put(key(region, jobId), job);
+ return true;
+ }
+ }
+
+ /**
+ * Probe-confirmed: a metrics job and a message-insights job over one window report the same
+ * ProcessedRecordsCount, so the number counts the stored rows the job examined rather than the
+ * rows it wrote.
+ */
+ private long processedRecords(ExportJob job) {
+ JsonNode source = readDataSource(job);
+ boolean metrics = ExportJob.SOURCE_METRICS.equals(job.getExportSourceType());
+ JsonNode window = metrics
+ ? source.path("MetricsDataSource") : source.path("MessageInsightsDataSource");
+ return SesExportPayloads.insightsRows(inWindow(sentEmailService.listInRegion(job.getRegion()),
+ date(window, "StartDate"), date(window, "EndDate"), !metrics)).size();
+ }
+
+ private byte[] render(ExportJob job) {
+ JsonNode source = readDataSource(job);
+ if (ExportJob.SOURCE_MESSAGE_INSIGHTS.equals(job.getExportSourceType())) {
+ JsonNode insights = source.path("MessageInsightsDataSource");
+ List<SentEmail> emails = inWindow(sentEmailService.listInRegion(job.getRegion()),
+ date(insights, "StartDate"), date(insights, "EndDate"), true);
+ List<SesExportPayloads.InsightsRow> rows =
+ SesExportFilters.apply(SesExportPayloads.insightsRows(emails), insights);
+ return ExportJob.FORMAT_CSV.equals(job.getDataFormat())
+ ? SesExportPayloads.insightsCsv(rows)
+ : SesExportPayloads.insightsJson(rows);
+ }
+ JsonNode metrics = source.path("MetricsDataSource");
+ // Windowed here so a dimension value seen only outside the range cannot produce a row.
+ List<SentEmail> emails = inWindow(sentEmailService.listInRegion(job.getRegion()),
+ date(metrics, "StartDate"), date(metrics, "EndDate"), false);
+ return SesExportMetrics.render(emails, metrics, job.getDataFormat());
+ }
+
+ private JsonNode readDataSource(ExportJob job) {
+ try {
+ return objectMapper.readTree(job.getDataSource());
+ } catch (JsonProcessingException e) {
+ throw new AwsException("BadRequestException", e.getMessage(), 400);
+ }
+ }
+
+ /**
+ * The model documents the message-insights end date as inclusive, while the metrics source
+ * aggregates whole UTC days and ends exclusively, as the {@code BatchGetMetricData} probe
+ * showed. A send with no timestamp is kept, as the batch query keeps it.
+ */
+ private static List<SentEmail> inWindow(List<SentEmail> emails, Instant start, Instant end,
+ boolean endInclusive) {
+ List<SentEmail> kept = new ArrayList<>();
+ for (SentEmail email : emails) {
+ Instant sentAt = email.getSentAt();
+ if (sentAt == null) {
+ kept.add(email);
+ continue;
+ }
+ boolean beforeEnd = endInclusive ? !sentAt.isAfter(end) : sentAt.isBefore(end);
+ if (!sentAt.isBefore(start) && beforeEnd) {
+ kept.add(email);
+ }
+ }
+ return kept;
+ }
+
+ public ExportJob getExportJob(String region, String jobId) {
+ return exportJobStore.get(key(region, jobId))
+ .orElseThrow(() -> new AwsException("BadRequestException", "Invalid JobId", 400));
+ }
+
+ /**
+ * The presigned URL a finished job's {@code ExportDestination} carries. Empty while the job is
+ * still running or once it failed, since there is no object to point at.
+ */
+ public Optional<String> presignedUrl(ExportJob job) {
+ if (job.getObjectKey() == null) {
+ return Optional.empty();
+ }
+ return Optional.of(preSignedUrlGenerator.generatePresignedUrl(baseUrl,
+ exportBucket(job.getAccountId(), job.getRegion()), job.getObjectKey(), "GET",
+ PRESIGNED_URL_EXPIRY_SECONDS));
+ }
+
+ public List<ExportJob> listExportJobs(String region, String sourceType, String jobStatus) {
+ if (sourceType != null && !SOURCE_TYPES.contains(sourceType)) {
+ throw new AwsException("BadRequestException",
+ "1 validation error detected: Value at 'exportSourceType' failed to satisfy "
+ + "constraint: Member must satisfy enum value set: "
+ + "[METRICS_DATA, MESSAGE_INSIGHTS]", 400);
+ }
+ if (jobStatus != null && !JOB_STATUSES.contains(jobStatus)) {
+ throw new AwsException("BadRequestException",
+ "1 validation error detected: Value at 'jobStatus' failed to satisfy "
+ + "constraint: Member must satisfy enum value set: "
+ + "[CREATED, PROCESSING, COMPLETED, FAILED, CANCELLED]", 400);
+ }
+ return exportJobStore.scan(k -> k.startsWith(keyPrefix(region))).stream()
+ .filter(job -> sourceType == null || sourceType.equals(job.getExportSourceType()))
+ .filter(job -> jobStatus == null || jobStatus.equals(job.getJobStatus()))
+ // Newest first, as AWS returns them.
+ .sorted(Comparator.comparing(ExportJob::getCreatedTimestamp).reversed()
+ .thenComparing(ExportJob::getJobId))
+ .toList();
+ }
+
+ public void cancelExportJob(String region, String jobId) {
+ synchronized (createLock) {
+ ExportJob job = getExportJob(region, jobId);
+ if (job.isTerminal()) {
+ throw new AwsException("BadRequestException",
+ "Cannot cancel an export job that has a <" + job.getJobStatus()
+ + "> status.", 400);
+ }
+ job.setJobStatus(ExportJob.STATUS_CANCELLED);
+ job.setCompletedTimestamp(Instant.now(clock));
+ exportJobStore.put(key(region, jobId), job);
+ }
+ }
+
+ /**
+ * Runs from the constructor, where no request scope is active, so the store has to be read
+ * across accounts: a plain scan would only ever sweep the default account's jobs.
+ */
+ private void recoverInterruptedJobs() {
+ if (!(exportJobStore instanceof AccountAwareStorageBackend<ExportJob> jobs)) {
+ return;
+ }
+ for (AccountAwareStorageBackend.AccountEntry<ExportJob> entry
+ : jobs.scanAllAccountEntries(k -> k.startsWith("export-job::"))) {
+ ExportJob job = entry.value();
+ if (job.isTerminal()) {
+ continue;
+ }
+ job.setJobStatus(ExportJob.STATUS_FAILED);
+ job.setErrorMessage(RESTART_FAILURE_MESSAGE);
+ job.setCompletedTimestamp(Instant.now(clock));
+ jobs.putForAccount(entry.accountId(), entry.key(), job);
+ LOG.warnv("SES export job {0} was interrupted by a restart and is now FAILED",
+ job.getJobId());
+ }
+ }
+
+ /**
+ * Stops the workers before the stores are wiped. Without this a worker outlived the reset and
+ * recreated the export bucket and object the wipe had just removed.
+ */
+ @Override
+ public void beforeReset() {
+ synchronized (createLock) {
+ resetting = true;
+ generation.incrementAndGet();
+ }
+ boolean interrupted = false;
+ long deadline = System.currentTimeMillis() + RESET_DRAIN_TIMEOUT_MILLIS;
+ synchronized (workerMonitor) {
+ while (activeWorkers > 0) {
+ long remaining = deadline - System.currentTimeMillis();
+ if (remaining <= 0) {
+ LOG.warnv("{0} SES export worker(s) still running after {1}ms; one object may "
+ + "be written after the reset", activeWorkers, RESET_DRAIN_TIMEOUT_MILLIS);
+ break;
+ }
+ try {
+ workerMonitor.wait(remaining);
+ } catch (InterruptedException e) {
+ // Keep draining to the deadline; the interrupt is re-asserted on the way out.
+ interrupted = true;
+ }
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ @Override
+ public void afterReset() {
+ synchronized (createLock) {
+ resetting = false;
+ createLock.notifyAll();
+ }
+ }
+
+ @Override
+ public void clear() {
+ exportJobStore.clear();
+ LOG.info("Cleared all SES export jobs");
+ }
+
+ /**
+ * Two workers can reach an empty region together, and outside us-east-1 the second
+ * {@code createBucket} answers {@code BucketAlreadyOwnedByYou}, which would fail that export.
+ */
+ private String ensureExportBucket(ExportJob job) {
+ String bucket = exportBucket(job.getAccountId(), job.getRegion());
+ if (s3Service.bucketExists(bucket)) {
+ return bucket;
+ }
+ try {
+ s3Service.createBucket(bucket, job.getRegion());
+ } catch (AwsException e) {
+ if (!"BucketAlreadyOwnedByYou".equals(e.getErrorCode())) {
+ throw e;
+ }
+ LOG.debugv("Export bucket {0} was created by a concurrent export job", bucket);
+ }
+ return bucket;
+ }
+
+ /**
+ * The account is part of the name because an object write resolves its bucket across accounts
+ * when {@code global-bucket-namespace} is on, and would then land one account's export in
+ * another account's partition, where its owner could read it.
+ */
+ static String exportBucket(String accountId, String region) {
+ return "floci-" + accountId + "-" + region + EXPORT_BUCKET_SUFFIX;
+ }
+
+ /**
+ * AWS names the two payloads differently, and the shape of each name is probe-confirmed.
+ */
+ private String objectKey(ExportJob job) {
+ String stamp = OBJECT_STAMP.format(Instant.now(clock));
+ String extension = ExportJob.FORMAT_CSV.equals(job.getDataFormat()) ? "csv" : "json";
+ if (ExportJob.SOURCE_METRICS.equals(job.getExportSourceType())) {
+ return "ses-metrics-" + stamp + "-" + job.getJobId() + "." + extension;
+ }
+ return stamp + "_" + job.getJobId().replace("-", "").substring(0, 10) + "." + extension;
+ }
+
+ private static String contentType(String dataFormat) {
+ return ExportJob.FORMAT_CSV.equals(dataFormat) ? "text/csv" : "application/json";
+ }
+
+ private static AwsException notNull(String member) {
+ return new AwsException("BadRequestException",
+ "1 validation error detected: Value at '" + member
+ + "' failed to satisfy constraint: Member must not be null", 400);
+ }
+
+ /**
+ * A union member of the wrong JSON type read as absent, so a request carrying a valid source
+ * plus a malformed one passed the exactly-one check and silently exported the valid half.
+ */
+ private static JsonNode objectMemberOrAbsent(JsonNode parent, String field) {
+ JsonNode member = parent.path(field);
+ if (!member.isMissingNode() && !member.isNull() && !member.isObject()) {
+ throw new AwsException("SerializationException", null, 400);
+ }
+ return member;
+ }
+
+ /**
+ * The model bounds {@code MaxResults} at 1 to 10000. An unchecked negative value reached
+ * {@code subList} inside the filter and surfaced as a 500.
+ */
+ private static void requireMaxResults(JsonNode insights) {
+ // intMemberOrAbsent answers SerializationException for a value that is not an integer, so
+ // a wrong type is not reported as a value out of range.
+ Integer maxResults = intMemberOrAbsent(insights, "MaxResults");
+ if (maxResults == null) {
+ return;
+ }
+ if (maxResults < 1 || maxResults > MAX_RESULTS) {
+ throw new AwsException("BadRequestException",
+ "1 validation error detected: Value at "
+ + "'exportDataSource.messageInsightsDataSource.maxResults' failed to "
+ + "satisfy constraint: Member must be between 1 and " + MAX_RESULTS
+ + ", inclusive", 400);
+ }
+ }
+
+ private static Instant requireDate(JsonNode source, String field) {
+ Instant value = date(source, field);
+ if (value == null) {
+ throw new AwsException("BadRequestException",
+ "1 validation error detected: Value at '" + field.substring(0, 1).toLowerCase(Locale.ROOT)
+ + field.substring(1) + "' failed to satisfy constraint: "
+ + "Member must not be null", 400);
+ }
+ return value;
+ }
+
+ private static Instant date(JsonNode source, String field) {
+ JsonNode node = source.path(field);
+ if (node.isNumber()) {
+ return Instant.ofEpochMilli(Math.round(node.doubleValue() * 1000.0));
+ }
+ if (node.isTextual()) {
+ try {
+ return Instant.parse(node.asText());
+ } catch (DateTimeParseException e) {
+ throw new AwsException("SerializationException", e.getMessage(), 400);
+ }
+ }
+ return null;
+ }
+
+ private static boolean isMidnight(Instant instant) {
+ return instant.equals(instant.truncatedTo(ChronoUnit.DAYS));
+ }
+
+ private static String keyPrefix(String region) {
+ return "export-job::" + region + "::";
+ }
+
+ private static String key(String region, String jobId) {
+ return keyPrefix(region) + jobId;
+ }
+
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesExportMetrics.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportMetrics.java
new file mode 100644
index 0000000000..d82db629f2
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportMetrics.java
@@ -0,0 +1,321 @@
+package io.github.hectorvent.floci.services.ses;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.ses.model.EmailInsights;
+import io.github.hectorvent.floci.services.ses.model.ExportJob;
+import io.github.hectorvent.floci.services.ses.model.SentEmail;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import static io.github.hectorvent.floci.services.ses.SesV2Json.stringArrayOrAbsent;
+import static io.github.hectorvent.floci.services.ses.SesV2Json.stringMemberOrAbsent;
+
+/**
+ * The metrics half of an export job. Every count comes from {@link SesMetricData}, the same
+ * aggregation {@code BatchGetMetricData} serves, so an export and a query of the same window
+ * cannot disagree.
+ *
+ * <p>One deviation, documented in {@code docs/services/ses.md}: real SES emits a row for every ISP
+ * it knows, zeros included, which is a catalogue Floci has no honest way to reproduce. Floci emits
+ * a row per dimension combination it has actually seen. A `RATE` aggregation is the metric's volume
+ * over the send volume for the same combination, which matches the definition AWS documents; the
+ * probe could not confirm it, because the probed account's rate export carried no data.
+ */
+final class SesExportMetrics {
+
+ private static final String PATH = "exportDataSource.metricsDataSource.";
+ private static final String METRIC_ENUM =
+ "[DELIVERY_COMPLAINT, CLICK, SEND, OPEN, COMPLAINT, DELIVERY, DELIVERY_OPEN, "
+ + "PERMANENT_BOUNCE, DELIVERY_CLICK, TRANSIENT_BOUNCE]";
+ private static final String DIMENSION_ENUM =
+ "[EMAIL_IDENTITY, CONFIGURATION_SET, ISP, TENANT_NAME]";
+ private static final String AGGREGATION_ENUM = "[RATE, VOLUME]";
+ private static final int MAX_DIMENSIONS = 3;
+ private static final int MAX_DIMENSION_VALUES = 10;
+ private static final int MAX_METRICS = 10;
+
+ private SesExportMetrics() {}
+
+ /**
+ * The model's constraints on a metrics data source: {@code Dimensions} (1 to 3 known names,
+ * each with 1 to 10 values), {@code Namespace} and {@code Metrics} (1 to 10) are required, and
+ * a metric's {@code Name} and {@code Aggregation} are checked against their enums when present,
+ * which is as far as the model goes: neither member is required, and an absent
+ * {@code Aggregation} keeps its {@code VOLUME} default. Without this an unknown dimension was
+ * dropped, an unknown metric aggregated to zero and an unrecognised aggregation was read as
+ * {@code VOLUME}, so a malformed request produced a successful, meaningless export.
+ *
+ * <p>The probe never sent a malformed source, so the wording and the position in the sequence
+ * are Floci's, chosen to match the sibling {@code BatchGetMetricData}. Dimension values are
+ * deliberately not checked: {@code *} is a legal value here and the export probe never showed
+ * the per-value rules the batch operation applies.
+ */
+ static void validate(JsonNode metrics) {
+ requireWireTypes(metrics);
+ List<String> violations = new ArrayList<>();
+ validateDimensions(metrics.path("Dimensions"), violations);
+ JsonNode namespace = metrics.path("Namespace");
+ if (!namespace.isTextual() || !"VDM".equals(namespace.textValue())) {
+ violations.add(constraint("namespace",
+ "Member must satisfy enum value set: [VDM]"));
+ }
+ validateMetrics(metrics.path("Metrics"), violations);
+ if (!violations.isEmpty()) {
+ String header = violations.size() == 1
+ ? "1 validation error detected: "
+ : violations.size() + " validation errors detected: ";
+ throw new AwsException("BadRequestException",
+ header + String.join("; ", violations), 400);
+ }
+ }
+
+ /**
+ * The deserialization-level checks, which come first and answer {@code SerializationException}
+ * like every other typed SES v2 member: a dimension value list of numbers was otherwise
+ * coerced to strings, and a metric that is a bare string was read as one with no members at
+ * all, both producing a successful but meaningless export.
+ */
+ private static void requireWireTypes(JsonNode metrics) {
+ JsonNode dimensions = metrics.path("Dimensions");
+ if (present(dimensions)) {
+ if (!dimensions.isObject()) {
+ throw serializationError();
+ }
+ Iterator<String> names = dimensions.fieldNames();
+ while (names.hasNext()) {
+ stringArrayOrAbsent(dimensions, names.next());
+ }
+ }
+ stringMemberOrAbsent(metrics, "Namespace");
+ JsonNode requested = metrics.path("Metrics");
+ if (present(requested)) {
+ if (!requested.isArray()) {
+ throw serializationError();
+ }
+ for (JsonNode metric : requested) {
+ if (!metric.isObject()) {
+ throw serializationError();
+ }
+ stringMemberOrAbsent(metric, "Name");
+ stringMemberOrAbsent(metric, "Aggregation");
+ }
+ }
+ }
+
+ private static AwsException serializationError() {
+ return new AwsException("SerializationException", null, 400);
+ }
+
+ private static void validateDimensions(JsonNode dimensions, List<String> violations) {
+ if (!dimensions.isObject()) {
+ violations.add(constraint("dimensions", "Member must not be null"));
+ return;
+ }
+ if (dimensions.isEmpty()) {
+ violations.add(constraint("dimensions",
+ "Member must have length greater than or equal to 1"));
+ return;
+ }
+ if (dimensions.size() > MAX_DIMENSIONS) {
+ violations.add(constraint("dimensions",
+ "Member must have length less than or equal to " + MAX_DIMENSIONS));
+ }
+ Iterator<Map.Entry<String, JsonNode>> fields = dimensions.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> field = fields.next();
+ if (!SesMetricData.DIMENSION_NAMES.contains(field.getKey())) {
+ violations.add(constraint("dimensions",
+ "Member must satisfy enum value set: " + DIMENSION_ENUM));
+ continue;
+ }
+ JsonNode values = field.getValue();
+ if (!values.isArray() || values.isEmpty()) {
+ violations.add(constraint("dimensions." + field.getKey(),
+ "Member must have length greater than or equal to 1"));
+ } else if (values.size() > MAX_DIMENSION_VALUES) {
+ violations.add(constraint("dimensions." + field.getKey(),
+ "Member must have length less than or equal to " + MAX_DIMENSION_VALUES));
+ }
+ }
+ }
+
+ private static void validateMetrics(JsonNode metrics, List<String> violations) {
+ if (!metrics.isArray()) {
+ violations.add(constraint("metrics", "Member must not be null"));
+ return;
+ }
+ if (metrics.isEmpty()) {
+ violations.add(constraint("metrics",
+ "Member must have length greater than or equal to 1"));
+ return;
+ }
+ if (metrics.size() > MAX_METRICS) {
+ violations.add(constraint("metrics",
+ "Member must have length less than or equal to " + MAX_METRICS));
+ }
+ for (int i = 0; i < metrics.size(); i++) {
+ JsonNode metric = metrics.get(i);
+ // Neither member is required by the model, so only a value that is present is
+ // checked: an absent Aggregation keeps its VOLUME default.
+ JsonNode name = metric.path("Name");
+ if (present(name) && !SesMetricData.METRICS.contains(name.asText())) {
+ violations.add(constraint("metrics." + (i + 1) + ".member.name",
+ "Member must satisfy enum value set: " + METRIC_ENUM));
+ }
+ JsonNode aggregation = metric.path("Aggregation");
+ if (present(aggregation)
+ && !("RATE".equals(aggregation.asText()) || "VOLUME".equals(aggregation.asText()))) {
+ violations.add(constraint("metrics." + (i + 1) + ".member.aggregation",
+ "Member must satisfy enum value set: " + AGGREGATION_ENUM));
+ }
+ }
+ }
+
+ private static boolean present(JsonNode node) {
+ return !node.isMissingNode() && !node.isNull();
+ }
+
+ private static String constraint(String member, String constraint) {
+ return "Value at '" + PATH + member + "' failed to satisfy constraint: " + constraint;
+ }
+
+ static byte[] render(List<SentEmail> emails, JsonNode metrics, String dataFormat) {
+ List<String> dimensionNames = new ArrayList<>();
+ Map<String, Set<String>> requested = new LinkedHashMap<>();
+ JsonNode dimensions = metrics.path("Dimensions");
+ Iterator<Map.Entry<String, JsonNode>> fields = dimensions.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> field = fields.next();
+ dimensionNames.add(field.getKey());
+ Set<String> values = new LinkedHashSet<>();
+ for (JsonNode value : field.getValue()) {
+ values.add(value.asText());
+ }
+ requested.put(field.getKey(), values);
+ }
+
+ List<String> metricColumns = new ArrayList<>();
+ List<String> metricNames = new ArrayList<>();
+ List<Boolean> asRate = new ArrayList<>();
+ for (JsonNode metric : metrics.path("Metrics")) {
+ String name = metric.path("Name").asText("");
+ String aggregation = metric.path("Aggregation").asText("VOLUME");
+ metricColumns.add(name + "_" + aggregation);
+ metricNames.add(name);
+ asRate.add("RATE".equals(aggregation));
+ }
+
+ Instant start = instant(metrics.path("StartDate"));
+ Instant end = instant(metrics.path("EndDate"));
+
+ List<List<String>> combinations = combinations(emails, dimensionNames, requested);
+ List<SesExportPayloads.MetricRow> rows = new ArrayList<>(combinations.size());
+ for (List<String> combination : combinations) {
+ Map<String, String> filter = new LinkedHashMap<>();
+ for (int i = 0; i < dimensionNames.size(); i++) {
+ filter.put(dimensionNames.get(i), combination.get(i));
+ }
+ List<Double> values = new ArrayList<>(metricNames.size());
+ for (int i = 0; i < metricNames.size(); i++) {
+ double volume = total(emails, metricNames.get(i), filter, start, end);
+ if (!asRate.get(i)) {
+ values.add(volume);
+ continue;
+ }
+ double sends = total(emails, "SEND", filter, start, end);
+ values.add(sends == 0.0 ? 0.0 : volume / sends);
+ }
+ rows.add(new SesExportPayloads.MetricRow(combination, values));
+ }
+
+ return ExportJob.FORMAT_CSV.equals(dataFormat)
+ ? SesExportPayloads.metricsCsv(dimensionNames, metricColumns, rows)
+ : SesExportPayloads.metricsJson(dimensionNames, metricColumns, rows);
+ }
+
+ private static double total(List<SentEmail> emails, String metric, Map<String, String> filter,
+ Instant start, Instant end) {
+ SesMetricData.Series series = SesMetricData.aggregate(emails,
+ new SesMetricData.Query("export", metric, filter, start, end));
+ double sum = 0.0;
+ for (Long value : series.values()) {
+ sum += value;
+ }
+ return sum;
+ }
+
+ /**
+ * The dimension combinations present in the stored messages, narrowed to the requested values.
+ * A requested value of {@code *} means every value, which is how AWS asks for the whole
+ * catalogue.
+ */
+ private static List<List<String>> combinations(List<SentEmail> emails, List<String> names,
+ Map<String, Set<String>> requested) {
+ Set<List<String>> seen = new LinkedHashSet<>();
+ for (SentEmail email : emails) {
+ List<EmailInsights> recipients = email.getInsights() == null
+ ? List.of() : email.getInsights();
+ for (EmailInsights recipient : recipients) {
+ List<String> combination = new ArrayList<>(names.size());
+ boolean usable = true;
+ for (String name : names) {
+ String value = dimensionValue(name, email, recipient);
+ if (value == null || !wants(name, requested.get(name), value)) {
+ usable = false;
+ break;
+ }
+ combination.add(value);
+ }
+ if (usable) {
+ seen.add(combination);
+ }
+ }
+ }
+ return new ArrayList<>(seen);
+ }
+
+ /**
+ * An {@code EMAIL_IDENTITY} value names either a sending address or its domain, so a requested
+ * domain has to match the addresses inside it. {@link SesMetricData} draws that line for
+ * {@code BatchGetMetricData}, and the two read the same sends, so they read it the same way.
+ */
+ private static boolean wants(String name, Set<String> wanted, String value) {
+ if (wanted.contains("*") || wanted.contains(value)) {
+ return true;
+ }
+ if (!"EMAIL_IDENTITY".equals(name)) {
+ return false;
+ }
+ return wanted.stream().anyMatch(identity -> SesMetricData.matchesIdentity(value, identity));
+ }
+
+ private static String dimensionValue(String name, SentEmail email, EmailInsights recipient) {
+ return switch (name) {
+ case "ISP" -> recipient.isp();
+ // The bare address, lower-cased, not the stored Source: a display name would key a row
+ // that the identity matching behind the counts then refuses, leaving every value at
+ // zero, and that matching ignores case, so two spellings would each count both sends.
+ case "EMAIL_IDENTITY" -> email.getSource() == null
+ ? null : SesService.extractEmailAddress(email.getSource()).toLowerCase(Locale.ROOT);
+ case "CONFIGURATION_SET" -> email.getConfigurationSetName();
+ case "TENANT_NAME" -> email.getTenantName();
+ default -> null;
+ };
+ }
+
+ private static Instant instant(JsonNode node) {
+ if (node.isNumber()) {
+ return Instant.ofEpochMilli(Math.round(node.doubleValue() * 1000.0));
+ }
+ return node.isTextual() ? Instant.parse(node.asText()) : Instant.EPOCH;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesExportPayloads.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportPayloads.java
new file mode 100644
index 0000000000..a586da3897
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportPayloads.java
@@ -0,0 +1,261 @@
+package io.github.hectorvent.floci.services.ses;
+
+import io.github.hectorvent.floci.services.ses.model.EmailInsights;
+import io.github.hectorvent.floci.services.ses.model.InsightsBounce;
+import io.github.hectorvent.floci.services.ses.model.InsightsEvent;
+import io.github.hectorvent.floci.services.ses.model.InsightsEventDetails;
+import io.github.hectorvent.floci.services.ses.model.SentEmail;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * Renders the two export payloads AWS produces, from the same stored timelines
+ * {@link SesMessageInsights} derives and {@link SesMetricData} aggregates.
+ *
+ * <p>Probe-confirmed against real SES (2026-09-23, us-east-1): the column names and their order,
+ * the quoting, the {@code yyyy-MM-dd HH:mm:ss.SSS UTC} timestamps, the trailing comma that leaves
+ * a thirteenth unnamed column on the message-insights CSV, the leading-comma JSON array, and the
+ * four-decimal metric values.
+ */
+final class SesExportPayloads {
+
+ private static final DateTimeFormatter EXPORT_TIMESTAMP = DateTimeFormatter
+ .ofPattern("yyyy-MM-dd HH:mm:ss.SSS 'UTC'", Locale.ROOT)
+ .withZone(ZoneOffset.UTC);
+
+ private static final List<String> INSIGHTS_COLUMNS = List.of(
+ "messageid", "sendtimestamp", "isp", "fromaddress", "destination", "subject",
+ "last_delivery_event", "last_engagement_event", "last_delivery_event_timestamp",
+ "last_engagement_event_timestamp", "bounce_sub_type", "diagnostic_code");
+
+ private SesExportPayloads() {}
+
+ /**
+ * One exported row: an envelope recipient of one stored message. The tenant sits beside the
+ * columns rather than among them, because {@code TenantName} is a filter member with no
+ * column of its own in the probed file.
+ */
+ record InsightsRow(Map<String, String> columns, String tenantName) {}
+
+ static List<InsightsRow> insightsRows(List<SentEmail> emails) {
+ List<InsightsRow> rows = new ArrayList<>();
+ for (SentEmail email : emails) {
+ for (EmailInsights recipient : orEmpty(email.getInsights())) {
+ rows.add(insightsRow(email, recipient));
+ }
+ }
+ return rows;
+ }
+
+ private static InsightsRow insightsRow(SentEmail email, EmailInsights recipient) {
+ Map<String, String> columns = new LinkedHashMap<>();
+ columns.put("messageid", email.getMessageId());
+ columns.put("sendtimestamp", format(email.getSentAt()));
+ columns.put("isp", recipient.isp());
+ columns.put("fromaddress", email.getSource());
+ columns.put("destination", recipient.destination());
+ columns.put("subject", email.getSubject());
+
+ InsightsEvent last = lastDeliveryEvent(recipient);
+ columns.put("last_delivery_event", last == null ? null : deliveryEventName(last));
+ columns.put("last_engagement_event", null);
+ columns.put("last_delivery_event_timestamp", last == null ? null : format(last.timestamp()));
+ columns.put("last_engagement_event_timestamp", null);
+ InsightsBounce bounce = bounceOf(last);
+ columns.put("bounce_sub_type", bounce == null ? null : bounce.bounceSubType());
+ columns.put("diagnostic_code", bounce == null ? null : bounce.diagnosticCode());
+ return new InsightsRow(columns, email.getTenantName());
+ }
+
+ /**
+ * The delivery-side enum splits a bounce by type, where the stored timeline carries one
+ * {@code BOUNCE} event plus its details. Floci only ever derives a permanent bounce, so the
+ * transient and undetermined values never appear.
+ */
+ private static String deliveryEventName(InsightsEvent event) {
+ if (!"BOUNCE".equals(event.type())) {
+ return event.type();
+ }
+ InsightsBounce bounce = bounceOf(event);
+ String bounceType = bounce == null ? null : bounce.bounceType();
+ if ("TRANSIENT".equals(bounceType)) {
+ return "TRANSIENT_BOUNCE";
+ }
+ if ("PERMANENT".equals(bounceType)) {
+ return "PERMANENT_BOUNCE";
+ }
+ return "UNDETERMINED_BOUNCE";
+ }
+
+ private static InsightsEvent lastDeliveryEvent(EmailInsights recipient) {
+ InsightsEvent last = null;
+ for (InsightsEvent event : orEmpty(recipient.events())) {
+ last = event;
+ }
+ return last;
+ }
+
+ private static InsightsBounce bounceOf(InsightsEvent event) {
+ if (event == null) {
+ return null;
+ }
+ InsightsEventDetails details = event.details();
+ return details == null ? null : details.bounce();
+ }
+
+ static byte[] insightsCsv(List<InsightsRow> rows) {
+ StringBuilder out = new StringBuilder();
+ for (String column : INSIGHTS_COLUMNS) {
+ out.append(quote(column)).append(',');
+ }
+ out.append('\n');
+ for (InsightsRow row : rows) {
+ for (String column : INSIGHTS_COLUMNS) {
+ out.append(quote(row.columns().get(column))).append(',');
+ }
+ out.append('\n');
+ }
+ return out.toString().getBytes(StandardCharsets.UTF_8);
+ }
+
+ static byte[] insightsJson(List<InsightsRow> rows) {
+ StringBuilder out = new StringBuilder("[\n");
+ boolean first = true;
+ for (InsightsRow row : rows) {
+ out.append(first ? "" : ",");
+ first = false;
+ out.append('{');
+ boolean firstColumn = true;
+ for (String column : INSIGHTS_COLUMNS) {
+ String value = row.columns().get(column);
+ if (value == null) {
+ continue;
+ }
+ if (!firstColumn) {
+ out.append(',');
+ }
+ firstColumn = false;
+ out.append(jsonString(column)).append(':').append(jsonString(value));
+ }
+ out.append("}\n");
+ }
+ out.append("]\n");
+ return out.toString().getBytes(StandardCharsets.UTF_8);
+ }
+
+ /** One exported metric row: the dimension values it is keyed by, then the requested metrics. */
+ record MetricRow(List<String> dimensionValues, List<Double> values) {}
+
+ static byte[] metricsCsv(List<String> dimensionNames, List<String> metricColumns,
+ List<MetricRow> rows) {
+ StringBuilder out = new StringBuilder(String.join(",", dimensionNames));
+ for (String column : metricColumns) {
+ out.append(',').append(column);
+ }
+ out.append('\n');
+ for (MetricRow row : rows) {
+ for (int i = 0; i < row.dimensionValues().size(); i++) {
+ if (i > 0) {
+ out.append(',');
+ }
+ out.append(csvField(row.dimensionValues().get(i)));
+ }
+ for (Double value : row.values()) {
+ out.append(',').append(decimal(value));
+ }
+ out.append('\n');
+ }
+ return out.toString().getBytes(StandardCharsets.UTF_8);
+ }
+
+ static byte[] metricsJson(List<String> dimensionNames, List<String> metricColumns,
+ List<MetricRow> rows) {
+ StringBuilder out = new StringBuilder("[");
+ boolean first = true;
+ for (MetricRow row : rows) {
+ if (!first) {
+ out.append(',');
+ }
+ first = false;
+ out.append('{');
+ for (int i = 0; i < dimensionNames.size(); i++) {
+ out.append(jsonString(dimensionNames.get(i).toLowerCase(Locale.ROOT))).append(':')
+ .append(jsonString(row.dimensionValues().get(i))).append(',');
+ }
+ out.append("\"metrics\":{");
+ for (int i = 0; i < metricColumns.size(); i++) {
+ if (i > 0) {
+ out.append(',');
+ }
+ out.append(jsonString(metricColumns.get(i))).append(':')
+ .append(decimal(row.values().get(i)));
+ }
+ out.append("}}");
+ }
+ out.append("]\n");
+ return out.toString().getBytes(StandardCharsets.UTF_8);
+ }
+
+ /**
+ * The probed metrics file leaves its values bare, so a value is quoted only when it has to be.
+ * A dimension value can carry a comma, and an unquoted one would shift every later column.
+ */
+ private static String csvField(String value) {
+ if (value == null) {
+ return "";
+ }
+ if (value.indexOf(',') < 0 && value.indexOf('"') < 0 && value.indexOf('\n') < 0) {
+ return value;
+ }
+ return quote(value);
+ }
+
+ private static String decimal(Double value) {
+ return String.format(Locale.ROOT, "%.4f", value == null ? 0.0 : value);
+ }
+
+ private static String format(Instant instant) {
+ return instant == null ? null : EXPORT_TIMESTAMP.format(instant);
+ }
+
+ private static String quote(String value) {
+ if (value == null) {
+ return "\"\"";
+ }
+ return '"' + value.replace("\"", "\"\"") + '"';
+ }
+
+ private static String jsonString(String value) {
+ StringBuilder out = new StringBuilder("\"");
+ for (int i = 0; i < value.length(); i++) {
+ char c = value.charAt(i);
+ switch (c) {
+ case '"' -> out.append("\\\"");
+ case '\\' -> out.append("\\\\");
+ case '\n' -> out.append("\\n");
+ case '\r' -> out.append("\\r");
+ case '\t' -> out.append("\\t");
+ default -> {
+ if (c < 0x20) {
+ out.append(String.format(Locale.ROOT, "\\u%04x", (int) c));
+ } else {
+ out.append(c);
+ }
+ }
+ }
+ }
+ return out.append('"').toString();
+ }
+
+ private static <T> List<T> orEmpty(List<T> values) {
+ return values == null ? List.of() : values;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesMetricData.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesMetricData.java
index a48c4dd804..11374b1c11 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ses/SesMetricData.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesMetricData.java
@@ -166,7 +166,7 @@ final class SesMetricData {
* An identity dimension names either the sending address or its domain, the way the identity
* itself can be either; the probe filtered a domain and matched sends from an address in it.
*/
- private static boolean matchesIdentity(String source, String identity) {
+ static boolean matchesIdentity(String source, String identity) {
if (source == null) {
return false;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/model/ExportJob.java b/src/main/java/io/github/hectorvent/floci/services/ses/model/ExportJob.java
new file mode 100644
index 0000000000..0d6537a5b4
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/model/ExportJob.java
@@ -0,0 +1,112 @@
+package io.github.hectorvent.floci.services.ses.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+
+/**
+ * A stored SES V2 export job. The request's {@code ExportDataSource} is kept verbatim as JSON
+ * because {@code GetExportJob} echoes it back, filters and all, and Floci has no reason to model
+ * every filter member just to hand it straight back.
+ */
+@RegisterForReflection
+public class ExportJob {
+
+ public static final String SOURCE_METRICS = "METRICS_DATA";
+ public static final String SOURCE_MESSAGE_INSIGHTS = "MESSAGE_INSIGHTS";
+
+ public static final String STATUS_CREATED = "CREATED";
+ public static final String STATUS_PROCESSING = "PROCESSING";
+ public static final String STATUS_COMPLETED = "COMPLETED";
+ public static final String STATUS_FAILED = "FAILED";
+ public static final String STATUS_CANCELLED = "CANCELLED";
+
+ public static final String FORMAT_CSV = "CSV";
+ public static final String FORMAT_JSON = "JSON";
+
+ @JsonProperty("JobId")
+ private String jobId;
+
+ @JsonProperty("Region")
+ private String region;
+
+ /** The account that created the job, which keeps its export bucket inside that account. */
+ @JsonProperty("AccountId")
+ private String accountId;
+
+ @JsonProperty("ExportSourceType")
+ private String exportSourceType;
+
+ @JsonProperty("JobStatus")
+ private String jobStatus;
+
+ @JsonProperty("DataFormat")
+ private String dataFormat;
+
+ @JsonProperty("DataSource")
+ private String dataSource;
+
+ /** The key of the produced object, once the worker has written it. */
+ @JsonProperty("ObjectKey")
+ private String objectKey;
+
+ @JsonProperty("CreatedTimestamp")
+ private Instant createdTimestamp;
+
+ @JsonProperty("CompletedTimestamp")
+ private Instant completedTimestamp;
+
+ @JsonProperty("ErrorMessage")
+ private String errorMessage;
+
+ /** How many stored recipient rows the job examined, which is AWS's ProcessedRecordsCount. */
+ @JsonProperty("ProcessedRecordsCount")
+ private Long processedRecordsCount;
+
+ public ExportJob() {}
+
+ public boolean isTerminal() {
+ return STATUS_COMPLETED.equals(jobStatus)
+ || STATUS_FAILED.equals(jobStatus)
+ || STATUS_CANCELLED.equals(jobStatus);
+ }
+
+ public String getJobId() { return jobId; }
+ public void setJobId(String jobId) { this.jobId = jobId; }
+
+ public String getRegion() { return region; }
+ public void setRegion(String region) { this.region = region; }
+
+ public String getAccountId() { return accountId; }
+ public void setAccountId(String accountId) { this.accountId = accountId; }
+
+ public String getExportSourceType() { return exportSourceType; }
+ public void setExportSourceType(String exportSourceType) { this.exportSourceType = exportSourceType; }
+
+ public String getJobStatus() { return jobStatus; }
+ public void setJobStatus(String jobStatus) { this.jobStatus = jobStatus; }
+
+ public String getDataFormat() { return dataFormat; }
+ public void setDataFormat(String dataFormat) { this.dataFormat = dataFormat; }
+
+ public String getDataSource() { return dataSource; }
+ public void setDataSource(String dataSource) { this.dataSource = dataSource; }
+
+ public String getObjectKey() { return objectKey; }
+ public void setObjectKey(String objectKey) { this.objectKey = objectKey; }
+
+ public Instant getCreatedTimestamp() { return createdTimestamp; }
+ public void setCreatedTimestamp(Instant createdTimestamp) { this.createdTimestamp = createdTimestamp; }
+
+ public Instant getCompletedTimestamp() { return completedTimestamp; }
+ public void setCompletedTimestamp(Instant completedTimestamp) { this.completedTimestamp = completedTimestamp; }
+
+ public String getErrorMessage() { return errorMessage; }
+ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }
+
+ public Long getProcessedRecordsCount() { return processedRecordsCount; }
+ public void setProcessedRecordsCount(Long processedRecordsCount) {
+ this.processedRecordsCount = processedRecordsCount;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesExportFiltersTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportFiltersTest.java
new file mode 100644
index 0000000000..6ee2915d4d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportFiltersTest.java
@@ -0,0 +1,101 @@
+package io.github.hectorvent.floci.services.ses;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class SesExportFiltersTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private static SesExportPayloads.InsightsRow row(String destination, String tenantName) {
+ Map<String, String> columns = new LinkedHashMap<>();
+ columns.put("destination", destination);
+ columns.put("isp", "UNKNOWN_ISP");
+ columns.put("last_delivery_event", "DELIVERY");
+ columns.put("last_engagement_event", null);
+ return new SesExportPayloads.InsightsRow(columns, tenantName);
+ }
+
+ private static JsonNode source(String json) {
+ try {
+ return MAPPER.readTree(json);
+ } catch (Exception e) {
+ throw new AssertionError("bad test json", e);
+ }
+ }
+
+ private static List<String> destinations(List<SesExportPayloads.InsightsRow> rows) {
+ return rows.stream().map(row -> row.columns().get("destination")).toList();
+ }
+
+ private static final List<SesExportPayloads.InsightsRow> ROWS = List.of(
+ row("[email protected]", "blue"),
+ row("[email protected]", "green"),
+ row("[email protected]", null));
+
+ @Test
+ void includeOnTenantNameKeepsOnlyThatTenant() {
+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,
+ source("{\"Include\": {\"TenantName\": [\"blue\"]}}"));
+
+ assertEquals(List.of("[email protected]"), destinations(kept));
+ }
+
+ @Test
+ void includeOnTenantNameAcceptsSeveralValues() {
+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,
+ source("{\"Include\": {\"TenantName\": [\"blue\", \"green\"]}}"));
+
+ assertEquals(List.of("[email protected]", "[email protected]"), destinations(kept));
+ }
+
+ @Test
+ void excludeOnTenantNameDropsThatTenantAndKeepsTheTenantlessSend() {
+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,
+ source("{\"Exclude\": {\"TenantName\": [\"blue\"]}}"));
+
+ assertEquals(List.of("[email protected]", "[email protected]"), destinations(kept));
+ }
+
+ @Test
+ void membersWithinOneIncludeAllHaveToMatch() {
+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,
+ source("{\"Include\": {\"TenantName\": [\"blue\"], \"Isp\": [\"GMAIL\"]}}"));
+
+ assertTrue(kept.isEmpty(), "the ISP member excludes the row the tenant member keeps");
+ }
+
+ @Test
+ void anIncludeOnLastEngagementEventSelectsNothing() {
+ // Floci derives no opens or clicks, so the column is always null and cannot match.
+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,
+ source("{\"Include\": {\"LastEngagementEvent\": [\"OPEN\"]}}"));
+
+ assertTrue(kept.isEmpty());
+ }
+
+ @Test
+ void anExcludeCarryingNoValuesKeepsEveryRow() {
+ // Every member is absent, so the block constrains nothing. Read as a match it would empty
+ // the file, which is the one direction where "no constraint" is not harmless.
+ assertEquals(3, SesExportFilters.apply(ROWS, source("{\"Exclude\": {}}")).size());
+ assertEquals(3, SesExportFilters.apply(ROWS,
+ source("{\"Exclude\": {\"TenantName\": []}}")).size());
+ }
+
+ @Test
+ void maxResultsTruncatesAfterFiltering() {
+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,
+ source("{\"Exclude\": {\"TenantName\": [\"blue\"]}, \"MaxResults\": 1}"));
+
+ assertEquals(List.of("[email protected]"), destinations(kept));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobServiceTest.java
new file mode 100644
index 0000000000..7722f15d8f
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobServiceTest.java
@@ -0,0 +1,361 @@
+package io.github.hectorvent.floci.services.ses;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.s3.PreSignedUrlGenerator;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.ses.model.EmailInsights;
+import io.github.hectorvent.floci.services.ses.model.ExportJob;
+import io.github.hectorvent.floci.services.ses.model.InsightsEvent;
+import io.github.hectorvent.floci.services.ses.model.SentEmail;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * The parts of {@link SesExportJobService} the integration test cannot pin down: what happens when
+ * a cancel lands in the window between the object being written and the job being marked complete.
+ * The worker runs on a captured task here, so that window is reachable.
+ */
+class SesExportJobServiceTest {
+
+ private static final String REGION = "eu-north-1";
+ private static final String ACCOUNT = "000000000000";
+
+ private final S3Service s3Service = mock(S3Service.class);
+ private final PreSignedUrlGenerator presigner = mock(PreSignedUrlGenerator.class);
+ private final SesSentEmailService sentEmailService =
+ new SesSentEmailService(new InMemoryStorage<>());
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private final List<Runnable> tasks = new ArrayList<>();
+
+ private SesExportJobService service;
+
+ @BeforeEach
+ void setUp() {
+ when(s3Service.bucketExists(anyString())).thenReturn(true);
+ service = new SesExportJobService(new InMemoryStorage<>(), sentEmailService, s3Service,
+ presigner, objectMapper, Clock.fixed(Instant.parse("2026-09-23T00:00:00Z"),
+ ZoneOffset.UTC),
+ tasks::add, "http://localhost:4566");
+ }
+
+ private JsonNode insightsSource() {
+ try {
+ return objectMapper.readTree("""
+ {"MessageInsightsDataSource": {"StartDate": 1758499200, "EndDate": 1758672000}}
+ """);
+ } catch (JsonProcessingException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ private ExportJob createJob() {
+ return service.createExportJob(REGION, ACCOUNT, insightsSource(), destination(), () -> { });
+ }
+
+ private JsonNode destination() {
+ return objectMapper.createObjectNode().put("DataFormat", ExportJob.FORMAT_CSV);
+ }
+
+ @Test
+ void aCancelDuringTheWriteLeavesNoObjectBehind() {
+ ExportJob job = createJob();
+ // The cancel lands while the worker is inside putObject, which is the only window in which
+ // an object can be written for a job that will never point at it.
+ when(s3Service.putObject(anyString(), anyString(), any(), anyString(), any()))
+ .thenAnswer(invocation -> {
+ service.cancelExportJob(REGION, job.getJobId());
+ return null;
+ });
+
+ tasks.forEach(Runnable::run);
+
+ ExportJob stored = service.getExportJob(REGION, job.getJobId());
+ assertEquals(ExportJob.STATUS_CANCELLED, stored.getJobStatus());
+ assertNull(stored.getObjectKey(), "a cancelled job points at nothing");
+ verify(s3Service).deleteObject(eq(SesExportJobService.exportBucket(ACCOUNT, REGION)), anyString());
+ }
+
+ @Test
+ void anUninterruptedJobKeepsItsObject() {
+ ExportJob job = createJob();
+
+ tasks.forEach(Runnable::run);
+
+ ExportJob stored = service.getExportJob(REGION, job.getJobId());
+ assertEquals(ExportJob.STATUS_COMPLETED, stored.getJobStatus());
+ verify(s3Service, never()).deleteObject(anyString(), anyString());
+ }
+
+ @Test
+ void cancellingBeforeTheWorkerRunsSkipsTheWriteEntirely() {
+ ExportJob job = createJob();
+ service.cancelExportJob(REGION, job.getJobId());
+
+ tasks.forEach(Runnable::run);
+
+ assertEquals(ExportJob.STATUS_CANCELLED,
+ service.getExportJob(REGION, job.getJobId()).getJobStatus());
+ verify(s3Service, never()).putObject(anyString(), anyString(), any(), anyString(),
+ any(Map.class));
+ }
+
+ @Test
+ void aResetAbandonsAJobThatHasNotStartedWriting() {
+ ExportJob job = createJob();
+
+ service.beforeReset();
+ tasks.forEach(Runnable::run);
+
+ verify(s3Service, never()).putObject(anyString(), anyString(), any(), anyString(),
+ any(Map.class));
+ assertEquals(ExportJob.STATUS_PROCESSING,
+ service.getExportJob(REGION, job.getJobId()).getJobStatus());
+ }
+
+ @Test
+ void aResetWaitsForAWorkerAlreadyInsideItsWrite() throws Exception {
+ CountDownLatch writing = new CountDownLatch(1);
+ CountDownLatch releaseWrite = new CountDownLatch(1);
+ AtomicBoolean workerFinishedTheWrite = new AtomicBoolean();
+ SesExportJobService threaded = new SesExportJobService(new InMemoryStorage<>(),
+ sentEmailService, s3Service, presigner, objectMapper,
+ Clock.fixed(Instant.parse("2026-09-23T00:00:00Z"), ZoneOffset.UTC),
+ task -> new Thread(task).start(), "http://localhost:4566");
+ when(s3Service.putObject(anyString(), anyString(), any(), anyString(), any()))
+ .thenAnswer(invocation -> {
+ writing.countDown();
+ releaseWrite.await(5, TimeUnit.SECONDS);
+ workerFinishedTheWrite.set(true);
+ return null;
+ });
+
+ threaded.createExportJob(REGION, ACCOUNT, insightsSource(), destination(), () -> { });
+ assertTrue(writing.await(5, TimeUnit.SECONDS), "the worker never reached its write");
+ Thread reset = new Thread(threaded::beforeReset);
+ reset.start();
+ awaitParked(reset, "the reset waiting on the in-flight write");
+ releaseWrite.countDown();
+ reset.join(5_000);
+
+ // join returns on timeout too, so without this the assertion below would hold even if
+ // beforeReset never came back.
+ assertFalse(reset.isAlive(), "the reset did not finish after the in-flight write");
+ assertTrue(workerFinishedTheWrite.get(),
+ "the reset returned before the in-flight write finished");
+ }
+
+ private void recordSend(Instant sentAt, String source) {
+ SentEmail email = new SentEmail();
+ email.setMessageId("m-" + sentAt.toEpochMilli() + "-" + source);
+ email.setSource(source);
+ email.setSentAt(sentAt);
+ email.setInsights(List.of(new EmailInsights("[email protected]", "UNKNOWN_ISP",
+ List.of(InsightsEvent.of(sentAt, "SEND")))));
+ sentEmailService.record(REGION, email.getMessageId(), email);
+ }
+
+ private String runMetricsExport(String dimensions, Instant start, Instant end) throws Exception {
+ JsonNode source = objectMapper.readTree("""
+ {"MetricsDataSource": {"Dimensions": %s, "Namespace": "VDM",
+ "Metrics": [{"Name": "SEND", "Aggregation": "VOLUME"}],
+ "StartDate": %d, "EndDate": %d}}
+ """.formatted(dimensions, start.getEpochSecond(), end.getEpochSecond()));
+ service.createExportJob(REGION, ACCOUNT, source, destination(), () -> { });
+ tasks.forEach(Runnable::run);
+ ArgumentCaptor<byte[]> payload = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service).putObject(anyString(), anyString(), payload.capture(), anyString(), any());
+ return new String(payload.getValue(), StandardCharsets.UTF_8);
+ }
+
+ @Test
+ void anIdentityRowIsKeyedByTheBareAddressAndCountsItsSend() throws Exception {
+ recordSend(Instant.parse("2026-09-23T10:00:00Z"), "\"Doe, Jane\" <[email protected]>");
+
+ String csv = runMetricsExport("{\"EMAIL_IDENTITY\": [\"*\"]}",
+ Instant.parse("2026-09-23T00:00:00Z"), Instant.parse("2026-09-24T00:00:00Z"));
+
+ // Keyed by the stored Source, the row is one the identity matching behind the counts then
+ // refuses, so it comes back at zero, and a comma in the display name splits the CSV row.
+ assertEquals("EMAIL_IDENTITY,SEND_VOLUME\[email protected],1.0000\n", csv);
+ }
+
+ @Test
+ void aDimensionValueSeenOnlyOutsideTheWindowProducesNoRow() throws Exception {
+ recordSend(Instant.parse("2026-09-23T10:00:00Z"), "[email protected]");
+ recordSend(Instant.parse("2026-09-20T10:00:00Z"), "[email protected]");
+
+ String csv = runMetricsExport("{\"EMAIL_IDENTITY\": [\"*\"]}",
+ Instant.parse("2026-09-23T00:00:00Z"), Instant.parse("2026-09-24T00:00:00Z"));
+
+ assertTrue(csv.contains("[email protected],1.0000"), csv);
+ assertFalse(csv.contains("[email protected]"), csv);
+ }
+
+ @Test
+ void anInsightsWindowIncludesASendOnItsEndInstant() throws Exception {
+ // The model calls MessageInsightsDataSource.EndDate inclusive, unlike the daily metrics
+ // buckets, whose exclusive end the BatchGetMetricData probe confirmed.
+ Instant sentAt = Instant.parse("2026-09-23T12:00:00Z");
+ recordSend(sentAt, "[email protected]");
+ JsonNode source = objectMapper.readTree("""
+ {"MessageInsightsDataSource": {"StartDate": %d, "EndDate": %d}}
+ """.formatted(sentAt.minusSeconds(3600).getEpochSecond(), sentAt.getEpochSecond()));
+
+ ExportJob job = service.createExportJob(REGION, ACCOUNT, source, destination(), () -> { });
+ tasks.forEach(Runnable::run);
+
+ assertEquals(1L, service.getExportJob(REGION, job.getJobId()).getProcessedRecordsCount());
+ }
+
+ @Test
+ void aBucketCreatedByAConcurrentJobDoesNotFailTheExport() {
+ when(s3Service.bucketExists(anyString())).thenReturn(false);
+ when(s3Service.createBucket(anyString(), anyString())).thenThrow(new AwsException(
+ "BucketAlreadyOwnedByYou",
+ "Your previous request to create the named bucket succeeded and you already own it.",
+ 409));
+ ExportJob job = createJob();
+
+ tasks.forEach(Runnable::run);
+
+ assertEquals(ExportJob.STATUS_COMPLETED,
+ service.getExportJob(REGION, job.getJobId()).getJobStatus());
+ }
+
+ @Test
+ void aCreateArrivingDuringAResetWaitsForItToFinish() throws Exception {
+ service.beforeReset();
+ AtomicBoolean admitted = new AtomicBoolean();
+ Thread creator = new Thread(() -> {
+ createJob();
+ admitted.set(true);
+ });
+
+ creator.start();
+ // Waiting for the thread to actually park proves the reset blocked it. A plain sleep would
+ // pass while the thread was merely unscheduled, and stay green with the blocking removed.
+ awaitParked(creator, "the create waiting on the reset");
+ assertFalse(admitted.get(), "a create must not be admitted while the reset is in progress");
+ service.afterReset();
+ creator.join(5_000);
+
+ assertTrue(admitted.get(), "afterReset must release the waiting create");
+ }
+
+ @Test
+ void theExportBucketIsScopedToTheJobsAccount() {
+ // An object write resolves its bucket across accounts under global-bucket-namespace, so a
+ // shared name would put this account's export in whichever account created the bucket.
+ createJob();
+
+ tasks.forEach(Runnable::run);
+
+ ArgumentCaptor<String> bucket = ArgumentCaptor.forClass(String.class);
+ verify(s3Service).putObject(bucket.capture(), anyString(), any(), anyString(), any());
+ assertEquals("floci-" + ACCOUNT + "-" + REGION + "-ses-export-files", bucket.getValue());
+ }
+
+ @Test
+ void identitiesDifferingOnlyInCaseShareOneRow() throws Exception {
+ // The aggregator matches identities without regard to case, so two rows would each count
+ // both sends and the export would disagree with a BatchGetMetricData query.
+ recordSend(Instant.parse("2026-09-23T10:00:00Z"), "[email protected]");
+ recordSend(Instant.parse("2026-09-23T11:00:00Z"), "[email protected]");
+
+ String csv = runMetricsExport("{\"EMAIL_IDENTITY\": [\"*\"]}",
+ Instant.parse("2026-09-23T00:00:00Z"), Instant.parse("2026-09-24T00:00:00Z"));
+
+ assertEquals("EMAIL_IDENTITY,SEND_VOLUME\[email protected],2.0000\n", csv);
+ }
+
+ @Test
+ void theConcurrentJobCeilingIsThePublishedQuota() {
+ // SES documents 20 concurrent export jobs, the same figure it gives import jobs. The
+ // workers are captured here, so every job stays PROCESSING and occupies a slot.
+ for (int i = 0; i < SesExportJobService.MAX_CONCURRENT_JOBS; i++) {
+ createJob();
+ }
+
+ AwsException thrown = assertThrows(AwsException.class, this::createJob);
+
+ assertEquals("LimitExceededException", thrown.getErrorCode());
+ assertEquals(20, SesExportJobService.MAX_CONCURRENT_JOBS);
+ }
+
+ /** Both waits here park in {@code Object.wait(timeout)}, so the state is timed, not WAITING. */
+ private static void awaitParked(Thread thread, String what) {
+ long deadline = System.currentTimeMillis() + 5_000;
+ while (System.currentTimeMillis() < deadline) {
+ Thread.State state = thread.getState();
+ if (state == Thread.State.TIMED_WAITING || state == Thread.State.WAITING) {
+ return;
+ }
+ Thread.onSpinWait();
+ }
+ throw new AssertionError(what + " never parked, it was " + thread.getState());
+ }
+
+ @Test
+ void recoverInterruptedJobsFailsAJobOfEveryAccount() {
+ String otherAccount = "111122223333";
+ AccountAwareStorageBackend<ExportJob> persisted =
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, ACCOUNT);
+ ExportJob stuck = new ExportJob();
+ stuck.setJobId("stuck");
+ stuck.setRegion(REGION);
+ stuck.setJobStatus(ExportJob.STATUS_PROCESSING);
+ stuck.setCreatedTimestamp(Instant.parse("2026-09-23T00:00:00Z"));
+ String key = "export-job::" + REGION + "::stuck";
+ persisted.putForAccount(otherAccount, key, stuck);
+
+ new SesExportJobService(persisted, sentEmailService, s3Service, presigner, objectMapper,
+ Clock.fixed(Instant.parse("2026-09-23T00:00:00Z"), ZoneOffset.UTC), tasks::add,
+ "http://localhost:4566");
+
+ ExportJob recovered = persisted.getForAccount(otherAccount, key).orElseThrow();
+ assertEquals(ExportJob.STATUS_FAILED, recovered.getJobStatus());
+ assertEquals(SesExportJobService.RESTART_FAILURE_MESSAGE, recovered.getErrorMessage());
+ }
+
+ @Test
+ void cancellingAFinishedJobIsRefused() {
+ ExportJob job = createJob();
+ tasks.forEach(Runnable::run);
+
+ AwsException thrown = assertThrows(AwsException.class,
+ () -> service.cancelExportJob(REGION, job.getJobId()));
+
+ assertEquals("Cannot cancel an export job that has a <COMPLETED> status.",
+ thrown.getMessage());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobV2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobV2IntegrationTest.java
new file mode 100644
index 0000000000..74fa24f509
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobV2IntegrationTest.java
@@ -0,0 +1,549 @@
+package io.github.hectorvent.floci.services.ses;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.everyItem;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Integration tests for the four SES v2 export-job operations.
+ *
+ * <p>Probe-confirmed against real SES (2026-09-23): a request naming its own {@code S3Url} is
+ * refused, exactly one data source is required, the metrics source insists on whole UTC days, an
+ * unknown job is a 400 rather than a 404, and a finished job cannot be cancelled. Uses an isolated
+ * signing region so no other test's sends reach the exported rows.
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class SesExportJobV2IntegrationTest {
+
+ private static final String AUTH =
+ "AWS4-HMAC-SHA256 Credential=AKID/20260101/eu-west-2/ses/aws4_request";
+ private static final String SENDER = "[email protected]";
+
+ private static long midnightToday() {
+ return Instant.now().truncatedTo(ChronoUnit.DAYS).getEpochSecond();
+ }
+
+ private static Response create(String body) {
+ return given().contentType("application/json").header("Authorization", AUTH)
+ .body(body)
+ .when().post("/v2/email/export-jobs");
+ }
+
+ private static String insightsSource() {
+ return """
+ {"ExportDataSource": {"MessageInsightsDataSource":
+ {"StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "CSV"}}
+ """.formatted(Instant.now().minus(Duration.ofDays(1)).getEpochSecond(),
+ Instant.now().plus(Duration.ofDays(1)).getEpochSecond());
+ }
+
+ private static String awaitTerminal(String jobId) {
+ for (int attempt = 0; attempt < 100; attempt++) {
+ String status = given().header("Authorization", AUTH)
+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(200)
+ .extract().path("JobStatus");
+ if (!"PROCESSING".equals(status) && !"CREATED".equals(status)) {
+ return status;
+ }
+ try {
+ Thread.sleep(50);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError("interrupted while waiting for the job", e);
+ }
+ }
+ throw new AssertionError("export job " + jobId + " never reached a terminal state");
+ }
+
+ @Test
+ @Order(0)
+ void createExportJob_vdmDisabled_isABadRequestNotANotFound() {
+ // The same sentence message insights and metric data use, but 400 here, not 404.
+ create(insightsSource()).then().statusCode(400)
+ .body("message", containsString(
+ "To use this feature you must enable Virtual Deliverability Manager"));
+ }
+
+ @Test
+ @Order(1)
+ void enableVdmAndVerifyTheSender() {
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"VdmAttributes\":{\"VdmEnabled\":\"ENABLED\"}}")
+ .when().put("/v2/email/account/vdm").then().statusCode(200);
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"EmailIdentity\":\"" + SENDER + "\"}")
+ .when().post("/v2/email/identities").then().statusCode(200);
+ }
+
+ @Test
+ @Order(2)
+ void createExportJob_refusesACallerChosenDestination() {
+ create("""
+ {"ExportDataSource": {"MessageInsightsDataSource":
+ {"StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "CSV", "S3Url": "s3://mine/prefix/"}}
+ """.formatted(midnightToday() - 86400, midnightToday()))
+ .then().statusCode(400)
+ .body("message", equalTo("Providing a custom S3 URL is not supported"));
+ }
+
+ @Test
+ @Order(3)
+ void createExportJob_checksTheDestinationInTheProbedOrder() {
+ // Probed: the data source outranks the destination's own constraints, which outrank the
+ // refusal of a caller-chosen S3Url, which outranks the VDM gate.
+ create("""
+ {"ExportDataSource": {"MessageInsightsDataSource":
+ {"StartDate": %d, "EndDate": %d}}}
+ """.formatted(midnightToday() - 86400, midnightToday()))
+ .then().statusCode(400).body("message", equalTo(
+ "1 validation error detected: Value at 'exportDestination' failed to satisfy "
+ + "constraint: Member must not be null"));
+
+ create("""
+ {"ExportDataSource": {"MessageInsightsDataSource":
+ {"StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"S3Url": "s3://mine/prefix/"}}
+ """.formatted(midnightToday() - 86400, midnightToday()))
+ .then().statusCode(400).body("message", equalTo(
+ "1 validation error detected: Value at 'exportDestination.dataFormat' failed to "
+ + "satisfy constraint: Member must not be null"));
+
+ create("""
+ {"ExportDataSource": {"MessageInsightsDataSource":
+ {"StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "PARQUET"}}
+ """.formatted(midnightToday() - 86400, midnightToday()))
+ .then().statusCode(400).body("message", containsString(
+ "Member must satisfy enum value set: [CSV, JSON]"));
+
+ // A missing data source beats even a caller-chosen destination.
+ create("""
+ {"ExportDataSource": {},
+ "ExportDestination": {"DataFormat": "CSV", "S3Url": "s3://mine/prefix/"}}
+ """)
+ .then().statusCode(400).body("message", equalTo("One data source must be provided"));
+ }
+
+ @Test
+ @Order(4)
+ void createExportJob_requiresExactlyOneDataSource() {
+ String both = """
+ {"ExportDataSource": {"MessageInsightsDataSource": {"StartDate": %d, "EndDate": %d},
+ "MetricsDataSource": {"Dimensions": {"ISP": ["*"]},
+ "Namespace": "VDM",
+ "Metrics": [{"Name": "SEND"}],
+ "StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "CSV"}}
+ """.formatted(midnightToday() - 86400, midnightToday(),
+ midnightToday() - 86400, midnightToday());
+ create(both).then().statusCode(400)
+ .body("message", equalTo("One data source must be provided"));
+
+ create("{\"ExportDataSource\": {}, \"ExportDestination\": {\"DataFormat\": \"CSV\"}}")
+ .then().statusCode(400).body("message", equalTo("One data source must be provided"));
+ }
+
+ @Test
+ @Order(5)
+ void createExportJob_checksTheDates() {
+ create("""
+ {"ExportDataSource": {"MessageInsightsDataSource":
+ {"StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "CSV"}}
+ """.formatted(midnightToday(), midnightToday() - 86400))
+ .then().statusCode(400).body("message", equalTo("Invalid date range"));
+
+ // Only the metrics source insists on whole days.
+ create("""
+ {"ExportDataSource": {"MetricsDataSource": {"Dimensions": {"ISP": ["*"]},
+ "Namespace": "VDM",
+ "Metrics": [{"Name": "SEND"}],
+ "StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "CSV"}}
+ """.formatted(midnightToday() - 86400 + 3600, midnightToday()))
+ .then().statusCode(400).body("message", containsString("midnight to midnight UTC"));
+ }
+
+ @Test
+ @Order(6)
+ void exportJob_writesTheInsightsFileAndServesItBehindAPresignedUrl() {
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("""
+ {"FromEmailAddress": "%s",
+ "Destination": {"ToAddresses": ["[email protected]"]},
+ "Content": {"Simple": {"Subject": {"Data": "exported"},
+ "Body": {"Text": {"Data": "hi"}}}}}
+ """.formatted(SENDER))
+ .when().post("/v2/email/outbound-emails").then().statusCode(200);
+
+ String jobId = create(insightsSource()).then().statusCode(200)
+ .extract().path("JobId");
+ assertEquals("COMPLETED", awaitTerminal(jobId));
+
+ Response job = given().header("Authorization", AUTH)
+ .when().get("/v2/email/export-jobs/" + jobId);
+ job.then().statusCode(200)
+ .body("ExportSourceType", equalTo("MESSAGE_INSIGHTS"))
+ .body("ExportDestination.DataFormat", equalTo("CSV"))
+ .body("ExportDestination.S3Url", notNullValue())
+ .body("ExportDataSource.MessageInsightsDataSource", notNullValue())
+ .body("CompletedTimestamp", notNullValue())
+ // Probed: AWS reports the rows examined and never ExportedRecordsCount.
+ .body("Statistics.ProcessedRecordsCount", equalTo(1))
+ .body("Statistics", not(hasKey("ExportedRecordsCount")));
+
+ // The URL is Floci's own S3 emulation, so the object is fetched back through it.
+ String url = job.path("ExportDestination.S3Url");
+ String path = url.substring(url.indexOf('/', url.indexOf("//") + 2));
+ String body = given().when().get(path).then().statusCode(200).extract().asString();
+ assertTrue(body.startsWith("\"messageid\",\"sendtimestamp\""), body.substring(0, 60));
+ assertTrue(body.contains("[email protected]"), "the send should be in the export");
+ }
+
+ @Test
+ @Order(7)
+ void exportJob_metricsSourceCountsTheSameSends() {
+ String jobId = create("""
+ {"ExportDataSource": {"MetricsDataSource": {"Dimensions": {"ISP": ["*"]},
+ "Namespace": "VDM",
+ "Metrics": [{"Name": "SEND",
+ "Aggregation": "VOLUME"}],
+ "StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "CSV"}}
+ """.formatted(midnightToday(), midnightToday() + 86400))
+ .then().statusCode(200).extract().path("JobId");
+ assertEquals("COMPLETED", awaitTerminal(jobId));
+
+ String url = given().header("Authorization", AUTH)
+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(200)
+ .extract().path("ExportDestination.S3Url");
+ String path = url.substring(url.indexOf('/', url.indexOf("//") + 2));
+ String body = given().when().get(path).then().statusCode(200).extract().asString();
+
+ assertTrue(body.startsWith("ISP,SEND_VOLUME\n"), body);
+ assertTrue(body.contains("UNKNOWN_ISP,1.0000"), body);
+ }
+
+ @Test
+ @Order(8)
+ void cancelExportJob_refusesAFinishedJob() {
+ // Only the refusal is reachable here: a Floci export finishes in milliseconds, so a job is
+ // already COMPLETED by the time a second request arrives. The accepted-cancel path is
+ // covered deterministically in SesExportJobServiceTest.
+ String finished = create(insightsSource()).then().statusCode(200).extract().path("JobId");
+ assertEquals("COMPLETED", awaitTerminal(finished));
+
+ given().contentType("application/json").header("Authorization", AUTH)
+ .when().put("/v2/email/export-jobs/" + finished + "/cancel").then().statusCode(400)
+ .body("message", equalTo(
+ "Cannot cancel an export job that has a <COMPLETED> status."));
+ }
+
+ @Test
+ @Order(9)
+ void getExportJob_unknownJobIsABadRequest() {
+ given().header("Authorization", AUTH)
+ .when().get("/v2/email/export-jobs/no-such-job").then().statusCode(400)
+ .body("message", equalTo("Invalid JobId"));
+ }
+
+ @Test
+ @Order(10)
+ void listExportJobs_filtersAndPaginates() {
+ given().contentType("application/json").header("Authorization", AUTH).body("{}")
+ .when().post("/v2/email/list-export-jobs").then().statusCode(200)
+ .body("ExportJobs.JobStatus", everyItem(equalTo("COMPLETED")));
+
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"ExportSourceType\": \"METRICS_DATA\"}")
+ .when().post("/v2/email/list-export-jobs").then().statusCode(200)
+ .body("ExportJobs", hasSize(1));
+
+ String nextToken = given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"PageSize\": 1}")
+ .when().post("/v2/email/list-export-jobs").then().statusCode(200)
+ .body("ExportJobs", hasSize(1))
+ .extract().path("NextToken");
+ assertNotNull(nextToken, "a truncated page carries a NextToken");
+ }
+
+ @Test
+ @Order(11)
+ void insightsExport_filtersOnTheTenantTheSendNamed() {
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"TenantName\": \"export-tenant\"}")
+ .when().post("/v2/email/tenants").then().statusCode(200);
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("""
+ {"TenantName": "export-tenant",
+ "ResourceArn": "arn:aws:ses:eu-west-2:000000000000:identity/%s"}
+ """.formatted(SENDER))
+ .when().post("/v2/email/tenants/resources").then().statusCode(200);
+
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("""
+ {"FromEmailAddress": "%s", "TenantName": "export-tenant",
+ "Destination": {"ToAddresses": ["[email protected]"]},
+ "Content": {"Simple": {"Subject": {"Data": "t"},
+ "Body": {"Text": {"Data": "hi"}}}}}
+ """.formatted(SENDER))
+ .when().post("/v2/email/outbound-emails").then().statusCode(200);
+
+ // [email protected] was sent at order 6 without a tenant, so it is the row the filter
+ // has to drop. An Include that matched everything would keep it.
+ assertTrue(insightsExport("\"Include\": {\"TenantName\": [\"export-tenant\"]},")
+ .contains("[email protected]"));
+ assertFalse(insightsExport("\"Include\": {\"TenantName\": [\"export-tenant\"]},")
+ .contains("[email protected]"));
+
+ String excluded = insightsExport("\"Exclude\": {\"TenantName\": [\"export-tenant\"]},");
+ assertTrue(excluded.contains("[email protected]"), excluded);
+ assertFalse(excluded.contains("[email protected]"), excluded);
+ }
+
+ /** Runs a message-insights export carrying the given filter members and returns the file. */
+ private static String insightsExport(String filters) {
+ String jobId = create("""
+ {"ExportDataSource": {"MessageInsightsDataSource":
+ {%s "StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "CSV"}}
+ """.formatted(filters, Instant.now().minus(Duration.ofDays(1)).getEpochSecond(),
+ Instant.now().plus(Duration.ofDays(1)).getEpochSecond()))
+ .then().statusCode(200).extract().path("JobId");
+ assertEquals("COMPLETED", awaitTerminal(jobId));
+
+ String url = given().header("Authorization", AUTH)
+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(200)
+ .extract().path("ExportDestination.S3Url");
+ String path = url.substring(url.indexOf('/', url.indexOf("//") + 2));
+ return given().when().get(path).then().statusCode(200).extract().asString();
+ }
+
+ @Test
+ @Order(12)
+ void createExportJob_validatesTheMetricsSourceMembers() {
+ // Without these an unknown dimension was dropped, an unknown metric aggregated to zero and
+ // an unrecognised aggregation was read as VOLUME, so the job completed with a meaningless
+ // file. The wording follows BatchGetMetricData; AWS's own was not observed.
+ metricsSource("{\"NOT_A_DIMENSION\": [\"*\"]}", "\"VDM\"", "[{\"Name\": \"SEND\"}]")
+ .then().statusCode(400).body("message", containsString(
+ "Value at 'exportDataSource.metricsDataSource.dimensions' failed to satisfy "
+ + "constraint: Member must satisfy enum value set: "
+ + "[EMAIL_IDENTITY, CONFIGURATION_SET, ISP, TENANT_NAME]"));
+
+ metricsSource("{\"ISP\": [\"*\"]}", "\"VDM\"", "[{\"Name\": \"NOT_A_METRIC\"}]")
+ .then().statusCode(400).body("message", containsString(
+ "'exportDataSource.metricsDataSource.metrics.1.member.name'"));
+
+ metricsSource("{\"ISP\": [\"*\"]}", "\"VDM\"",
+ "[{\"Name\": \"SEND\", \"Aggregation\": \"AVERAGE\"}]")
+ .then().statusCode(400).body("message", containsString(
+ "'exportDataSource.metricsDataSource.metrics.1.member.aggregation'"));
+
+ metricsSource("{}", "\"SES\"", "[]")
+ .then().statusCode(400).body("message", containsString("3 validation errors"));
+
+ metricsSource("{\"ISP\": []}", "\"VDM\"", "[{\"Name\": \"SEND\"}]")
+ .then().statusCode(400).body("message", containsString(
+ "'exportDataSource.metricsDataSource.dimensions.ISP'"));
+ }
+
+ @Test
+ @Order(13)
+ void metricsExport_matchesAnIdentityDomainTheWayTheBatchQueryDoes() {
+ // The sender is an address; a domain names the same identity, so it has to select the row.
+ String jobId = metricsSource("{\"EMAIL_IDENTITY\": [\"example.com\"]}", "\"VDM\"",
+ "[{\"Name\": \"SEND\", \"Aggregation\": \"VOLUME\"}]")
+ .then().statusCode(200).extract().path("JobId");
+ assertEquals("COMPLETED", awaitTerminal(jobId));
+
+ String url = given().header("Authorization", AUTH)
+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(200)
+ .extract().path("ExportDestination.S3Url");
+ String path = url.substring(url.indexOf('/', url.indexOf("//") + 2));
+ String body = given().when().get(path).then().statusCode(200).extract().asString();
+
+ assertTrue(body.startsWith("EMAIL_IDENTITY,SEND_VOLUME\n"), body);
+ assertTrue(body.contains(SENDER), "a domain filter has to match the addresses inside it");
+ }
+
+ @Test
+ @Order(14)
+ void malformedPagingAndResultLimitsAreRefused() {
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"PageSize\": \"ten\"}")
+ .when().post("/v2/email/list-export-jobs").then().statusCode(400);
+
+ create("""
+ {"ExportDataSource": {"MessageInsightsDataSource":
+ {"MaxResults": -1, "StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "CSV"}}
+ """.formatted(midnightToday(), midnightToday() + 86400))
+ .then().statusCode(400).body("message", containsString("Member must be between 1 and 10000"));
+ }
+
+ /** A metrics export request with the three members the service validates. */
+ private static Response metricsSource(String dimensions, String namespace, String metrics) {
+ return create("""
+ {"ExportDataSource": {"MetricsDataSource": {"Dimensions": %s,
+ "Namespace": %s,
+ "Metrics": %s,
+ "StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "CSV"}}
+ """.formatted(dimensions, namespace, metrics, midnightToday(),
+ midnightToday() + 86400));
+ }
+
+ @Test
+ @Order(15)
+ void exportJob_runsUnderTheCallersAccount() {
+ // The worker runs on its own thread, where account-aware storage falls back to the default
+ // account unless the caller's is carried over: the job would then never be found again.
+ String otherAccount = "AWS4-HMAC-SHA256 Credential=111122223333/20260101/eu-west-2/ses/"
+ + "aws4_request";
+ given().contentType("application/json").header("Authorization", otherAccount)
+ .body("{\"VdmAttributes\":{\"VdmEnabled\":\"ENABLED\"}}")
+ .when().put("/v2/email/account/vdm").then().statusCode(200);
+
+ String jobId = given().contentType("application/json").header("Authorization", otherAccount)
+ .body(insightsSource())
+ .when().post("/v2/email/export-jobs").then().statusCode(200).extract().path("JobId");
+
+ String status = null;
+ for (int attempt = 0; attempt < 100 && !"COMPLETED".equals(status); attempt++) {
+ status = given().header("Authorization", otherAccount)
+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(200)
+ .extract().path("JobStatus");
+ if ("PROCESSING".equals(status) || "CREATED".equals(status)) {
+ status = null;
+ }
+ }
+ assertEquals("COMPLETED", status, "the job never finished under its own account");
+
+ // The default account cannot see it.
+ given().header("Authorization", AUTH)
+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(400);
+
+ given().contentType("application/json").header("Authorization", otherAccount)
+ .body("{\"VdmAttributes\":{\"VdmEnabled\":\"DISABLED\"}}")
+ .when().put("/v2/email/account/vdm");
+ }
+
+ @Test
+ @Order(16)
+ void listExportJobs_acceptsAPageSizeAboveTheDefault() {
+ // PageSize carries no bound in the model, so a large page is a normal request.
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"PageSize\": 250}")
+ .when().post("/v2/email/list-export-jobs").then().statusCode(200);
+
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"PageSize\": 0}")
+ .when().post("/v2/email/list-export-jobs").then().statusCode(400);
+
+ // A page size at the int ceiling is clamped, not refused, and must not report a further
+ // page: the paginator adds the page size to the offset as an int.
+ String token = given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"PageSize\": 1}")
+ .when().post("/v2/email/list-export-jobs").then().statusCode(200)
+ .extract().path("NextToken");
+ assertNotNull(token, "the fixture needs more than one job for this to mean anything");
+
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"PageSize\": 2147483647, \"NextToken\": \"%s\"}".formatted(token))
+ .when().post("/v2/email/list-export-jobs").then().statusCode(200)
+ .body("$", not(hasKey("NextToken")));
+ }
+
+ @Test
+ @Order(17)
+ void createExportJob_rejectsMembersOfTheWrongType() {
+ // Each of these was read as absent or coerced, so the job was created and produced a
+ // successful but meaningless file. They are deserialization errors, like every other
+ // typed SES v2 member.
+ create("""
+ {"ExportDataSource": {"MessageInsightsDataSource": {"StartDate": %d, "EndDate": %d},
+ "MetricsDataSource": []},
+ "ExportDestination": {"DataFormat": "CSV"}}
+ """.formatted(midnightToday(), midnightToday() + 86400))
+ .then().statusCode(400).body("__type", equalTo("SerializationException"));
+
+ create("""
+ {"ExportDataSource": {"MessageInsightsDataSource":
+ {"Include": {"TenantName": "blue"}, "StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "CSV"}}
+ """.formatted(midnightToday(), midnightToday() + 86400))
+ .then().statusCode(400).body("__type", equalTo("SerializationException"));
+
+ metricsSource("{\"ISP\": [1]}", "\"VDM\"", "[{\"Name\": \"SEND\"}]")
+ .then().statusCode(400).body("__type", equalTo("SerializationException"));
+
+ metricsSource("{\"ISP\": [\"*\"]}", "\"VDM\"", "[\"SEND\"]")
+ .then().statusCode(400).body("__type", equalTo("SerializationException"));
+
+ // A MaxResults of the wrong type is a deserialization error, not a value out of range.
+ insightsWith("\"MaxResults\": \"100\",")
+ .then().statusCode(400).body("__type", equalTo("SerializationException"));
+ }
+
+ @Test
+ @Order(18)
+ void createExportJob_holdsEachFilterMemberToItsLength() {
+ // The model allows one Subject and two LastEngagementEvent values.
+ insightsWith("\"Include\": {\"Subject\": [\"one\", \"two\"]},")
+ .then().statusCode(400).body("message", equalTo(
+ "1 validation error detected: Value at 'exportDataSource."
+ + "messageInsightsDataSource.include.subject' failed to satisfy "
+ + "constraint: Member must have length less than or equal to 1"));
+
+ insightsWith("\"Exclude\": {\"LastEngagementEvent\": [\"OPEN\", \"CLICK\", \"OPEN\"]},")
+ .then().statusCode(400).body("message", containsString(
+ "exclude.lastEngagementEvent' failed to satisfy constraint: Member must "
+ + "have length less than or equal to 2"));
+
+ insightsWith("\"Include\": {\"Isp\": [\"a\", \"b\", \"c\", \"d\", \"e\"]},")
+ .then().statusCode(200);
+ }
+
+ /** A message-insights export request carrying the given extra members. */
+ private static Response insightsWith(String members) {
+ return create("""
+ {"ExportDataSource": {"MessageInsightsDataSource":
+ {%s "StartDate": %d, "EndDate": %d}},
+ "ExportDestination": {"DataFormat": "CSV"}}
+ """.formatted(members, midnightToday(), midnightToday() + 86400));
+ }
+
+ @Test
+ @Order(19)
+ void cleanUpVdmAndIdentity() {
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"VdmAttributes\":{\"VdmEnabled\":\"DISABLED\"}}")
+ .when().put("/v2/email/account/vdm");
+ given().header("Authorization", AUTH)
+ .when().delete("/v2/email/identities/" + SENDER);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesExportPayloadsTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportPayloadsTest.java
new file mode 100644
index 0000000000..d4feba8bff
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportPayloadsTest.java
@@ -0,0 +1,136 @@
+package io.github.hectorvent.floci.services.ses;
+
+import io.github.hectorvent.floci.services.ses.model.EmailInsights;
+import io.github.hectorvent.floci.services.ses.model.InsightsBounce;
+import io.github.hectorvent.floci.services.ses.model.InsightsEvent;
+import io.github.hectorvent.floci.services.ses.model.InsightsEventDetails;
+import io.github.hectorvent.floci.services.ses.model.SentEmail;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The export payloads. Every column name, the quoting, the timestamp pattern and the trailing
+ * comma are probe-confirmed against real SES, so this test pins the wire format rather than
+ * Floci's preference.
+ */
+class SesExportPayloadsTest {
+
+ private static final Instant SENT = Instant.parse("2026-09-21T01:02:03.123Z");
+
+ private static final String HEADER =
+ "\"messageid\",\"sendtimestamp\",\"isp\",\"fromaddress\",\"destination\",\"subject\","
+ + "\"last_delivery_event\",\"last_engagement_event\","
+ + "\"last_delivery_event_timestamp\",\"last_engagement_event_timestamp\","
+ + "\"bounce_sub_type\",\"diagnostic_code\",";
+
+ private static SentEmail email(String subject, EmailInsights... recipients) {
+ SentEmail email = new SentEmail();
+ email.setMessageId("message-1");
+ email.setSource("[email protected]");
+ email.setSubject(subject);
+ email.setSentAt(SENT);
+ email.setInsights(List.of(recipients));
+ return email;
+ }
+
+ private static EmailInsights delivered(String destination) {
+ return new EmailInsights(destination, "UNKNOWN_ISP", List.of(
+ InsightsEvent.of(SENT, "SEND"), InsightsEvent.of(SENT, "DELIVERY")));
+ }
+
+ private static EmailInsights bounced(String destination) {
+ return new EmailInsights(destination, "UNKNOWN_ISP", List.of(
+ InsightsEvent.of(SENT, "SEND"),
+ new InsightsEvent(SENT, "BOUNCE", new InsightsEventDetails(
+ new InsightsBounce("PERMANENT", "General", "smtp; 550 user unknown"), null))));
+ }
+
+ private static String csv(List<SentEmail> emails) {
+ return new String(SesExportPayloads.insightsCsv(SesExportPayloads.insightsRows(emails)),
+ StandardCharsets.UTF_8);
+ }
+
+ @Test
+ void insightsCsvCarriesTheProbedHeaderIncludingItsTrailingComma() {
+ String[] lines = csv(List.of(email("hello", delivered("[email protected]")))).split("\n");
+
+ assertEquals(HEADER, lines[0]);
+ assertEquals("\"message-1\",\"2026-09-21 01:02:03.123 UTC\",\"UNKNOWN_ISP\","
+ + "\"[email protected]\",\"[email protected]\",\"hello\",\"DELIVERY\",\"\","
+ + "\"2026-09-21 01:02:03.123 UTC\",\"\",\"\",\"\",", lines[1]);
+ }
+
+ @Test
+ void oneRowPerRecipientNotPerMessage() {
+ String[] lines = csv(List.of(email("hello",
+ delivered("[email protected]"), delivered("[email protected]")))).split("\n");
+
+ assertEquals(3, lines.length);
+ }
+
+ @Test
+ void aBounceReportsItsTypeAndCarriesTheDetails() {
+ String[] lines = csv(List.of(email("hello", bounced("[email protected]")))).split("\n");
+
+ assertTrue(lines[1].contains("\"PERMANENT_BOUNCE\""), lines[1]);
+ assertTrue(lines[1].contains("\"General\""), lines[1]);
+ assertTrue(lines[1].contains("\"smtp; 550 user unknown\""), lines[1]);
+ }
+
+ @Test
+ void quotesInsideAValueAreDoubled() {
+ String row = csv(List.of(email("say \"hi\"", delivered("[email protected]")))).split("\n")[1];
+
+ assertTrue(row.contains("\"say \"\"hi\"\"\""), row);
+ }
+
+ @Test
+ void insightsJsonIsOneArrayWithTheCommaLeadingEachRowAfterTheFirst() {
+ String json = new String(SesExportPayloads.insightsJson(SesExportPayloads.insightsRows(
+ List.of(email("hello", delivered("[email protected]"), delivered("[email protected]"))))),
+ StandardCharsets.UTF_8);
+ String[] lines = json.split("\n");
+
+ assertEquals("[", lines[0]);
+ assertTrue(lines[1].startsWith("{\"messageid\":\"message-1\""), lines[1]);
+ assertTrue(lines[2].startsWith(",{"), lines[2]);
+ assertEquals("]", lines[3]);
+ }
+
+ @Test
+ void metricsCsvNamesTheDimensionThenEachMetricAndAggregation() {
+ String csv = new String(SesExportPayloads.metricsCsv(List.of("ISP"),
+ List.of("SEND_VOLUME", "DELIVERY_VOLUME"),
+ List.of(new SesExportPayloads.MetricRow(List.of("UNKNOWN_ISP"), List.of(3.0, 2.0)))),
+ StandardCharsets.UTF_8);
+
+ assertEquals("ISP,SEND_VOLUME,DELIVERY_VOLUME\nUNKNOWN_ISP,3.0000,2.0000\n", csv);
+ }
+
+ @Test
+ void metricsCsvQuotesOnlyTheValuesThatNeedIt() {
+ // The probed file leaves values bare, but an unquoted comma would shift every later column.
+ String csv = new String(SesExportPayloads.metricsCsv(List.of("EMAIL_IDENTITY"),
+ List.of("SEND_VOLUME"),
+ List.of(new SesExportPayloads.MetricRow(List.of("a,[email protected]"), List.of(1.0)))),
+ StandardCharsets.UTF_8);
+
+ assertEquals("EMAIL_IDENTITY,SEND_VOLUME\n\"a,[email protected]\",1.0000\n", csv);
+ }
+
+ @Test
+ void metricsJsonKeysTheDimensionInLowerCaseAndNestsTheMetrics() {
+ String json = new String(SesExportPayloads.metricsJson(List.of("ISP"),
+ List.of("DELIVERY_RATE"),
+ List.of(new SesExportPayloads.MetricRow(List.of("UNKNOWN_ISP"), List.of(0.5)))),
+ StandardCharsets.UTF_8);
+
+ assertEquals("[{\"isp\":\"UNKNOWN_ISP\",\"metrics\":{\"DELIVERY_RATE\":0.5000}}]\n", json);
+ }
+}