Light, fluffy, and always free - The AWS Local Emulator alternative
feat(ses): add v2 export jobs (#4327)
Kenshin Okinaka08c6ce0768f7
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesExportJobTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesExportJobTest.javanew file mode 100644index 0000000000..1873ff0e30--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesExportJobTest.java@@ -0,0 +1,233 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import software.amazon.awssdk.services.sesv2.SesV2Client;+import software.amazon.awssdk.services.sesv2.model.BadRequestException;+import software.amazon.awssdk.services.sesv2.model.Body;+import software.amazon.awssdk.services.sesv2.model.CancelExportJobRequest;+import software.amazon.awssdk.services.sesv2.model.Content;+import software.amazon.awssdk.services.sesv2.model.CreateEmailIdentityRequest;+import software.amazon.awssdk.services.sesv2.model.CreateExportJobRequest;+import software.amazon.awssdk.services.sesv2.model.DataFormat;+import software.amazon.awssdk.services.sesv2.model.DeleteEmailIdentityRequest;+import software.amazon.awssdk.services.sesv2.model.Destination;+import software.amazon.awssdk.services.sesv2.model.EmailContent;+import software.amazon.awssdk.services.sesv2.model.ExportDataSource;+import software.amazon.awssdk.services.sesv2.model.ExportDestination;+import software.amazon.awssdk.services.sesv2.model.ExportSourceType;+import software.amazon.awssdk.services.sesv2.model.GetAccountRequest;+import software.amazon.awssdk.services.sesv2.model.GetAccountResponse;+import software.amazon.awssdk.services.sesv2.model.GetExportJobRequest;+import software.amazon.awssdk.services.sesv2.model.GetExportJobResponse;+import software.amazon.awssdk.services.sesv2.model.ListExportJobsRequest;+import software.amazon.awssdk.services.sesv2.model.ListExportJobsResponse;+import software.amazon.awssdk.services.sesv2.model.Message;+import software.amazon.awssdk.services.sesv2.model.MessageInsightsDataSource;+import software.amazon.awssdk.services.sesv2.model.PutAccountVdmAttributesRequest;+import software.amazon.awssdk.services.sesv2.model.SendEmailRequest;+import software.amazon.awssdk.services.sesv2.model.VdmAttributes;++import java.time.Duration;+import java.time.Instant;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;++/**+ * SDK compatibility test for the SES v2 export jobs against a live Floci instance: the Virtual+ * Deliverability Manager gate, which answers BadRequestException here rather than the+ * NotFoundException the other two gated operations use, the refusal of a caller-chosen S3Url, a+ * job followed to completion with its presigned URL and its echoed data source, and the two+ * server-side checks the SDK cannot catch on the client.+ */+@DisplayName("SES v2 Export Jobs")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class SesExportJobTest {++ private static final String SENDER = "[email protected]";++ private static SesV2Client sesV2;+ private static VdmAttributes originalVdm;+ private static String completedJobId;++ @BeforeAll+ static void setup() {+ sesV2 = TestFixtures.sesV2Client();+ GetAccountResponse account = sesV2.getAccount(GetAccountRequest.builder().build());+ originalVdm = account.vdmAttributes();+ quietly(() -> sesV2.deleteEmailIdentity(+ DeleteEmailIdentityRequest.builder().emailIdentity(SENDER).build()));+ setVdmEnabled(false);+ sesV2.createEmailIdentity(CreateEmailIdentityRequest.builder().emailIdentity(SENDER).build());+ }++ @AfterAll+ static void cleanup() {+ if (sesV2 == null) {+ return;+ }+ quietly(() -> sesV2.deleteEmailIdentity(+ DeleteEmailIdentityRequest.builder().emailIdentity(SENDER).build()));+ quietly(SesExportJobTest::restoreVdm);+ sesV2.close();+ }++ @Test+ @Order(1)+ @DisplayName("the VDM gate is a BadRequest here, not the NotFound the other gated operations use")+ void vdmGateIsABadRequest() {+ assertThatThrownBy(() -> sesV2.createExportJob(insightsExport(null)))+ .isInstanceOf(BadRequestException.class)+ .hasMessageContaining("Virtual Deliverability Manager");+ }++ @Test+ @Order(2)+ @DisplayName("a caller-chosen destination is refused")+ void customDestinationIsRefused() {+ setVdmEnabled(true);++ assertThatThrownBy(() -> sesV2.createExportJob(insightsExport("s3://mine/prefix/")))+ .isInstanceOf(BadRequestException.class)+ .hasMessageContaining("Providing a custom S3 URL is not supported");+ }++ @Test+ @Order(3)+ @DisplayName("a job runs to completion and carries a presigned URL and its own data source")+ void jobCompletesAndCarriesItsDestination() {+ send();+ completedJobId = sesV2.createExportJob(insightsExport(null)).jobId();+ assertThat(completedJobId).isNotBlank();++ GetExportJobResponse job = awaitTerminal(completedJobId);++ assertThat(job.jobStatusAsString()).isEqualTo("COMPLETED");+ assertThat(job.exportSourceType()).isEqualTo(ExportSourceType.MESSAGE_INSIGHTS);+ assertThat(job.exportDestination().dataFormat()).isEqualTo(DataFormat.CSV);+ assertThat(job.exportDestination().s3Url()).isNotBlank();+ // The union round-trips through the SDK, which is what an integration test asserting raw+ // JSON cannot show.+ assertThat(job.exportDataSource().messageInsightsDataSource()).isNotNull();+ assertThat(job.exportDataSource().metricsDataSource()).isNull();+ assertThat(job.createdTimestamp()).isNotNull();+ assertThat(job.completedTimestamp()).isNotNull();+ }++ @Test+ @Order(4)+ @DisplayName("the list reports the job and paginates")+ void listReportsTheJob() {+ ListExportJobsResponse all = sesV2.listExportJobs(ListExportJobsRequest.builder()+ .exportSourceType(ExportSourceType.MESSAGE_INSIGHTS).build());+ assertThat(all.exportJobs()).isNotEmpty();+ assertThat(all.exportJobs()).allSatisfy(summary ->+ assertThat(summary.exportSourceType()).isEqualTo(ExportSourceType.MESSAGE_INSIGHTS));++ ListExportJobsResponse firstPage = sesV2.listExportJobs(+ ListExportJobsRequest.builder().pageSize(1).build());+ assertThat(firstPage.exportJobs()).hasSize(1);+ }++ @Test+ @Order(5)+ @DisplayName("a finished job cannot be cancelled")+ void finishedJobCannotBeCancelled() {+ assertThatThrownBy(() -> sesV2.cancelExportJob(+ CancelExportJobRequest.builder().jobId(completedJobId).build()))+ .isInstanceOf(BadRequestException.class)+ .hasMessageContaining("Cannot cancel an export job that has a <COMPLETED> status.");+ }++ @Test+ @Order(6)+ @DisplayName("an unknown job id is a BadRequest rather than a NotFound")+ void unknownJobIsABadRequest() {+ assertThatThrownBy(() -> sesV2.getExportJob(+ GetExportJobRequest.builder().jobId("no-such-job").build()))+ .isInstanceOf(BadRequestException.class)+ .hasMessageContaining("Invalid JobId");+ }++ private static CreateExportJobRequest insightsExport(String s3Url) {+ ExportDestination.Builder destination = ExportDestination.builder().dataFormat(DataFormat.CSV);+ if (s3Url != null) {+ destination.s3Url(s3Url);+ }+ return CreateExportJobRequest.builder()+ .exportDataSource(ExportDataSource.builder()+ .messageInsightsDataSource(MessageInsightsDataSource.builder()+ .startDate(Instant.now().minus(Duration.ofDays(1)))+ .endDate(Instant.now().plus(Duration.ofDays(1)))+ .build())+ .build())+ .exportDestination(destination.build())+ .build();+ }++ private static GetExportJobResponse awaitTerminal(String jobId) {+ for (int attempt = 0; attempt < 100; attempt++) {+ GetExportJobResponse job = sesV2.getExportJob(+ GetExportJobRequest.builder().jobId(jobId).build());+ String status = job.jobStatusAsString();+ if (!"PROCESSING".equals(status) && !"CREATED".equals(status)) {+ return job;+ }+ try {+ Thread.sleep(50);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError("interrupted while waiting for the job", e);+ }+ }+ throw new AssertionError("export job " + jobId + " never reached a terminal state");+ }++ private static void send() {+ sesV2.sendEmail(SendEmailRequest.builder()+ .fromEmailAddress(SENDER)+ .destination(Destination.builder().toAddresses("[email protected]").build())+ .content(EmailContent.builder()+ .simple(Message.builder()+ .subject(Content.builder().data("compat-export").build())+ .body(Body.builder()+ .text(Content.builder().data("hi").build())+ .build())+ .build())+ .build())+ .build());+ }++ private static void setVdmEnabled(boolean enabled) {+ sesV2.putAccountVdmAttributes(PutAccountVdmAttributesRequest.builder()+ .vdmAttributes(VdmAttributes.builder()+ .vdmEnabled(enabled ? "ENABLED" : "DISABLED")+ .build())+ .build());+ }++ private static void restoreVdm() {+ if (originalVdm == null || originalVdm.vdmEnabled() == null) {+ setVdmEnabled(false);+ return;+ }+ sesV2.putAccountVdmAttributes(PutAccountVdmAttributesRequest.builder()+ .vdmAttributes(originalVdm)+ .build());+ }++ private static void quietly(Runnable action) {+ try {+ action.run();+ } catch (RuntimeException expected) {+ // Best effort teardown: the identity or the VDM state may already be gone.+ }+ }+}diff --git a/docs/services/ses.md b/docs/services/ses.mdindex 807f99f83d..63ec1c4fff 100644--- a/docs/services/ses.md+++ b/docs/services/ses.md@@ -379,6 +379,10 @@ Alongside the classic Query API, Floci implements a subset of the SES v2 REST JS| `POST` | `/v2/email/import-jobs/list` | `ListImportJobs` || `GET` | `/v2/email/insights/{MessageId}` | `GetMessageInsights` || `POST` | `/v2/email/metrics/batch` | `BatchGetMetricData` |+| `POST` | `/v2/email/export-jobs` | `CreateExportJob` |+| `GET` | `/v2/email/export-jobs/{JobId}` | `GetExportJob` |+| `POST` | `/v2/email/list-export-jobs` | `ListExportJobs` |+| `PUT` | `/v2/email/export-jobs/{JobId}/cancel` | `CancelExportJob` |Floci models no leased dedicated IPs: `GetDedicatedIps` is empty and IP-targeted operations return `NotFoundException`, as real AWS does for an account with no leased IPs, with required request members validated first (`BadRequestException`). `PutDedicatedIpPoolScalingAttributes` rejects downgrading a `MANAGED` pool to `STANDARD`, and `PutAccountDedicatedIpWarmupAttributes` stores the flag behind `GetAccount.DedicatedIpAutoWarmupEnabled` (default `true`).@@ -422,6 +426,8 @@ Suppressed recipients are filtered out of the SMTP relay step (non-suppressed re`BatchGetMetricData` reads those same stored timelines, so it and `GetMessageInsights` cannot disagree, and it shares the VDM gate. Counts are per recipient, in whole UTC days, and sparse: a day with no data is absent, not zero. `OPEN`, `CLICK` and their delivery variants stay empty because they need a real mailbox, and `TRANSIENT_BOUNCE` because every bounce Floci derives is permanent. Dimensions filter the send: `EMAIL_IDENTITY` on the address or its domain, `CONFIGURATION_SET` on the set it resolved to, `TENANT_NAME` on the tenant it named, and `ISP` only ever on `UNKNOWN_ISP`. Each value is validated with AWS's own rule and message: the configuration set, the tenant and the ISP take ASCII letters, digits, `_` and `-`, and an identity is a bare name or `local@domain`. Where AWS's validation order could not be observed, Floci ranks the member constraints below a duplicate `Id` and the interval cap below the start-before-end check, an absent date fails with a null-constraint message, and a query with two invalid dimension values reports the first in `EMAIL_IDENTITY`, `ISP`, `CONFIGURATION_SET`, `TENANT_NAME` order.+Export jobs render those same timelines as CSV or JSON. Floci writes the file into its own S3 emulation, under a bucket named `floci-<account>-<region>-ses-export-files`, so the presigned URL on a finished job resolves against Floci. The metrics export emits one row per dimension combination Floci has seen, where real SES emits its whole ISP catalogue with zeros, and a `RATE` is the metric's volume over the send volume for that combination. `LastEngagementEvent` filters select nothing, since Floci derives no opens or clicks. An export finishes in milliseconds, so `CancelExportJob` rarely finds a running job to catch. With S3 authentication enforcement on, the `S3Url` of a finished job answers 403: the shared presigner signs with Floci's own HMAC, which S3 then verifies as SigV4.+Import jobs read their source from Floci's own S3 emulation, so the object has to be uploaded there rather than to real S3; the CSV and newline-delimited JSON record formats are AWS's own ([suppression list](https://docs.aws.amazon.com/ses/latest/dg/sending-email-suppression-list.html#sending-email-suppression-list-manual-add-bulk), [contact list](https://docs.aws.amazon.com/ses/latest/dg/sending-email-list-management.html#configuring-list-management-bulk-import)). A job still running when the emulator restarts is marked `FAILED` on the next start. Not emulated: `FailureInfo.FailedRecordsS3Url` (Floci writes no failed-record file), the per-file record limits, the same-region bucket rule, and `ListImportJobs` pagination, which returns every job in one page.`SendEmail` honors `ListManagementOptions` (`ContactListName`, optional `TopicName`). When present, each recipient is matched against the named contact list and suppressed as a `Bounce` when opted out — reusing the same relay-exclusion and Bounce-event path as suppression-list filtering, matching AWS ("SES will issue a bounce event for a message that is sent to an unsubscribed contact"). A contact is opted out when `UnsubscribeAll` is set; with a `TopicName`, an explicit `OPT_OUT` preference for that topic (or, absent an explicit preference, the topic's `DefaultSubscriptionStatus` being `OPT_OUT`) suppresses; without a `TopicName`, only `UnsubscribeAll` contacts are suppressed. A recipient that is not yet a contact is created on the list automatically (as on AWS) and then evaluated. Referencing a contact list that does not exist fails the send with `NotFoundException`. The topic-default fallback at send time is an intentional deviation — it is not documented by AWS and mirrors the effective-status model AWS uses for `ListContacts`.diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.javaindex a51f50f03c..f1da8511e7 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java@@ -38,6 +38,7 @@ import io.github.hectorvent.floci.services.ses.SesConfigurationSetController;import io.github.hectorvent.floci.services.ses.SesContactController;import io.github.hectorvent.floci.services.ses.SesCvetController;import io.github.hectorvent.floci.services.ses.SesDedicatedIpController;+import io.github.hectorvent.floci.services.ses.SesExportJobController;import io.github.hectorvent.floci.services.ses.SesIdentityController;import io.github.hectorvent.floci.services.ses.SesImportJobController;import io.github.hectorvent.floci.services.ses.SesInsightsController;@@ -307,7 +308,8 @@ public class ResolvedServiceCatalog {Set.of(), Set.of("email", "ses", "sesv2"), Set.of(),Set.of(SesAccountController.class, SesConfigurationSetController.class,SesContactController.class, SesCvetController.class,- SesDedicatedIpController.class, SesIdentityController.class,+ SesDedicatedIpController.class, SesExportJobController.class,+ SesIdentityController.class,SesInsightsController.class, SesMetricsController.class,SesSendController.class,SesSuppressionController.class, SesTagController.class,diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesAccountService.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesAccountService.javaindex 220d08e575..acd302f72c 100644--- a/src/main/java/io/github/hectorvent/floci/services/ses/SesAccountService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesAccountService.java@@ -87,10 +87,19 @@ public class SesAccountService {/** The probe-confirmed gate the VDM-only operations check before reading the request. */public void requireVdmEnabled(String region) {+ requireVdmEnabled(region, "NotFoundException", 404);+ }++ /**+ * The same gate with the caller's error shape. Message insights and metric data answer+ * {@code NotFoundException} 404, while export jobs answer {@code BadRequestException} 400 with+ * the identical sentence, all probe-confirmed.+ */+ public void requireVdmEnabled(String region, String errorCode, int status) {Optional<AccountVdmAttributes> vdm = findAccountVdmAttributes(region);if (vdm.isEmpty() || !vdm.get().vdmEnabled()) {- throw new AwsException("NotFoundException",- "To use this feature you must enable Virtual Deliverability Manager", 404);+ throw new AwsException(errorCode,+ "To use this feature you must enable Virtual Deliverability Manager", status);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesExportFilters.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportFilters.javanew file mode 100644index 0000000000..cd883f9b90--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportFilters.java@@ -0,0 +1,157 @@+package io.github.hectorvent.floci.services.ses;++import com.fasterxml.jackson.databind.JsonNode;+import io.github.hectorvent.floci.core.common.AwsException;++import java.util.ArrayList;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Locale;+import java.util.Set;+import java.util.function.Function;++import static io.github.hectorvent.floci.services.ses.SesV2Json.stringArrayOrAbsent;++/**+ * The {@code Include} / {@code Exclude} filters and {@code MaxResults} of a message-insights+ * export. A row has to match every member present in {@code Include} and no member present in+ * {@code Exclude}; within one member the values are alternatives.+ *+ * <p>{@code TenantName} matches the tenant the send named, which is carried beside the row's+ * columns because the exported file has no tenant column.+ *+ * <p>{@code LastEngagementEvent} filters on opens and clicks, which Floci never derives, so an+ * include on it selects nothing and an exclude on it removes nothing. That is stated in+ * {@code docs/services/ses.md} rather than treated as an error, because the request is valid.+ */+final class SesExportFilters {++ private static final int NO_LIMIT = -1;++ private SesExportFilters() {}++ static List<SesExportPayloads.InsightsRow> apply(List<SesExportPayloads.InsightsRow> rows,+ JsonNode insights) {+ JsonNode include = insights.path("Include");+ JsonNode exclude = insights.path("Exclude");+ List<SesExportPayloads.InsightsRow> kept = new ArrayList<>();+ for (SesExportPayloads.InsightsRow row : rows) {+ if (include.isObject() && !matches(row, include)) {+ continue;+ }+ if (exclude.isObject() && hasConstraint(exclude) && matches(row, exclude)) {+ continue;+ }+ kept.add(row);+ }+ // The service validated the range at create time; this stays defensive so a stored job+ // written by an older build cannot slice with a negative bound.+ JsonNode maxResults = insights.path("MaxResults");+ if (maxResults.isInt() && maxResults.intValue() > 0 && kept.size() > maxResults.intValue()) {+ return kept.subList(0, maxResults.intValue());+ }+ return kept;+ }++ /**+ * The filter members, each paired with the value it reads off a row, so validating a request+ * and filtering its rows can never drift apart.+ */+ private enum Member {+ FROM_EMAIL_ADDRESS("FromEmailAddress", 5, row -> row.columns().get("fromaddress")),+ DESTINATION("Destination", 5, row -> row.columns().get("destination")),+ SUBJECT("Subject", 1, row -> row.columns().get("subject")),+ ISP("Isp", 5, row -> row.columns().get("isp")),+ LAST_DELIVERY_EVENT("LastDeliveryEvent", 5,+ row -> row.columns().get("last_delivery_event")),+ LAST_ENGAGEMENT_EVENT("LastEngagementEvent", 2,+ row -> row.columns().get("last_engagement_event")),+ // The tenant filter arrived with the 2026-09 tenant launch and is absent from the pinned+ // botocore model, so its list has no length to enforce yet.+ TENANT_NAME("TenantName", NO_LIMIT, SesExportPayloads.InsightsRow::tenantName);++ private final String wireName;+ private final int maxValues;+ private final Function<SesExportPayloads.InsightsRow, String> value;++ Member(String wireName, int maxValues,+ Function<SesExportPayloads.InsightsRow, String> value) {+ this.wireName = wireName;+ this.maxValues = maxValues;+ this.value = value;+ }+ }++ /**+ * Rejects a filter block that is not an object and a member that is not a list of strings,+ * rather than reading it as no filter at all and quietly returning rows the caller excluded,+ * then holds each member to the length the model gives it.+ */+ static void validate(JsonNode insights) {+ List<String> violations = new ArrayList<>();+ requireFilterBlock(insights.path("Include"), "include", violations);+ requireFilterBlock(insights.path("Exclude"), "exclude", violations);+ if (!violations.isEmpty()) {+ String header = violations.size() == 1+ ? "1 validation error detected: "+ : violations.size() + " validation errors detected: ";+ throw new AwsException("BadRequestException",+ header + String.join("; ", violations), 400);+ }+ }++ private static void requireFilterBlock(JsonNode filters, String path, List<String> violations) {+ if (filters.isMissingNode() || filters.isNull()) {+ return;+ }+ if (!filters.isObject()) {+ throw new AwsException("SerializationException", null, 400);+ }+ for (Member member : Member.values()) {+ List<String> values = stringArrayOrAbsent(filters, member.wireName);+ if (values != null && member.maxValues != NO_LIMIT && values.size() > member.maxValues) {+ violations.add("Value at 'exportDataSource.messageInsightsDataSource." + path + "."+ + member.wireName.substring(0, 1).toLowerCase(Locale.ROOT)+ + member.wireName.substring(1)+ + "' failed to satisfy constraint: Member must have length less than or "+ + "equal to " + member.maxValues);+ }+ }+ }++ /**+ * A member with no values constrains nothing, so a block carrying none matches every row. That+ * reading is harmless for {@code Include} and empties the file for {@code Exclude}, which is+ * why the exclude side asks first whether there is anything to exclude on.+ */+ private static boolean hasConstraint(JsonNode filters) {+ for (Member member : Member.values()) {+ JsonNode values = filters.path(member.wireName);+ if (values.isArray() && !values.isEmpty()) {+ return true;+ }+ }+ return false;+ }++ private static boolean matches(SesExportPayloads.InsightsRow row, JsonNode filters) {+ for (Member member : Member.values()) {+ if (!matchesMember(filters, member.wireName, member.value.apply(row))) {+ return false;+ }+ }+ return true;+ }++ private static boolean matchesMember(JsonNode filters, String member, String actual) {+ JsonNode values = filters.path(member);+ if (!values.isArray() || values.isEmpty()) {+ return true;+ }+ Set<String> wanted = new LinkedHashSet<>();+ for (JsonNode value : values) {+ wanted.add(value.asText());+ }+ return actual != null && wanted.contains(actual);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobController.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobController.javanew file mode 100644index 0000000000..1c09e70582--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobController.java@@ -0,0 +1,171 @@+package io.github.hectorvent.floci.services.ses;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.PaginatedResult;+import io.github.hectorvent.floci.core.common.Pagination;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.ses.model.ExportJob;+import jakarta.inject.Inject;+import jakarta.ws.rs.Consumes;+import jakarta.ws.rs.GET;+import jakarta.ws.rs.POST;+import jakarta.ws.rs.PUT;+import jakarta.ws.rs.Path;+import jakarta.ws.rs.PathParam;+import jakarta.ws.rs.Produces;+import jakarta.ws.rs.core.Context;+import jakarta.ws.rs.core.HttpHeaders;+import jakarta.ws.rs.core.MediaType;+import jakarta.ws.rs.core.Response;++import java.time.Instant;+import java.util.Optional;++import static io.github.hectorvent.floci.services.ses.SesV2Json.intMemberOrAbsent;+import static io.github.hectorvent.floci.services.ses.SesV2Json.readOptionBody;+import static io.github.hectorvent.floci.services.ses.SesV2Json.stringMemberOrAbsent;++/**+ * SES V2 export jobs ({@code /v2/email/export-jobs}, {@code /v2/email/list-export-jobs}).+ *+ * <p>Probe-confirmed against real SES (2026-09-23): a request that names its own {@code S3Url} is+ * refused before the Virtual Deliverability Manager gate is reached, which is why the gate is+ * passed into the service rather than checked first as it is for message insights and metric data.+ * The gate itself answers {@code BadRequestException} 400 here, where those two answer+ * {@code NotFoundException} 404 with the same sentence.+ */+@Path("/v2/email")+@Produces(MediaType.APPLICATION_JSON)+@Consumes(MediaType.APPLICATION_JSON)+public class SesExportJobController {++ private static final int DEFAULT_PAGE_SIZE = 100;+ /**+ * A page no emulator will ever fill, so a larger {@code PageSize} can be clamped instead of+ * refused. The paginator adds this to the page offset as an int, so {@code Integer.MAX_VALUE}+ * would overflow that sum and report a further page that does not exist.+ */+ private static final int MAX_PAGE_SIZE = 1_000_000;++ private final SesExportJobService exportJobService;+ private final SesAccountService accountService;+ private final RegionResolver regionResolver;+ private final ObjectMapper objectMapper;++ @Inject+ public SesExportJobController(SesExportJobService exportJobService,+ SesAccountService accountService,+ RegionResolver regionResolver, ObjectMapper objectMapper) {+ this.exportJobService = exportJobService;+ this.accountService = accountService;+ this.regionResolver = regionResolver;+ this.objectMapper = objectMapper;+ }++ @POST+ @Path("/export-jobs")+ public Response createExportJob(@Context HttpHeaders headers, String body) {+ String region = regionResolver.resolveRegion(headers);+ JsonNode request = readOptionBody(objectMapper, body);+ JsonNode destination = request.path("ExportDestination");+ JsonNode dataSource = request.path("ExportDataSource");+ ExportJob job = exportJobService.createExportJob(region, regionResolver.getAccountId(),+ dataSource, destination,+ () -> accountService.requireVdmEnabled(region, "BadRequestException", 400));+ ObjectNode result = objectMapper.createObjectNode();+ result.put("JobId", job.getJobId());+ return Response.ok(result).build();+ }++ @GET+ @Path("/export-jobs/{jobId}")+ public Response getExportJob(@Context HttpHeaders headers, @PathParam("jobId") String jobId) {+ String region = regionResolver.resolveRegion(headers);+ ExportJob job = exportJobService.getExportJob(region, jobId);+ ObjectNode result = objectMapper.createObjectNode();+ result.put("JobId", job.getJobId());+ result.put("ExportSourceType", job.getExportSourceType());+ result.put("JobStatus", job.getJobStatus());++ ObjectNode destination = result.putObject("ExportDestination");+ destination.put("DataFormat", job.getDataFormat());+ exportJobService.presignedUrl(job).ifPresent(url -> destination.put("S3Url", url));+ result.set("ExportDataSource", readOptionBody(objectMapper, job.getDataSource()));++ putTimestamp(result, "CreatedTimestamp", job.getCreatedTimestamp());+ putTimestamp(result, "CompletedTimestamp", job.getCompletedTimestamp());+ if (job.getProcessedRecordsCount() != null) {+ // AWS returns only ProcessedRecordsCount, never ExportedRecordsCount, so neither does+ // Floci even though the model carries both.+ result.putObject("Statistics")+ .put("ProcessedRecordsCount", job.getProcessedRecordsCount());+ }+ if (job.getErrorMessage() != null) {+ result.putObject("FailureInfo").put("ErrorMessage", job.getErrorMessage());+ }+ return Response.ok(result).build();+ }++ @POST+ @Path("/list-export-jobs")+ public Response listExportJobs(@Context HttpHeaders headers, String body) {+ String region = regionResolver.resolveRegion(headers);+ JsonNode request = readOptionBody(objectMapper, body);+ String sourceType = stringMemberOrAbsent(request, "ExportSourceType");+ String jobStatus = stringMemberOrAbsent(request, "JobStatus");+ String nextToken = stringMemberOrAbsent(request, "NextToken");+ // PageSize carries no bound in the model, so a large one is clamped rather than refused;+ // only a value below 1 is an error, and the default applies when it is absent.+ Integer pageSize = intMemberOrAbsent(request, "PageSize");+ if (pageSize != null && pageSize > MAX_PAGE_SIZE) {+ pageSize = MAX_PAGE_SIZE;+ }++ PaginatedResult<ExportJob> page = Pagination.paginate(+ exportJobService.listExportJobs(region, sourceType, jobStatus),+ SesExportJobController::cursor, pageSize, nextToken, DEFAULT_PAGE_SIZE,+ MAX_PAGE_SIZE, "BadRequestException");++ ObjectNode result = objectMapper.createObjectNode();+ ArrayNode jobs = result.putArray("ExportJobs");+ for (ExportJob job : page.items()) {+ ObjectNode node = jobs.addObject();+ node.put("JobId", job.getJobId());+ node.put("ExportSourceType", job.getExportSourceType());+ node.put("JobStatus", job.getJobStatus());+ putTimestamp(node, "CreatedTimestamp", job.getCreatedTimestamp());+ putTimestamp(node, "CompletedTimestamp", job.getCompletedTimestamp());+ }+ if (page.nextToken() != null) {+ result.put("NextToken", page.nextToken());+ }+ return Response.ok(result).build();+ }++ @PUT+ @Path("/export-jobs/{jobId}/cancel")+ public Response cancelExportJob(@Context HttpHeaders headers, @PathParam("jobId") String jobId) {+ String region = regionResolver.resolveRegion(headers);+ exportJobService.cancelExportJob(region, jobId);+ return Response.ok(objectMapper.createObjectNode()).build();+ }++ /**+ * The shared paginator orders by the cursor ascending, and AWS returns the newest job first, so+ * the cursor counts time backwards. The job id breaks ties between jobs created in the same+ * millisecond.+ */+ private static String cursor(ExportJob job) {+ Instant created = job.getCreatedTimestamp();+ long descending = Long.MAX_VALUE - (created == null ? 0L : created.toEpochMilli());+ return String.format("%019d#%s", descending, job.getJobId());+ }++ private static void putTimestamp(ObjectNode node, String field, Instant value) {+ Optional.ofNullable(value)+ .ifPresent(instant -> node.put(field, instant.toEpochMilli() / 1000.0));+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobService.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobService.javanew file mode 100644index 0000000000..522a4de558--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportJobService.java@@ -0,0 +1,607 @@+package io.github.hectorvent.floci.services.ses;++import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.core.common.Resettable;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.s3.PreSignedUrlGenerator;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.ses.model.ExportJob;+import io.github.hectorvent.floci.services.ses.model.SentEmail;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.time.Clock;+import java.time.Instant;+import java.time.ZoneOffset;+import java.time.format.DateTimeFormatter;+import java.time.format.DateTimeParseException;+import java.time.temporal.ChronoUnit;+import java.util.ArrayList;+import java.util.Comparator;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.UUID;+import java.util.concurrent.Executor;+import java.util.concurrent.atomic.AtomicInteger;++import static io.github.hectorvent.floci.services.ses.SesV2Json.intMemberOrAbsent;+import static io.github.hectorvent.floci.services.ses.SesV2Json.stringMemberOrAbsent;++/**+ * SES V2 export jobs. A job reads the timelines already stored at send time, renders them through+ * {@link SesExportPayloads} and writes one object, which {@code GetExportJob} hands back as a+ * presigned URL.+ *+ * <p>Probe-confirmed against real SES (2026-09-23, us-east-1): the caller cannot choose the+ * destination, a request carrying {@code S3Url} is refused before anything else is checked, and+ * exactly one data source must be present. The metrics source insists on whole UTC days where the+ * message-insights source does not. A job is already {@code PROCESSING} on the first read, so+ * {@code CREATED} is never observed from outside.+ *+ * <p>Deviations, all documented in {@code docs/services/ses.md}: real SES writes into a bucket it+ * owns, while Floci writes into its own S3 emulation so the presigned URL resolves; the metrics+ * export lists only the dimension values Floci has seen rather than AWS's fixed ISP catalogue; and+ * the concurrent-job ceiling is the published quota of 20, since the probe hit AWS's own+ * limit before it could measure one.+ */+@ApplicationScoped+public class SesExportJobService implements Resettable {++ private static final Logger LOG = Logger.getLogger(SesExportJobService.class);++ // The published quota, the same figure SES gives import jobs:+ // https://docs.aws.amazon.com/ses/latest/dg/quotas.html+ static final int MAX_CONCURRENT_JOBS = 20;+ static final int MAX_RESULTS = 10_000;+ static final long RESET_DRAIN_TIMEOUT_MILLIS = 5_000L;+ private static final long RESET_ADMISSION_WAIT_MILLIS = 30_000L;+ static final String RESTART_FAILURE_MESSAGE = "The emulator restarted before the export finished";+ static final String EXPORT_BUCKET_SUFFIX = "-ses-export-files";++ private static final Set<String> DATA_FORMATS = Set.of(ExportJob.FORMAT_CSV, ExportJob.FORMAT_JSON);+ private static final Set<String> SOURCE_TYPES =+ Set.of(ExportJob.SOURCE_METRICS, ExportJob.SOURCE_MESSAGE_INSIGHTS);+ private static final Set<String> JOB_STATUSES = Set.of(ExportJob.STATUS_CREATED,+ ExportJob.STATUS_PROCESSING, ExportJob.STATUS_COMPLETED, ExportJob.STATUS_FAILED,+ ExportJob.STATUS_CANCELLED);+ private static final DateTimeFormatter OBJECT_STAMP =+ DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss", Locale.ROOT).withZone(ZoneOffset.UTC);+ private static final int PRESIGNED_URL_EXPIRY_SECONDS = 3600;++ private final StorageBackend<String, ExportJob> exportJobStore;+ private final SesSentEmailService sentEmailService;+ private final S3Service s3Service;+ private final PreSignedUrlGenerator preSignedUrlGenerator;+ private final ObjectMapper objectMapper;+ private final Clock clock;+ private final Executor workers;+ private final String baseUrl;+ private final Object createLock = new Object();+ // Reset support, following SesImportJobService: beforeReset bumps the generation so a running+ // worker stops before it writes its object, then waits for the workers already inside one. The+ // wait is bounded, so a reset still returns when a worker is wedged.+ private final AtomicInteger generation = new AtomicInteger();+ private boolean resetting;+ private final Object workerMonitor = new Object();+ private int activeWorkers;++ @Inject+ public SesExportJobService(StorageFactory storageFactory, SesSentEmailService sentEmailService,+ S3Service s3Service, PreSignedUrlGenerator preSignedUrlGenerator,+ ObjectMapper objectMapper, Clock clock, EmulatorConfig config) {+ this(storageFactory.create("ses", "ses-export-jobs.json",+ new TypeReference<Map<String, ExportJob>>() {}),+ sentEmailService, s3Service, preSignedUrlGenerator, objectMapper, clock,+ task -> Thread.ofVirtual().start(task), config.effectiveBaseUrl());+ }++ SesExportJobService(StorageBackend<String, ExportJob> exportJobStore,+ SesSentEmailService sentEmailService, S3Service s3Service,+ PreSignedUrlGenerator preSignedUrlGenerator, ObjectMapper objectMapper,+ Clock clock, Executor workers, String baseUrl) {+ this.exportJobStore = exportJobStore;+ this.sentEmailService = sentEmailService;+ this.s3Service = s3Service;+ this.preSignedUrlGenerator = preSignedUrlGenerator;+ this.objectMapper = objectMapper;+ this.clock = clock;+ this.workers = workers;+ this.baseUrl = baseUrl;+ recoverInterruptedJobs();+ }++ /**+ * The request's already-parsed {@code ExportDataSource} and {@code ExportDestination}. The+ * controller keeps the JSON shape checks; the service owns the order the service-level rules+ * fire in.+ */+ public ExportJob createExportJob(String region, String accountId, JsonNode exportDataSource,+ JsonNode exportDestination, Runnable vdmGate) {+ JsonNode metrics = objectMemberOrAbsent(exportDataSource, "MetricsDataSource");+ JsonNode insights = objectMemberOrAbsent(exportDataSource, "MessageInsightsDataSource");+ if (metrics.isObject() == insights.isObject()) {+ throw new AwsException("BadRequestException", "One data source must be provided", 400);+ }+ if (!exportDestination.isObject()) {+ throw notNull("exportDestination");+ }+ String dataFormat = stringMemberOrAbsent(exportDestination, "DataFormat");+ if (dataFormat == null) {+ throw notNull("exportDestination.dataFormat");+ }+ if (!DATA_FORMATS.contains(dataFormat)) {+ throw new AwsException("BadRequestException",+ "1 validation error detected: Value at 'exportDestination.dataFormat' failed to "+ + "satisfy constraint: Member must satisfy enum value set: [CSV, JSON]", 400);+ }+ // Probe-confirmed order: the data source and the destination's own constraints are checked+ // first, then the refusal of a caller-chosen destination, then the VDM gate.+ if (exportDestination.hasNonNull("S3Url")) {+ throw new AwsException("BadRequestException",+ "Providing a custom S3 URL is not supported", 400);+ }+ vdmGate.run();+ if (metrics.isObject()) {+ SesExportMetrics.validate(metrics);+ } else {+ SesExportFilters.validate(insights);+ requireMaxResults(insights);+ }++ JsonNode source = metrics.isObject() ? metrics : insights;+ Instant start = requireDate(source, "StartDate");+ Instant end = requireDate(source, "EndDate");+ if (!start.isBefore(end)) {+ throw new AwsException("BadRequestException", "Invalid date range", 400);+ }+ if (metrics.isObject() && (!isMidnight(start) || !isMidnight(end))) {+ throw new AwsException("BadRequestException",+ "To get daily aggregated data you must not specify partial-day timestamps. "+ + "Please make your interval go from midnight to midnight UTC.", 400);+ }++ ExportJob job = new ExportJob();+ job.setJobId(UUID.randomUUID().toString());+ job.setRegion(region);+ job.setAccountId(accountId);+ job.setExportSourceType(metrics.isObject()+ ? ExportJob.SOURCE_METRICS : ExportJob.SOURCE_MESSAGE_INSIGHTS);+ job.setDataFormat(dataFormat);+ job.setDataSource(exportDataSource.toString());+ job.setJobStatus(ExportJob.STATUS_PROCESSING);+ job.setCreatedTimestamp(Instant.now(clock));+ return start(job, accountId);+ }++ private ExportJob start(ExportJob job, String accountId) {+ synchronized (createLock) {+ awaitResetFinished();+ long active = exportJobStore.scan(k -> k.startsWith(keyPrefix(job.getRegion()))).stream()+ .filter(candidate -> !candidate.isTerminal())+ .count();+ if (active >= MAX_CONCURRENT_JOBS) {+ throw new AwsException("LimitExceededException",+ "There are too many <" + job.getExportSourceType() + "> export jobs running"+ + " - either cancel one of the jobs or wait for it to complete"+ + " before resubmitting.", 400);+ }+ exportJobStore.put(key(job.getRegion(), job.getJobId()), job);+ // Persist and dispatch under the lock beforeReset takes, so a job is either admitted+ // before a reset and then abandoned by its stale generation, or admitted after it.+ int startedIn = generation.get();+ // The worker runs on a fresh thread, where account-aware storage would otherwise fall+ // back to the default account and never find this job again.+ workers.execute(() -> RequestScopes.runAs(accountId,+ () -> run(job.getRegion(), job.getJobId(), startedIn)));+ }+ return job;+ }++ // Holds createLock. A create arriving while a reset drains would capture the already-bumped+ // generation, so its worker would write an object after the wipe. It waits for afterReset.+ private void awaitResetFinished() {+ long deadline = System.currentTimeMillis() + RESET_ADMISSION_WAIT_MILLIS;+ while (resetting) {+ long remaining = deadline - System.currentTimeMillis();+ if (remaining <= 0) {+ throw resettingNow();+ }+ try {+ createLock.wait(remaining);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw resettingNow();+ }+ }+ }++ private static AwsException resettingNow() {+ return new AwsException("TooManyRequestsException",+ "Export jobs are not accepted while the emulator is resetting.", 429);+ }++ private void run(String region, String jobId, int startedIn) {+ synchronized (workerMonitor) {+ activeWorkers++;+ }+ try {+ runJob(region, jobId, startedIn);+ } finally {+ synchronized (workerMonitor) {+ activeWorkers--;+ workerMonitor.notifyAll();+ }+ }+ }++ private void runJob(String region, String jobId, int startedIn) {+ try {+ Optional<ExportJob> current = exportJobStore.get(key(region, jobId));+ if (current.isEmpty() || current.get().isTerminal()) {+ // Cancelled between the create and the worker starting.+ return;+ }+ ExportJob job = current.get();+ long processed = processedRecords(job);+ byte[] payload = render(job);+ if (generation.get() != startedIn) {+ // A reset wiped the stores while this job was rendering. Writing the object now+ // would put a bucket back that the reset removed.+ LOG.infov("SES export job {0} abandoned by an emulator reset", jobId);+ return;+ }+ String bucket = ensureExportBucket(job);+ String objectKey = objectKey(job);+ s3Service.putObject(bucket, objectKey, payload, contentType(job.getDataFormat()),+ Map.of());+ if (!finish(region, jobId, ExportJob.STATUS_COMPLETED, objectKey, null, processed)) {+ // Cancelled while the object was being written. The job will never point at it,+ // so the object goes too rather than sitting in the bucket unreachable.+ s3Service.deleteObject(bucket, objectKey);+ }+ } catch (RuntimeException e) {+ LOG.warnv("SES export job {0} failed: {1}", jobId, e.getMessage());+ finish(region, jobId, ExportJob.STATUS_FAILED, null, e.getMessage(), null);+ }+ }++ /** False when the job already reached a terminal state, which means a cancel won the race. */+ private boolean finish(String region, String jobId, String status, String objectKey,+ String error, Long processedRecords) {+ synchronized (createLock) {+ Optional<ExportJob> current = exportJobStore.get(key(region, jobId));+ if (current.isEmpty() || current.get().isTerminal()) {+ return false;+ }+ ExportJob job = current.get();+ job.setJobStatus(status);+ job.setObjectKey(objectKey);+ job.setErrorMessage(error);+ job.setProcessedRecordsCount(processedRecords);+ job.setCompletedTimestamp(Instant.now(clock));+ exportJobStore.put(key(region, jobId), job);+ return true;+ }+ }++ /**+ * Probe-confirmed: a metrics job and a message-insights job over one window report the same+ * ProcessedRecordsCount, so the number counts the stored rows the job examined rather than the+ * rows it wrote.+ */+ private long processedRecords(ExportJob job) {+ JsonNode source = readDataSource(job);+ boolean metrics = ExportJob.SOURCE_METRICS.equals(job.getExportSourceType());+ JsonNode window = metrics+ ? source.path("MetricsDataSource") : source.path("MessageInsightsDataSource");+ return SesExportPayloads.insightsRows(inWindow(sentEmailService.listInRegion(job.getRegion()),+ date(window, "StartDate"), date(window, "EndDate"), !metrics)).size();+ }++ private byte[] render(ExportJob job) {+ JsonNode source = readDataSource(job);+ if (ExportJob.SOURCE_MESSAGE_INSIGHTS.equals(job.getExportSourceType())) {+ JsonNode insights = source.path("MessageInsightsDataSource");+ List<SentEmail> emails = inWindow(sentEmailService.listInRegion(job.getRegion()),+ date(insights, "StartDate"), date(insights, "EndDate"), true);+ List<SesExportPayloads.InsightsRow> rows =+ SesExportFilters.apply(SesExportPayloads.insightsRows(emails), insights);+ return ExportJob.FORMAT_CSV.equals(job.getDataFormat())+ ? SesExportPayloads.insightsCsv(rows)+ : SesExportPayloads.insightsJson(rows);+ }+ JsonNode metrics = source.path("MetricsDataSource");+ // Windowed here so a dimension value seen only outside the range cannot produce a row.+ List<SentEmail> emails = inWindow(sentEmailService.listInRegion(job.getRegion()),+ date(metrics, "StartDate"), date(metrics, "EndDate"), false);+ return SesExportMetrics.render(emails, metrics, job.getDataFormat());+ }++ private JsonNode readDataSource(ExportJob job) {+ try {+ return objectMapper.readTree(job.getDataSource());+ } catch (JsonProcessingException e) {+ throw new AwsException("BadRequestException", e.getMessage(), 400);+ }+ }++ /**+ * The model documents the message-insights end date as inclusive, while the metrics source+ * aggregates whole UTC days and ends exclusively, as the {@code BatchGetMetricData} probe+ * showed. A send with no timestamp is kept, as the batch query keeps it.+ */+ private static List<SentEmail> inWindow(List<SentEmail> emails, Instant start, Instant end,+ boolean endInclusive) {+ List<SentEmail> kept = new ArrayList<>();+ for (SentEmail email : emails) {+ Instant sentAt = email.getSentAt();+ if (sentAt == null) {+ kept.add(email);+ continue;+ }+ boolean beforeEnd = endInclusive ? !sentAt.isAfter(end) : sentAt.isBefore(end);+ if (!sentAt.isBefore(start) && beforeEnd) {+ kept.add(email);+ }+ }+ return kept;+ }++ public ExportJob getExportJob(String region, String jobId) {+ return exportJobStore.get(key(region, jobId))+ .orElseThrow(() -> new AwsException("BadRequestException", "Invalid JobId", 400));+ }++ /**+ * The presigned URL a finished job's {@code ExportDestination} carries. Empty while the job is+ * still running or once it failed, since there is no object to point at.+ */+ public Optional<String> presignedUrl(ExportJob job) {+ if (job.getObjectKey() == null) {+ return Optional.empty();+ }+ return Optional.of(preSignedUrlGenerator.generatePresignedUrl(baseUrl,+ exportBucket(job.getAccountId(), job.getRegion()), job.getObjectKey(), "GET",+ PRESIGNED_URL_EXPIRY_SECONDS));+ }++ public List<ExportJob> listExportJobs(String region, String sourceType, String jobStatus) {+ if (sourceType != null && !SOURCE_TYPES.contains(sourceType)) {+ throw new AwsException("BadRequestException",+ "1 validation error detected: Value at 'exportSourceType' failed to satisfy "+ + "constraint: Member must satisfy enum value set: "+ + "[METRICS_DATA, MESSAGE_INSIGHTS]", 400);+ }+ if (jobStatus != null && !JOB_STATUSES.contains(jobStatus)) {+ throw new AwsException("BadRequestException",+ "1 validation error detected: Value at 'jobStatus' failed to satisfy "+ + "constraint: Member must satisfy enum value set: "+ + "[CREATED, PROCESSING, COMPLETED, FAILED, CANCELLED]", 400);+ }+ return exportJobStore.scan(k -> k.startsWith(keyPrefix(region))).stream()+ .filter(job -> sourceType == null || sourceType.equals(job.getExportSourceType()))+ .filter(job -> jobStatus == null || jobStatus.equals(job.getJobStatus()))+ // Newest first, as AWS returns them.+ .sorted(Comparator.comparing(ExportJob::getCreatedTimestamp).reversed()+ .thenComparing(ExportJob::getJobId))+ .toList();+ }++ public void cancelExportJob(String region, String jobId) {+ synchronized (createLock) {+ ExportJob job = getExportJob(region, jobId);+ if (job.isTerminal()) {+ throw new AwsException("BadRequestException",+ "Cannot cancel an export job that has a <" + job.getJobStatus()+ + "> status.", 400);+ }+ job.setJobStatus(ExportJob.STATUS_CANCELLED);+ job.setCompletedTimestamp(Instant.now(clock));+ exportJobStore.put(key(region, jobId), job);+ }+ }++ /**+ * Runs from the constructor, where no request scope is active, so the store has to be read+ * across accounts: a plain scan would only ever sweep the default account's jobs.+ */+ private void recoverInterruptedJobs() {+ if (!(exportJobStore instanceof AccountAwareStorageBackend<ExportJob> jobs)) {+ return;+ }+ for (AccountAwareStorageBackend.AccountEntry<ExportJob> entry+ : jobs.scanAllAccountEntries(k -> k.startsWith("export-job::"))) {+ ExportJob job = entry.value();+ if (job.isTerminal()) {+ continue;+ }+ job.setJobStatus(ExportJob.STATUS_FAILED);+ job.setErrorMessage(RESTART_FAILURE_MESSAGE);+ job.setCompletedTimestamp(Instant.now(clock));+ jobs.putForAccount(entry.accountId(), entry.key(), job);+ LOG.warnv("SES export job {0} was interrupted by a restart and is now FAILED",+ job.getJobId());+ }+ }++ /**+ * Stops the workers before the stores are wiped. Without this a worker outlived the reset and+ * recreated the export bucket and object the wipe had just removed.+ */+ @Override+ public void beforeReset() {+ synchronized (createLock) {+ resetting = true;+ generation.incrementAndGet();+ }+ boolean interrupted = false;+ long deadline = System.currentTimeMillis() + RESET_DRAIN_TIMEOUT_MILLIS;+ synchronized (workerMonitor) {+ while (activeWorkers > 0) {+ long remaining = deadline - System.currentTimeMillis();+ if (remaining <= 0) {+ LOG.warnv("{0} SES export worker(s) still running after {1}ms; one object may "+ + "be written after the reset", activeWorkers, RESET_DRAIN_TIMEOUT_MILLIS);+ break;+ }+ try {+ workerMonitor.wait(remaining);+ } catch (InterruptedException e) {+ // Keep draining to the deadline; the interrupt is re-asserted on the way out.+ interrupted = true;+ }+ }+ }+ if (interrupted) {+ Thread.currentThread().interrupt();+ }+ }++ @Override+ public void afterReset() {+ synchronized (createLock) {+ resetting = false;+ createLock.notifyAll();+ }+ }++ @Override+ public void clear() {+ exportJobStore.clear();+ LOG.info("Cleared all SES export jobs");+ }++ /**+ * Two workers can reach an empty region together, and outside us-east-1 the second+ * {@code createBucket} answers {@code BucketAlreadyOwnedByYou}, which would fail that export.+ */+ private String ensureExportBucket(ExportJob job) {+ String bucket = exportBucket(job.getAccountId(), job.getRegion());+ if (s3Service.bucketExists(bucket)) {+ return bucket;+ }+ try {+ s3Service.createBucket(bucket, job.getRegion());+ } catch (AwsException e) {+ if (!"BucketAlreadyOwnedByYou".equals(e.getErrorCode())) {+ throw e;+ }+ LOG.debugv("Export bucket {0} was created by a concurrent export job", bucket);+ }+ return bucket;+ }++ /**+ * The account is part of the name because an object write resolves its bucket across accounts+ * when {@code global-bucket-namespace} is on, and would then land one account's export in+ * another account's partition, where its owner could read it.+ */+ static String exportBucket(String accountId, String region) {+ return "floci-" + accountId + "-" + region + EXPORT_BUCKET_SUFFIX;+ }++ /**+ * AWS names the two payloads differently, and the shape of each name is probe-confirmed.+ */+ private String objectKey(ExportJob job) {+ String stamp = OBJECT_STAMP.format(Instant.now(clock));+ String extension = ExportJob.FORMAT_CSV.equals(job.getDataFormat()) ? "csv" : "json";+ if (ExportJob.SOURCE_METRICS.equals(job.getExportSourceType())) {+ return "ses-metrics-" + stamp + "-" + job.getJobId() + "." + extension;+ }+ return stamp + "_" + job.getJobId().replace("-", "").substring(0, 10) + "." + extension;+ }++ private static String contentType(String dataFormat) {+ return ExportJob.FORMAT_CSV.equals(dataFormat) ? "text/csv" : "application/json";+ }++ private static AwsException notNull(String member) {+ return new AwsException("BadRequestException",+ "1 validation error detected: Value at '" + member+ + "' failed to satisfy constraint: Member must not be null", 400);+ }++ /**+ * A union member of the wrong JSON type read as absent, so a request carrying a valid source+ * plus a malformed one passed the exactly-one check and silently exported the valid half.+ */+ private static JsonNode objectMemberOrAbsent(JsonNode parent, String field) {+ JsonNode member = parent.path(field);+ if (!member.isMissingNode() && !member.isNull() && !member.isObject()) {+ throw new AwsException("SerializationException", null, 400);+ }+ return member;+ }++ /**+ * The model bounds {@code MaxResults} at 1 to 10000. An unchecked negative value reached+ * {@code subList} inside the filter and surfaced as a 500.+ */+ private static void requireMaxResults(JsonNode insights) {+ // intMemberOrAbsent answers SerializationException for a value that is not an integer, so+ // a wrong type is not reported as a value out of range.+ Integer maxResults = intMemberOrAbsent(insights, "MaxResults");+ if (maxResults == null) {+ return;+ }+ if (maxResults < 1 || maxResults > MAX_RESULTS) {+ throw new AwsException("BadRequestException",+ "1 validation error detected: Value at "+ + "'exportDataSource.messageInsightsDataSource.maxResults' failed to "+ + "satisfy constraint: Member must be between 1 and " + MAX_RESULTS+ + ", inclusive", 400);+ }+ }++ private static Instant requireDate(JsonNode source, String field) {+ Instant value = date(source, field);+ if (value == null) {+ throw new AwsException("BadRequestException",+ "1 validation error detected: Value at '" + field.substring(0, 1).toLowerCase(Locale.ROOT)+ + field.substring(1) + "' failed to satisfy constraint: "+ + "Member must not be null", 400);+ }+ return value;+ }++ private static Instant date(JsonNode source, String field) {+ JsonNode node = source.path(field);+ if (node.isNumber()) {+ return Instant.ofEpochMilli(Math.round(node.doubleValue() * 1000.0));+ }+ if (node.isTextual()) {+ try {+ return Instant.parse(node.asText());+ } catch (DateTimeParseException e) {+ throw new AwsException("SerializationException", e.getMessage(), 400);+ }+ }+ return null;+ }++ private static boolean isMidnight(Instant instant) {+ return instant.equals(instant.truncatedTo(ChronoUnit.DAYS));+ }++ private static String keyPrefix(String region) {+ return "export-job::" + region + "::";+ }++ private static String key(String region, String jobId) {+ return keyPrefix(region) + jobId;+ }++}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesExportMetrics.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportMetrics.javanew file mode 100644index 0000000000..d82db629f2--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportMetrics.java@@ -0,0 +1,321 @@+package io.github.hectorvent.floci.services.ses;++import com.fasterxml.jackson.databind.JsonNode;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.ses.model.EmailInsights;+import io.github.hectorvent.floci.services.ses.model.ExportJob;+import io.github.hectorvent.floci.services.ses.model.SentEmail;++import java.time.Instant;+import java.util.ArrayList;+import java.util.Iterator;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.Set;++import static io.github.hectorvent.floci.services.ses.SesV2Json.stringArrayOrAbsent;+import static io.github.hectorvent.floci.services.ses.SesV2Json.stringMemberOrAbsent;++/**+ * The metrics half of an export job. Every count comes from {@link SesMetricData}, the same+ * aggregation {@code BatchGetMetricData} serves, so an export and a query of the same window+ * cannot disagree.+ *+ * <p>One deviation, documented in {@code docs/services/ses.md}: real SES emits a row for every ISP+ * it knows, zeros included, which is a catalogue Floci has no honest way to reproduce. Floci emits+ * a row per dimension combination it has actually seen. A `RATE` aggregation is the metric's volume+ * over the send volume for the same combination, which matches the definition AWS documents; the+ * probe could not confirm it, because the probed account's rate export carried no data.+ */+final class SesExportMetrics {++ private static final String PATH = "exportDataSource.metricsDataSource.";+ private static final String METRIC_ENUM =+ "[DELIVERY_COMPLAINT, CLICK, SEND, OPEN, COMPLAINT, DELIVERY, DELIVERY_OPEN, "+ + "PERMANENT_BOUNCE, DELIVERY_CLICK, TRANSIENT_BOUNCE]";+ private static final String DIMENSION_ENUM =+ "[EMAIL_IDENTITY, CONFIGURATION_SET, ISP, TENANT_NAME]";+ private static final String AGGREGATION_ENUM = "[RATE, VOLUME]";+ private static final int MAX_DIMENSIONS = 3;+ private static final int MAX_DIMENSION_VALUES = 10;+ private static final int MAX_METRICS = 10;++ private SesExportMetrics() {}++ /**+ * The model's constraints on a metrics data source: {@code Dimensions} (1 to 3 known names,+ * each with 1 to 10 values), {@code Namespace} and {@code Metrics} (1 to 10) are required, and+ * a metric's {@code Name} and {@code Aggregation} are checked against their enums when present,+ * which is as far as the model goes: neither member is required, and an absent+ * {@code Aggregation} keeps its {@code VOLUME} default. Without this an unknown dimension was+ * dropped, an unknown metric aggregated to zero and an unrecognised aggregation was read as+ * {@code VOLUME}, so a malformed request produced a successful, meaningless export.+ *+ * <p>The probe never sent a malformed source, so the wording and the position in the sequence+ * are Floci's, chosen to match the sibling {@code BatchGetMetricData}. Dimension values are+ * deliberately not checked: {@code *} is a legal value here and the export probe never showed+ * the per-value rules the batch operation applies.+ */+ static void validate(JsonNode metrics) {+ requireWireTypes(metrics);+ List<String> violations = new ArrayList<>();+ validateDimensions(metrics.path("Dimensions"), violations);+ JsonNode namespace = metrics.path("Namespace");+ if (!namespace.isTextual() || !"VDM".equals(namespace.textValue())) {+ violations.add(constraint("namespace",+ "Member must satisfy enum value set: [VDM]"));+ }+ validateMetrics(metrics.path("Metrics"), violations);+ if (!violations.isEmpty()) {+ String header = violations.size() == 1+ ? "1 validation error detected: "+ : violations.size() + " validation errors detected: ";+ throw new AwsException("BadRequestException",+ header + String.join("; ", violations), 400);+ }+ }++ /**+ * The deserialization-level checks, which come first and answer {@code SerializationException}+ * like every other typed SES v2 member: a dimension value list of numbers was otherwise+ * coerced to strings, and a metric that is a bare string was read as one with no members at+ * all, both producing a successful but meaningless export.+ */+ private static void requireWireTypes(JsonNode metrics) {+ JsonNode dimensions = metrics.path("Dimensions");+ if (present(dimensions)) {+ if (!dimensions.isObject()) {+ throw serializationError();+ }+ Iterator<String> names = dimensions.fieldNames();+ while (names.hasNext()) {+ stringArrayOrAbsent(dimensions, names.next());+ }+ }+ stringMemberOrAbsent(metrics, "Namespace");+ JsonNode requested = metrics.path("Metrics");+ if (present(requested)) {+ if (!requested.isArray()) {+ throw serializationError();+ }+ for (JsonNode metric : requested) {+ if (!metric.isObject()) {+ throw serializationError();+ }+ stringMemberOrAbsent(metric, "Name");+ stringMemberOrAbsent(metric, "Aggregation");+ }+ }+ }++ private static AwsException serializationError() {+ return new AwsException("SerializationException", null, 400);+ }++ private static void validateDimensions(JsonNode dimensions, List<String> violations) {+ if (!dimensions.isObject()) {+ violations.add(constraint("dimensions", "Member must not be null"));+ return;+ }+ if (dimensions.isEmpty()) {+ violations.add(constraint("dimensions",+ "Member must have length greater than or equal to 1"));+ return;+ }+ if (dimensions.size() > MAX_DIMENSIONS) {+ violations.add(constraint("dimensions",+ "Member must have length less than or equal to " + MAX_DIMENSIONS));+ }+ Iterator<Map.Entry<String, JsonNode>> fields = dimensions.fields();+ while (fields.hasNext()) {+ Map.Entry<String, JsonNode> field = fields.next();+ if (!SesMetricData.DIMENSION_NAMES.contains(field.getKey())) {+ violations.add(constraint("dimensions",+ "Member must satisfy enum value set: " + DIMENSION_ENUM));+ continue;+ }+ JsonNode values = field.getValue();+ if (!values.isArray() || values.isEmpty()) {+ violations.add(constraint("dimensions." + field.getKey(),+ "Member must have length greater than or equal to 1"));+ } else if (values.size() > MAX_DIMENSION_VALUES) {+ violations.add(constraint("dimensions." + field.getKey(),+ "Member must have length less than or equal to " + MAX_DIMENSION_VALUES));+ }+ }+ }++ private static void validateMetrics(JsonNode metrics, List<String> violations) {+ if (!metrics.isArray()) {+ violations.add(constraint("metrics", "Member must not be null"));+ return;+ }+ if (metrics.isEmpty()) {+ violations.add(constraint("metrics",+ "Member must have length greater than or equal to 1"));+ return;+ }+ if (metrics.size() > MAX_METRICS) {+ violations.add(constraint("metrics",+ "Member must have length less than or equal to " + MAX_METRICS));+ }+ for (int i = 0; i < metrics.size(); i++) {+ JsonNode metric = metrics.get(i);+ // Neither member is required by the model, so only a value that is present is+ // checked: an absent Aggregation keeps its VOLUME default.+ JsonNode name = metric.path("Name");+ if (present(name) && !SesMetricData.METRICS.contains(name.asText())) {+ violations.add(constraint("metrics." + (i + 1) + ".member.name",+ "Member must satisfy enum value set: " + METRIC_ENUM));+ }+ JsonNode aggregation = metric.path("Aggregation");+ if (present(aggregation)+ && !("RATE".equals(aggregation.asText()) || "VOLUME".equals(aggregation.asText()))) {+ violations.add(constraint("metrics." + (i + 1) + ".member.aggregation",+ "Member must satisfy enum value set: " + AGGREGATION_ENUM));+ }+ }+ }++ private static boolean present(JsonNode node) {+ return !node.isMissingNode() && !node.isNull();+ }++ private static String constraint(String member, String constraint) {+ return "Value at '" + PATH + member + "' failed to satisfy constraint: " + constraint;+ }++ static byte[] render(List<SentEmail> emails, JsonNode metrics, String dataFormat) {+ List<String> dimensionNames = new ArrayList<>();+ Map<String, Set<String>> requested = new LinkedHashMap<>();+ JsonNode dimensions = metrics.path("Dimensions");+ Iterator<Map.Entry<String, JsonNode>> fields = dimensions.fields();+ while (fields.hasNext()) {+ Map.Entry<String, JsonNode> field = fields.next();+ dimensionNames.add(field.getKey());+ Set<String> values = new LinkedHashSet<>();+ for (JsonNode value : field.getValue()) {+ values.add(value.asText());+ }+ requested.put(field.getKey(), values);+ }++ List<String> metricColumns = new ArrayList<>();+ List<String> metricNames = new ArrayList<>();+ List<Boolean> asRate = new ArrayList<>();+ for (JsonNode metric : metrics.path("Metrics")) {+ String name = metric.path("Name").asText("");+ String aggregation = metric.path("Aggregation").asText("VOLUME");+ metricColumns.add(name + "_" + aggregation);+ metricNames.add(name);+ asRate.add("RATE".equals(aggregation));+ }++ Instant start = instant(metrics.path("StartDate"));+ Instant end = instant(metrics.path("EndDate"));++ List<List<String>> combinations = combinations(emails, dimensionNames, requested);+ List<SesExportPayloads.MetricRow> rows = new ArrayList<>(combinations.size());+ for (List<String> combination : combinations) {+ Map<String, String> filter = new LinkedHashMap<>();+ for (int i = 0; i < dimensionNames.size(); i++) {+ filter.put(dimensionNames.get(i), combination.get(i));+ }+ List<Double> values = new ArrayList<>(metricNames.size());+ for (int i = 0; i < metricNames.size(); i++) {+ double volume = total(emails, metricNames.get(i), filter, start, end);+ if (!asRate.get(i)) {+ values.add(volume);+ continue;+ }+ double sends = total(emails, "SEND", filter, start, end);+ values.add(sends == 0.0 ? 0.0 : volume / sends);+ }+ rows.add(new SesExportPayloads.MetricRow(combination, values));+ }++ return ExportJob.FORMAT_CSV.equals(dataFormat)+ ? SesExportPayloads.metricsCsv(dimensionNames, metricColumns, rows)+ : SesExportPayloads.metricsJson(dimensionNames, metricColumns, rows);+ }++ private static double total(List<SentEmail> emails, String metric, Map<String, String> filter,+ Instant start, Instant end) {+ SesMetricData.Series series = SesMetricData.aggregate(emails,+ new SesMetricData.Query("export", metric, filter, start, end));+ double sum = 0.0;+ for (Long value : series.values()) {+ sum += value;+ }+ return sum;+ }++ /**+ * The dimension combinations present in the stored messages, narrowed to the requested values.+ * A requested value of {@code *} means every value, which is how AWS asks for the whole+ * catalogue.+ */+ private static List<List<String>> combinations(List<SentEmail> emails, List<String> names,+ Map<String, Set<String>> requested) {+ Set<List<String>> seen = new LinkedHashSet<>();+ for (SentEmail email : emails) {+ List<EmailInsights> recipients = email.getInsights() == null+ ? List.of() : email.getInsights();+ for (EmailInsights recipient : recipients) {+ List<String> combination = new ArrayList<>(names.size());+ boolean usable = true;+ for (String name : names) {+ String value = dimensionValue(name, email, recipient);+ if (value == null || !wants(name, requested.get(name), value)) {+ usable = false;+ break;+ }+ combination.add(value);+ }+ if (usable) {+ seen.add(combination);+ }+ }+ }+ return new ArrayList<>(seen);+ }++ /**+ * An {@code EMAIL_IDENTITY} value names either a sending address or its domain, so a requested+ * domain has to match the addresses inside it. {@link SesMetricData} draws that line for+ * {@code BatchGetMetricData}, and the two read the same sends, so they read it the same way.+ */+ private static boolean wants(String name, Set<String> wanted, String value) {+ if (wanted.contains("*") || wanted.contains(value)) {+ return true;+ }+ if (!"EMAIL_IDENTITY".equals(name)) {+ return false;+ }+ return wanted.stream().anyMatch(identity -> SesMetricData.matchesIdentity(value, identity));+ }++ private static String dimensionValue(String name, SentEmail email, EmailInsights recipient) {+ return switch (name) {+ case "ISP" -> recipient.isp();+ // The bare address, lower-cased, not the stored Source: a display name would key a row+ // that the identity matching behind the counts then refuses, leaving every value at+ // zero, and that matching ignores case, so two spellings would each count both sends.+ case "EMAIL_IDENTITY" -> email.getSource() == null+ ? null : SesService.extractEmailAddress(email.getSource()).toLowerCase(Locale.ROOT);+ case "CONFIGURATION_SET" -> email.getConfigurationSetName();+ case "TENANT_NAME" -> email.getTenantName();+ default -> null;+ };+ }++ private static Instant instant(JsonNode node) {+ if (node.isNumber()) {+ return Instant.ofEpochMilli(Math.round(node.doubleValue() * 1000.0));+ }+ return node.isTextual() ? Instant.parse(node.asText()) : Instant.EPOCH;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesExportPayloads.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportPayloads.javanew file mode 100644index 0000000000..a586da3897--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesExportPayloads.java@@ -0,0 +1,261 @@+package io.github.hectorvent.floci.services.ses;++import io.github.hectorvent.floci.services.ses.model.EmailInsights;+import io.github.hectorvent.floci.services.ses.model.InsightsBounce;+import io.github.hectorvent.floci.services.ses.model.InsightsEvent;+import io.github.hectorvent.floci.services.ses.model.InsightsEventDetails;+import io.github.hectorvent.floci.services.ses.model.SentEmail;++import java.nio.charset.StandardCharsets;+import java.time.Instant;+import java.time.ZoneOffset;+import java.time.format.DateTimeFormatter;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Locale;+import java.util.Map;++/**+ * Renders the two export payloads AWS produces, from the same stored timelines+ * {@link SesMessageInsights} derives and {@link SesMetricData} aggregates.+ *+ * <p>Probe-confirmed against real SES (2026-09-23, us-east-1): the column names and their order,+ * the quoting, the {@code yyyy-MM-dd HH:mm:ss.SSS UTC} timestamps, the trailing comma that leaves+ * a thirteenth unnamed column on the message-insights CSV, the leading-comma JSON array, and the+ * four-decimal metric values.+ */+final class SesExportPayloads {++ private static final DateTimeFormatter EXPORT_TIMESTAMP = DateTimeFormatter+ .ofPattern("yyyy-MM-dd HH:mm:ss.SSS 'UTC'", Locale.ROOT)+ .withZone(ZoneOffset.UTC);++ private static final List<String> INSIGHTS_COLUMNS = List.of(+ "messageid", "sendtimestamp", "isp", "fromaddress", "destination", "subject",+ "last_delivery_event", "last_engagement_event", "last_delivery_event_timestamp",+ "last_engagement_event_timestamp", "bounce_sub_type", "diagnostic_code");++ private SesExportPayloads() {}++ /**+ * One exported row: an envelope recipient of one stored message. The tenant sits beside the+ * columns rather than among them, because {@code TenantName} is a filter member with no+ * column of its own in the probed file.+ */+ record InsightsRow(Map<String, String> columns, String tenantName) {}++ static List<InsightsRow> insightsRows(List<SentEmail> emails) {+ List<InsightsRow> rows = new ArrayList<>();+ for (SentEmail email : emails) {+ for (EmailInsights recipient : orEmpty(email.getInsights())) {+ rows.add(insightsRow(email, recipient));+ }+ }+ return rows;+ }++ private static InsightsRow insightsRow(SentEmail email, EmailInsights recipient) {+ Map<String, String> columns = new LinkedHashMap<>();+ columns.put("messageid", email.getMessageId());+ columns.put("sendtimestamp", format(email.getSentAt()));+ columns.put("isp", recipient.isp());+ columns.put("fromaddress", email.getSource());+ columns.put("destination", recipient.destination());+ columns.put("subject", email.getSubject());++ InsightsEvent last = lastDeliveryEvent(recipient);+ columns.put("last_delivery_event", last == null ? null : deliveryEventName(last));+ columns.put("last_engagement_event", null);+ columns.put("last_delivery_event_timestamp", last == null ? null : format(last.timestamp()));+ columns.put("last_engagement_event_timestamp", null);+ InsightsBounce bounce = bounceOf(last);+ columns.put("bounce_sub_type", bounce == null ? null : bounce.bounceSubType());+ columns.put("diagnostic_code", bounce == null ? null : bounce.diagnosticCode());+ return new InsightsRow(columns, email.getTenantName());+ }++ /**+ * The delivery-side enum splits a bounce by type, where the stored timeline carries one+ * {@code BOUNCE} event plus its details. Floci only ever derives a permanent bounce, so the+ * transient and undetermined values never appear.+ */+ private static String deliveryEventName(InsightsEvent event) {+ if (!"BOUNCE".equals(event.type())) {+ return event.type();+ }+ InsightsBounce bounce = bounceOf(event);+ String bounceType = bounce == null ? null : bounce.bounceType();+ if ("TRANSIENT".equals(bounceType)) {+ return "TRANSIENT_BOUNCE";+ }+ if ("PERMANENT".equals(bounceType)) {+ return "PERMANENT_BOUNCE";+ }+ return "UNDETERMINED_BOUNCE";+ }++ private static InsightsEvent lastDeliveryEvent(EmailInsights recipient) {+ InsightsEvent last = null;+ for (InsightsEvent event : orEmpty(recipient.events())) {+ last = event;+ }+ return last;+ }++ private static InsightsBounce bounceOf(InsightsEvent event) {+ if (event == null) {+ return null;+ }+ InsightsEventDetails details = event.details();+ return details == null ? null : details.bounce();+ }++ static byte[] insightsCsv(List<InsightsRow> rows) {+ StringBuilder out = new StringBuilder();+ for (String column : INSIGHTS_COLUMNS) {+ out.append(quote(column)).append(',');+ }+ out.append('\n');+ for (InsightsRow row : rows) {+ for (String column : INSIGHTS_COLUMNS) {+ out.append(quote(row.columns().get(column))).append(',');+ }+ out.append('\n');+ }+ return out.toString().getBytes(StandardCharsets.UTF_8);+ }++ static byte[] insightsJson(List<InsightsRow> rows) {+ StringBuilder out = new StringBuilder("[\n");+ boolean first = true;+ for (InsightsRow row : rows) {+ out.append(first ? "" : ",");+ first = false;+ out.append('{');+ boolean firstColumn = true;+ for (String column : INSIGHTS_COLUMNS) {+ String value = row.columns().get(column);+ if (value == null) {+ continue;+ }+ if (!firstColumn) {+ out.append(',');+ }+ firstColumn = false;+ out.append(jsonString(column)).append(':').append(jsonString(value));+ }+ out.append("}\n");+ }+ out.append("]\n");+ return out.toString().getBytes(StandardCharsets.UTF_8);+ }++ /** One exported metric row: the dimension values it is keyed by, then the requested metrics. */+ record MetricRow(List<String> dimensionValues, List<Double> values) {}++ static byte[] metricsCsv(List<String> dimensionNames, List<String> metricColumns,+ List<MetricRow> rows) {+ StringBuilder out = new StringBuilder(String.join(",", dimensionNames));+ for (String column : metricColumns) {+ out.append(',').append(column);+ }+ out.append('\n');+ for (MetricRow row : rows) {+ for (int i = 0; i < row.dimensionValues().size(); i++) {+ if (i > 0) {+ out.append(',');+ }+ out.append(csvField(row.dimensionValues().get(i)));+ }+ for (Double value : row.values()) {+ out.append(',').append(decimal(value));+ }+ out.append('\n');+ }+ return out.toString().getBytes(StandardCharsets.UTF_8);+ }++ static byte[] metricsJson(List<String> dimensionNames, List<String> metricColumns,+ List<MetricRow> rows) {+ StringBuilder out = new StringBuilder("[");+ boolean first = true;+ for (MetricRow row : rows) {+ if (!first) {+ out.append(',');+ }+ first = false;+ out.append('{');+ for (int i = 0; i < dimensionNames.size(); i++) {+ out.append(jsonString(dimensionNames.get(i).toLowerCase(Locale.ROOT))).append(':')+ .append(jsonString(row.dimensionValues().get(i))).append(',');+ }+ out.append("\"metrics\":{");+ for (int i = 0; i < metricColumns.size(); i++) {+ if (i > 0) {+ out.append(',');+ }+ out.append(jsonString(metricColumns.get(i))).append(':')+ .append(decimal(row.values().get(i)));+ }+ out.append("}}");+ }+ out.append("]\n");+ return out.toString().getBytes(StandardCharsets.UTF_8);+ }++ /**+ * The probed metrics file leaves its values bare, so a value is quoted only when it has to be.+ * A dimension value can carry a comma, and an unquoted one would shift every later column.+ */+ private static String csvField(String value) {+ if (value == null) {+ return "";+ }+ if (value.indexOf(',') < 0 && value.indexOf('"') < 0 && value.indexOf('\n') < 0) {+ return value;+ }+ return quote(value);+ }++ private static String decimal(Double value) {+ return String.format(Locale.ROOT, "%.4f", value == null ? 0.0 : value);+ }++ private static String format(Instant instant) {+ return instant == null ? null : EXPORT_TIMESTAMP.format(instant);+ }++ private static String quote(String value) {+ if (value == null) {+ return "\"\"";+ }+ return '"' + value.replace("\"", "\"\"") + '"';+ }++ private static String jsonString(String value) {+ StringBuilder out = new StringBuilder("\"");+ for (int i = 0; i < value.length(); i++) {+ char c = value.charAt(i);+ switch (c) {+ case '"' -> out.append("\\\"");+ case '\\' -> out.append("\\\\");+ case '\n' -> out.append("\\n");+ case '\r' -> out.append("\\r");+ case '\t' -> out.append("\\t");+ default -> {+ if (c < 0x20) {+ out.append(String.format(Locale.ROOT, "\\u%04x", (int) c));+ } else {+ out.append(c);+ }+ }+ }+ }+ return out.append('"').toString();+ }++ private static <T> List<T> orEmpty(List<T> values) {+ return values == null ? List.of() : values;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesMetricData.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesMetricData.javaindex a48c4dd804..11374b1c11 100644--- a/src/main/java/io/github/hectorvent/floci/services/ses/SesMetricData.java+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesMetricData.java@@ -166,7 +166,7 @@ final class SesMetricData {* An identity dimension names either the sending address or its domain, the way the identity* itself can be either; the probe filtered a domain and matched sends from an address in it.*/- private static boolean matchesIdentity(String source, String identity) {+ static boolean matchesIdentity(String source, String identity) {if (source == null) {return false;}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/model/ExportJob.java b/src/main/java/io/github/hectorvent/floci/services/ses/model/ExportJob.javanew file mode 100644index 0000000000..0d6537a5b4--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/model/ExportJob.java@@ -0,0 +1,112 @@+package io.github.hectorvent.floci.services.ses.model;++import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;++/**+ * A stored SES V2 export job. The request's {@code ExportDataSource} is kept verbatim as JSON+ * because {@code GetExportJob} echoes it back, filters and all, and Floci has no reason to model+ * every filter member just to hand it straight back.+ */+@RegisterForReflection+public class ExportJob {++ public static final String SOURCE_METRICS = "METRICS_DATA";+ public static final String SOURCE_MESSAGE_INSIGHTS = "MESSAGE_INSIGHTS";++ public static final String STATUS_CREATED = "CREATED";+ public static final String STATUS_PROCESSING = "PROCESSING";+ public static final String STATUS_COMPLETED = "COMPLETED";+ public static final String STATUS_FAILED = "FAILED";+ public static final String STATUS_CANCELLED = "CANCELLED";++ public static final String FORMAT_CSV = "CSV";+ public static final String FORMAT_JSON = "JSON";++ @JsonProperty("JobId")+ private String jobId;++ @JsonProperty("Region")+ private String region;++ /** The account that created the job, which keeps its export bucket inside that account. */+ @JsonProperty("AccountId")+ private String accountId;++ @JsonProperty("ExportSourceType")+ private String exportSourceType;++ @JsonProperty("JobStatus")+ private String jobStatus;++ @JsonProperty("DataFormat")+ private String dataFormat;++ @JsonProperty("DataSource")+ private String dataSource;++ /** The key of the produced object, once the worker has written it. */+ @JsonProperty("ObjectKey")+ private String objectKey;++ @JsonProperty("CreatedTimestamp")+ private Instant createdTimestamp;++ @JsonProperty("CompletedTimestamp")+ private Instant completedTimestamp;++ @JsonProperty("ErrorMessage")+ private String errorMessage;++ /** How many stored recipient rows the job examined, which is AWS's ProcessedRecordsCount. */+ @JsonProperty("ProcessedRecordsCount")+ private Long processedRecordsCount;++ public ExportJob() {}++ public boolean isTerminal() {+ return STATUS_COMPLETED.equals(jobStatus)+ || STATUS_FAILED.equals(jobStatus)+ || STATUS_CANCELLED.equals(jobStatus);+ }++ public String getJobId() { return jobId; }+ public void setJobId(String jobId) { this.jobId = jobId; }++ public String getRegion() { return region; }+ public void setRegion(String region) { this.region = region; }++ public String getAccountId() { return accountId; }+ public void setAccountId(String accountId) { this.accountId = accountId; }++ public String getExportSourceType() { return exportSourceType; }+ public void setExportSourceType(String exportSourceType) { this.exportSourceType = exportSourceType; }++ public String getJobStatus() { return jobStatus; }+ public void setJobStatus(String jobStatus) { this.jobStatus = jobStatus; }++ public String getDataFormat() { return dataFormat; }+ public void setDataFormat(String dataFormat) { this.dataFormat = dataFormat; }++ public String getDataSource() { return dataSource; }+ public void setDataSource(String dataSource) { this.dataSource = dataSource; }++ public String getObjectKey() { return objectKey; }+ public void setObjectKey(String objectKey) { this.objectKey = objectKey; }++ public Instant getCreatedTimestamp() { return createdTimestamp; }+ public void setCreatedTimestamp(Instant createdTimestamp) { this.createdTimestamp = createdTimestamp; }++ public Instant getCompletedTimestamp() { return completedTimestamp; }+ public void setCompletedTimestamp(Instant completedTimestamp) { this.completedTimestamp = completedTimestamp; }++ public String getErrorMessage() { return errorMessage; }+ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }++ public Long getProcessedRecordsCount() { return processedRecordsCount; }+ public void setProcessedRecordsCount(Long processedRecordsCount) {+ this.processedRecordsCount = processedRecordsCount;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesExportFiltersTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportFiltersTest.javanew file mode 100644index 0000000000..6ee2915d4d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportFiltersTest.java@@ -0,0 +1,101 @@+package io.github.hectorvent.floci.services.ses;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import org.junit.jupiter.api.Test;++import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++class SesExportFiltersTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();++ private static SesExportPayloads.InsightsRow row(String destination, String tenantName) {+ Map<String, String> columns = new LinkedHashMap<>();+ columns.put("destination", destination);+ columns.put("isp", "UNKNOWN_ISP");+ columns.put("last_delivery_event", "DELIVERY");+ columns.put("last_engagement_event", null);+ return new SesExportPayloads.InsightsRow(columns, tenantName);+ }++ private static JsonNode source(String json) {+ try {+ return MAPPER.readTree(json);+ } catch (Exception e) {+ throw new AssertionError("bad test json", e);+ }+ }++ private static List<String> destinations(List<SesExportPayloads.InsightsRow> rows) {+ return rows.stream().map(row -> row.columns().get("destination")).toList();+ }++ private static final List<SesExportPayloads.InsightsRow> ROWS = List.of(+ row("[email protected]", "blue"),+ row("[email protected]", "green"),+ row("[email protected]", null));++ @Test+ void includeOnTenantNameKeepsOnlyThatTenant() {+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,+ source("{\"Include\": {\"TenantName\": [\"blue\"]}}"));++ assertEquals(List.of("[email protected]"), destinations(kept));+ }++ @Test+ void includeOnTenantNameAcceptsSeveralValues() {+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,+ source("{\"Include\": {\"TenantName\": [\"blue\", \"green\"]}}"));++ assertEquals(List.of("[email protected]", "[email protected]"), destinations(kept));+ }++ @Test+ void excludeOnTenantNameDropsThatTenantAndKeepsTheTenantlessSend() {+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,+ source("{\"Exclude\": {\"TenantName\": [\"blue\"]}}"));++ assertEquals(List.of("[email protected]", "[email protected]"), destinations(kept));+ }++ @Test+ void membersWithinOneIncludeAllHaveToMatch() {+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,+ source("{\"Include\": {\"TenantName\": [\"blue\"], \"Isp\": [\"GMAIL\"]}}"));++ assertTrue(kept.isEmpty(), "the ISP member excludes the row the tenant member keeps");+ }++ @Test+ void anIncludeOnLastEngagementEventSelectsNothing() {+ // Floci derives no opens or clicks, so the column is always null and cannot match.+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,+ source("{\"Include\": {\"LastEngagementEvent\": [\"OPEN\"]}}"));++ assertTrue(kept.isEmpty());+ }++ @Test+ void anExcludeCarryingNoValuesKeepsEveryRow() {+ // Every member is absent, so the block constrains nothing. Read as a match it would empty+ // the file, which is the one direction where "no constraint" is not harmless.+ assertEquals(3, SesExportFilters.apply(ROWS, source("{\"Exclude\": {}}")).size());+ assertEquals(3, SesExportFilters.apply(ROWS,+ source("{\"Exclude\": {\"TenantName\": []}}")).size());+ }++ @Test+ void maxResultsTruncatesAfterFiltering() {+ List<SesExportPayloads.InsightsRow> kept = SesExportFilters.apply(ROWS,+ source("{\"Exclude\": {\"TenantName\": [\"blue\"]}, \"MaxResults\": 1}"));++ assertEquals(List.of("[email protected]"), destinations(kept));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobServiceTest.javanew file mode 100644index 0000000000..7722f15d8f--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobServiceTest.java@@ -0,0 +1,361 @@+package io.github.hectorvent.floci.services.ses;++import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.s3.PreSignedUrlGenerator;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.ses.model.EmailInsights;+import io.github.hectorvent.floci.services.ses.model.ExportJob;+import io.github.hectorvent.floci.services.ses.model.InsightsEvent;+import io.github.hectorvent.floci.services.ses.model.SentEmail;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;++import java.nio.charset.StandardCharsets;+import java.time.Clock;+import java.time.Instant;+import java.time.ZoneOffset;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * The parts of {@link SesExportJobService} the integration test cannot pin down: what happens when+ * a cancel lands in the window between the object being written and the job being marked complete.+ * The worker runs on a captured task here, so that window is reachable.+ */+class SesExportJobServiceTest {++ private static final String REGION = "eu-north-1";+ private static final String ACCOUNT = "000000000000";++ private final S3Service s3Service = mock(S3Service.class);+ private final PreSignedUrlGenerator presigner = mock(PreSignedUrlGenerator.class);+ private final SesSentEmailService sentEmailService =+ new SesSentEmailService(new InMemoryStorage<>());+ private final ObjectMapper objectMapper = new ObjectMapper();+ private final List<Runnable> tasks = new ArrayList<>();++ private SesExportJobService service;++ @BeforeEach+ void setUp() {+ when(s3Service.bucketExists(anyString())).thenReturn(true);+ service = new SesExportJobService(new InMemoryStorage<>(), sentEmailService, s3Service,+ presigner, objectMapper, Clock.fixed(Instant.parse("2026-09-23T00:00:00Z"),+ ZoneOffset.UTC),+ tasks::add, "http://localhost:4566");+ }++ private JsonNode insightsSource() {+ try {+ return objectMapper.readTree("""+ {"MessageInsightsDataSource": {"StartDate": 1758499200, "EndDate": 1758672000}}+ """);+ } catch (JsonProcessingException e) {+ throw new AssertionError(e);+ }+ }++ private ExportJob createJob() {+ return service.createExportJob(REGION, ACCOUNT, insightsSource(), destination(), () -> { });+ }++ private JsonNode destination() {+ return objectMapper.createObjectNode().put("DataFormat", ExportJob.FORMAT_CSV);+ }++ @Test+ void aCancelDuringTheWriteLeavesNoObjectBehind() {+ ExportJob job = createJob();+ // The cancel lands while the worker is inside putObject, which is the only window in which+ // an object can be written for a job that will never point at it.+ when(s3Service.putObject(anyString(), anyString(), any(), anyString(), any()))+ .thenAnswer(invocation -> {+ service.cancelExportJob(REGION, job.getJobId());+ return null;+ });++ tasks.forEach(Runnable::run);++ ExportJob stored = service.getExportJob(REGION, job.getJobId());+ assertEquals(ExportJob.STATUS_CANCELLED, stored.getJobStatus());+ assertNull(stored.getObjectKey(), "a cancelled job points at nothing");+ verify(s3Service).deleteObject(eq(SesExportJobService.exportBucket(ACCOUNT, REGION)), anyString());+ }++ @Test+ void anUninterruptedJobKeepsItsObject() {+ ExportJob job = createJob();++ tasks.forEach(Runnable::run);++ ExportJob stored = service.getExportJob(REGION, job.getJobId());+ assertEquals(ExportJob.STATUS_COMPLETED, stored.getJobStatus());+ verify(s3Service, never()).deleteObject(anyString(), anyString());+ }++ @Test+ void cancellingBeforeTheWorkerRunsSkipsTheWriteEntirely() {+ ExportJob job = createJob();+ service.cancelExportJob(REGION, job.getJobId());++ tasks.forEach(Runnable::run);++ assertEquals(ExportJob.STATUS_CANCELLED,+ service.getExportJob(REGION, job.getJobId()).getJobStatus());+ verify(s3Service, never()).putObject(anyString(), anyString(), any(), anyString(),+ any(Map.class));+ }++ @Test+ void aResetAbandonsAJobThatHasNotStartedWriting() {+ ExportJob job = createJob();++ service.beforeReset();+ tasks.forEach(Runnable::run);++ verify(s3Service, never()).putObject(anyString(), anyString(), any(), anyString(),+ any(Map.class));+ assertEquals(ExportJob.STATUS_PROCESSING,+ service.getExportJob(REGION, job.getJobId()).getJobStatus());+ }++ @Test+ void aResetWaitsForAWorkerAlreadyInsideItsWrite() throws Exception {+ CountDownLatch writing = new CountDownLatch(1);+ CountDownLatch releaseWrite = new CountDownLatch(1);+ AtomicBoolean workerFinishedTheWrite = new AtomicBoolean();+ SesExportJobService threaded = new SesExportJobService(new InMemoryStorage<>(),+ sentEmailService, s3Service, presigner, objectMapper,+ Clock.fixed(Instant.parse("2026-09-23T00:00:00Z"), ZoneOffset.UTC),+ task -> new Thread(task).start(), "http://localhost:4566");+ when(s3Service.putObject(anyString(), anyString(), any(), anyString(), any()))+ .thenAnswer(invocation -> {+ writing.countDown();+ releaseWrite.await(5, TimeUnit.SECONDS);+ workerFinishedTheWrite.set(true);+ return null;+ });++ threaded.createExportJob(REGION, ACCOUNT, insightsSource(), destination(), () -> { });+ assertTrue(writing.await(5, TimeUnit.SECONDS), "the worker never reached its write");+ Thread reset = new Thread(threaded::beforeReset);+ reset.start();+ awaitParked(reset, "the reset waiting on the in-flight write");+ releaseWrite.countDown();+ reset.join(5_000);++ // join returns on timeout too, so without this the assertion below would hold even if+ // beforeReset never came back.+ assertFalse(reset.isAlive(), "the reset did not finish after the in-flight write");+ assertTrue(workerFinishedTheWrite.get(),+ "the reset returned before the in-flight write finished");+ }++ private void recordSend(Instant sentAt, String source) {+ SentEmail email = new SentEmail();+ email.setMessageId("m-" + sentAt.toEpochMilli() + "-" + source);+ email.setSource(source);+ email.setSentAt(sentAt);+ email.setInsights(List.of(new EmailInsights("[email protected]", "UNKNOWN_ISP",+ List.of(InsightsEvent.of(sentAt, "SEND")))));+ sentEmailService.record(REGION, email.getMessageId(), email);+ }++ private String runMetricsExport(String dimensions, Instant start, Instant end) throws Exception {+ JsonNode source = objectMapper.readTree("""+ {"MetricsDataSource": {"Dimensions": %s, "Namespace": "VDM",+ "Metrics": [{"Name": "SEND", "Aggregation": "VOLUME"}],+ "StartDate": %d, "EndDate": %d}}+ """.formatted(dimensions, start.getEpochSecond(), end.getEpochSecond()));+ service.createExportJob(REGION, ACCOUNT, source, destination(), () -> { });+ tasks.forEach(Runnable::run);+ ArgumentCaptor<byte[]> payload = ArgumentCaptor.forClass(byte[].class);+ verify(s3Service).putObject(anyString(), anyString(), payload.capture(), anyString(), any());+ return new String(payload.getValue(), StandardCharsets.UTF_8);+ }++ @Test+ void anIdentityRowIsKeyedByTheBareAddressAndCountsItsSend() throws Exception {+ recordSend(Instant.parse("2026-09-23T10:00:00Z"), "\"Doe, Jane\" <[email protected]>");++ String csv = runMetricsExport("{\"EMAIL_IDENTITY\": [\"*\"]}",+ Instant.parse("2026-09-23T00:00:00Z"), Instant.parse("2026-09-24T00:00:00Z"));++ // Keyed by the stored Source, the row is one the identity matching behind the counts then+ // refuses, so it comes back at zero, and a comma in the display name splits the CSV row.+ assertEquals("EMAIL_IDENTITY,SEND_VOLUME\[email protected],1.0000\n", csv);+ }++ @Test+ void aDimensionValueSeenOnlyOutsideTheWindowProducesNoRow() throws Exception {+ recordSend(Instant.parse("2026-09-23T10:00:00Z"), "[email protected]");+ recordSend(Instant.parse("2026-09-20T10:00:00Z"), "[email protected]");++ String csv = runMetricsExport("{\"EMAIL_IDENTITY\": [\"*\"]}",+ Instant.parse("2026-09-23T00:00:00Z"), Instant.parse("2026-09-24T00:00:00Z"));++ assertTrue(csv.contains("[email protected],1.0000"), csv);+ assertFalse(csv.contains("[email protected]"), csv);+ }++ @Test+ void anInsightsWindowIncludesASendOnItsEndInstant() throws Exception {+ // The model calls MessageInsightsDataSource.EndDate inclusive, unlike the daily metrics+ // buckets, whose exclusive end the BatchGetMetricData probe confirmed.+ Instant sentAt = Instant.parse("2026-09-23T12:00:00Z");+ recordSend(sentAt, "[email protected]");+ JsonNode source = objectMapper.readTree("""+ {"MessageInsightsDataSource": {"StartDate": %d, "EndDate": %d}}+ """.formatted(sentAt.minusSeconds(3600).getEpochSecond(), sentAt.getEpochSecond()));++ ExportJob job = service.createExportJob(REGION, ACCOUNT, source, destination(), () -> { });+ tasks.forEach(Runnable::run);++ assertEquals(1L, service.getExportJob(REGION, job.getJobId()).getProcessedRecordsCount());+ }++ @Test+ void aBucketCreatedByAConcurrentJobDoesNotFailTheExport() {+ when(s3Service.bucketExists(anyString())).thenReturn(false);+ when(s3Service.createBucket(anyString(), anyString())).thenThrow(new AwsException(+ "BucketAlreadyOwnedByYou",+ "Your previous request to create the named bucket succeeded and you already own it.",+ 409));+ ExportJob job = createJob();++ tasks.forEach(Runnable::run);++ assertEquals(ExportJob.STATUS_COMPLETED,+ service.getExportJob(REGION, job.getJobId()).getJobStatus());+ }++ @Test+ void aCreateArrivingDuringAResetWaitsForItToFinish() throws Exception {+ service.beforeReset();+ AtomicBoolean admitted = new AtomicBoolean();+ Thread creator = new Thread(() -> {+ createJob();+ admitted.set(true);+ });++ creator.start();+ // Waiting for the thread to actually park proves the reset blocked it. A plain sleep would+ // pass while the thread was merely unscheduled, and stay green with the blocking removed.+ awaitParked(creator, "the create waiting on the reset");+ assertFalse(admitted.get(), "a create must not be admitted while the reset is in progress");+ service.afterReset();+ creator.join(5_000);++ assertTrue(admitted.get(), "afterReset must release the waiting create");+ }++ @Test+ void theExportBucketIsScopedToTheJobsAccount() {+ // An object write resolves its bucket across accounts under global-bucket-namespace, so a+ // shared name would put this account's export in whichever account created the bucket.+ createJob();++ tasks.forEach(Runnable::run);++ ArgumentCaptor<String> bucket = ArgumentCaptor.forClass(String.class);+ verify(s3Service).putObject(bucket.capture(), anyString(), any(), anyString(), any());+ assertEquals("floci-" + ACCOUNT + "-" + REGION + "-ses-export-files", bucket.getValue());+ }++ @Test+ void identitiesDifferingOnlyInCaseShareOneRow() throws Exception {+ // The aggregator matches identities without regard to case, so two rows would each count+ // both sends and the export would disagree with a BatchGetMetricData query.+ recordSend(Instant.parse("2026-09-23T10:00:00Z"), "[email protected]");+ recordSend(Instant.parse("2026-09-23T11:00:00Z"), "[email protected]");++ String csv = runMetricsExport("{\"EMAIL_IDENTITY\": [\"*\"]}",+ Instant.parse("2026-09-23T00:00:00Z"), Instant.parse("2026-09-24T00:00:00Z"));++ assertEquals("EMAIL_IDENTITY,SEND_VOLUME\[email protected],2.0000\n", csv);+ }++ @Test+ void theConcurrentJobCeilingIsThePublishedQuota() {+ // SES documents 20 concurrent export jobs, the same figure it gives import jobs. The+ // workers are captured here, so every job stays PROCESSING and occupies a slot.+ for (int i = 0; i < SesExportJobService.MAX_CONCURRENT_JOBS; i++) {+ createJob();+ }++ AwsException thrown = assertThrows(AwsException.class, this::createJob);++ assertEquals("LimitExceededException", thrown.getErrorCode());+ assertEquals(20, SesExportJobService.MAX_CONCURRENT_JOBS);+ }++ /** Both waits here park in {@code Object.wait(timeout)}, so the state is timed, not WAITING. */+ private static void awaitParked(Thread thread, String what) {+ long deadline = System.currentTimeMillis() + 5_000;+ while (System.currentTimeMillis() < deadline) {+ Thread.State state = thread.getState();+ if (state == Thread.State.TIMED_WAITING || state == Thread.State.WAITING) {+ return;+ }+ Thread.onSpinWait();+ }+ throw new AssertionError(what + " never parked, it was " + thread.getState());+ }++ @Test+ void recoverInterruptedJobsFailsAJobOfEveryAccount() {+ String otherAccount = "111122223333";+ AccountAwareStorageBackend<ExportJob> persisted =+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, ACCOUNT);+ ExportJob stuck = new ExportJob();+ stuck.setJobId("stuck");+ stuck.setRegion(REGION);+ stuck.setJobStatus(ExportJob.STATUS_PROCESSING);+ stuck.setCreatedTimestamp(Instant.parse("2026-09-23T00:00:00Z"));+ String key = "export-job::" + REGION + "::stuck";+ persisted.putForAccount(otherAccount, key, stuck);++ new SesExportJobService(persisted, sentEmailService, s3Service, presigner, objectMapper,+ Clock.fixed(Instant.parse("2026-09-23T00:00:00Z"), ZoneOffset.UTC), tasks::add,+ "http://localhost:4566");++ ExportJob recovered = persisted.getForAccount(otherAccount, key).orElseThrow();+ assertEquals(ExportJob.STATUS_FAILED, recovered.getJobStatus());+ assertEquals(SesExportJobService.RESTART_FAILURE_MESSAGE, recovered.getErrorMessage());+ }++ @Test+ void cancellingAFinishedJobIsRefused() {+ ExportJob job = createJob();+ tasks.forEach(Runnable::run);++ AwsException thrown = assertThrows(AwsException.class,+ () -> service.cancelExportJob(REGION, job.getJobId()));++ assertEquals("Cannot cancel an export job that has a <COMPLETED> status.",+ thrown.getMessage());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobV2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobV2IntegrationTest.javanew file mode 100644index 0000000000..74fa24f509--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportJobV2IntegrationTest.java@@ -0,0 +1,549 @@+package io.github.hectorvent.floci.services.ses;++import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import java.time.Duration;+import java.time.Instant;+import java.time.temporal.ChronoUnit;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.everyItem;+import static org.hamcrest.Matchers.hasKey;+import static org.hamcrest.Matchers.hasSize;+import static org.hamcrest.Matchers.not;+import static org.hamcrest.Matchers.notNullValue;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Integration tests for the four SES v2 export-job operations.+ *+ * <p>Probe-confirmed against real SES (2026-09-23): a request naming its own {@code S3Url} is+ * refused, exactly one data source is required, the metrics source insists on whole UTC days, an+ * unknown job is a 400 rather than a 404, and a finished job cannot be cancelled. Uses an isolated+ * signing region so no other test's sends reach the exported rows.+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class SesExportJobV2IntegrationTest {++ private static final String AUTH =+ "AWS4-HMAC-SHA256 Credential=AKID/20260101/eu-west-2/ses/aws4_request";+ private static final String SENDER = "[email protected]";++ private static long midnightToday() {+ return Instant.now().truncatedTo(ChronoUnit.DAYS).getEpochSecond();+ }++ private static Response create(String body) {+ return given().contentType("application/json").header("Authorization", AUTH)+ .body(body)+ .when().post("/v2/email/export-jobs");+ }++ private static String insightsSource() {+ return """+ {"ExportDataSource": {"MessageInsightsDataSource":+ {"StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "CSV"}}+ """.formatted(Instant.now().minus(Duration.ofDays(1)).getEpochSecond(),+ Instant.now().plus(Duration.ofDays(1)).getEpochSecond());+ }++ private static String awaitTerminal(String jobId) {+ for (int attempt = 0; attempt < 100; attempt++) {+ String status = given().header("Authorization", AUTH)+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(200)+ .extract().path("JobStatus");+ if (!"PROCESSING".equals(status) && !"CREATED".equals(status)) {+ return status;+ }+ try {+ Thread.sleep(50);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError("interrupted while waiting for the job", e);+ }+ }+ throw new AssertionError("export job " + jobId + " never reached a terminal state");+ }++ @Test+ @Order(0)+ void createExportJob_vdmDisabled_isABadRequestNotANotFound() {+ // The same sentence message insights and metric data use, but 400 here, not 404.+ create(insightsSource()).then().statusCode(400)+ .body("message", containsString(+ "To use this feature you must enable Virtual Deliverability Manager"));+ }++ @Test+ @Order(1)+ void enableVdmAndVerifyTheSender() {+ given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"VdmAttributes\":{\"VdmEnabled\":\"ENABLED\"}}")+ .when().put("/v2/email/account/vdm").then().statusCode(200);+ given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"EmailIdentity\":\"" + SENDER + "\"}")+ .when().post("/v2/email/identities").then().statusCode(200);+ }++ @Test+ @Order(2)+ void createExportJob_refusesACallerChosenDestination() {+ create("""+ {"ExportDataSource": {"MessageInsightsDataSource":+ {"StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "CSV", "S3Url": "s3://mine/prefix/"}}+ """.formatted(midnightToday() - 86400, midnightToday()))+ .then().statusCode(400)+ .body("message", equalTo("Providing a custom S3 URL is not supported"));+ }++ @Test+ @Order(3)+ void createExportJob_checksTheDestinationInTheProbedOrder() {+ // Probed: the data source outranks the destination's own constraints, which outrank the+ // refusal of a caller-chosen S3Url, which outranks the VDM gate.+ create("""+ {"ExportDataSource": {"MessageInsightsDataSource":+ {"StartDate": %d, "EndDate": %d}}}+ """.formatted(midnightToday() - 86400, midnightToday()))+ .then().statusCode(400).body("message", equalTo(+ "1 validation error detected: Value at 'exportDestination' failed to satisfy "+ + "constraint: Member must not be null"));++ create("""+ {"ExportDataSource": {"MessageInsightsDataSource":+ {"StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"S3Url": "s3://mine/prefix/"}}+ """.formatted(midnightToday() - 86400, midnightToday()))+ .then().statusCode(400).body("message", equalTo(+ "1 validation error detected: Value at 'exportDestination.dataFormat' failed to "+ + "satisfy constraint: Member must not be null"));++ create("""+ {"ExportDataSource": {"MessageInsightsDataSource":+ {"StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "PARQUET"}}+ """.formatted(midnightToday() - 86400, midnightToday()))+ .then().statusCode(400).body("message", containsString(+ "Member must satisfy enum value set: [CSV, JSON]"));++ // A missing data source beats even a caller-chosen destination.+ create("""+ {"ExportDataSource": {},+ "ExportDestination": {"DataFormat": "CSV", "S3Url": "s3://mine/prefix/"}}+ """)+ .then().statusCode(400).body("message", equalTo("One data source must be provided"));+ }++ @Test+ @Order(4)+ void createExportJob_requiresExactlyOneDataSource() {+ String both = """+ {"ExportDataSource": {"MessageInsightsDataSource": {"StartDate": %d, "EndDate": %d},+ "MetricsDataSource": {"Dimensions": {"ISP": ["*"]},+ "Namespace": "VDM",+ "Metrics": [{"Name": "SEND"}],+ "StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "CSV"}}+ """.formatted(midnightToday() - 86400, midnightToday(),+ midnightToday() - 86400, midnightToday());+ create(both).then().statusCode(400)+ .body("message", equalTo("One data source must be provided"));++ create("{\"ExportDataSource\": {}, \"ExportDestination\": {\"DataFormat\": \"CSV\"}}")+ .then().statusCode(400).body("message", equalTo("One data source must be provided"));+ }++ @Test+ @Order(5)+ void createExportJob_checksTheDates() {+ create("""+ {"ExportDataSource": {"MessageInsightsDataSource":+ {"StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "CSV"}}+ """.formatted(midnightToday(), midnightToday() - 86400))+ .then().statusCode(400).body("message", equalTo("Invalid date range"));++ // Only the metrics source insists on whole days.+ create("""+ {"ExportDataSource": {"MetricsDataSource": {"Dimensions": {"ISP": ["*"]},+ "Namespace": "VDM",+ "Metrics": [{"Name": "SEND"}],+ "StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "CSV"}}+ """.formatted(midnightToday() - 86400 + 3600, midnightToday()))+ .then().statusCode(400).body("message", containsString("midnight to midnight UTC"));+ }++ @Test+ @Order(6)+ void exportJob_writesTheInsightsFileAndServesItBehindAPresignedUrl() {+ given().contentType("application/json").header("Authorization", AUTH)+ .body("""+ {"FromEmailAddress": "%s",+ "Destination": {"ToAddresses": ["[email protected]"]},+ "Content": {"Simple": {"Subject": {"Data": "exported"},+ "Body": {"Text": {"Data": "hi"}}}}}+ """.formatted(SENDER))+ .when().post("/v2/email/outbound-emails").then().statusCode(200);++ String jobId = create(insightsSource()).then().statusCode(200)+ .extract().path("JobId");+ assertEquals("COMPLETED", awaitTerminal(jobId));++ Response job = given().header("Authorization", AUTH)+ .when().get("/v2/email/export-jobs/" + jobId);+ job.then().statusCode(200)+ .body("ExportSourceType", equalTo("MESSAGE_INSIGHTS"))+ .body("ExportDestination.DataFormat", equalTo("CSV"))+ .body("ExportDestination.S3Url", notNullValue())+ .body("ExportDataSource.MessageInsightsDataSource", notNullValue())+ .body("CompletedTimestamp", notNullValue())+ // Probed: AWS reports the rows examined and never ExportedRecordsCount.+ .body("Statistics.ProcessedRecordsCount", equalTo(1))+ .body("Statistics", not(hasKey("ExportedRecordsCount")));++ // The URL is Floci's own S3 emulation, so the object is fetched back through it.+ String url = job.path("ExportDestination.S3Url");+ String path = url.substring(url.indexOf('/', url.indexOf("//") + 2));+ String body = given().when().get(path).then().statusCode(200).extract().asString();+ assertTrue(body.startsWith("\"messageid\",\"sendtimestamp\""), body.substring(0, 60));+ assertTrue(body.contains("[email protected]"), "the send should be in the export");+ }++ @Test+ @Order(7)+ void exportJob_metricsSourceCountsTheSameSends() {+ String jobId = create("""+ {"ExportDataSource": {"MetricsDataSource": {"Dimensions": {"ISP": ["*"]},+ "Namespace": "VDM",+ "Metrics": [{"Name": "SEND",+ "Aggregation": "VOLUME"}],+ "StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "CSV"}}+ """.formatted(midnightToday(), midnightToday() + 86400))+ .then().statusCode(200).extract().path("JobId");+ assertEquals("COMPLETED", awaitTerminal(jobId));++ String url = given().header("Authorization", AUTH)+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(200)+ .extract().path("ExportDestination.S3Url");+ String path = url.substring(url.indexOf('/', url.indexOf("//") + 2));+ String body = given().when().get(path).then().statusCode(200).extract().asString();++ assertTrue(body.startsWith("ISP,SEND_VOLUME\n"), body);+ assertTrue(body.contains("UNKNOWN_ISP,1.0000"), body);+ }++ @Test+ @Order(8)+ void cancelExportJob_refusesAFinishedJob() {+ // Only the refusal is reachable here: a Floci export finishes in milliseconds, so a job is+ // already COMPLETED by the time a second request arrives. The accepted-cancel path is+ // covered deterministically in SesExportJobServiceTest.+ String finished = create(insightsSource()).then().statusCode(200).extract().path("JobId");+ assertEquals("COMPLETED", awaitTerminal(finished));++ given().contentType("application/json").header("Authorization", AUTH)+ .when().put("/v2/email/export-jobs/" + finished + "/cancel").then().statusCode(400)+ .body("message", equalTo(+ "Cannot cancel an export job that has a <COMPLETED> status."));+ }++ @Test+ @Order(9)+ void getExportJob_unknownJobIsABadRequest() {+ given().header("Authorization", AUTH)+ .when().get("/v2/email/export-jobs/no-such-job").then().statusCode(400)+ .body("message", equalTo("Invalid JobId"));+ }++ @Test+ @Order(10)+ void listExportJobs_filtersAndPaginates() {+ given().contentType("application/json").header("Authorization", AUTH).body("{}")+ .when().post("/v2/email/list-export-jobs").then().statusCode(200)+ .body("ExportJobs.JobStatus", everyItem(equalTo("COMPLETED")));++ given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"ExportSourceType\": \"METRICS_DATA\"}")+ .when().post("/v2/email/list-export-jobs").then().statusCode(200)+ .body("ExportJobs", hasSize(1));++ String nextToken = given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"PageSize\": 1}")+ .when().post("/v2/email/list-export-jobs").then().statusCode(200)+ .body("ExportJobs", hasSize(1))+ .extract().path("NextToken");+ assertNotNull(nextToken, "a truncated page carries a NextToken");+ }++ @Test+ @Order(11)+ void insightsExport_filtersOnTheTenantTheSendNamed() {+ given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"TenantName\": \"export-tenant\"}")+ .when().post("/v2/email/tenants").then().statusCode(200);+ given().contentType("application/json").header("Authorization", AUTH)+ .body("""+ {"TenantName": "export-tenant",+ "ResourceArn": "arn:aws:ses:eu-west-2:000000000000:identity/%s"}+ """.formatted(SENDER))+ .when().post("/v2/email/tenants/resources").then().statusCode(200);++ given().contentType("application/json").header("Authorization", AUTH)+ .body("""+ {"FromEmailAddress": "%s", "TenantName": "export-tenant",+ "Destination": {"ToAddresses": ["[email protected]"]},+ "Content": {"Simple": {"Subject": {"Data": "t"},+ "Body": {"Text": {"Data": "hi"}}}}}+ """.formatted(SENDER))+ .when().post("/v2/email/outbound-emails").then().statusCode(200);++ // [email protected] was sent at order 6 without a tenant, so it is the row the filter+ // has to drop. An Include that matched everything would keep it.+ assertTrue(insightsExport("\"Include\": {\"TenantName\": [\"export-tenant\"]},")+ .contains("[email protected]"));+ assertFalse(insightsExport("\"Include\": {\"TenantName\": [\"export-tenant\"]},")+ .contains("[email protected]"));++ String excluded = insightsExport("\"Exclude\": {\"TenantName\": [\"export-tenant\"]},");+ assertTrue(excluded.contains("[email protected]"), excluded);+ assertFalse(excluded.contains("[email protected]"), excluded);+ }++ /** Runs a message-insights export carrying the given filter members and returns the file. */+ private static String insightsExport(String filters) {+ String jobId = create("""+ {"ExportDataSource": {"MessageInsightsDataSource":+ {%s "StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "CSV"}}+ """.formatted(filters, Instant.now().minus(Duration.ofDays(1)).getEpochSecond(),+ Instant.now().plus(Duration.ofDays(1)).getEpochSecond()))+ .then().statusCode(200).extract().path("JobId");+ assertEquals("COMPLETED", awaitTerminal(jobId));++ String url = given().header("Authorization", AUTH)+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(200)+ .extract().path("ExportDestination.S3Url");+ String path = url.substring(url.indexOf('/', url.indexOf("//") + 2));+ return given().when().get(path).then().statusCode(200).extract().asString();+ }++ @Test+ @Order(12)+ void createExportJob_validatesTheMetricsSourceMembers() {+ // Without these an unknown dimension was dropped, an unknown metric aggregated to zero and+ // an unrecognised aggregation was read as VOLUME, so the job completed with a meaningless+ // file. The wording follows BatchGetMetricData; AWS's own was not observed.+ metricsSource("{\"NOT_A_DIMENSION\": [\"*\"]}", "\"VDM\"", "[{\"Name\": \"SEND\"}]")+ .then().statusCode(400).body("message", containsString(+ "Value at 'exportDataSource.metricsDataSource.dimensions' failed to satisfy "+ + "constraint: Member must satisfy enum value set: "+ + "[EMAIL_IDENTITY, CONFIGURATION_SET, ISP, TENANT_NAME]"));++ metricsSource("{\"ISP\": [\"*\"]}", "\"VDM\"", "[{\"Name\": \"NOT_A_METRIC\"}]")+ .then().statusCode(400).body("message", containsString(+ "'exportDataSource.metricsDataSource.metrics.1.member.name'"));++ metricsSource("{\"ISP\": [\"*\"]}", "\"VDM\"",+ "[{\"Name\": \"SEND\", \"Aggregation\": \"AVERAGE\"}]")+ .then().statusCode(400).body("message", containsString(+ "'exportDataSource.metricsDataSource.metrics.1.member.aggregation'"));++ metricsSource("{}", "\"SES\"", "[]")+ .then().statusCode(400).body("message", containsString("3 validation errors"));++ metricsSource("{\"ISP\": []}", "\"VDM\"", "[{\"Name\": \"SEND\"}]")+ .then().statusCode(400).body("message", containsString(+ "'exportDataSource.metricsDataSource.dimensions.ISP'"));+ }++ @Test+ @Order(13)+ void metricsExport_matchesAnIdentityDomainTheWayTheBatchQueryDoes() {+ // The sender is an address; a domain names the same identity, so it has to select the row.+ String jobId = metricsSource("{\"EMAIL_IDENTITY\": [\"example.com\"]}", "\"VDM\"",+ "[{\"Name\": \"SEND\", \"Aggregation\": \"VOLUME\"}]")+ .then().statusCode(200).extract().path("JobId");+ assertEquals("COMPLETED", awaitTerminal(jobId));++ String url = given().header("Authorization", AUTH)+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(200)+ .extract().path("ExportDestination.S3Url");+ String path = url.substring(url.indexOf('/', url.indexOf("//") + 2));+ String body = given().when().get(path).then().statusCode(200).extract().asString();++ assertTrue(body.startsWith("EMAIL_IDENTITY,SEND_VOLUME\n"), body);+ assertTrue(body.contains(SENDER), "a domain filter has to match the addresses inside it");+ }++ @Test+ @Order(14)+ void malformedPagingAndResultLimitsAreRefused() {+ given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"PageSize\": \"ten\"}")+ .when().post("/v2/email/list-export-jobs").then().statusCode(400);++ create("""+ {"ExportDataSource": {"MessageInsightsDataSource":+ {"MaxResults": -1, "StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "CSV"}}+ """.formatted(midnightToday(), midnightToday() + 86400))+ .then().statusCode(400).body("message", containsString("Member must be between 1 and 10000"));+ }++ /** A metrics export request with the three members the service validates. */+ private static Response metricsSource(String dimensions, String namespace, String metrics) {+ return create("""+ {"ExportDataSource": {"MetricsDataSource": {"Dimensions": %s,+ "Namespace": %s,+ "Metrics": %s,+ "StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "CSV"}}+ """.formatted(dimensions, namespace, metrics, midnightToday(),+ midnightToday() + 86400));+ }++ @Test+ @Order(15)+ void exportJob_runsUnderTheCallersAccount() {+ // The worker runs on its own thread, where account-aware storage falls back to the default+ // account unless the caller's is carried over: the job would then never be found again.+ String otherAccount = "AWS4-HMAC-SHA256 Credential=111122223333/20260101/eu-west-2/ses/"+ + "aws4_request";+ given().contentType("application/json").header("Authorization", otherAccount)+ .body("{\"VdmAttributes\":{\"VdmEnabled\":\"ENABLED\"}}")+ .when().put("/v2/email/account/vdm").then().statusCode(200);++ String jobId = given().contentType("application/json").header("Authorization", otherAccount)+ .body(insightsSource())+ .when().post("/v2/email/export-jobs").then().statusCode(200).extract().path("JobId");++ String status = null;+ for (int attempt = 0; attempt < 100 && !"COMPLETED".equals(status); attempt++) {+ status = given().header("Authorization", otherAccount)+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(200)+ .extract().path("JobStatus");+ if ("PROCESSING".equals(status) || "CREATED".equals(status)) {+ status = null;+ }+ }+ assertEquals("COMPLETED", status, "the job never finished under its own account");++ // The default account cannot see it.+ given().header("Authorization", AUTH)+ .when().get("/v2/email/export-jobs/" + jobId).then().statusCode(400);++ given().contentType("application/json").header("Authorization", otherAccount)+ .body("{\"VdmAttributes\":{\"VdmEnabled\":\"DISABLED\"}}")+ .when().put("/v2/email/account/vdm");+ }++ @Test+ @Order(16)+ void listExportJobs_acceptsAPageSizeAboveTheDefault() {+ // PageSize carries no bound in the model, so a large page is a normal request.+ given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"PageSize\": 250}")+ .when().post("/v2/email/list-export-jobs").then().statusCode(200);++ given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"PageSize\": 0}")+ .when().post("/v2/email/list-export-jobs").then().statusCode(400);++ // A page size at the int ceiling is clamped, not refused, and must not report a further+ // page: the paginator adds the page size to the offset as an int.+ String token = given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"PageSize\": 1}")+ .when().post("/v2/email/list-export-jobs").then().statusCode(200)+ .extract().path("NextToken");+ assertNotNull(token, "the fixture needs more than one job for this to mean anything");++ given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"PageSize\": 2147483647, \"NextToken\": \"%s\"}".formatted(token))+ .when().post("/v2/email/list-export-jobs").then().statusCode(200)+ .body("$", not(hasKey("NextToken")));+ }++ @Test+ @Order(17)+ void createExportJob_rejectsMembersOfTheWrongType() {+ // Each of these was read as absent or coerced, so the job was created and produced a+ // successful but meaningless file. They are deserialization errors, like every other+ // typed SES v2 member.+ create("""+ {"ExportDataSource": {"MessageInsightsDataSource": {"StartDate": %d, "EndDate": %d},+ "MetricsDataSource": []},+ "ExportDestination": {"DataFormat": "CSV"}}+ """.formatted(midnightToday(), midnightToday() + 86400))+ .then().statusCode(400).body("__type", equalTo("SerializationException"));++ create("""+ {"ExportDataSource": {"MessageInsightsDataSource":+ {"Include": {"TenantName": "blue"}, "StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "CSV"}}+ """.formatted(midnightToday(), midnightToday() + 86400))+ .then().statusCode(400).body("__type", equalTo("SerializationException"));++ metricsSource("{\"ISP\": [1]}", "\"VDM\"", "[{\"Name\": \"SEND\"}]")+ .then().statusCode(400).body("__type", equalTo("SerializationException"));++ metricsSource("{\"ISP\": [\"*\"]}", "\"VDM\"", "[\"SEND\"]")+ .then().statusCode(400).body("__type", equalTo("SerializationException"));++ // A MaxResults of the wrong type is a deserialization error, not a value out of range.+ insightsWith("\"MaxResults\": \"100\",")+ .then().statusCode(400).body("__type", equalTo("SerializationException"));+ }++ @Test+ @Order(18)+ void createExportJob_holdsEachFilterMemberToItsLength() {+ // The model allows one Subject and two LastEngagementEvent values.+ insightsWith("\"Include\": {\"Subject\": [\"one\", \"two\"]},")+ .then().statusCode(400).body("message", equalTo(+ "1 validation error detected: Value at 'exportDataSource."+ + "messageInsightsDataSource.include.subject' failed to satisfy "+ + "constraint: Member must have length less than or equal to 1"));++ insightsWith("\"Exclude\": {\"LastEngagementEvent\": [\"OPEN\", \"CLICK\", \"OPEN\"]},")+ .then().statusCode(400).body("message", containsString(+ "exclude.lastEngagementEvent' failed to satisfy constraint: Member must "+ + "have length less than or equal to 2"));++ insightsWith("\"Include\": {\"Isp\": [\"a\", \"b\", \"c\", \"d\", \"e\"]},")+ .then().statusCode(200);+ }++ /** A message-insights export request carrying the given extra members. */+ private static Response insightsWith(String members) {+ return create("""+ {"ExportDataSource": {"MessageInsightsDataSource":+ {%s "StartDate": %d, "EndDate": %d}},+ "ExportDestination": {"DataFormat": "CSV"}}+ """.formatted(members, midnightToday(), midnightToday() + 86400));+ }++ @Test+ @Order(19)+ void cleanUpVdmAndIdentity() {+ given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"VdmAttributes\":{\"VdmEnabled\":\"DISABLED\"}}")+ .when().put("/v2/email/account/vdm");+ given().header("Authorization", AUTH)+ .when().delete("/v2/email/identities/" + SENDER);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesExportPayloadsTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportPayloadsTest.javanew file mode 100644index 0000000000..d4feba8bff--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesExportPayloadsTest.java@@ -0,0 +1,136 @@+package io.github.hectorvent.floci.services.ses;++import io.github.hectorvent.floci.services.ses.model.EmailInsights;+import io.github.hectorvent.floci.services.ses.model.InsightsBounce;+import io.github.hectorvent.floci.services.ses.model.InsightsEvent;+import io.github.hectorvent.floci.services.ses.model.InsightsEventDetails;+import io.github.hectorvent.floci.services.ses.model.SentEmail;+import org.junit.jupiter.api.Test;++import java.nio.charset.StandardCharsets;+import java.time.Instant;+import java.util.List;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * The export payloads. Every column name, the quoting, the timestamp pattern and the trailing+ * comma are probe-confirmed against real SES, so this test pins the wire format rather than+ * Floci's preference.+ */+class SesExportPayloadsTest {++ private static final Instant SENT = Instant.parse("2026-09-21T01:02:03.123Z");++ private static final String HEADER =+ "\"messageid\",\"sendtimestamp\",\"isp\",\"fromaddress\",\"destination\",\"subject\","+ + "\"last_delivery_event\",\"last_engagement_event\","+ + "\"last_delivery_event_timestamp\",\"last_engagement_event_timestamp\","+ + "\"bounce_sub_type\",\"diagnostic_code\",";++ private static SentEmail email(String subject, EmailInsights... recipients) {+ SentEmail email = new SentEmail();+ email.setMessageId("message-1");+ email.setSource("[email protected]");+ email.setSubject(subject);+ email.setSentAt(SENT);+ email.setInsights(List.of(recipients));+ return email;+ }++ private static EmailInsights delivered(String destination) {+ return new EmailInsights(destination, "UNKNOWN_ISP", List.of(+ InsightsEvent.of(SENT, "SEND"), InsightsEvent.of(SENT, "DELIVERY")));+ }++ private static EmailInsights bounced(String destination) {+ return new EmailInsights(destination, "UNKNOWN_ISP", List.of(+ InsightsEvent.of(SENT, "SEND"),+ new InsightsEvent(SENT, "BOUNCE", new InsightsEventDetails(+ new InsightsBounce("PERMANENT", "General", "smtp; 550 user unknown"), null))));+ }++ private static String csv(List<SentEmail> emails) {+ return new String(SesExportPayloads.insightsCsv(SesExportPayloads.insightsRows(emails)),+ StandardCharsets.UTF_8);+ }++ @Test+ void insightsCsvCarriesTheProbedHeaderIncludingItsTrailingComma() {+ String[] lines = csv(List.of(email("hello", delivered("[email protected]")))).split("\n");++ assertEquals(HEADER, lines[0]);+ assertEquals("\"message-1\",\"2026-09-21 01:02:03.123 UTC\",\"UNKNOWN_ISP\","+ + "\"[email protected]\",\"[email protected]\",\"hello\",\"DELIVERY\",\"\","+ + "\"2026-09-21 01:02:03.123 UTC\",\"\",\"\",\"\",", lines[1]);+ }++ @Test+ void oneRowPerRecipientNotPerMessage() {+ String[] lines = csv(List.of(email("hello",+ delivered("[email protected]"), delivered("[email protected]")))).split("\n");++ assertEquals(3, lines.length);+ }++ @Test+ void aBounceReportsItsTypeAndCarriesTheDetails() {+ String[] lines = csv(List.of(email("hello", bounced("[email protected]")))).split("\n");++ assertTrue(lines[1].contains("\"PERMANENT_BOUNCE\""), lines[1]);+ assertTrue(lines[1].contains("\"General\""), lines[1]);+ assertTrue(lines[1].contains("\"smtp; 550 user unknown\""), lines[1]);+ }++ @Test+ void quotesInsideAValueAreDoubled() {+ String row = csv(List.of(email("say \"hi\"", delivered("[email protected]")))).split("\n")[1];++ assertTrue(row.contains("\"say \"\"hi\"\"\""), row);+ }++ @Test+ void insightsJsonIsOneArrayWithTheCommaLeadingEachRowAfterTheFirst() {+ String json = new String(SesExportPayloads.insightsJson(SesExportPayloads.insightsRows(+ List.of(email("hello", delivered("[email protected]"), delivered("[email protected]"))))),+ StandardCharsets.UTF_8);+ String[] lines = json.split("\n");++ assertEquals("[", lines[0]);+ assertTrue(lines[1].startsWith("{\"messageid\":\"message-1\""), lines[1]);+ assertTrue(lines[2].startsWith(",{"), lines[2]);+ assertEquals("]", lines[3]);+ }++ @Test+ void metricsCsvNamesTheDimensionThenEachMetricAndAggregation() {+ String csv = new String(SesExportPayloads.metricsCsv(List.of("ISP"),+ List.of("SEND_VOLUME", "DELIVERY_VOLUME"),+ List.of(new SesExportPayloads.MetricRow(List.of("UNKNOWN_ISP"), List.of(3.0, 2.0)))),+ StandardCharsets.UTF_8);++ assertEquals("ISP,SEND_VOLUME,DELIVERY_VOLUME\nUNKNOWN_ISP,3.0000,2.0000\n", csv);+ }++ @Test+ void metricsCsvQuotesOnlyTheValuesThatNeedIt() {+ // The probed file leaves values bare, but an unquoted comma would shift every later column.+ String csv = new String(SesExportPayloads.metricsCsv(List.of("EMAIL_IDENTITY"),+ List.of("SEND_VOLUME"),+ List.of(new SesExportPayloads.MetricRow(List.of("a,[email protected]"), List.of(1.0)))),+ StandardCharsets.UTF_8);++ assertEquals("EMAIL_IDENTITY,SEND_VOLUME\n\"a,[email protected]\",1.0000\n", csv);+ }++ @Test+ void metricsJsonKeysTheDimensionInLowerCaseAndNestsTheMetrics() {+ String json = new String(SesExportPayloads.metricsJson(List.of("ISP"),+ List.of("DELIVERY_RATE"),+ List.of(new SesExportPayloads.MetricRow(List.of("UNKNOWN_ISP"), List.of(0.5)))),+ StandardCharsets.UTF_8);++ assertEquals("[{\"isp\":\"UNKNOWN_ISP\",\"metrics\":{\"DELIVERY_RATE\":0.5000}}]\n", json);+ }+}