Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(ses): add v2 import jobs for the suppression list and contact lists (#3892)

Kenshin Okinaka8ca748355df9
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesImportJobTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesImportJobTest.java
new file mode 100644
index 0000000000..b5435d829e
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesImportJobTest.java
@@ -0,0 +1,269 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.sesv2.SesV2Client;
+import software.amazon.awssdk.services.sesv2.model.BadRequestException;
+import software.amazon.awssdk.services.sesv2.model.ContactListDestination;
+import software.amazon.awssdk.services.sesv2.model.ContactListImportAction;
+import software.amazon.awssdk.services.sesv2.model.CreateContactListRequest;
+import software.amazon.awssdk.services.sesv2.model.CreateImportJobRequest;
+import software.amazon.awssdk.services.sesv2.model.CreateImportJobResponse;
+import software.amazon.awssdk.services.sesv2.model.DataFormat;
+import software.amazon.awssdk.services.sesv2.model.DeleteContactListRequest;
+import software.amazon.awssdk.services.sesv2.model.DeleteSuppressedDestinationRequest;
+import software.amazon.awssdk.services.sesv2.model.GetContactRequest;
+import software.amazon.awssdk.services.sesv2.model.GetContactResponse;
+import software.amazon.awssdk.services.sesv2.model.GetImportJobRequest;
+import software.amazon.awssdk.services.sesv2.model.GetImportJobResponse;
+import software.amazon.awssdk.services.sesv2.model.ImportDataSource;
+import software.amazon.awssdk.services.sesv2.model.ImportDestination;
+import software.amazon.awssdk.services.sesv2.model.ImportDestinationType;
+import software.amazon.awssdk.services.sesv2.model.ImportJobSummary;
+import software.amazon.awssdk.services.sesv2.model.JobStatus;
+import software.amazon.awssdk.services.sesv2.model.ListImportJobsRequest;
+import software.amazon.awssdk.services.sesv2.model.ListImportJobsResponse;
+import software.amazon.awssdk.services.sesv2.model.ListSuppressedDestinationsRequest;
+import software.amazon.awssdk.services.sesv2.model.NotFoundException;
+import software.amazon.awssdk.services.sesv2.model.SubscriptionStatus;
+import software.amazon.awssdk.services.sesv2.model.SuppressionListDestination;
+import software.amazon.awssdk.services.sesv2.model.SuppressionListImportAction;
+import software.amazon.awssdk.services.sesv2.model.SuppressedDestinationSummary;
+import software.amazon.awssdk.services.sesv2.model.Topic;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * SDK compatibility test for the SES v2 import-job APIs: CreateImportJob / GetImportJob /
+ * ListImportJobs marshalling, the asynchronous job lifecycle polled through GetImportJob, and the
+ * imported rows read back through the suppression-list and contact APIs, against a live Floci
+ * instance. The source objects are written through the S3 SDK.
+ */
+@DisplayName("SES v2 Import Jobs")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class SesImportJobTest {
+
+ private static final String BUCKET = "floci-ses-import-job-sdk-test";
+ private static final String LIST = "compat-import-job-list";
+ private static final String SUPPRESSION_KEY = "suppression.csv";
+ private static final String CONTACTS_KEY = "contacts.json";
+
+ private static SesV2Client sesV2;
+ private static S3Client s3;
+ private static String suppressionJobId;
+ private static String contactJobId;
+
+ @BeforeAll
+ static void setup() {
+ sesV2 = TestFixtures.sesV2Client();
+ s3 = TestFixtures.s3Client();
+ cleanupQuietly();
+ s3.createBucket(CreateBucketRequest.builder().bucket(BUCKET).build());
+ s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(SUPPRESSION_KEY).build(),
+ RequestBody.fromString("[email protected],BOUNCE\[email protected],COMPLAINT\[email protected],SPAM\n"));
+ s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(CONTACTS_KEY).build(),
+ RequestBody.fromString("""
+ {"emailAddress":"[email protected]","unsubscribeAll":false,"attributesData":"{\\"Name\\":\\"Carol\\"}","topicPreferences":[{"topicName":"Sports","subscriptionStatus":"OPT_IN"}]}
+ {"emailAddress":"[email protected]","unsubscribeAll":true}
+ """));
+ sesV2.createContactList(CreateContactListRequest.builder()
+ .contactListName(LIST)
+ .topics(Topic.builder().topicName("Sports").displayName("Sports")
+ .defaultSubscriptionStatus(SubscriptionStatus.OPT_OUT).build())
+ .build());
+ }
+
+ @AfterAll
+ static void cleanup() {
+ cleanupQuietly();
+ if (sesV2 != null) {
+ sesV2.close();
+ }
+ if (s3 != null) {
+ s3.close();
+ }
+ }
+
+ private static void cleanupQuietly() {
+ for (String address : List.of("[email protected]", "[email protected]")) {
+ quietly(() -> sesV2.deleteSuppressedDestination(
+ DeleteSuppressedDestinationRequest.builder().emailAddress(address).build()));
+ }
+ quietly(() -> sesV2.deleteContactList(DeleteContactListRequest.builder().contactListName(LIST).build()));
+ for (String key : List.of(SUPPRESSION_KEY, CONTACTS_KEY)) {
+ quietly(() -> s3.deleteObject(DeleteObjectRequest.builder().bucket(BUCKET).key(key).build()));
+ }
+ quietly(() -> s3.deleteBucket(DeleteBucketRequest.builder().bucket(BUCKET).build()));
+ }
+
+ // Best-effort teardown: the resource is usually already gone (a fresh Floci, or the previous
+ // run cleaned up), so a NotFound here is the expected state and the next call must still run.
+ private static void quietly(Runnable cleanup) {
+ try {
+ cleanup.run();
+ } catch (Exception ignored) {
+ // see above: an absent resource is exactly what cleanup wants
+ }
+ }
+
+ @Test
+ @Order(1)
+ @DisplayName("CreateImportJob into the suppression list completes and lists the addresses")
+ void createImportJob_suppressionList() throws InterruptedException {
+ CreateImportJobResponse created = sesV2.createImportJob(CreateImportJobRequest.builder()
+ .importDataSource(ImportDataSource.builder()
+ .s3Url("s3://" + BUCKET + "/" + SUPPRESSION_KEY).dataFormat(DataFormat.CSV).build())
+ .importDestination(ImportDestination.builder()
+ .suppressionListDestination(SuppressionListDestination.builder()
+ .suppressionListImportAction(SuppressionListImportAction.PUT).build())
+ .build())
+ .build());
+ suppressionJobId = created.jobId();
+ assertThat(suppressionJobId).isNotBlank();
+
+ GetImportJobResponse job = pollUntilTerminal(suppressionJobId);
+ assertThat(job.jobStatus()).isEqualTo(JobStatus.COMPLETED);
+ assertThat(job.importDataSource().s3Url()).isEqualTo("s3://" + BUCKET + "/" + SUPPRESSION_KEY);
+ assertThat(job.importDataSource().dataFormat()).isEqualTo(DataFormat.CSV);
+ assertThat(job.importDestination().suppressionListDestination().suppressionListImportAction())
+ .isEqualTo(SuppressionListImportAction.PUT);
+ assertThat(job.importDestination().contactListDestination()).isNull();
+ assertThat(job.processedRecordsCount()).isEqualTo(3);
+ assertThat(job.failedRecordsCount()).isEqualTo(1);
+ assertThat(job.createdTimestamp()).isNotNull();
+ assertThat(job.completedTimestamp()).isNotNull();
+ assertThat(job.failureInfo()).isNull();
+
+ List<String> suppressed = sesV2.listSuppressedDestinations(ListSuppressedDestinationsRequest.builder().build())
+ .suppressedDestinationSummaries().stream()
+ .map(SuppressedDestinationSummary::emailAddress)
+ .toList();
+ assertThat(suppressed).contains("[email protected]", "[email protected]");
+ }
+
+ @Test
+ @Order(2)
+ @DisplayName("CreateImportJob into a contact list creates the contacts")
+ void createImportJob_contactList() throws InterruptedException {
+ CreateImportJobResponse created = sesV2.createImportJob(CreateImportJobRequest.builder()
+ .importDataSource(ImportDataSource.builder()
+ .s3Url("s3://" + BUCKET + "/" + CONTACTS_KEY).dataFormat(DataFormat.JSON).build())
+ .importDestination(ImportDestination.builder()
+ .contactListDestination(ContactListDestination.builder()
+ .contactListName(LIST)
+ .contactListImportAction(ContactListImportAction.PUT).build())
+ .build())
+ .build());
+ contactJobId = created.jobId();
+
+ GetImportJobResponse job = pollUntilTerminal(contactJobId);
+ assertThat(job.jobStatus()).isEqualTo(JobStatus.COMPLETED);
+ assertThat(job.importDestination().contactListDestination().contactListName()).isEqualTo(LIST);
+ assertThat(job.processedRecordsCount()).isEqualTo(2);
+ assertThat(job.failedRecordsCount()).isEqualTo(0);
+
+ GetContactResponse carol = sesV2.getContact(GetContactRequest.builder()
+ .contactListName(LIST).emailAddress("[email protected]").build());
+ assertThat(carol.attributesData()).isEqualTo("{\"Name\":\"Carol\"}");
+ assertThat(carol.topicPreferences()).hasSize(1);
+ assertThat(carol.topicPreferences().get(0).subscriptionStatus()).isEqualTo(SubscriptionStatus.OPT_IN);
+ GetContactResponse dave = sesV2.getContact(GetContactRequest.builder()
+ .contactListName(LIST).emailAddress("[email protected]").build());
+ assertThat(dave.unsubscribeAll()).isTrue();
+ }
+
+ @Test
+ @Order(3)
+ @DisplayName("ListImportJobs returns both jobs and filters by destination type")
+ void listImportJobs() {
+ ListImportJobsResponse all = sesV2.listImportJobs(ListImportJobsRequest.builder().build());
+ assertThat(all.importJobs()).extracting(ImportJobSummary::jobId).contains(suppressionJobId, contactJobId);
+ assertThat(all.nextToken()).isNull();
+
+ ListImportJobsResponse contactOnly = sesV2.listImportJobs(ListImportJobsRequest.builder()
+ .importDestinationType(ImportDestinationType.CONTACT_LIST).pageSize(10).build());
+ assertThat(contactOnly.importJobs()).extracting(ImportJobSummary::jobId)
+ .contains(contactJobId).doesNotContain(suppressionJobId);
+ assertThat(contactOnly.importJobs().get(0).jobStatus()).isEqualTo(JobStatus.COMPLETED);
+ }
+
+ @Test
+ @Order(4)
+ @DisplayName("CreateImportJob with a missing object completes as FAILED with an ErrorMessage")
+ void createImportJob_missingObject_fails() throws InterruptedException {
+ String jobId = sesV2.createImportJob(CreateImportJobRequest.builder()
+ .importDataSource(ImportDataSource.builder()
+ .s3Url("s3://" + BUCKET + "/missing.csv").dataFormat(DataFormat.CSV).build())
+ .importDestination(ImportDestination.builder()
+ .suppressionListDestination(SuppressionListDestination.builder()
+ .suppressionListImportAction(SuppressionListImportAction.DELETE).build())
+ .build())
+ .build()).jobId();
+
+ GetImportJobResponse job = pollUntilTerminal(jobId);
+ assertThat(job.jobStatus()).isEqualTo(JobStatus.FAILED);
+ assertThat(job.failureInfo().errorMessage()).contains("does not exist");
+ }
+
+ @Test
+ @Order(5)
+ @DisplayName("CreateImportJob rejects an invalid S3Url and an unknown contact list")
+ void createImportJob_validation() {
+ assertThatThrownBy(() -> sesV2.createImportJob(CreateImportJobRequest.builder()
+ .importDataSource(ImportDataSource.builder().s3Url("https://example.com/x.csv")
+ .dataFormat(DataFormat.CSV).build())
+ .importDestination(ImportDestination.builder()
+ .suppressionListDestination(SuppressionListDestination.builder()
+ .suppressionListImportAction(SuppressionListImportAction.PUT).build())
+ .build())
+ .build()))
+ .isInstanceOf(BadRequestException.class);
+
+ assertThatThrownBy(() -> sesV2.createImportJob(CreateImportJobRequest.builder()
+ .importDataSource(ImportDataSource.builder().s3Url("s3://" + BUCKET + "/" + CONTACTS_KEY)
+ .dataFormat(DataFormat.JSON).build())
+ .importDestination(ImportDestination.builder()
+ .contactListDestination(ContactListDestination.builder()
+ .contactListName("no-such-list")
+ .contactListImportAction(ContactListImportAction.PUT).build())
+ .build())
+ .build()))
+ .isInstanceOf(NotFoundException.class);
+ }
+
+ @Test
+ @Order(6)
+ @DisplayName("GetImportJob for an unknown id throws NotFoundException")
+ void getImportJob_unknown() {
+ assertThatThrownBy(() -> sesV2.getImportJob(GetImportJobRequest.builder().jobId("no-such-job").build()))
+ .isInstanceOf(NotFoundException.class);
+ }
+
+ private static GetImportJobResponse pollUntilTerminal(String jobId) throws InterruptedException {
+ long deadline = System.currentTimeMillis() + 10_000;
+ GetImportJobResponse job;
+ do {
+ job = sesV2.getImportJob(GetImportJobRequest.builder().jobId(jobId).build());
+ if (job.jobStatus() == JobStatus.COMPLETED || job.jobStatus() == JobStatus.FAILED) {
+ return job;
+ }
+ Thread.sleep(100);
+ } while (System.currentTimeMillis() < deadline);
+ return job;
+ }
+}
diff --git a/docs/services/ses.md b/docs/services/ses.md
index 5bb05ffa85..bdc5406a57 100644
--- a/docs/services/ses.md
+++ b/docs/services/ses.md
@@ -372,6 +372,9 @@ Alongside the classic Query API, Floci implements a subset of the SES v2 REST JS
| `POST` | `/v2/email/tags` | `TagResource` |
| `DELETE` | `/v2/email/tags?ResourceArn=...&TagKeys=...` | `UntagResource` |
| `GET` | `/v2/email/tags?ResourceArn=...` | `ListTagsForResource` |
+| `POST` | `/v2/email/import-jobs` | `CreateImportJob` |
+| `GET` | `/v2/email/import-jobs/{JobId}` | `GetImportJob` |
+| `POST` | `/v2/email/import-jobs/list` | `ListImportJobs` |
Floci models no leased dedicated IPs: `GetDedicatedIps` is empty and IP-targeted operations return `NotFoundException`, as real AWS does for an account with no leased IPs, with required request members validated first (`BadRequestException`). `PutDedicatedIpPoolScalingAttributes` rejects downgrading a `MANAGED` pool to `STANDARD`, and `PutAccountDedicatedIpWarmupAttributes` stores the flag behind `GetAccount.DedicatedIpAutoWarmupEnabled` (default `true`).
@@ -408,6 +411,8 @@ Suppression list entries are stored per region with `Reason` ∈ {`BOUNCE`, `COM
Suppressed recipients are filtered out of the SMTP relay step (non-suppressed recipients on the same send still reach the relay normally), and the configuration set's event destinations receive a synthetic `Bounce` or `Complaint` event alongside the always-emitted `Send` event. The `SendEmail` API response (`200` + `MessageId`), the stored `SentEmail` visible at `GET /_aws/ses`, and the published event's `mail.destination` all retain the original recipient list — matching the AWS contract that the message is "accepted, just not sent" for suppressed addresses.
+Import jobs read their source from Floci's own S3 emulation, so the object has to be uploaded there rather than to real S3; the CSV and newline-delimited JSON record formats are AWS's own ([suppression list](https://docs.aws.amazon.com/ses/latest/dg/sending-email-suppression-list.html#sending-email-suppression-list-manual-add-bulk), [contact list](https://docs.aws.amazon.com/ses/latest/dg/sending-email-list-management.html#configuring-list-management-bulk-import)). A job still running when the emulator restarts is marked `FAILED` on the next start. Not emulated: `FailureInfo.FailedRecordsS3Url` (Floci writes no failed-record file), the per-file record limits, the same-region bucket rule, and `ListImportJobs` pagination, which returns every job in one page.
+
`SendEmail` honors `ListManagementOptions` (`ContactListName`, optional `TopicName`). When present, each recipient is matched against the named contact list and suppressed as a `Bounce` when opted out — reusing the same relay-exclusion and Bounce-event path as suppression-list filtering, matching AWS ("SES will issue a bounce event for a message that is sent to an unsubscribed contact"). A contact is opted out when `UnsubscribeAll` is set; with a `TopicName`, an explicit `OPT_OUT` preference for that topic (or, absent an explicit preference, the topic's `DefaultSubscriptionStatus` being `OPT_OUT`) suppresses; without a `TopicName`, only `UnsubscribeAll` contacts are suppressed. A recipient that is not yet a contact is created on the list automatically (as on AWS) and then evaluated. Referencing a contact list that does not exist fails the send with `NotFoundException`. The topic-default fallback at send time is an intentional deviation — it is not documented by AWS and mirrors the effective-status model AWS uses for `ListContacts`.
For a **single-recipient** list-managed send, Floci injects a functional unsubscribe link (matching AWS, which only does this for one recipient): the `{{amazonSESUnsubscribeUrl}}` body placeholder is replaced (up to twice) and a `List-Unsubscribe` header plus `List-Unsubscribe-Post: List-Unsubscribe=One-Click` are added (applied to the relayed message and shown under `Headers` at `GET /_aws/ses`; a caller-supplied `List-Unsubscribe` is overridden, matching AWS). The `{{amazonSESUnsubscribeUrl}}` placeholder is also preserved through template rendering so a templated body can carry it. On a send without `ListManagementOptions`, the placeholder is left in the body verbatim (neither replaced nor stripped) — this specific behavior is Floci's choice and is not verified against real SES. Unlike AWS's opaque hosted URL, the link points at Floci's own `/_aws/ses/unsubscribe?region=…&contactList=…&address=…&topic=…` endpoint. `GET` (a browser click) only renders a confirmation page and changes nothing — matching AWS's landing-page behavior and avoiding the RFC 8058 hazard where a client or bot that prefetches the link would silently unsubscribe the contact — while `POST` (the one-click request the confirmation form submits) applies the opt-out (a topic → `OPT_OUT` for that topic; no topic → `UnsubscribeAll`), auto-creating the contact if needed. Two deviations from AWS here: the link is carried as readable query parameters rather than an opaque token (so, like the rest of Floci, the endpoint is unauthenticated and only safe on a trusted dev/test network — a `POST` can opt out any contact), and only the one-click URL entry is emitted — its scheme follows Floci's base URL (`http` unless TLS is enabled) — whereas AWS also includes a `mailto:` entry, which Floci can't service since it has no inbound mail endpoint. Raw (MIME) sends do not get link injection yet.
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
index 3d4f49f184..0d55ab6dc2 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
@@ -39,6 +39,7 @@ import io.github.hectorvent.floci.services.ses.SesContactController;
import io.github.hectorvent.floci.services.ses.SesCvetController;
import io.github.hectorvent.floci.services.ses.SesDedicatedIpController;
import io.github.hectorvent.floci.services.ses.SesIdentityController;
+import io.github.hectorvent.floci.services.ses.SesImportJobController;
import io.github.hectorvent.floci.services.ses.SesSendController;
import io.github.hectorvent.floci.services.ses.SesSuppressionController;
import io.github.hectorvent.floci.services.ses.SesTagController;
@@ -301,7 +302,7 @@ public class ResolvedServiceCatalog {
SesDedicatedIpController.class, SesIdentityController.class,
SesSendController.class, SesSuppressionController.class,
SesTagController.class, SesTemplateController.class,
- SesTenantController.class)),
+ SesImportJobController.class, SesTenantController.class)),
descriptor("es", "opensearch", config.services().opensearch().enabled(), true,
"opensearch", storageMode(config.storage().services().opensearch().mode(), config.storage().mode()),
config.storage().services().opensearch().flushIntervalMs(), null, ServiceProtocol.REST_JSON,
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesContactService.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesContactService.java
index 87855be41b..897e1827fa 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ses/SesContactService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesContactService.java
@@ -386,6 +386,32 @@ public class SesContactService {
return existing;
}
+ /**
+ * Replaces a contact outright, as a contact-list import job's {@code PUT} action does. Probed
+ * against real AWS: an imported record is the whole contact, so preferences, attributes, and
+ * the unsubscribe flag it omits are cleared rather than merged, unlike
+ * {@link #updateContact}, whose topic preferences merge by name.
+ */
+ public Contact replaceContact(String listName, String emailAddress, List<TopicPreference> topicPreferences,
+ Boolean unsubscribeAll, String attributesData, String region) {
+ validateContactInput(listName, emailAddress, topicPreferences, region);
+ String key = contactKey(region, listName, emailAddress);
+ Contact existing;
+ // Same lock as create/update: a concurrent deleteContactList must not slip between the
+ // validation and the write.
+ synchronized (contactMutationLock) {
+ getContactList(listName, region);
+ existing = contactStore.get(key).orElseThrow(() -> contactNotFound(emailAddress));
+ existing.setTopicPreferences(topicPreferences);
+ existing.setUnsubscribeAll(unsubscribeAll != null && unsubscribeAll);
+ existing.setAttributesData(attributesData);
+ existing.setLastUpdatedTimestamp(Instant.now(clock));
+ contactStore.put(key, existing);
+ }
+ LOG.infov("Replaced SES contact {0} in list {1}", emailAddress, listName);
+ return existing;
+ }
+
public void deleteContact(String listName, String emailAddress, String region) {
validateEmailAddress(emailAddress);
String key = contactKey(region, listName, emailAddress);
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobController.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobController.java
new file mode 100644
index 0000000000..a0e12938e7
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobController.java
@@ -0,0 +1,177 @@
+package io.github.hectorvent.floci.services.ses;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.services.ses.model.ImportJob;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.HttpHeaders;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+import org.jboss.logging.Logger;
+
+import java.util.List;
+
+import static io.github.hectorvent.floci.services.ses.SesV2Json.intMemberOrAbsent;
+import static io.github.hectorvent.floci.services.ses.SesV2Json.readRequiredStringField;
+import static io.github.hectorvent.floci.services.ses.SesV2Json.remapV1Exception;
+import static io.github.hectorvent.floci.services.ses.SesV2Json.requireJsonObject;
+import static io.github.hectorvent.floci.services.ses.SesV2Json.requireObjectOrAbsent;
+import static io.github.hectorvent.floci.services.ses.SesV2Json.stringMemberOrAbsent;
+
+/**
+ * SES V2 import job endpoints ({@code /v2/email/import-jobs}). Every operation is a single-domain
+ * call on {@link SesImportJobService}; the JSON shape of {@code ImportDestination} /
+ * {@code ImportDataSource} is parsed and rendered here.
+ */
+@Path("/v2/email")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public class SesImportJobController {
+
+ private static final Logger LOG = Logger.getLogger(SesImportJobController.class);
+
+ private final SesImportJobService importJobService;
+ private final RegionResolver regionResolver;
+ private final ObjectMapper objectMapper;
+ // FAIL_ON_TRAILING_TOKENS so "{...} garbage" is rejected instead of creating a job from the
+ // leading object, as the JSON protocol controllers do.
+ private final ObjectReader strictReader;
+
+ @Inject
+ public SesImportJobController(SesImportJobService importJobService, RegionResolver regionResolver,
+ ObjectMapper objectMapper) {
+ this.importJobService = importJobService;
+ this.regionResolver = regionResolver;
+ this.objectMapper = objectMapper;
+ this.strictReader = objectMapper.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
+ }
+
+ @POST
+ @Path("/import-jobs")
+ public Response createImportJob(@Context HttpHeaders headers, String body) {
+ String region = regionResolver.resolveRegion(headers);
+ try {
+ if (body == null || body.isBlank()) {
+ throw new AwsException("BadRequestException", "Request body is required.", 400);
+ }
+ JsonNode request = strictReader.readTree(body);
+ requireJsonObject(request);
+ JsonNode source = requireObjectOrAbsent(request, "ImportDataSource");
+ if (!source.isObject()) {
+ throw new AwsException("BadRequestException", "ImportDataSource is required.", 400);
+ }
+ String s3Url = readRequiredStringField(source, "S3Url");
+ String dataFormat = readRequiredStringField(source, "DataFormat");
+ JsonNode destination = requireObjectOrAbsent(request, "ImportDestination");
+ if (!destination.isObject()) {
+ throw new AwsException("BadRequestException", "ImportDestination is required.", 400);
+ }
+ JsonNode suppression = requireObjectOrAbsent(destination, "SuppressionListDestination");
+ JsonNode contactList = requireObjectOrAbsent(destination, "ContactListDestination");
+ if (suppression.isObject() == contactList.isObject()) {
+ throw new AwsException("BadRequestException",
+ "ImportDestination must contain exactly one of SuppressionListDestination "
+ + "or ContactListDestination.", 400);
+ }
+ String accountId = regionResolver.getAccountId();
+ ImportJob job;
+ if (suppression.isObject()) {
+ String action = readRequiredStringField(suppression, "SuppressionListImportAction");
+ job = importJobService.createSuppressionListImportJob(region, accountId, s3Url, dataFormat, action);
+ } else {
+ String listName = readRequiredStringField(contactList, "ContactListName");
+ String action = readRequiredStringField(contactList, "ContactListImportAction");
+ job = importJobService.createContactListImportJob(region, accountId, s3Url, dataFormat,
+ listName, action);
+ }
+ LOG.infov("SES V2 CreateImportJob: {0}", job.getJobId());
+ ObjectNode result = objectMapper.createObjectNode();
+ result.put("JobId", job.getJobId());
+ return Response.ok(result).build();
+ } catch (AwsException e) {
+ throw remapV1Exception(e);
+ } catch (JsonProcessingException e) {
+ throw new AwsException("BadRequestException", e.getMessage(), 400);
+ }
+ }
+
+ @GET
+ @Path("/import-jobs/{jobId}")
+ public Response getImportJob(@Context HttpHeaders headers, @PathParam("jobId") String jobId) {
+ String region = regionResolver.resolveRegion(headers);
+ ImportJob job = importJobService.getImportJob(region, jobId);
+ ObjectNode result = objectMapper.createObjectNode();
+ writeSummary(result, job);
+ ObjectNode source = result.putObject("ImportDataSource");
+ source.put("S3Url", job.getS3Url());
+ source.put("DataFormat", job.getDataFormat());
+ if (job.getCompletedTimestamp() != null) {
+ result.put("CompletedTimestamp", job.getCompletedTimestamp().toEpochMilli() / 1000.0);
+ }
+ if (job.getErrorMessage() != null) {
+ result.putObject("FailureInfo").put("ErrorMessage", job.getErrorMessage());
+ }
+ return Response.ok(result).build();
+ }
+
+ @POST
+ @Path("/import-jobs/list")
+ public Response listImportJobs(@Context HttpHeaders headers, String body) {
+ String region = regionResolver.resolveRegion(headers);
+ try {
+ JsonNode request = (body == null || body.isBlank())
+ ? objectMapper.createObjectNode()
+ : strictReader.readTree(body);
+ requireJsonObject(request);
+ String destinationType = stringMemberOrAbsent(request, "ImportDestinationType");
+ Integer pageSize = intMemberOrAbsent(request, "PageSize");
+ String nextToken = stringMemberOrAbsent(request, "NextToken");
+ List<ImportJob> jobs = importJobService.listImportJobs(region, destinationType, pageSize, nextToken);
+ ObjectNode result = objectMapper.createObjectNode();
+ ArrayNode items = result.putArray("ImportJobs");
+ for (ImportJob job : jobs) {
+ writeSummary(items.addObject(), job);
+ }
+ // AWS renders NextToken as an explicit null on the last (here: only) page.
+ result.putNull("NextToken");
+ return Response.ok(result).build();
+ } catch (AwsException e) {
+ throw remapV1Exception(e);
+ } catch (JsonProcessingException e) {
+ throw new AwsException("BadRequestException", e.getMessage(), 400);
+ }
+ }
+
+ // The ImportJobSummary members, shared by GetImportJob (which adds the source, completion time
+ // and failure info on top).
+ private static void writeSummary(ObjectNode node, ImportJob job) {
+ node.put("JobId", job.getJobId());
+ ObjectNode destination = node.putObject("ImportDestination");
+ if (ImportJob.DESTINATION_SUPPRESSION_LIST.equals(job.getDestinationType())) {
+ destination.putObject("SuppressionListDestination")
+ .put("SuppressionListImportAction", job.getImportAction());
+ } else {
+ ObjectNode contactList = destination.putObject("ContactListDestination");
+ contactList.put("ContactListName", job.getContactListName());
+ contactList.put("ContactListImportAction", job.getImportAction());
+ }
+ node.put("JobStatus", job.getJobStatus());
+ node.put("CreatedTimestamp", job.getCreatedTimestamp().toEpochMilli() / 1000.0);
+ node.put("ProcessedRecordsCount", job.getProcessedRecordsCount());
+ node.put("FailedRecordsCount", job.getFailedRecordsCount());
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobService.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobService.java
new file mode 100644
index 0000000000..e8700a23e5
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobService.java
@@ -0,0 +1,474 @@
+package io.github.hectorvent.floci.services.ses;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.core.common.Resettable;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.ses.imports.ImportRecord;
+import io.github.hectorvent.floci.services.ses.imports.RecordReader;
+import io.github.hectorvent.floci.services.ses.imports.RecordReaderFactory;
+import io.github.hectorvent.floci.services.ses.model.ImportJob;
+import io.quarkus.runtime.StartupEvent;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * SES V2 import jobs: bulk-loads a CSV or newline-delimited JSON object from S3 into the account
+ * suppression list or a contact list. A job is stored as CREATED and returned immediately; a
+ * worker then flips it to PROCESSING and applies the records one by one through
+ * {@link SesSuppressionService} / {@link SesContactService}, so the imported data lives in those
+ * services' stores and only the job record lives here. A record that fails validation is counted
+ * in {@code FailedRecordsCount} and the job still completes; the job itself fails when the source
+ * cannot be read or the contact list is gone.
+ */
+@ApplicationScoped
+public class SesImportJobService implements Resettable {
+
+ private static final Logger LOG = Logger.getLogger(SesImportJobService.class);
+
+ static final int MAX_CONCURRENT_JOBS = 20;
+ // AWS reports an import's counters as it runs, so the worker republishes them as it goes.
+ // Every record would mean serializing the job per row, so progress lands in batches.
+ static final int PROGRESS_RECORD_INTERVAL = 100;
+ static final String RESTART_FAILURE_MESSAGE = "The emulator restarted before the import finished";
+ private static final long RESET_ADMISSION_WAIT_MILLIS = 30_000L;
+ // The drain only ever waits for the record being applied right now plus the read of the next
+ // one, which is a store get/put and a line off a local file, so this deadline is a safety net
+ // that a healthy import never reaches. 5s is what every other worker wait in Floci uses
+ // (WalStorage, HybridStorage, CloudWatchLogsMetricFilterService, CodePipelineService). Raising
+ // it is the wrong direction: a reset walks every Resettable service in turn, so per-service
+ // deadlines add up.
+ static final long RESET_DRAIN_TIMEOUT_MILLIS = 5_000L;
+
+ private static final Pattern S3_URL = Pattern.compile("^s3://([^/]+)/(.*?([^/]+)/?)$");
+ private static final Set<String> DATA_FORMATS = Set.of(ImportJob.FORMAT_CSV, ImportJob.FORMAT_JSON);
+ private static final Set<String> IMPORT_ACTIONS = Set.of(ImportJob.ACTION_PUT, ImportJob.ACTION_DELETE);
+ private static final Set<String> DESTINATION_TYPES =
+ Set.of(ImportJob.DESTINATION_SUPPRESSION_LIST, ImportJob.DESTINATION_CONTACT_LIST);
+
+ private final StorageBackend<String, ImportJob> importJobStore;
+ private final S3Service s3Service;
+ private final SesSuppressionService suppressionService;
+ private final SesContactService contactService;
+ private final ObjectMapper objectMapper;
+ private final RecordReaderFactory readerFactory;
+ private final Clock clock;
+ private final Executor workers;
+ private final long resetDrainTimeoutMillis;
+ // Admission lock: serializes the concurrent-job count against the store write (so two creates
+ // cannot both pass the limit check) and against reset, which flips {@code resetting} under it.
+ private final Object createLock = new Object();
+ private boolean resetting;
+ // Reset support: beforeReset bumps the generation so every running worker stops before its
+ // next write, then waits on workerMonitor for the workers still inside one. That wait is
+ // bounded (see RESET_DRAIN_TIMEOUT_MILLIS), so a worker wedged past the deadline can still
+ // land the record it is applying right now; it is logged when that happens.
+ private final AtomicInteger generation = new AtomicInteger();
+ private final Object workerMonitor = new Object();
+ private int activeWorkers;
+
+ @Inject
+ public SesImportJobService(StorageFactory storageFactory, S3Service s3Service,
+ SesSuppressionService suppressionService, SesContactService contactService,
+ ObjectMapper objectMapper, Clock clock) {
+ this(storageFactory.create("ses", "ses-import-jobs.json",
+ new TypeReference<Map<String, ImportJob>>() {}),
+ s3Service, suppressionService, contactService, objectMapper, clock,
+ task -> Thread.ofVirtual().start(task));
+ }
+
+ SesImportJobService(StorageBackend<String, ImportJob> importJobStore, S3Service s3Service,
+ SesSuppressionService suppressionService, SesContactService contactService,
+ ObjectMapper objectMapper, Clock clock, Executor workers) {
+ this(importJobStore, s3Service, suppressionService, contactService, objectMapper, clock, workers,
+ RESET_DRAIN_TIMEOUT_MILLIS);
+ }
+
+ SesImportJobService(StorageBackend<String, ImportJob> importJobStore, S3Service s3Service,
+ SesSuppressionService suppressionService, SesContactService contactService,
+ ObjectMapper objectMapper, Clock clock, Executor workers,
+ long resetDrainTimeoutMillis) {
+ this.resetDrainTimeoutMillis = resetDrainTimeoutMillis;
+ this.importJobStore = importJobStore;
+ this.s3Service = s3Service;
+ this.suppressionService = suppressionService;
+ this.contactService = contactService;
+ this.objectMapper = objectMapper;
+ this.readerFactory = new RecordReaderFactory(objectMapper);
+ this.clock = clock;
+ this.workers = workers;
+ recoverInterruptedJobs();
+ }
+
+ public ImportJob createSuppressionListImportJob(String region, String accountId, String s3Url,
+ String dataFormat, String importAction) {
+ ImportJob job = newJob(region, s3Url, dataFormat, importAction);
+ job.setDestinationType(ImportJob.DESTINATION_SUPPRESSION_LIST);
+ return start(job, accountId);
+ }
+
+ public ImportJob createContactListImportJob(String region, String accountId, String s3Url,
+ String dataFormat, String contactListName,
+ String importAction) {
+ ImportJob job = newJob(region, s3Url, dataFormat, importAction);
+ job.setDestinationType(ImportJob.DESTINATION_CONTACT_LIST);
+ job.setContactListName(contactListName);
+ requireContactList(job);
+ return start(job, accountId);
+ }
+
+ /**
+ * Probed against real AWS: an import job checks the contact list when it is created, answering
+ * NotFoundException even though the model lists only BadRequest / LimitExceeded /
+ * TooManyRequests for CreateImportJob. This path also words it differently from the contact-list
+ * APIs ("ContactList <name> doesn't exist" rather than "List with name: <name> doesn't exist."),
+ * so the message is rewritten here and the contact service keeps its own.
+ */
+ private void requireContactList(ImportJob job) {
+ try {
+ contactService.getContactList(job.getContactListName(), job.getRegion());
+ } catch (AwsException e) {
+ if (!"NotFoundException".equals(e.getErrorCode())) {
+ throw e;
+ }
+ throw new AwsException("NotFoundException",
+ "ContactList <" + job.getContactListName() + "> doesn't exist", 404);
+ }
+ }
+
+ private ImportJob newJob(String region, String s3Url, String dataFormat, String importAction) {
+ if (s3Url == null || !S3_URL.matcher(s3Url).matches()) {
+ throw validationError("importDataSource.s3Url",
+ "Member must satisfy regular expression pattern: " + S3_URL.pattern());
+ }
+ if (dataFormat == null || !DATA_FORMATS.contains(dataFormat)) {
+ throw validationError("importDataSource.dataFormat",
+ "Member must satisfy enum value set: [CSV, JSON]");
+ }
+ if (importAction == null || !IMPORT_ACTIONS.contains(importAction)) {
+ throw validationError("importDestination", "Member must satisfy enum value set: [DELETE, PUT]");
+ }
+ ImportJob job = new ImportJob();
+ job.setJobId(UUID.randomUUID().toString());
+ job.setRegion(region);
+ job.setS3Url(s3Url);
+ job.setDataFormat(dataFormat);
+ job.setImportAction(importAction);
+ job.setJobStatus(ImportJob.STATUS_CREATED);
+ job.setCreatedTimestamp(Instant.now(clock));
+ return job;
+ }
+
+ private ImportJob start(ImportJob job, String accountId) {
+ // The worker gets its own copy so the record serialized into the Create response never
+ // changes underneath the handler, and runs as the request account so its store writes
+ // land in that account's partition.
+ ImportJob worker = objectMapper.convertValue(job, ImportJob.class);
+ synchronized (createLock) {
+ awaitResetFinished();
+ long active = importJobStore.scan(k -> k.startsWith(keyPrefix(job.getRegion()))).stream()
+ .filter(j -> !j.isTerminal())
+ .count();
+ if (active >= MAX_CONCURRENT_JOBS) {
+ throw new AwsException("LimitExceededException",
+ "The maximum number of concurrent import jobs (" + MAX_CONCURRENT_JOBS
+ + ") has been reached.", 400);
+ }
+ publish(key(job.getRegion(), job.getJobId()), job);
+ // Persist, capture the generation and dispatch under the same lock beforeReset takes,
+ // so a job is either fully admitted before a reset (and then abandoned by its stale
+ // generation) or admitted only after the reset has finished.
+ int startedIn = generation.get();
+ workers.execute(() -> RequestScopes.runAs(accountId, () -> runImport(worker, startedIn)));
+ }
+ LOG.infov("Created SES import job {0} in region {1}: {2} {3} from {4}", job.getJobId(),
+ job.getRegion(), job.getImportAction(), job.getDestinationType(), job.getS3Url());
+ return job;
+ }
+
+ // Holds createLock. A create that arrives while a reset is in progress waits for afterReset
+ // rather than persisting a job that the wipe would then race.
+ private void awaitResetFinished() {
+ long deadline = System.currentTimeMillis() + RESET_ADMISSION_WAIT_MILLIS;
+ while (resetting) {
+ long remaining = deadline - System.currentTimeMillis();
+ if (remaining <= 0) {
+ throw new AwsException("TooManyRequestsException",
+ "Import jobs are not accepted while the emulator is resetting.", 429);
+ }
+ try {
+ createLock.wait(remaining);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AwsException("TooManyRequestsException",
+ "Import jobs are not accepted while the emulator is resetting.", 429);
+ }
+ }
+ }
+
+ private void runImport(ImportJob job, int startedIn) {
+ synchronized (workerMonitor) {
+ activeWorkers++;
+ }
+ try {
+ runImportInGeneration(job, startedIn);
+ } finally {
+ synchronized (workerMonitor) {
+ activeWorkers--;
+ workerMonitor.notifyAll();
+ }
+ }
+ }
+
+ private void runImportInGeneration(ImportJob job, int startedIn) {
+ String storeKey = key(job.getRegion(), job.getJobId());
+ if (generation.get() != startedIn) {
+ return;
+ }
+ job.setJobStatus(ImportJob.STATUS_PROCESSING);
+ publish(storeKey, job);
+ try {
+ Matcher url = S3_URL.matcher(job.getS3Url());
+ if (!url.matches()) {
+ throw new AwsException("BadRequestException", "Invalid S3 URL: " + job.getS3Url(), 400);
+ }
+ if (ImportJob.DESTINATION_CONTACT_LIST.equals(job.getDestinationType())) {
+ requireContactList(job);
+ }
+ // Streamed and applied one record at a time, so an import never holds more than the
+ // current record in memory whatever the object's size.
+ try (InputStream raw = s3Service.openObjectStream(url.group(1), url.group(2), null);
+ BufferedReader source = new BufferedReader(new InputStreamReader(raw, StandardCharsets.UTF_8))) {
+ applyRecords(job, storeKey, startedIn, readerFactory.forJob(job, source));
+ }
+ job.setJobStatus(ImportJob.STATUS_COMPLETED);
+ } catch (AwsException e) {
+ job.setJobStatus(ImportJob.STATUS_FAILED);
+ job.setErrorMessage(e.getMessage());
+ } catch (IOException | RuntimeException e) {
+ LOG.errorv(e, "SES import job {0} failed", job.getJobId());
+ job.setJobStatus(ImportJob.STATUS_FAILED);
+ job.setErrorMessage(e.getMessage());
+ }
+ job.setCompletedTimestamp(Instant.now(clock));
+ if (generation.get() != startedIn) {
+ LOG.infov("SES import job {0} abandoned by an emulator reset", job.getJobId());
+ return;
+ }
+ publish(storeKey, job);
+ LOG.infov("SES import job {0} {1}: processed={2}, failed={3}", job.getJobId(), job.getJobStatus(),
+ job.getProcessedRecordsCount(), job.getFailedRecordsCount());
+ }
+
+ // The worker goes on mutating its own ImportJob after each write, so every store write publishes
+ // a copy: a backend that hands back the stored instance must never let a poll observe a
+ // half-applied transition, such as COMPLETED before the completion timestamp is set.
+ private void publish(String storeKey, ImportJob job) {
+ importJobStore.put(storeKey, objectMapper.convertValue(job, ImportJob.class));
+ }
+
+ private void applyRecords(ImportJob job, String storeKey, int startedIn,
+ RecordReader reader)
+ throws IOException {
+ for (ImportRecord record = reader.next(); record != null; record = reader.next()) {
+ if (generation.get() != startedIn) {
+ LOG.infov("SES import job {0} abandoned by an emulator reset", job.getJobId());
+ return;
+ }
+ job.setProcessedRecordsCount(job.getProcessedRecordsCount() + 1);
+ applyRecord(job, record);
+ if (job.getProcessedRecordsCount() % PROGRESS_RECORD_INTERVAL == 0) {
+ publish(storeKey, job);
+ }
+ }
+ }
+
+ private void applyRecord(ImportJob job, ImportRecord record) {
+ if (record.error() != null) {
+ failRecord(job, record.line(), record.error());
+ return;
+ }
+ try {
+ apply(job, record);
+ } catch (AwsException e) {
+ // A NotFound for a contact-list job is a missing contact (a failed record) unless the
+ // list itself has gone, which fails the whole job like the pre-run check.
+ if (ImportJob.DESTINATION_CONTACT_LIST.equals(job.getDestinationType())
+ && "NotFoundException".equals(e.getErrorCode())) {
+ requireContactList(job);
+ }
+ failRecord(job, record.line(), e.getMessage());
+ }
+ }
+
+ private static void failRecord(ImportJob job, int line, String reason) {
+ job.setFailedRecordsCount(job.getFailedRecordsCount() + 1);
+ LOG.debugv("SES import job {0} skipped record at line {1}: {2}", job.getJobId(), line, reason);
+ }
+
+ private void apply(ImportJob job, ImportRecord record) {
+ boolean put = ImportJob.ACTION_PUT.equals(job.getImportAction());
+ if (ImportJob.DESTINATION_SUPPRESSION_LIST.equals(job.getDestinationType())) {
+ if (put) {
+ suppressionService.putSuppressedDestination(job.getRegion(), record.emailAddress(), record.reason());
+ } else {
+ suppressionService.deleteSuppressedDestination(job.getRegion(), record.emailAddress());
+ }
+ return;
+ }
+ String list = job.getContactListName();
+ if (!put) {
+ contactService.deleteContact(list, record.emailAddress(), job.getRegion());
+ return;
+ }
+ // PUT over an existing contact replaces it outright rather than merging: probed against real
+ // AWS, an import record that omits a topic, the attributes, or the unsubscribe flag clears
+ // them. That is not UpdateContact's contract, so it has its own service path.
+ try {
+ contactService.createContact(list, record.emailAddress(), record.topicPreferences(),
+ record.unsubscribeAll(), record.attributesData(), job.getRegion());
+ } catch (AwsException e) {
+ if (!"AlreadyExistsException".equals(e.getErrorCode())) {
+ throw e;
+ }
+ contactService.replaceContact(list, record.emailAddress(), record.topicPreferences(),
+ record.unsubscribeAll(), record.attributesData(), job.getRegion());
+ }
+ }
+
+ public ImportJob getImportJob(String region, String jobId) {
+ return importJobStore.get(key(region, jobId))
+ .orElseThrow(() -> new AwsException("NotFoundException",
+ "Import job " + jobId + " does not exist.", 404));
+ }
+
+ public List<ImportJob> listImportJobs(String region, String destinationType, Integer pageSize,
+ String nextToken) {
+ if (destinationType != null && !DESTINATION_TYPES.contains(destinationType)) {
+ throw validationError("importDestinationType",
+ "Member must satisfy enum value set: [SUPPRESSION_LIST, CONTACT_LIST]");
+ }
+ SesTenantService.validateListPaging(pageSize, nextToken);
+ return importJobStore.scan(k -> k.startsWith(keyPrefix(region))).stream()
+ .filter(j -> destinationType == null || destinationType.equals(j.getDestinationType()))
+ .sorted(Comparator.comparing(ImportJob::getCreatedTimestamp)
+ .thenComparing(ImportJob::getJobId))
+ .toList();
+ }
+
+ // ──────────────────────────── Lifecycle ────────────────────────────
+
+ // Recovery happens in the constructor, and a normal-scoped bean is only built on first use, so
+ // without this the jobs left non-terminal by a restart would not be failed until an import
+ // endpoint was called. Observing startup makes "marked FAILED on the next start" true of the
+ // boot itself, which is what the docs promise.
+ void onStart(@Observes StartupEvent ignored) {
+ LOG.debugv("SES import job service ready");
+ }
+
+ @Override
+ public void beforeReset() {
+ synchronized (createLock) {
+ resetting = true;
+ generation.incrementAndGet();
+ }
+ // Drain, but bounded: a reset has to return even when a worker is wedged in a slow read.
+ // Every worker re-checks the generation before its next write, so the wait normally covers
+ // only the record being applied right now. If the deadline passes anyway that one record
+ // can still land after the wipe, which the warning below is there to explain.
+ boolean interrupted = false;
+ long deadline = System.currentTimeMillis() + resetDrainTimeoutMillis;
+ synchronized (workerMonitor) {
+ while (activeWorkers > 0) {
+ long remaining = deadline - System.currentTimeMillis();
+ if (remaining <= 0) {
+ LOG.warnv("{0} SES import worker(s) still running after {1}ms; a record in flight may "
+ + "write after the reset", activeWorkers, resetDrainTimeoutMillis);
+ break;
+ }
+ try {
+ workerMonitor.wait(remaining);
+ } catch (InterruptedException e) {
+ // Keep draining to the deadline; the interrupt is re-asserted on the way out.
+ interrupted = true;
+ }
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ @Override
+ public void afterReset() {
+ synchronized (createLock) {
+ resetting = false;
+ createLock.notifyAll();
+ }
+ }
+
+ @Override
+ public void clear() {
+ // The job store is wiped by StorageFactory; the imported rows belong to the suppression and
+ // contact services, which clear their own stores.
+ }
+
+
+ // A job interrupted by a restart would otherwise stay CREATED/PROCESSING forever, since no
+ // worker survives the process.
+ private void recoverInterruptedJobs() {
+ if (!(importJobStore instanceof AccountAwareStorageBackend<ImportJob> jobs)) {
+ return;
+ }
+ Instant now = Instant.now(clock);
+ for (AccountAwareStorageBackend.AccountEntry<ImportJob> entry : jobs.scanAllAccountEntries(k -> true)) {
+ ImportJob job = entry.value();
+ if (job.isTerminal()) {
+ continue;
+ }
+ job.setJobStatus(ImportJob.STATUS_FAILED);
+ job.setErrorMessage(RESTART_FAILURE_MESSAGE);
+ job.setCompletedTimestamp(now);
+ jobs.putForAccount(entry.accountId(), entry.key(), job);
+ LOG.warnv("SES import job {0} was interrupted by a restart and is now FAILED", job.getJobId());
+ }
+ }
+
+ private static AwsException validationError(String path, String constraint) {
+ return new AwsException("BadRequestException",
+ "1 validation error detected: Value at '" + path + "' failed to satisfy constraint: "
+ + constraint, 400);
+ }
+
+ static String key(String region, String jobId) {
+ return keyPrefix(region) + jobId;
+ }
+
+ private static String keyPrefix(String region) {
+ return "importJob::" + region + "::";
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReader.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReader.java
new file mode 100644
index 0000000000..293d7644a5
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReader.java
@@ -0,0 +1,261 @@
+package io.github.hectorvent.floci.services.ses.imports;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.ses.model.ImportJob;
+import io.github.hectorvent.floci.services.ses.model.TopicPreference;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Reads RFC 4180 records one at a time: a record ends at an unquoted line break (a quoted
+ * attributesData may span lines), the first record of a contact-list file is the header, and
+ * a row whose width or quoting is wrong is a failed record. Records are numbered by position.
+ *
+ * <p>Deliberately not {@link io.github.hectorvent.floci.core.common.CsvParser}, which parses a
+ * whole document held as a {@code String} and is permissive by design (a quote inside an
+ * unquoted field is literal, an unterminated quoted field simply ends at EOF, and
+ * {@code parseAll} applies the backslash escapes S3 Select, Athena, and the Step Functions
+ * ItemReader expect). An import reads its source straight off an S3 stream so a large file is
+ * never materialized, and it has to report malformed quoting rather than absorb it, since a bad
+ * row has to land in {@code FailedRecordsCount} or fail the job. Nothing else in Floci streams
+ * CSV out of S3 yet: DynamoDB's {@code ImportTable} sidesteps it by rejecting the CSV
+ * {@code InputFormat} outright, so this is the first of its kind, and the place to grow a
+ * shared streaming reader from if that changes.
+ */
+final class CsvRecordReader implements RecordReader {
+
+ private static final String TOPIC_PREFERENCE_COLUMN_PREFIX = "topicPreferences.";
+ private final Reader source;
+ private final ImportJob job;
+ private final boolean contactList;
+ private int pending = -1;
+ private boolean bomChecked;
+ private int recordNumber;
+ private Map<String, Integer> columns;
+
+ CsvRecordReader(Reader source, ImportJob job) {
+ this.source = source;
+ this.job = job;
+ this.contactList = ImportJob.DESTINATION_CONTACT_LIST.equals(job.getDestinationType());
+ }
+
+ @Override
+ public ImportRecord next() throws IOException {
+ String raw;
+ while ((raw = nextRawRecord()) != null) {
+ recordNumber++;
+ if (raw.isBlank()) {
+ continue;
+ }
+ List<String> fields;
+ try {
+ fields = splitCsvLine(raw);
+ } catch (InvalidRecordException e) {
+ if (contactList && columns == null) {
+ throw new AwsException("BadRequestException", "Malformed CSV header: " + e.getMessage(), 400);
+ }
+ return ImportRecord.invalid(recordNumber, e.getMessage());
+ }
+ if (contactList && columns == null) {
+ columns = readContactHeader(fields);
+ continue;
+ }
+ if (!contactList) {
+ // The documented formats are exactly "email,reason" for PUT and "email" for DELETE.
+ int expected = ImportJob.ACTION_PUT.equals(job.getImportAction()) ? 2 : 1;
+ if (fields.size() != expected) {
+ return ImportRecord.invalid(recordNumber,
+ "Expected " + expected + " column(s) but found " + fields.size() + ".");
+ }
+ return new ImportRecord(recordNumber, fields.get(0), expected == 2 ? fields.get(1) : null,
+ null, null, null, null);
+ }
+ if (fields.size() != columns.size()) {
+ return ImportRecord.invalid(recordNumber,
+ "Expected " + columns.size() + " column(s) but found " + fields.size() + ".");
+ }
+ return contactRecordFromCsv(recordNumber, fields, columns);
+ }
+ if (contactList && columns == null) {
+ throw new AwsException("BadRequestException",
+ "The CSV file must start with a header line naming the emailAddress column.", 400);
+ }
+ return null;
+ }
+
+ // Only a quote at the start of a field opens a quoted section (a bare quote elsewhere is
+ // left for splitCsvLine to reject as a malformed row), so one bad row cannot swallow the
+ // rest of the file; a doubled quote inside a quoted field stays inside it.
+ private String nextRawRecord() throws IOException {
+ int start = read();
+ if (start < 0) {
+ return null;
+ }
+ pushBack(start);
+ StringBuilder current = new StringBuilder();
+ boolean quoted = false;
+ boolean fieldStart = true;
+ int next;
+ while ((next = read()) >= 0) {
+ char c = (char) next;
+ if (quoted) {
+ if (c == '"') {
+ int peek = read();
+ if (peek == '"') {
+ current.append(c);
+ } else {
+ quoted = false;
+ pushBack(peek);
+ }
+ }
+ current.append(c);
+ } else if (c == '\n' || c == '\r') {
+ if (c == '\r') {
+ int peek = read();
+ if (peek != '\n') {
+ pushBack(peek);
+ }
+ }
+ return current.toString();
+ } else {
+ if (c == '"' && fieldStart) {
+ quoted = true;
+ }
+ fieldStart = c == ',';
+ current.append(c);
+ }
+ }
+ if (quoted) {
+ throw new AwsException("BadRequestException", "Malformed CSV: unterminated quoted field.", 400);
+ }
+ return current.toString();
+ }
+
+ private int read() throws IOException {
+ int c;
+ if (pending >= 0) {
+ c = pending;
+ pending = -1;
+ } else {
+ c = source.read();
+ }
+ if (!bomChecked) {
+ bomChecked = true;
+ if (c == '\uFEFF') {
+ return read();
+ }
+ }
+ return c;
+ }
+
+ private void pushBack(int c) {
+ if (c >= 0) {
+ pending = c;
+ }
+ }
+
+ private static Map<String, Integer> readContactHeader(List<String> header) {
+ Map<String, Integer> columns = new LinkedHashMap<>();
+ for (int i = 0; i < header.size(); i++) {
+ columns.put(header.get(i).trim(), i);
+ }
+ if (!columns.containsKey("emailAddress")) {
+ throw new AwsException("BadRequestException",
+ "The CSV header line must contain an emailAddress column.", 400);
+ }
+ return columns;
+ }
+
+ private static ImportRecord contactRecordFromCsv(int line, List<String> fields, Map<String, Integer> columns) {
+ String email = column(fields, columns, "emailAddress");
+ Boolean unsubscribeAll = null;
+ String unsubscribeText = column(fields, columns, "unsubscribeAll");
+ if (unsubscribeText != null && !unsubscribeText.isEmpty()) {
+ if (unsubscribeText.equalsIgnoreCase("true")) {
+ unsubscribeAll = true;
+ } else if (unsubscribeText.equalsIgnoreCase("false")) {
+ unsubscribeAll = false;
+ } else {
+ return ImportRecord.invalid(line, "unsubscribeAll must be true or false.");
+ }
+ }
+ String attributesData = column(fields, columns, "attributesData");
+ if (attributesData != null && attributesData.isEmpty()) {
+ attributesData = null;
+ }
+ List<TopicPreference> prefs = null;
+ for (Map.Entry<String, Integer> entry : columns.entrySet()) {
+ if (!entry.getKey().startsWith(TOPIC_PREFERENCE_COLUMN_PREFIX)) {
+ continue;
+ }
+ String status = fields.get(entry.getValue()).trim();
+ if (status.isEmpty()) {
+ continue;
+ }
+ if (prefs == null) {
+ prefs = new ArrayList<>();
+ }
+ prefs.add(new TopicPreference(entry.getKey().substring(TOPIC_PREFERENCE_COLUMN_PREFIX.length()),
+ status));
+ }
+ return new ImportRecord(line, email, null, unsubscribeAll, attributesData, prefs, null);
+ }
+
+ private static String column(List<String> fields, Map<String, Integer> columns, String name) {
+ Integer index = columns.get(name);
+ return index == null ? null : fields.get(index);
+ }
+
+ // RFC 4180 field splitting: a quoted field may contain commas and line breaks, and a doubled
+ // quote inside a quoted field is a literal quote. A quote that does not open a field is data,
+ // not an error: the developer guide's own contact row writes attributesData unquoted as
+ // {"Name": "John"}, and core/common/CsvParser keeps such a quote literal as well. Anything but
+ // a delimiter after a closing quote is still malformed, since that row cannot be read back.
+ static List<String> splitCsvLine(String line) {
+ List<String> fields = new ArrayList<>();
+ StringBuilder current = new StringBuilder();
+ boolean quoted = false;
+ boolean afterClosingQuote = false;
+ for (int i = 0; i < line.length(); i++) {
+ char c = line.charAt(i);
+ if (quoted) {
+ if (c == '"') {
+ if (i + 1 < line.length() && line.charAt(i + 1) == '"') {
+ current.append('"');
+ i++;
+ } else {
+ quoted = false;
+ afterClosingQuote = true;
+ }
+ } else {
+ current.append(c);
+ }
+ } else if (afterClosingQuote) {
+ if (c != ',') {
+ throw new InvalidRecordException("Malformed CSV: unexpected character after a closing quote.");
+ }
+ fields.add(current.toString());
+ current.setLength(0);
+ afterClosingQuote = false;
+ } else if (c == '"' && current.isEmpty()) {
+ quoted = true;
+ } else if (c == ',') {
+ fields.add(current.toString());
+ current.setLength(0);
+ } else {
+ current.append(c);
+ }
+ }
+ if (quoted) {
+ throw new InvalidRecordException("Malformed CSV: unterminated quoted field.");
+ }
+ fields.add(current.toString());
+ return fields;
+ }
+
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/ImportRecord.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/ImportRecord.java
new file mode 100644
index 0000000000..70c50e7bdb
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/ImportRecord.java
@@ -0,0 +1,18 @@
+package io.github.hectorvent.floci.services.ses.imports;
+
+import io.github.hectorvent.floci.services.ses.model.TopicPreference;
+
+import java.util.List;
+
+/**
+ * One parsed source record. {@code error} is set when the record could not be turned into an
+ * import (it still counts as processed and failed, and the job carries on), mirroring how AWS
+ * reports a partially failed import as COMPLETED with a FailedRecordsCount.
+ */
+public record ImportRecord(int line, String emailAddress, String reason, Boolean unsubscribeAll,
+ String attributesData, List<TopicPreference> topicPreferences, String error) {
+
+ public static ImportRecord invalid(int line, String error) {
+ return new ImportRecord(line, null, null, null, null, null, error);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/InvalidRecordException.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/InvalidRecordException.java
new file mode 100644
index 0000000000..ac239cb8d6
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/InvalidRecordException.java
@@ -0,0 +1,13 @@
+package io.github.hectorvent.floci.services.ses.imports;
+
+
+/**
+ * A source record that cannot be turned into an import. Caught by the reader that raised it and
+ * reported as a failed record, so one bad row never fails the whole job.
+ */
+final class InvalidRecordException extends RuntimeException {
+
+ InvalidRecordException(String message) {
+ super(message);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReader.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReader.java
new file mode 100644
index 0000000000..ef8d152d15
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReader.java
@@ -0,0 +1,102 @@
+package io.github.hectorvent.floci.services.ses.imports;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectReader;
+import io.github.hectorvent.floci.services.ses.model.ImportJob;
+import io.github.hectorvent.floci.services.ses.model.TopicPreference;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Reads newline-delimited JSON: one complete object per line. */
+final class JsonLineReader implements RecordReader {
+
+ private static final String BOM = "\uFEFF";
+ private final BufferedReader source;
+ private final ImportJob job;
+ private final ObjectReader lineReader;
+ private int lineNumber;
+
+ JsonLineReader(BufferedReader source, ImportJob job, ObjectReader lineReader) {
+ this.source = source;
+ this.job = job;
+ this.lineReader = lineReader;
+ }
+
+ @Override
+ public ImportRecord next() throws IOException {
+ String read;
+ while ((read = source.readLine()) != null) {
+ lineNumber++;
+ String line = (lineNumber == 1 ? stripBom(read) : read).trim();
+ if (line.isEmpty()) {
+ continue;
+ }
+ JsonNode node;
+ try {
+ node = lineReader.readTree(line);
+ } catch (JsonProcessingException e) {
+ return ImportRecord.invalid(lineNumber, "Malformed JSON: " + e.getOriginalMessage());
+ }
+ try {
+ return jsonRecord(lineNumber, node, job);
+ } catch (InvalidRecordException e) {
+ return ImportRecord.invalid(lineNumber, e.getMessage());
+ }
+ }
+ return null;
+ }
+
+ private static ImportRecord jsonRecord(int line, JsonNode node, ImportJob job) {
+ if (!node.isObject()) {
+ throw new InvalidRecordException("Each line must be a JSON object.");
+ }
+ String email = textMember(node, "emailAddress");
+ if (ImportJob.DESTINATION_SUPPRESSION_LIST.equals(job.getDestinationType())) {
+ return new ImportRecord(line, email, textMember(node, "reason"), null, null, null, null);
+ }
+ Boolean unsubscribeAll = null;
+ JsonNode unsubscribeNode = node.get("unsubscribeAll");
+ if (unsubscribeNode != null && !unsubscribeNode.isNull()) {
+ if (!unsubscribeNode.isBoolean()) {
+ throw new InvalidRecordException("unsubscribeAll must be a boolean.");
+ }
+ unsubscribeAll = unsubscribeNode.booleanValue();
+ }
+ List<TopicPreference> prefs = null;
+ JsonNode prefsNode = node.get("topicPreferences");
+ if (prefsNode != null && !prefsNode.isNull()) {
+ if (!prefsNode.isArray()) {
+ throw new InvalidRecordException("topicPreferences must be an array.");
+ }
+ prefs = new ArrayList<>();
+ for (JsonNode p : prefsNode) {
+ if (!p.isObject()) {
+ throw new InvalidRecordException("Each topicPreferences element must be a JSON object.");
+ }
+ prefs.add(new TopicPreference(textMember(p, "topicName"), textMember(p, "subscriptionStatus")));
+ }
+ }
+ return new ImportRecord(line, email, null, unsubscribeAll, textMember(node, "attributesData"), prefs, null);
+ }
+
+ // A present member of the wrong JSON type is an invalid record, not coerced text, matching the
+ // SerializationException the CreateContact / PutSuppressedDestination bodies get.
+ private static String textMember(JsonNode node, String field) {
+ JsonNode member = node.get(field);
+ if (member == null || member.isNull()) {
+ return null;
+ }
+ if (!member.isTextual()) {
+ throw new InvalidRecordException(field + " must be a JSON string.");
+ }
+ return member.textValue();
+ }
+
+ private static String stripBom(String text) {
+ return text.startsWith(BOM) ? text.substring(BOM.length()) : text;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReader.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReader.java
new file mode 100644
index 0000000000..da239be0d6
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReader.java
@@ -0,0 +1,13 @@
+package io.github.hectorvent.floci.services.ses.imports;
+
+import java.io.IOException;
+
+/**
+ * Yields an import job's source records lazily, one at a time, so a large object is never held in
+ * memory. {@code null} ends the source; a problem with the file as a whole rather than with one
+ * record is an {@code AwsException} that fails the job.
+ */
+public interface RecordReader {
+
+ ImportRecord next() throws IOException;
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReaderFactory.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReaderFactory.java
new file mode 100644
index 0000000000..c5a6ba2f22
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReaderFactory.java
@@ -0,0 +1,36 @@
+package io.github.hectorvent.floci.services.ses.imports;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+import io.github.hectorvent.floci.services.ses.model.ImportJob;
+
+import java.io.BufferedReader;
+
+/**
+ * Builds the reader for an import job's DataFormat. Takes the mapper rather than a ready-made
+ * {@link ObjectReader} on purpose: reading a JSON line strictly is a requirement of the format, not
+ * a caller's preference, so the setting lives here instead of leaking to whoever wires this up,
+ * where a plain reader would compile and quietly accept malformed lines.
+ */
+public final class RecordReaderFactory {
+
+ /**
+ * FAIL_ON_TRAILING_TOKENS so a line like {@code {"emailAddress":"x"} garbage} is a malformed
+ * record rather than a silently accepted object, as the JSON protocol controllers read their
+ * bodies. Built once: an import reads thousands of lines through it.
+ */
+ private final ObjectReader lineReader;
+
+ public RecordReaderFactory(ObjectMapper objectMapper) {
+ this.lineReader = objectMapper.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
+ }
+
+ /** CSV is parsed without Jackson, so only the JSON reader is handed the line reader. */
+ public RecordReader forJob(ImportJob job, BufferedReader source) {
+ return ImportJob.FORMAT_JSON.equals(job.getDataFormat())
+ ? new JsonLineReader(source, job, lineReader)
+ : new CsvRecordReader(source, job);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/model/ImportJob.java b/src/main/java/io/github/hectorvent/floci/services/ses/model/ImportJob.java
new file mode 100644
index 0000000000..ffb6b6ad74
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/model/ImportJob.java
@@ -0,0 +1,92 @@
+package io.github.hectorvent.floci.services.ses.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+
+/**
+ * A SES V2 import job: the S3 source, the suppression-list or contact-list destination, and the
+ * job's progress. The destination is flattened (type + action + optional contact list name) since
+ * a job has exactly one; the controller rebuilds the AWS {@code ImportDestination} shape from it.
+ */
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ImportJob {
+
+ public static final String DESTINATION_SUPPRESSION_LIST = "SUPPRESSION_LIST";
+ public static final String DESTINATION_CONTACT_LIST = "CONTACT_LIST";
+ public static final String ACTION_PUT = "PUT";
+ public static final String ACTION_DELETE = "DELETE";
+ public static final String FORMAT_CSV = "CSV";
+ public static final String FORMAT_JSON = "JSON";
+ public static final String STATUS_CREATED = "CREATED";
+ public static final String STATUS_PROCESSING = "PROCESSING";
+ public static final String STATUS_COMPLETED = "COMPLETED";
+ public static final String STATUS_FAILED = "FAILED";
+
+ private String jobId;
+ private String region;
+ private String destinationType;
+ private String importAction;
+ private String contactListName;
+ private String s3Url;
+ private String dataFormat;
+ private String jobStatus;
+ private Instant createdTimestamp;
+ private Instant completedTimestamp;
+ private long processedRecordsCount;
+ private long failedRecordsCount;
+ private String errorMessage;
+
+ public ImportJob() {}
+
+ public String getJobId() { return jobId; }
+ public void setJobId(String jobId) { this.jobId = jobId; }
+
+ public String getRegion() { return region; }
+ public void setRegion(String region) { this.region = region; }
+
+ public String getDestinationType() { return destinationType; }
+ public void setDestinationType(String destinationType) { this.destinationType = destinationType; }
+
+ public String getImportAction() { return importAction; }
+ public void setImportAction(String importAction) { this.importAction = importAction; }
+
+ public String getContactListName() { return contactListName; }
+ public void setContactListName(String contactListName) { this.contactListName = contactListName; }
+
+ public String getS3Url() { return s3Url; }
+ public void setS3Url(String s3Url) { this.s3Url = s3Url; }
+
+ public String getDataFormat() { return dataFormat; }
+ public void setDataFormat(String dataFormat) { this.dataFormat = dataFormat; }
+
+ public String getJobStatus() { return jobStatus; }
+ public void setJobStatus(String jobStatus) { this.jobStatus = jobStatus; }
+
+ public Instant getCreatedTimestamp() { return createdTimestamp; }
+ public void setCreatedTimestamp(Instant createdTimestamp) { this.createdTimestamp = createdTimestamp; }
+
+ public Instant getCompletedTimestamp() { return completedTimestamp; }
+ public void setCompletedTimestamp(Instant completedTimestamp) { this.completedTimestamp = completedTimestamp; }
+
+ public long getProcessedRecordsCount() { return processedRecordsCount; }
+ public void setProcessedRecordsCount(long processedRecordsCount) {
+ this.processedRecordsCount = processedRecordsCount;
+ }
+
+ public long getFailedRecordsCount() { return failedRecordsCount; }
+ public void setFailedRecordsCount(long failedRecordsCount) { this.failedRecordsCount = failedRecordsCount; }
+
+ public String getErrorMessage() { return errorMessage; }
+ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }
+
+ @JsonIgnore
+ public boolean isTerminal() {
+ return STATUS_COMPLETED.equals(jobStatus) || STATUS_FAILED.equals(jobStatus);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobServiceTest.java
new file mode 100644
index 0000000000..2934073fe2
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobServiceTest.java
@@ -0,0 +1,656 @@
+package io.github.hectorvent.floci.services.ses;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.ses.model.Contact;
+import io.github.hectorvent.floci.services.ses.model.ImportJob;
+import io.github.hectorvent.floci.services.ses.model.Topic;
+import io.github.hectorvent.floci.services.ses.model.TopicPreference;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit test for the import-job domain: request validation, the CSV / newline-delimited JSON record
+ * formats, per-record failure counting versus whole-job failure, and the concurrency limit. The
+ * worker executor runs inline so a job is terminal by the time create returns; S3 is a mock, the
+ * suppression and contact services are real ones over in-memory stores so the imported rows can be
+ * read back.
+ */
+class SesImportJobServiceTest {
+
+ private static final String REGION = "eu-west-3";
+ private static final String ACCOUNT = "000000000000";
+ private static final String BUCKET = "imports";
+ private static final String LIST = "newsletter";
+
+ private S3Service s3Service;
+ private SesSuppressionService suppressionService;
+ private SesContactService contactService;
+ private InMemoryStorage<String, ImportJob> jobStore;
+ private SesImportJobService service;
+
+ @BeforeEach
+ void setUp() {
+ s3Service = mock(S3Service.class);
+ Clock clock = Clock.fixed(Instant.parse("2026-09-18T00:00:00Z"), ZoneOffset.UTC);
+ suppressionService = new SesSuppressionService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new InMemoryStorage<>());
+ contactService = new SesContactService(new InMemoryStorage<>(), new InMemoryStorage<>(), clock);
+ contactService.createContactList(LIST, null,
+ List.of(new Topic("Sports", "Sports", "OPT_OUT", null),
+ new Topic("Cycling", "Cycling", "OPT_IN", null)),
+ null, REGION);
+ jobStore = new InMemoryStorage<>();
+ service = newService(Runnable::run);
+ }
+
+ private SesImportJobService newService(Executor workers) {
+ return new SesImportJobService(jobStore, s3Service, suppressionService, contactService,
+ new ObjectMapper().findAndRegisterModules(), Clock.systemUTC(), workers);
+ }
+
+ private void stubObject(String key, String content) {
+ when(s3Service.openObjectStream(BUCKET, key, null)).thenAnswer(
+ invocation -> new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)));
+ }
+
+ private static String url(String key) {
+ return "s3://" + BUCKET + "/" + key;
+ }
+
+ // ──────────────────────────── Suppression list ────────────────────────────
+
+ @Test
+ void suppressionCsvPut_addsEveryRowAndCompletes() {
+ stubObject("add.csv", "[email protected],BOUNCE\[email protected],COMPLAINT\n");
+
+ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("add.csv"), "CSV", "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());
+ assertEquals(2, stored.getProcessedRecordsCount());
+ assertEquals(0, stored.getFailedRecordsCount());
+ assertNotNull(stored.getCompletedTimestamp());
+ assertNull(stored.getErrorMessage());
+ assertEquals("BOUNCE", suppressionService.getSuppressedDestination(REGION, "[email protected]").getReason());
+ assertEquals("COMPLAINT", suppressionService.getSuppressedDestination(REGION, "[email protected]").getReason());
+ }
+
+ @Test
+ void suppressionJsonPut_countsInvalidRowsAsFailedButCompletes() {
+ stubObject("add.json", """
+ {"emailAddress":"[email protected]","reason":"BOUNCE"}
+
+ {"emailAddress":"[email protected]","reason":"SPAM"}
+ not json at all
+ {"emailAddress":"[email protected]","reason":"COMPLAINT"}
+ """);
+
+ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("add.json"), "JSON", "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());
+ assertEquals(4, stored.getProcessedRecordsCount());
+ assertEquals(2, stored.getFailedRecordsCount());
+ assertEquals(2, suppressionService.listSuppressedDestinations(REGION, null).size());
+ }
+
+ @Test
+ void suppressionCsvDelete_removesListedAddresses_missingOnesFail() {
+ suppressionService.putSuppressedDestination(REGION, "[email protected]", "BOUNCE");
+ stubObject("remove.csv", "[email protected]\[email protected]\n");
+
+ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("remove.csv"), "CSV", "DELETE");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());
+ assertEquals(2, stored.getProcessedRecordsCount());
+ assertEquals(1, stored.getFailedRecordsCount());
+ assertTrue(suppressionService.listSuppressedDestinations(REGION, null).isEmpty());
+ }
+
+ // ──────────────────────────── Contact list ────────────────────────────
+
+ @Test
+ void contactCsvPut_readsHeaderTopicColumnsAndQuotedAttributes() {
+ stubObject("contacts.csv", """
+ emailAddress,unsubscribeAll,attributesData,topicPreferences.Sports,topicPreferences.Cycling
+ [email protected],false,"{""Name"":""Alice"",""City"":""Paris""}",OPT_IN,OPT_OUT
+ [email protected],true,,OPT_OUT,
+ """);
+
+ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());
+ assertEquals(2, stored.getProcessedRecordsCount());
+ assertEquals(0, stored.getFailedRecordsCount());
+ Contact alice = contactService.getContact(LIST, "[email protected]", REGION).contact();
+ assertEquals("{\"Name\":\"Alice\",\"City\":\"Paris\"}", alice.getAttributesData());
+ assertEquals(Map.of("Sports", "OPT_IN", "Cycling", "OPT_OUT"), preferences(alice));
+ Contact bob = contactService.getContact(LIST, "[email protected]", REGION).contact();
+ assertTrue(bob.isUnsubscribeAll());
+ assertNull(bob.getAttributesData());
+ assertEquals(Map.of("Sports", "OPT_OUT"), preferences(bob));
+ }
+
+ @Test
+ void contactJsonPut_overridesExistingContact() {
+ contactService.createContact(LIST, "[email protected]",
+ List.of(new TopicPreference("Sports", "OPT_OUT")), false, "{\"old\":true}", REGION);
+ stubObject("contacts.json", """
+ {"emailAddress":"[email protected]","unsubscribeAll":true,"attributesData":"{\\"new\\":true}",
+ "topicPreferences":[{"topicName":"Cycling","subscriptionStatus":"OPT_IN"}]}
+ """.replace("\n", ""));
+
+ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.json"), "JSON", LIST, "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());
+ assertEquals(0, stored.getFailedRecordsCount());
+ Contact alice = contactService.getContact(LIST, "[email protected]", REGION).contact();
+ assertTrue(alice.isUnsubscribeAll());
+ assertEquals("{\"new\":true}", alice.getAttributesData());
+ // Probed against real AWS: a PUT import replaces the contact outright, so the Sports
+ // preference this record omits is cleared rather than merged.
+ assertEquals(Map.of("Cycling", "OPT_IN"), preferences(alice));
+
+ // A record carrying only the address clears the rest for the same reason.
+ stubObject("bare.json", "{\"emailAddress\":\"[email protected]\"}\n");
+ service.createContactListImportJob(REGION, ACCOUNT, url("bare.json"), "JSON", LIST, "PUT");
+
+ Contact cleared = contactService.getContact(LIST, "[email protected]", REGION).contact();
+ assertEquals(Map.of(), preferences(cleared));
+ assertFalse(cleared.isUnsubscribeAll());
+ assertNull(cleared.getAttributesData());
+ }
+
+ @Test
+ void contactCsv_unknownTopicAndBadBooleanAreFailedRecords() {
+ stubObject("contacts.csv", """
+ emailAddress,unsubscribeAll,topicPreferences.Chess
+ [email protected],false,OPT_IN
+ """);
+
+ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());
+ assertEquals(3, stored.getProcessedRecordsCount());
+ assertEquals(2, stored.getFailedRecordsCount());
+ assertEquals(1, contactService.listContacts(LIST, REGION).contacts().size());
+ }
+
+ @Test
+ void contactJsonDelete_removesContacts() {
+ contactService.createContact(LIST, "[email protected]", null, false, null, REGION);
+ stubObject("remove.json", "{\"emailAddress\":\"[email protected]\"}\n{\"emailAddress\":\"[email protected]\"}\n");
+
+ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("remove.json"), "JSON", LIST, "DELETE");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());
+ assertEquals(1, stored.getFailedRecordsCount());
+ assertTrue(contactService.listContacts(LIST, REGION).contacts().isEmpty());
+ }
+
+ @Test
+ void contactCsv_withoutEmailAddressHeader_failsTheJob() {
+ stubObject("contacts.csv", "email,unsubscribeAll\[email protected],false\n");
+
+ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_FAILED, stored.getJobStatus());
+ assertTrue(stored.getErrorMessage().contains("emailAddress"));
+ assertNotNull(stored.getCompletedTimestamp());
+ }
+
+ // ──────────────────────────── Whole-job failure ────────────────────────────
+
+ @Test
+ void missingObject_failsTheJobWithTheS3Message() {
+ when(s3Service.openObjectStream(eq(BUCKET), anyString(), eq(null))).thenThrow(
+ new AwsException("NoSuchKey", "The specified key does not exist.", 404));
+
+ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("missing.csv"), "CSV", "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_FAILED, stored.getJobStatus());
+ assertEquals("The specified key does not exist.", stored.getErrorMessage());
+ assertEquals(0, stored.getProcessedRecordsCount());
+ }
+
+ @Test
+ void contactListDeletedBeforeTheWorkerRuns_failsTheJob() {
+ stubObject("contacts.csv", "emailAddress\[email protected]\n");
+ // Hold the worker so the list can be deleted between create and run.
+ List<Runnable> queued = new ArrayList<>();
+ SesImportJobService deferred = newService(queued::add);
+
+ ImportJob job = deferred.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");
+ assertEquals(ImportJob.STATUS_CREATED, deferred.getImportJob(REGION, job.getJobId()).getJobStatus());
+ contactService.deleteContactList(LIST, REGION);
+ queued.forEach(Runnable::run);
+
+ ImportJob stored = deferred.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_FAILED, stored.getJobStatus());
+ assertNotNull(stored.getErrorMessage());
+ }
+
+ // ──────────────────────────── Validation ────────────────────────────
+
+ @Test
+ void create_rejectsBadUrlFormatAndAction() {
+ assertBadRequest(() -> service.createSuppressionListImportJob(REGION, ACCOUNT, "https://x/y", "CSV", "PUT"));
+ assertBadRequest(() -> service.createSuppressionListImportJob(REGION, ACCOUNT, "s3://bucket", "CSV", "PUT"));
+ assertBadRequest(() -> service.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "XML", "PUT"));
+ assertBadRequest(() -> service.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "UPSERT"));
+ assertTrue(service.listImportJobs(REGION, null, null, null).isEmpty());
+ }
+
+ @Test
+ void create_unknownContactList_isNotFound() {
+ AwsException e = assertThrows(AwsException.class,
+ () -> service.createContactListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "missing", "PUT"));
+ assertEquals("NotFoundException", e.getErrorCode());
+ // Probed against real AWS: this path words it differently from the contact-list APIs.
+ assertEquals("ContactList <missing> doesn't exist", e.getMessage());
+ assertTrue(service.listImportJobs(REGION, null, null, null).isEmpty());
+ }
+
+ @Test
+ void create_rejectsMoreThanTwentyInProgressJobs() {
+ stubObject("a.csv", "[email protected],BOUNCE\n");
+ SesImportJobService parked = newService(task -> { });
+ for (int i = 0; i < SesImportJobService.MAX_CONCURRENT_JOBS; i++) {
+ parked.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT");
+ }
+
+ AwsException e = assertThrows(AwsException.class,
+ () -> parked.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT"));
+ assertEquals("LimitExceededException", e.getErrorCode());
+ // Jobs in another region do not count against this region's limit.
+ assertEquals(ImportJob.STATUS_CREATED,
+ parked.createSuppressionListImportJob("us-east-1", ACCOUNT, url("a.csv"), "CSV", "PUT").getJobStatus());
+ }
+
+ @Test
+ void getImportJob_unknownId_isNotFound() {
+ AwsException e = assertThrows(AwsException.class, () -> service.getImportJob(REGION, "nope"));
+ assertEquals("NotFoundException", e.getErrorCode());
+ assertEquals(404, e.getHttpStatus());
+ }
+
+ @Test
+ void listImportJobs_filtersByDestinationTypeAndRejectsBadPaging() {
+ stubObject("a.csv", "[email protected],BOUNCE\n");
+ stubObject("c.csv", "emailAddress\[email protected]\n");
+ service.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT");
+ service.createContactListImportJob(REGION, ACCOUNT, url("c.csv"), "CSV", LIST, "PUT");
+
+ assertEquals(2, service.listImportJobs(REGION, null, null, null).size());
+ assertEquals(1, service.listImportJobs(REGION, "CONTACT_LIST", 10, null).size());
+ assertEquals(1, service.listImportJobs(REGION, "SUPPRESSION_LIST", null, null).size());
+ assertTrue(service.listImportJobs("us-east-1", null, null, null).isEmpty());
+ assertBadRequest(() -> service.listImportJobs(REGION, "EXPORT", null, null));
+ assertBadRequest(() -> service.listImportJobs(REGION, null, 0, null));
+ assertBadRequest(() -> service.listImportJobs(REGION, null, null, "token"));
+ }
+
+ @Test
+ void jsonLine_withTrailingTokensOrWrongMemberTypes_isAFailedRecord() {
+ stubObject("contacts.json", """
+ {"emailAddress":"[email protected]"} garbage
+ {"emailAddress":"[email protected]","attributesData":123}
+ {"emailAddress":"[email protected]","topicPreferences":[{"topicName":"Sports","subscriptionStatus":true}]}
+ {"emailAddress":"[email protected]","topicPreferences":["Sports"]}
+ {"emailAddress":"[email protected]"}
+ """);
+
+ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.json"), "JSON", LIST, "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());
+ assertEquals(5, stored.getProcessedRecordsCount());
+ assertEquals(4, stored.getFailedRecordsCount());
+ assertEquals(1, contactService.listContacts(LIST, REGION).contacts().size());
+ }
+
+ @Test
+ void contactCsv_quotedFieldMayContainLineBreaks() {
+ stubObject("contacts.csv", "emailAddress,attributesData\r\n"
+ + "[email protected],\"{\r\n \"\"Name\"\": \"\"Alice\"\"\r\n}\"\r\n"
+ + "[email protected],\r\n");
+
+ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());
+ assertEquals(2, stored.getProcessedRecordsCount());
+ assertEquals(0, stored.getFailedRecordsCount());
+ assertEquals("{\r\n \"Name\": \"Alice\"\r\n}",
+ contactService.getContact(LIST, "[email protected]", REGION).contact().getAttributesData());
+ }
+
+ @Test
+ void create_isSerializedAgainstTheConcurrencyLimit() throws InterruptedException {
+ stubObject("a.csv", "[email protected],BOUNCE\n");
+ SesImportJobService parked = newService(task -> { });
+ int attempts = SesImportJobService.MAX_CONCURRENT_JOBS * 2;
+ CountDownLatch start = new CountDownLatch(1);
+ AtomicInteger rejected = new AtomicInteger();
+ List<Thread> threads = new ArrayList<>();
+ for (int i = 0; i < attempts; i++) {
+ Thread t = new Thread(() -> {
+ try {
+ start.await();
+ parked.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT");
+ } catch (AwsException e) {
+ rejected.incrementAndGet();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
+ threads.add(t);
+ t.start();
+ }
+ start.countDown();
+ for (Thread t : threads) {
+ t.join();
+ }
+
+ assertEquals(SesImportJobService.MAX_CONCURRENT_JOBS, rejected.get());
+ assertEquals(SesImportJobService.MAX_CONCURRENT_JOBS, parked.listImportJobs(REGION, null, null, null).size());
+ }
+
+ @Test
+ void beforeReset_stopsAQueuedWorkerBeforeItWrites() {
+ stubObject("a.csv", "[email protected],BOUNCE\n");
+ List<Runnable> queued = new ArrayList<>();
+ SesImportJobService deferred = newService(queued::add);
+ ImportJob job = deferred.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT");
+
+ deferred.beforeReset();
+ queued.forEach(Runnable::run);
+ deferred.afterReset();
+
+ assertEquals(ImportJob.STATUS_CREATED, deferred.getImportJob(REGION, job.getJobId()).getJobStatus());
+ assertTrue(suppressionService.listSuppressedDestinations(REGION, null).isEmpty());
+ }
+
+ @Test
+ void beforeReset_waitsForARunningWorkerAndBlocksCreatesUntilAfterReset() throws Exception {
+ CountDownLatch s3Blocked = new CountDownLatch(1);
+ CountDownLatch releaseS3 = new CountDownLatch(1);
+ when(s3Service.openObjectStream(BUCKET, "slow.csv", null)).thenAnswer(invocation -> {
+ s3Blocked.countDown();
+ releaseS3.await();
+ return new ByteArrayInputStream("[email protected],BOUNCE\n".getBytes(StandardCharsets.UTF_8));
+ });
+ stubObject("a.csv", "[email protected],BOUNCE\n");
+ SesImportJobService threaded = newService(task -> new Thread(task).start());
+ ImportJob slow = threaded.createSuppressionListImportJob(REGION, ACCOUNT, url("slow.csv"), "CSV", "PUT");
+ assertTrue(s3Blocked.await(5, TimeUnit.SECONDS));
+
+ Thread reset = new Thread(threaded::beforeReset);
+ reset.start();
+ reset.join(300);
+ assertTrue(reset.isAlive(), "beforeReset must wait for the running worker");
+ // The create has to race the drain, not follow it: a job admitted between beforeReset and
+ // the storage wipe is exactly what the admission gate exists to stop.
+ AtomicInteger created = new AtomicInteger();
+ Thread create = new Thread(() -> {
+ threaded.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT");
+ created.incrementAndGet();
+ });
+ create.start();
+ create.join(300);
+ assertEquals(0, created.get(), "a create arriving while the drain runs waits");
+
+ releaseS3.countDown();
+ reset.join(5_000);
+ assertFalse(reset.isAlive(), "beforeReset returns once the worker has left");
+ create.join(300);
+ assertEquals(0, created.get(), "and keeps waiting until the reset finishes");
+
+ threaded.afterReset();
+ create.join(5_000);
+ assertEquals(1, created.get());
+ // The slow worker was abandoned: no write after the reset started.
+ assertEquals(ImportJob.STATUS_PROCESSING, threaded.getImportJob(REGION, slow.getJobId()).getJobStatus());
+ assertTrue(suppressionService.findSuppressedDestination(REGION, "[email protected]").isEmpty());
+ }
+
+ @Test
+ void beforeReset_givesUpOnAWedgedWorkerInsteadOfHangingTheReset() throws Exception {
+ CountDownLatch blocked = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ when(s3Service.openObjectStream(BUCKET, "wedged.csv", null)).thenAnswer(invocation -> {
+ blocked.countDown();
+ release.await();
+ return new ByteArrayInputStream("[email protected],BOUNCE\n".getBytes(StandardCharsets.UTF_8));
+ });
+ SesImportJobService threaded = new SesImportJobService(jobStore, s3Service, suppressionService,
+ contactService, new ObjectMapper().findAndRegisterModules(), Clock.systemUTC(),
+ task -> new Thread(task).start(), 200);
+ threaded.createSuppressionListImportJob(REGION, ACCOUNT, url("wedged.csv"), "CSV", "PUT");
+ assertTrue(blocked.await(5, TimeUnit.SECONDS));
+
+ // The worker never finishes, so the drain has to time out rather than block the reset.
+ long started = System.currentTimeMillis();
+ threaded.beforeReset();
+ long waited = System.currentTimeMillis() - started;
+
+ assertTrue(waited >= 200, "the drain should wait for its deadline, waited " + waited + "ms");
+ assertTrue(waited < 5_000, "beforeReset must return on the deadline, waited " + waited + "ms");
+ release.countDown();
+ threaded.afterReset();
+ }
+
+ @Test
+ void storedJobIsASnapshot_soAnEarlierPollIsNeverMutatedUnderTheReader() throws Exception {
+ CountDownLatch blocked = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ when(s3Service.openObjectStream(BUCKET, "slow.csv", null)).thenAnswer(invocation -> {
+ blocked.countDown();
+ release.await();
+ return new ByteArrayInputStream("[email protected],BOUNCE\n".getBytes(StandardCharsets.UTF_8));
+ });
+ SesImportJobService threaded = newService(task -> new Thread(task).start());
+ ImportJob job = threaded.createSuppressionListImportJob(REGION, ACCOUNT, url("slow.csv"), "CSV", "PUT");
+ assertTrue(blocked.await(5, TimeUnit.SECONDS));
+
+ ImportJob midRun = threaded.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_PROCESSING, midRun.getJobStatus());
+ release.countDown();
+ ImportJob finished = awaitTerminal(threaded, job.getJobId());
+
+ assertEquals(ImportJob.STATUS_COMPLETED, finished.getJobStatus());
+ assertNotNull(finished.getCompletedTimestamp());
+ assertEquals(1, finished.getProcessedRecordsCount());
+ // The worker keeps mutating its own copy, so the record the earlier poll returned is frozen:
+ // a poll can never catch a transition half applied (COMPLETED with no CompletedTimestamp).
+ assertEquals(ImportJob.STATUS_PROCESSING, midRun.getJobStatus());
+ assertEquals(0, midRun.getProcessedRecordsCount());
+ assertNull(midRun.getCompletedTimestamp());
+ }
+
+ private static ImportJob awaitTerminal(SesImportJobService service, String jobId) throws InterruptedException {
+ for (int i = 0; i < 250; i++) {
+ ImportJob job = service.getImportJob(REGION, jobId);
+ if (job.isTerminal()) {
+ return job;
+ }
+ Thread.sleep(20);
+ }
+ throw new AssertionError("import job " + jobId + " never reached a terminal status");
+ }
+
+ @Test
+ void longImportRepublishesItsCountersAsItRuns() {
+ int records = SesImportJobService.PROGRESS_RECORD_INTERVAL * 2 + 5;
+ StringBuilder csv = new StringBuilder();
+ for (int i = 0; i < records; i++) {
+ csv.append("user").append(i).append("@example.com,BOUNCE\n");
+ }
+ stubObject("bulk.csv", csv.toString());
+ List<Long> publishedCounts = new ArrayList<>();
+ StorageBackend<String, ImportJob> recording = new InMemoryStorage<>() {
+ @Override
+ public void put(String key, ImportJob value) {
+ publishedCounts.add(value.getProcessedRecordsCount());
+ super.put(key, value);
+ }
+ };
+ SesImportJobService service = new SesImportJobService(recording, s3Service, suppressionService,
+ contactService, new ObjectMapper().findAndRegisterModules(), Clock.systemUTC(), Runnable::run);
+
+ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("bulk.csv"), "CSV", "PUT");
+
+ assertEquals(records, service.getImportJob(REGION, job.getJobId()).getProcessedRecordsCount());
+ // A poll partway through sees real progress rather than 0 until the job ends.
+ assertEquals(List.of(0L, 0L, 100L, 200L, (long) records), publishedCounts);
+ }
+
+ @Test
+ void suppressionCsv_wrongColumnCountIsAFailedRecord() {
+ stubObject("mixed.csv", "[email protected],BOUNCE,extra\[email protected]\[email protected],COMPLAINT\n");
+
+ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("mixed.csv"), "CSV", "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(3, stored.getProcessedRecordsCount());
+ assertEquals(2, stored.getFailedRecordsCount());
+ assertEquals(1, suppressionService.listSuppressedDestinations(REGION, null).size());
+ }
+
+ @Test
+ void csv_unterminatedQuote_failsTheJob() {
+ stubObject("broken.csv", "emailAddress,attributesData\[email protected],\"{unterminated\n");
+
+ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("broken.csv"), "CSV", LIST, "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_FAILED, stored.getJobStatus());
+ assertTrue(stored.getErrorMessage().contains("unterminated"));
+ assertTrue(contactService.listContacts(LIST, REGION).contacts().isEmpty());
+ }
+
+ @Test
+ void contactListDeletedDuringTheRun_failsTheJob() {
+ stubObject("contacts.csv", "emailAddress\[email protected]\[email protected]\n");
+ SesContactService deletingService = mock(SesContactService.class);
+ when(deletingService.getContactList(LIST, REGION)).thenReturn(contactService.getContactList(LIST, REGION));
+ when(deletingService.createContact(eq(LIST), anyString(), any(),
+ any(), any(), eq(REGION)))
+ .thenAnswer(invocation -> {
+ when(deletingService.getContactList(LIST, REGION))
+ .thenThrow(new AwsException("NotFoundException", "List " + LIST + " not found.", 404));
+ throw new AwsException("NotFoundException", "List " + LIST + " not found.", 404);
+ });
+ SesImportJobService racing = new SesImportJobService(jobStore, s3Service, suppressionService,
+ deletingService, new ObjectMapper().findAndRegisterModules(), Clock.systemUTC(), Runnable::run);
+
+ ImportJob job = racing.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");
+
+ ImportJob stored = racing.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_FAILED, stored.getJobStatus());
+ assertEquals("ContactList <" + LIST + "> doesn't exist", stored.getErrorMessage());
+ assertEquals(1, stored.getProcessedRecordsCount());
+ }
+
+ @Test
+ void contactCsv_wrongRowWidthOrBadQuotingIsAFailedRecord() {
+ stubObject("contacts.csv", """
+ emailAddress,attributesData
+ [email protected],"{}",extra
+ """);
+
+ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");
+
+ ImportJob stored = service.getImportJob(REGION, job.getJobId());
+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());
+ assertEquals(4, stored.getProcessedRecordsCount());
+ // The wrong-width row and the one with data after a closing quote fail; carol's bare quote
+ // is data, as the developer guide's own unquoted attributesData shows.
+ assertEquals(2, stored.getFailedRecordsCount());
+ assertEquals(2, contactService.listContacts(LIST, REGION).contacts().size());
+ }
+
+ @Test
+ void recoverInterruptedJobs_failsNonTerminalJobsLeftInAnAccountAwareStore() {
+ AccountAwareStorageBackend<ImportJob> persisted =
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, ACCOUNT);
+ ImportJob stuck = new ImportJob();
+ stuck.setJobId("stuck");
+ stuck.setRegion(REGION);
+ stuck.setDestinationType(ImportJob.DESTINATION_SUPPRESSION_LIST);
+ stuck.setImportAction("PUT");
+ stuck.setJobStatus(ImportJob.STATUS_PROCESSING);
+ stuck.setCreatedTimestamp(Instant.now());
+ persisted.putForAccount(ACCOUNT, SesImportJobService.key(REGION, "stuck"), stuck);
+ ImportJob done = new ImportJob();
+ done.setJobId("done");
+ done.setRegion(REGION);
+ done.setDestinationType(ImportJob.DESTINATION_SUPPRESSION_LIST);
+ done.setImportAction("PUT");
+ done.setJobStatus(ImportJob.STATUS_COMPLETED);
+ done.setCreatedTimestamp(Instant.now());
+ persisted.putForAccount(ACCOUNT, SesImportJobService.key(REGION, "done"), done);
+
+ SesImportJobService restarted = new SesImportJobService(persisted, s3Service, suppressionService,
+ contactService, new ObjectMapper().findAndRegisterModules(), Clock.systemUTC(), Runnable::run);
+
+ ImportJob recovered = restarted.getImportJob(REGION, "stuck");
+ assertEquals(ImportJob.STATUS_FAILED, recovered.getJobStatus());
+ assertEquals(SesImportJobService.RESTART_FAILURE_MESSAGE, recovered.getErrorMessage());
+ assertNotNull(recovered.getCompletedTimestamp());
+ assertEquals(ImportJob.STATUS_COMPLETED, restarted.getImportJob(REGION, "done").getJobStatus());
+ }
+
+ private static void assertBadRequest(Runnable call) {
+ AwsException e = assertThrows(AwsException.class, call::run);
+ assertEquals("BadRequestException", e.getErrorCode());
+ }
+
+ private static Map<String, String> preferences(Contact contact) {
+ return contact.getTopicPreferences().stream()
+ .collect(Collectors.toMap(TopicPreference::getTopicName, TopicPreference::getSubscriptionStatus));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobV2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobV2IntegrationTest.java
new file mode 100644
index 0000000000..0cee1ac41b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobV2IntegrationTest.java
@@ -0,0 +1,309 @@
+package io.github.hectorvent.floci.services.ses;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.util.function.Supplier;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Integration tests for the SES V2 import-job endpoints: {@code POST /v2/email/import-jobs}
+ * (CreateImportJob), {@code GET /v2/email/import-jobs/{JobId}} (GetImportJob) and
+ * {@code POST /v2/email/import-jobs/list} (ListImportJobs), end to end through the local S3
+ * emulation into the suppression list and a contact list. The job runs on a background worker as
+ * on AWS, so the tests poll GetImportJob until it is terminal. Uses its own region so the
+ * suppression list and the one-per-region contact list do not collide with other SES tests.
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class SesImportJobV2IntegrationTest {
+
+ private static final String AUTH =
+ "AWS4-HMAC-SHA256 Credential=AKID/20260101/eu-west-3/ses/aws4_request";
+ private static final String BUCKET = "ses-import-job-it";
+ private static final String LIST = "import-job-list";
+
+ private static String suppressionJobId;
+ private static String contactJobId;
+
+ @Test
+ @Order(0)
+ void setupBucketAndContactList() {
+ given().when().put("/" + BUCKET).then().statusCode(200);
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("""
+ {"ContactListName":"%s","Topics":[
+ {"TopicName":"Sports","DisplayName":"Sports","DefaultSubscriptionStatus":"OPT_OUT"},
+ {"TopicName":"Cycling","DisplayName":"Cycling","DefaultSubscriptionStatus":"OPT_IN"}]}
+ """.formatted(LIST))
+ .when().post("/v2/email/contact-lists").then().statusCode(200);
+ }
+
+ @Test
+ @Order(1)
+ void createImportJob_suppressionCsv_runsToCompletedAndFillsTheList() {
+ putObject("suppress.csv", "[email protected],BOUNCE\[email protected],COMPLAINT\[email protected],SPAM\n");
+
+ suppressionJobId = given().contentType("application/json").header("Authorization", AUTH)
+ .body("""
+ {"ImportDataSource":{"S3Url":"s3://%s/suppress.csv","DataFormat":"CSV"},
+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"}}}
+ """.formatted(BUCKET))
+ .when().post("/v2/email/import-jobs").then().statusCode(200)
+ .body("JobId", notNullValue())
+ .extract().jsonPath().getString("JobId");
+
+ assertEquals("COMPLETED", pollUntilTerminal(suppressionJobId));
+ given().header("Authorization", AUTH)
+ .when().get("/v2/email/import-jobs/" + suppressionJobId).then().statusCode(200)
+ .body("JobId", equalTo(suppressionJobId))
+ .body("JobStatus", equalTo("COMPLETED"))
+ .body("ImportDataSource.S3Url", equalTo("s3://" + BUCKET + "/suppress.csv"))
+ .body("ImportDataSource.DataFormat", equalTo("CSV"))
+ .body("ImportDestination.SuppressionListDestination.SuppressionListImportAction", equalTo("PUT"))
+ .body("ImportDestination.ContactListDestination", nullValue())
+ .body("ProcessedRecordsCount", equalTo(3))
+ .body("FailedRecordsCount", equalTo(1))
+ .body("CreatedTimestamp", notNullValue())
+ .body("CompletedTimestamp", notNullValue())
+ .body("FailureInfo", nullValue());
+
+ given().header("Authorization", AUTH)
+ .when().get("/v2/email/suppression/addresses").then().statusCode(200)
+ .body("SuppressedDestinationSummaries.EmailAddress",
+ .body("SuppressedDestinationSummaries", hasSize(2));
+ }
+
+ @Test
+ @Order(2)
+ void createImportJob_contactJson_addsContactsWithPreferences() {
+ putObject("contacts.json", """
+ {"emailAddress":"[email protected]","unsubscribeAll":false,"attributesData":"{\\"Name\\":\\"Carol\\"}","topicPreferences":[{"topicName":"Sports","subscriptionStatus":"OPT_IN"}]}
+ {"emailAddress":"[email protected]","unsubscribeAll":true}
+ """);
+
+ contactJobId = given().contentType("application/json").header("Authorization", AUTH)
+ .body("""
+ {"ImportDataSource":{"S3Url":"s3://%s/contacts.json","DataFormat":"JSON"},
+ "ImportDestination":{"ContactListDestination":{"ContactListName":"%s","ContactListImportAction":"PUT"}}}
+ """.formatted(BUCKET, LIST))
+ .when().post("/v2/email/import-jobs").then().statusCode(200)
+ .extract().jsonPath().getString("JobId");
+
+ assertEquals("COMPLETED", pollUntilTerminal(contactJobId));
+ given().header("Authorization", AUTH)
+ .when().get("/v2/email/import-jobs/" + contactJobId).then().statusCode(200)
+ .body("ImportDestination.ContactListDestination.ContactListName", equalTo(LIST))
+ .body("ImportDestination.ContactListDestination.ContactListImportAction", equalTo("PUT"))
+ .body("ProcessedRecordsCount", equalTo(2))
+ .body("FailedRecordsCount", equalTo(0));
+
+ given().header("Authorization", AUTH)
+ .when().get("/v2/email/contact-lists/" + LIST + "/contacts/[email protected]").then().statusCode(200)
+ .body("AttributesData", equalTo("{\"Name\":\"Carol\"}"))
+ .body("TopicPreferences[0].TopicName", equalTo("Sports"))
+ .body("TopicPreferences[0].SubscriptionStatus", equalTo("OPT_IN"));
+ given().header("Authorization", AUTH)
+ .when().get("/v2/email/contact-lists/" + LIST + "/contacts/[email protected]").then().statusCode(200)
+ .body("UnsubscribeAll", equalTo(true));
+ }
+
+ @Test
+ @Order(3)
+ void listImportJobs_returnsSummariesAndFiltersByDestinationType() {
+ given().contentType("application/json").header("Authorization", AUTH)
+ .when().post("/v2/email/import-jobs/list").then().statusCode(200)
+ .body("ImportJobs.JobId", hasItems(suppressionJobId, contactJobId))
+ .body("ImportJobs.find { it.JobId == '" + contactJobId + "' }.JobStatus", equalTo("COMPLETED"))
+ .body("ImportJobs.find { it.JobId == '" + contactJobId + "' }.ImportDataSource", nullValue())
+ .body("NextToken", nullValue());
+
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"ImportDestinationType\":\"SUPPRESSION_LIST\",\"PageSize\":10}")
+ .when().post("/v2/email/import-jobs/list").then().statusCode(200)
+ .body("ImportJobs.JobId", hasItems(suppressionJobId))
+ .body("ImportJobs.JobId", not(hasItems(contactJobId)));
+
+ given().contentType("application/json").header("Authorization", AUTH)
+ .body("{\"ImportDestinationType\":\"EXPORT\"}")
+ .when().post("/v2/email/import-jobs/list").then().statusCode(400)
+ .body("__type", equalTo("BadRequestException"));
+ }
+
+ @Test
+ @Order(4)
+ void createImportJob_missingObject_failsTheJob() {
+ String jobId = given().contentType("application/json").header("Authorization", AUTH)
+ .body("""
+ {"ImportDataSource":{"S3Url":"s3://%s/does-not-exist.csv","DataFormat":"CSV"},
+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"DELETE"}}}
+ """.formatted(BUCKET))
+ .when().post("/v2/email/import-jobs").then().statusCode(200)
+ .extract().jsonPath().getString("JobId");
+
+ assertEquals("FAILED", pollUntilTerminal(jobId));
+ given().header("Authorization", AUTH)
+ .when().get("/v2/email/import-jobs/" + jobId).then().statusCode(200)
+ .body("FailureInfo.ErrorMessage", containsString("does not exist"))
+ .body("ProcessedRecordsCount", equalTo(0));
+ }
+
+ @Test
+ @Order(5)
+ void createImportJob_validationErrors() {
+ String badFormat = """
+ {"ImportDataSource":{"S3Url":"s3://%s/x.csv","DataFormat":"XML"},
+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"}}}
+ """.formatted(BUCKET);
+ String bothDestinations = """
+ {"ImportDataSource":{"S3Url":"s3://%s/x.csv","DataFormat":"CSV"},
+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"},
+ "ContactListDestination":{"ContactListName":"%s","ContactListImportAction":"PUT"}}}
+ """.formatted(BUCKET, LIST);
+ String noDataSource = """
+ {"ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"}}}
+ """;
+ String unknownList = """
+ {"ImportDataSource":{"S3Url":"s3://%s/x.csv","DataFormat":"CSV"},
+ "ImportDestination":{"ContactListDestination":{"ContactListName":"nope","ContactListImportAction":"PUT"}}}
+ """.formatted(BUCKET);
+
+ String trailingTokens = """
+ {"ImportDataSource":{"S3Url":"s3://%s/x.csv","DataFormat":"CSV"},
+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"}}} garbage
+ """.formatted(BUCKET);
+
+ for (String body : new String[] {badFormat, bothDestinations, noDataSource, trailingTokens}) {
+ given().contentType("application/json").header("Authorization", AUTH).body(body)
+ .when().post("/v2/email/import-jobs").then().statusCode(400)
+ .body("__type", equalTo("BadRequestException"));
+ }
+ given().contentType("application/json").header("Authorization", AUTH)
+ .when().post("/v2/email/import-jobs").then().statusCode(400)
+ .body("__type", equalTo("BadRequestException"));
+ given().contentType("application/json").header("Authorization", AUTH).body(unknownList)
+ .when().post("/v2/email/import-jobs").then().statusCode(404)
+ .body("__type", equalTo("NotFoundException"));
+ }
+
+ @Test
+ @Order(6)
+ void getImportJob_unknownId_isNotFound() {
+ given().header("Authorization", AUTH)
+ .when().get("/v2/email/import-jobs/00000000-0000-0000-0000-000000000000").then().statusCode(404)
+ .body("__type", equalTo("NotFoundException"));
+ }
+
+ @Test
+ @Order(7)
+ void importJobsAndTheirRecordsAreScopedToTheCallingAccount() {
+ // A 12-digit access key id in the credential becomes the account (AccountContextFilter), so
+ // these two headers are two accounts hitting the same region, neither of them the default
+ // account the rest of this class runs as.
+ String authA = accountAuth("111111111111", "ses");
+ String authB = accountAuth("222222222222", "ses");
+ // The credential's service scope routes the request, so the S3 calls need an s3-scoped one.
+ String s3AuthA = accountAuth("111111111111", "s3");
+ String bucketA = BUCKET + "-a";
+
+ given().header("Authorization", s3AuthA).when().put("/" + bucketA).then().statusCode(200);
+ given().header("Authorization", s3AuthA).contentType("text/plain").body("[email protected],BOUNCE\n")
+ .when().put("/" + bucketA + "/suppress.csv").then().statusCode(200);
+
+ String jobA = given().contentType("application/json").header("Authorization", authA)
+ .body("""
+ {"ImportDataSource":{"S3Url":"s3://%s/suppress.csv","DataFormat":"CSV"},
+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"}}}
+ """.formatted(bucketA))
+ .when().post("/v2/email/import-jobs").then().statusCode(200)
+ .extract().jsonPath().getString("JobId");
+
+ // Reaching COMPLETED under account A also proves the worker's terminal write landed in A's
+ // partition: RequestScopes.runAs carries the account onto the worker thread, and without it
+ // the write would go to the default account and this poll would never leave PROCESSING.
+ assertEquals("COMPLETED", pollUntilTerminal(jobA, authA));
+
+ given().header("Authorization", authB)
+ .when().get("/v2/email/import-jobs/" + jobA).then().statusCode(404)
+ .body("__type", equalTo("NotFoundException"));
+ given().contentType("application/json").header("Authorization", authB)
+ .when().post("/v2/email/import-jobs/list").then().statusCode(200)
+ .body("ImportJobs.JobId", not(hasItems(jobA)));
+ given().contentType("application/json").header("Authorization", authA)
+ .when().post("/v2/email/import-jobs/list").then().statusCode(200)
+ .body("ImportJobs.JobId", hasItems(jobA));
+
+ // The imported rows are account-scoped too, not just the job record.
+ given().header("Authorization", authA)
+ .when().get("/v2/email/suppression/addresses").then().statusCode(200)
+ .body("SuppressedDestinationSummaries.EmailAddress", hasItems("[email protected]"));
+ given().header("Authorization", authB)
+ .when().get("/v2/email/suppression/addresses").then().statusCode(200)
+ .body("SuppressedDestinationSummaries.EmailAddress", not(hasItems("[email protected]")));
+
+ given().header("Authorization", authA)
+ .when().delete("/v2/email/suppression/addresses/[email protected]").then().statusCode(200);
+ given().header("Authorization", s3AuthA).when().delete("/" + bucketA + "/suppress.csv").then().statusCode(204);
+ given().header("Authorization", s3AuthA).when().delete("/" + bucketA).then().statusCode(204);
+ }
+
+ private static String accountAuth(String accountId, String service) {
+ return "AWS4-HMAC-SHA256 Credential=" + accountId + "/20260101/eu-west-3/" + service + "/aws4_request";
+ }
+
+ @Test
+ @Order(8)
+ void cleanup() {
+ given().header("Authorization", AUTH)
+ .when().delete("/v2/email/contact-lists/" + LIST).then().statusCode(200);
+ for (String address : new String[] {"[email protected]", "[email protected]"}) {
+ given().header("Authorization", AUTH)
+ .when().delete("/v2/email/suppression/addresses/" + address).then().statusCode(200);
+ }
+ }
+
+ private static void putObject(String key, String content) {
+ given().contentType("text/plain").body(content)
+ .when().put("/" + BUCKET + "/" + key).then().statusCode(200);
+ }
+
+ private static String pollUntilTerminal(String jobId) {
+ return pollUntilTerminal(jobId, AUTH);
+ }
+
+ private static String pollUntilTerminal(String jobId, String auth) {
+ Supplier<Response> call = () -> given().header("Authorization", auth)
+ .when().get("/v2/email/import-jobs/" + jobId);
+ long deadline = System.currentTimeMillis() + 10_000;
+ while (System.currentTimeMillis() < deadline) {
+ String status = call.get().then().statusCode(200).extract().jsonPath().getString("JobStatus");
+ if ("COMPLETED".equals(status) || "FAILED".equals(status)) {
+ return status;
+ }
+ sleep(100);
+ }
+ return "TIMEOUT";
+ }
+
+ private static void sleep(long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReaderTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReaderTest.java
new file mode 100644
index 0000000000..260a052f6e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReaderTest.java
@@ -0,0 +1,196 @@
+package io.github.hectorvent.floci.services.ses.imports;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.ses.model.ImportJob;
+import io.github.hectorvent.floci.services.ses.model.TopicPreference;
+import org.junit.jupiter.api.Test;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Unit test for the CSV reader: where a record ends, how a field is split, and which malformed
+ * input fails one record versus the whole file. Reading is driven through the factory, the way the
+ * import worker reaches it.
+ */
+class CsvRecordReaderTest {
+
+ private static ImportJob job(String destinationType, String action, String contactList) {
+ ImportJob job = new ImportJob();
+ job.setDataFormat(ImportJob.FORMAT_CSV);
+ job.setDestinationType(destinationType);
+ job.setImportAction(action);
+ job.setContactListName(contactList);
+ return job;
+ }
+
+ private static ImportJob suppression(String action) {
+ return job(ImportJob.DESTINATION_SUPPRESSION_LIST, action, null);
+ }
+
+ private static ImportJob contacts() {
+ return job(ImportJob.DESTINATION_CONTACT_LIST, ImportJob.ACTION_PUT, "newsletter");
+ }
+
+ private static RecordReader reader(ImportJob job, String content) {
+ return new RecordReaderFactory(new ObjectMapper())
+ .forJob(job, new BufferedReader(new StringReader(content)));
+ }
+
+ private static List<ImportRecord> readAll(ImportJob job, String content) throws IOException {
+ RecordReader reader = reader(job, content);
+ List<ImportRecord> records = new ArrayList<>();
+ for (ImportRecord record = reader.next(); record != null; record = reader.next()) {
+ records.add(record);
+ }
+ return records;
+ }
+
+ private static Map<String, String> preferences(ImportRecord record) {
+ return record.topicPreferences().stream()
+ .collect(Collectors.toMap(TopicPreference::getTopicName, TopicPreference::getSubscriptionStatus));
+ }
+
+ @Test
+ void suppressionRows_carryTheAddressReasonAndLineNumber() throws IOException {
+ List<ImportRecord> records = readAll(suppression(ImportJob.ACTION_PUT),
+ "[email protected],BOUNCE\[email protected],COMPLAINT\n");
+
+ assertEquals(2, records.size());
+ assertEquals("[email protected]", records.get(0).emailAddress());
+ assertEquals("BOUNCE", records.get(0).reason());
+ assertEquals(1, records.get(0).line());
+ assertEquals(2, records.get(1).line());
+ }
+
+ @Test
+ void deleteRowsCarryNoReason_andAnExtraColumnFailsTheRecord() throws IOException {
+ List<ImportRecord> records = readAll(suppression(ImportJob.ACTION_DELETE),
+
+ assertNull(records.get(0).reason());
+ assertNull(records.get(0).error());
+ assertTrue(records.get(1).error().contains("1 column(s) but found 2"));
+ }
+
+ @Test
+ void quotedFieldMaySpanLines_andTheNextRecordKeepsCounting() throws IOException {
+ List<ImportRecord> records = readAll(contacts(), "emailAddress,attributesData\r\n"
+ + "[email protected],\"{\r\n \"\"Name\"\": \"\"Alice\"\"\r\n}\"\r\n"
+ + "[email protected],\r\n");
+
+ assertEquals(2, records.size());
+ assertEquals("{\r\n \"Name\": \"Alice\"\r\n}", records.get(0).attributesData());
+ // The header is record 1 and the multi-line row is record 2, so the row after it is 3:
+ // records are numbered by position, not by physical line.
+ assertEquals(2, records.get(0).line());
+ assertEquals(3, records.get(1).line());
+ assertEquals("[email protected]", records.get(1).emailAddress());
+ }
+
+ @Test
+ void headerTopicColumnsBecomePreferences_andBlankCellsAreLeftOut() throws IOException {
+ List<ImportRecord> records = readAll(contacts(),
+ "emailAddress,unsubscribeAll,topicPreferences.Sports,topicPreferences.Cycling\n"
+ + "[email protected],true,OPT_IN,\n");
+
+ ImportRecord alice = records.get(0);
+ assertEquals(Boolean.TRUE, alice.unsubscribeAll());
+ assertEquals(Map.of("Sports", "OPT_IN"), preferences(alice));
+ }
+
+ @Test
+ void aBomAndBlankLinesDoNotBecomeRecords() throws IOException {
+ List<ImportRecord> records = readAll(suppression(ImportJob.ACTION_DELETE),
+
+ assertEquals(2, records.size());
+ assertEquals("[email protected]", records.get(0).emailAddress());
+ // Blank lines are skipped but still counted, so the reported line matches the file.
+ assertEquals(4, records.get(1).line());
+ }
+
+ @Test
+ void theDeveloperGuidesOwnContactRowIsAccepted() throws IOException {
+ // Straight out of the SES list-management guide: attributesData is written unquoted, so the
+ // quotes inside it are data. Rejecting them would refuse the documented format.
+ List<ImportRecord> records = readAll(contacts(),
+ "emailAddress,unsubscribeAll,attributesData,topicPreferences.Sports,topicPreferences.Cycling\n"
+ + "[email protected],false,{\"Name\": \"John\"},OPT_IN,OPT_OUT\n"
+ + "[email protected],true,,OPT_OUT,OPT_OUT\n");
+
+ assertEquals(2, records.size());
+ assertNull(records.get(0).error());
+ assertEquals("{\"Name\": \"John\"}", records.get(0).attributesData());
+ assertEquals(Map.of("Sports", "OPT_IN", "Cycling", "OPT_OUT"), preferences(records.get(0)));
+ assertEquals(Boolean.TRUE, records.get(1).unsubscribeAll());
+ }
+
+ @Test
+ void garbageAfterAClosingQuoteFailsOneRecordAndTheReaderCarriesOn() throws IOException {
+ List<ImportRecord> records = readAll(contacts(), "emailAddress,attributesData\n"
+ + "[email protected],\"x\"junk\n"
+ + "[email protected],ab\"c\n"
+ + "[email protected],fine\n");
+
+ // A field that opened a quoted run has to end at the delimiter; anything else cannot be
+ // read back unambiguously.
+ assertTrue(records.get(0).error().contains("closing quote"));
+ // A quote that never opened a run is data, so this row is fine.
+ assertNull(records.get(1).error());
+ assertEquals("ab\"c", records.get(1).attributesData());
+ assertEquals("[email protected]", records.get(2).emailAddress());
+ assertNull(records.get(2).error());
+ }
+
+ @Test
+ void anUnterminatedQuoteFailsTheWholeFile() {
+ RecordReader reader = reader(contacts(), "emailAddress,attributesData\[email protected],\"{oops\n");
+
+ AwsException e = assertThrows(AwsException.class, () -> {
+ for (ImportRecord record = reader.next(); record != null; record = reader.next()) {
+ // drain until the scanner reaches the end of the unterminated field
+ }
+ });
+ assertEquals("BadRequestException", e.getErrorCode());
+ assertTrue(e.getMessage().contains("unterminated"));
+ }
+
+ @Test
+ void aContactFileWithoutAnEmailAddressHeaderFailsTheWholeFile() {
+ RecordReader reader = reader(contacts(), "email,unsubscribeAll\[email protected],false\n");
+
+ AwsException e = assertThrows(AwsException.class, reader::next);
+ assertEquals("BadRequestException", e.getErrorCode());
+ assertTrue(e.getMessage().contains("emailAddress"));
+ }
+
+ @Test
+ void anEmptyContactFileFailsForTheMissingHeader() {
+ AwsException e = assertThrows(AwsException.class, () -> reader(contacts(), "").next());
+ assertTrue(e.getMessage().contains("header line"));
+ }
+
+ @Test
+ void anEmptySuppressionFileYieldsNoRecords() throws IOException {
+ assertTrue(readAll(suppression(ImportJob.ACTION_PUT), "").isEmpty());
+ }
+
+ @Test
+ void splitCsvLine_handlesQuotesCommasAndEscapedQuotes() {
+ assertEquals(List.of("a", "b,c", "say \"hi\"", ""),
+ CsvRecordReader.splitCsvLine("a,\"b,c\",\"say \"\"hi\"\"\","));
+ assertEquals(List.of(""), CsvRecordReader.splitCsvLine(""));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReaderTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReaderTest.java
new file mode 100644
index 0000000000..27bf2a997d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReaderTest.java
@@ -0,0 +1,126 @@
+package io.github.hectorvent.floci.services.ses.imports;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.services.ses.model.ImportJob;
+import io.github.hectorvent.floci.services.ses.model.TopicPreference;
+import org.junit.jupiter.api.Test;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Unit test for the newline-delimited JSON reader: one object per line, and which malformed lines
+ * become failed records. Reading is driven through the factory, the way the import worker reaches
+ * it, so the strict parse the factory configures is exercised too.
+ */
+class JsonLineReaderTest {
+
+ private static ImportJob job(String destinationType) {
+ ImportJob job = new ImportJob();
+ job.setDataFormat(ImportJob.FORMAT_JSON);
+ job.setDestinationType(destinationType);
+ job.setImportAction(ImportJob.ACTION_PUT);
+ job.setContactListName("newsletter");
+ return job;
+ }
+
+ private static List<ImportRecord> readAll(ImportJob job, String content) throws IOException {
+ RecordReader reader = new RecordReaderFactory(new ObjectMapper())
+ .forJob(job, new BufferedReader(new StringReader(content)));
+ List<ImportRecord> records = new ArrayList<>();
+ for (ImportRecord record = reader.next(); record != null; record = reader.next()) {
+ records.add(record);
+ }
+ return records;
+ }
+
+ private static Map<String, String> preferences(ImportRecord record) {
+ return record.topicPreferences().stream()
+ .collect(Collectors.toMap(TopicPreference::getTopicName, TopicPreference::getSubscriptionStatus));
+ }
+
+ @Test
+ void suppressionLinesCarryTheAddressAndReason() throws IOException {
+ List<ImportRecord> records = readAll(job(ImportJob.DESTINATION_SUPPRESSION_LIST), """
+ {"emailAddress":"[email protected]","reason":"BOUNCE"}
+ {"emailAddress":"[email protected]"}
+ """);
+
+ assertEquals(2, records.size());
+ assertEquals("[email protected]", records.get(0).emailAddress());
+ assertEquals("BOUNCE", records.get(0).reason());
+ assertNull(records.get(1).reason());
+ }
+
+ @Test
+ void contactLinesCarryPreferencesAttributesAndTheUnsubscribeFlag() throws IOException {
+ List<ImportRecord> records = readAll(job(ImportJob.DESTINATION_CONTACT_LIST), """
+ {"emailAddress":"[email protected]","unsubscribeAll":false,"attributesData":"{\\"Name\\":\\"Alice\\"}",\
+ "topicPreferences":[{"topicName":"Sports","subscriptionStatus":"OPT_IN"},\
+ {"topicName":"Cycling","subscriptionStatus":"OPT_OUT"}]}
+ """);
+
+ ImportRecord alice = records.get(0);
+ assertEquals(Boolean.FALSE, alice.unsubscribeAll());
+ assertEquals("{\"Name\":\"Alice\"}", alice.attributesData());
+ assertEquals(Map.of("Sports", "OPT_IN", "Cycling", "OPT_OUT"), preferences(alice));
+ }
+
+ @Test
+ void aBomAndBlankLinesDoNotBecomeRecords() throws IOException {
+ List<ImportRecord> records = readAll(job(ImportJob.DESTINATION_SUPPRESSION_LIST),
+ "{\"emailAddress\":\"[email protected]\"}\n\n \n{\"emailAddress\":\"[email protected]\"}\n");
+
+ assertEquals(2, records.size());
+ assertEquals("[email protected]", records.get(0).emailAddress());
+ // Blank lines are skipped but still counted, so the reported line matches the file.
+ assertEquals(4, records.get(1).line());
+ }
+
+ @Test
+ void malformedJsonAndTrailingTokensAreFailedRecords() throws IOException {
+ List<ImportRecord> records = readAll(job(ImportJob.DESTINATION_SUPPRESSION_LIST), """
+ not json at all
+ {"emailAddress":"[email protected]"} garbage
+ ["emailAddress"]
+ {"emailAddress":"[email protected]"}
+ """);
+
+ assertTrue(records.get(0).error().contains("Malformed JSON"));
+ // The factory configures FAIL_ON_TRAILING_TOKENS, so the leading object is not accepted.
+ assertTrue(records.get(1).error().contains("Malformed JSON"));
+ assertTrue(records.get(2).error().contains("JSON object"));
+ assertNull(records.get(3).error());
+ }
+
+ @Test
+ void membersOfTheWrongTypeAreFailedRecordsRatherThanCoerced() throws IOException {
+ List<ImportRecord> records = readAll(job(ImportJob.DESTINATION_CONTACT_LIST), """
+ {"emailAddress":"[email protected]","attributesData":123}
+ {"emailAddress":"[email protected]","unsubscribeAll":"yes"}
+ {"emailAddress":"[email protected]","topicPreferences":"Sports"}
+ {"emailAddress":"[email protected]","topicPreferences":["Sports"]}
+ {"emailAddress":"[email protected]","topicPreferences":[{"topicName":"Sports","subscriptionStatus":true}]}
+ """);
+
+ assertTrue(records.get(0).error().contains("attributesData"));
+ assertTrue(records.get(1).error().contains("unsubscribeAll"));
+ assertTrue(records.get(2).error().contains("topicPreferences"));
+ assertTrue(records.get(3).error().contains("JSON object"));
+ assertTrue(records.get(4).error().contains("subscriptionStatus"));
+ }
+
+ @Test
+ void anEmptySourceYieldsNoRecords() throws IOException {
+ assertTrue(readAll(job(ImportJob.DESTINATION_SUPPRESSION_LIST), "").isEmpty());
+ }
+}