Light, fluffy, and always free - The AWS Local Emulator alternative
feat(ses): add v2 import jobs for the suppression list and contact lists (#3892)
Kenshin Okinaka8ca748355df9
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesImportJobTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesImportJobTest.javanew file mode 100644index 0000000000..b5435d829e--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesImportJobTest.java@@ -0,0 +1,269 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import software.amazon.awssdk.core.sync.RequestBody;+import software.amazon.awssdk.services.s3.S3Client;+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;+import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;+import software.amazon.awssdk.services.s3.model.PutObjectRequest;+import software.amazon.awssdk.services.sesv2.SesV2Client;+import software.amazon.awssdk.services.sesv2.model.BadRequestException;+import software.amazon.awssdk.services.sesv2.model.ContactListDestination;+import software.amazon.awssdk.services.sesv2.model.ContactListImportAction;+import software.amazon.awssdk.services.sesv2.model.CreateContactListRequest;+import software.amazon.awssdk.services.sesv2.model.CreateImportJobRequest;+import software.amazon.awssdk.services.sesv2.model.CreateImportJobResponse;+import software.amazon.awssdk.services.sesv2.model.DataFormat;+import software.amazon.awssdk.services.sesv2.model.DeleteContactListRequest;+import software.amazon.awssdk.services.sesv2.model.DeleteSuppressedDestinationRequest;+import software.amazon.awssdk.services.sesv2.model.GetContactRequest;+import software.amazon.awssdk.services.sesv2.model.GetContactResponse;+import software.amazon.awssdk.services.sesv2.model.GetImportJobRequest;+import software.amazon.awssdk.services.sesv2.model.GetImportJobResponse;+import software.amazon.awssdk.services.sesv2.model.ImportDataSource;+import software.amazon.awssdk.services.sesv2.model.ImportDestination;+import software.amazon.awssdk.services.sesv2.model.ImportDestinationType;+import software.amazon.awssdk.services.sesv2.model.ImportJobSummary;+import software.amazon.awssdk.services.sesv2.model.JobStatus;+import software.amazon.awssdk.services.sesv2.model.ListImportJobsRequest;+import software.amazon.awssdk.services.sesv2.model.ListImportJobsResponse;+import software.amazon.awssdk.services.sesv2.model.ListSuppressedDestinationsRequest;+import software.amazon.awssdk.services.sesv2.model.NotFoundException;+import software.amazon.awssdk.services.sesv2.model.SubscriptionStatus;+import software.amazon.awssdk.services.sesv2.model.SuppressionListDestination;+import software.amazon.awssdk.services.sesv2.model.SuppressionListImportAction;+import software.amazon.awssdk.services.sesv2.model.SuppressedDestinationSummary;+import software.amazon.awssdk.services.sesv2.model.Topic;++import java.util.List;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;++/**+ * SDK compatibility test for the SES v2 import-job APIs: CreateImportJob / GetImportJob /+ * ListImportJobs marshalling, the asynchronous job lifecycle polled through GetImportJob, and the+ * imported rows read back through the suppression-list and contact APIs, against a live Floci+ * instance. The source objects are written through the S3 SDK.+ */+@DisplayName("SES v2 Import Jobs")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class SesImportJobTest {++ private static final String BUCKET = "floci-ses-import-job-sdk-test";+ private static final String LIST = "compat-import-job-list";+ private static final String SUPPRESSION_KEY = "suppression.csv";+ private static final String CONTACTS_KEY = "contacts.json";++ private static SesV2Client sesV2;+ private static S3Client s3;+ private static String suppressionJobId;+ private static String contactJobId;++ @BeforeAll+ static void setup() {+ sesV2 = TestFixtures.sesV2Client();+ s3 = TestFixtures.s3Client();+ cleanupQuietly();+ s3.createBucket(CreateBucketRequest.builder().bucket(BUCKET).build());+ s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(SUPPRESSION_KEY).build(),+ RequestBody.fromString("[email protected],BOUNCE\[email protected],COMPLAINT\[email protected],SPAM\n"));+ s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(CONTACTS_KEY).build(),+ RequestBody.fromString("""+ {"emailAddress":"[email protected]","unsubscribeAll":false,"attributesData":"{\\"Name\\":\\"Carol\\"}","topicPreferences":[{"topicName":"Sports","subscriptionStatus":"OPT_IN"}]}+ {"emailAddress":"[email protected]","unsubscribeAll":true}+ """));+ sesV2.createContactList(CreateContactListRequest.builder()+ .contactListName(LIST)+ .topics(Topic.builder().topicName("Sports").displayName("Sports")+ .defaultSubscriptionStatus(SubscriptionStatus.OPT_OUT).build())+ .build());+ }++ @AfterAll+ static void cleanup() {+ cleanupQuietly();+ if (sesV2 != null) {+ sesV2.close();+ }+ if (s3 != null) {+ s3.close();+ }+ }++ private static void cleanupQuietly() {+ for (String address : List.of("[email protected]", "[email protected]")) {+ quietly(() -> sesV2.deleteSuppressedDestination(+ DeleteSuppressedDestinationRequest.builder().emailAddress(address).build()));+ }+ quietly(() -> sesV2.deleteContactList(DeleteContactListRequest.builder().contactListName(LIST).build()));+ for (String key : List.of(SUPPRESSION_KEY, CONTACTS_KEY)) {+ quietly(() -> s3.deleteObject(DeleteObjectRequest.builder().bucket(BUCKET).key(key).build()));+ }+ quietly(() -> s3.deleteBucket(DeleteBucketRequest.builder().bucket(BUCKET).build()));+ }++ // Best-effort teardown: the resource is usually already gone (a fresh Floci, or the previous+ // run cleaned up), so a NotFound here is the expected state and the next call must still run.+ private static void quietly(Runnable cleanup) {+ try {+ cleanup.run();+ } catch (Exception ignored) {+ // see above: an absent resource is exactly what cleanup wants+ }+ }++ @Test+ @Order(1)+ @DisplayName("CreateImportJob into the suppression list completes and lists the addresses")+ void createImportJob_suppressionList() throws InterruptedException {+ CreateImportJobResponse created = sesV2.createImportJob(CreateImportJobRequest.builder()+ .importDataSource(ImportDataSource.builder()+ .s3Url("s3://" + BUCKET + "/" + SUPPRESSION_KEY).dataFormat(DataFormat.CSV).build())+ .importDestination(ImportDestination.builder()+ .suppressionListDestination(SuppressionListDestination.builder()+ .suppressionListImportAction(SuppressionListImportAction.PUT).build())+ .build())+ .build());+ suppressionJobId = created.jobId();+ assertThat(suppressionJobId).isNotBlank();++ GetImportJobResponse job = pollUntilTerminal(suppressionJobId);+ assertThat(job.jobStatus()).isEqualTo(JobStatus.COMPLETED);+ assertThat(job.importDataSource().s3Url()).isEqualTo("s3://" + BUCKET + "/" + SUPPRESSION_KEY);+ assertThat(job.importDataSource().dataFormat()).isEqualTo(DataFormat.CSV);+ assertThat(job.importDestination().suppressionListDestination().suppressionListImportAction())+ .isEqualTo(SuppressionListImportAction.PUT);+ assertThat(job.importDestination().contactListDestination()).isNull();+ assertThat(job.processedRecordsCount()).isEqualTo(3);+ assertThat(job.failedRecordsCount()).isEqualTo(1);+ assertThat(job.createdTimestamp()).isNotNull();+ assertThat(job.completedTimestamp()).isNotNull();+ assertThat(job.failureInfo()).isNull();++ List<String> suppressed = sesV2.listSuppressedDestinations(ListSuppressedDestinationsRequest.builder().build())+ .suppressedDestinationSummaries().stream()+ .map(SuppressedDestinationSummary::emailAddress)+ .toList();+ assertThat(suppressed).contains("[email protected]", "[email protected]");+ }++ @Test+ @Order(2)+ @DisplayName("CreateImportJob into a contact list creates the contacts")+ void createImportJob_contactList() throws InterruptedException {+ CreateImportJobResponse created = sesV2.createImportJob(CreateImportJobRequest.builder()+ .importDataSource(ImportDataSource.builder()+ .s3Url("s3://" + BUCKET + "/" + CONTACTS_KEY).dataFormat(DataFormat.JSON).build())+ .importDestination(ImportDestination.builder()+ .contactListDestination(ContactListDestination.builder()+ .contactListName(LIST)+ .contactListImportAction(ContactListImportAction.PUT).build())+ .build())+ .build());+ contactJobId = created.jobId();++ GetImportJobResponse job = pollUntilTerminal(contactJobId);+ assertThat(job.jobStatus()).isEqualTo(JobStatus.COMPLETED);+ assertThat(job.importDestination().contactListDestination().contactListName()).isEqualTo(LIST);+ assertThat(job.processedRecordsCount()).isEqualTo(2);+ assertThat(job.failedRecordsCount()).isEqualTo(0);++ GetContactResponse carol = sesV2.getContact(GetContactRequest.builder()+ .contactListName(LIST).emailAddress("[email protected]").build());+ assertThat(carol.attributesData()).isEqualTo("{\"Name\":\"Carol\"}");+ assertThat(carol.topicPreferences()).hasSize(1);+ assertThat(carol.topicPreferences().get(0).subscriptionStatus()).isEqualTo(SubscriptionStatus.OPT_IN);+ GetContactResponse dave = sesV2.getContact(GetContactRequest.builder()+ .contactListName(LIST).emailAddress("[email protected]").build());+ assertThat(dave.unsubscribeAll()).isTrue();+ }++ @Test+ @Order(3)+ @DisplayName("ListImportJobs returns both jobs and filters by destination type")+ void listImportJobs() {+ ListImportJobsResponse all = sesV2.listImportJobs(ListImportJobsRequest.builder().build());+ assertThat(all.importJobs()).extracting(ImportJobSummary::jobId).contains(suppressionJobId, contactJobId);+ assertThat(all.nextToken()).isNull();++ ListImportJobsResponse contactOnly = sesV2.listImportJobs(ListImportJobsRequest.builder()+ .importDestinationType(ImportDestinationType.CONTACT_LIST).pageSize(10).build());+ assertThat(contactOnly.importJobs()).extracting(ImportJobSummary::jobId)+ .contains(contactJobId).doesNotContain(suppressionJobId);+ assertThat(contactOnly.importJobs().get(0).jobStatus()).isEqualTo(JobStatus.COMPLETED);+ }++ @Test+ @Order(4)+ @DisplayName("CreateImportJob with a missing object completes as FAILED with an ErrorMessage")+ void createImportJob_missingObject_fails() throws InterruptedException {+ String jobId = sesV2.createImportJob(CreateImportJobRequest.builder()+ .importDataSource(ImportDataSource.builder()+ .s3Url("s3://" + BUCKET + "/missing.csv").dataFormat(DataFormat.CSV).build())+ .importDestination(ImportDestination.builder()+ .suppressionListDestination(SuppressionListDestination.builder()+ .suppressionListImportAction(SuppressionListImportAction.DELETE).build())+ .build())+ .build()).jobId();++ GetImportJobResponse job = pollUntilTerminal(jobId);+ assertThat(job.jobStatus()).isEqualTo(JobStatus.FAILED);+ assertThat(job.failureInfo().errorMessage()).contains("does not exist");+ }++ @Test+ @Order(5)+ @DisplayName("CreateImportJob rejects an invalid S3Url and an unknown contact list")+ void createImportJob_validation() {+ assertThatThrownBy(() -> sesV2.createImportJob(CreateImportJobRequest.builder()+ .importDataSource(ImportDataSource.builder().s3Url("https://example.com/x.csv")+ .dataFormat(DataFormat.CSV).build())+ .importDestination(ImportDestination.builder()+ .suppressionListDestination(SuppressionListDestination.builder()+ .suppressionListImportAction(SuppressionListImportAction.PUT).build())+ .build())+ .build()))+ .isInstanceOf(BadRequestException.class);++ assertThatThrownBy(() -> sesV2.createImportJob(CreateImportJobRequest.builder()+ .importDataSource(ImportDataSource.builder().s3Url("s3://" + BUCKET + "/" + CONTACTS_KEY)+ .dataFormat(DataFormat.JSON).build())+ .importDestination(ImportDestination.builder()+ .contactListDestination(ContactListDestination.builder()+ .contactListName("no-such-list")+ .contactListImportAction(ContactListImportAction.PUT).build())+ .build())+ .build()))+ .isInstanceOf(NotFoundException.class);+ }++ @Test+ @Order(6)+ @DisplayName("GetImportJob for an unknown id throws NotFoundException")+ void getImportJob_unknown() {+ assertThatThrownBy(() -> sesV2.getImportJob(GetImportJobRequest.builder().jobId("no-such-job").build()))+ .isInstanceOf(NotFoundException.class);+ }++ private static GetImportJobResponse pollUntilTerminal(String jobId) throws InterruptedException {+ long deadline = System.currentTimeMillis() + 10_000;+ GetImportJobResponse job;+ do {+ job = sesV2.getImportJob(GetImportJobRequest.builder().jobId(jobId).build());+ if (job.jobStatus() == JobStatus.COMPLETED || job.jobStatus() == JobStatus.FAILED) {+ return job;+ }+ Thread.sleep(100);+ } while (System.currentTimeMillis() < deadline);+ return job;+ }+}diff --git a/docs/services/ses.md b/docs/services/ses.mdindex 5bb05ffa85..bdc5406a57 100644--- a/docs/services/ses.md+++ b/docs/services/ses.md@@ -372,6 +372,9 @@ Alongside the classic Query API, Floci implements a subset of the SES v2 REST JS| `POST` | `/v2/email/tags` | `TagResource` || `DELETE` | `/v2/email/tags?ResourceArn=...&TagKeys=...` | `UntagResource` || `GET` | `/v2/email/tags?ResourceArn=...` | `ListTagsForResource` |+| `POST` | `/v2/email/import-jobs` | `CreateImportJob` |+| `GET` | `/v2/email/import-jobs/{JobId}` | `GetImportJob` |+| `POST` | `/v2/email/import-jobs/list` | `ListImportJobs` |Floci models no leased dedicated IPs: `GetDedicatedIps` is empty and IP-targeted operations return `NotFoundException`, as real AWS does for an account with no leased IPs, with required request members validated first (`BadRequestException`). `PutDedicatedIpPoolScalingAttributes` rejects downgrading a `MANAGED` pool to `STANDARD`, and `PutAccountDedicatedIpWarmupAttributes` stores the flag behind `GetAccount.DedicatedIpAutoWarmupEnabled` (default `true`).@@ -408,6 +411,8 @@ Suppression list entries are stored per region with `Reason` ∈ {`BOUNCE`, `COMSuppressed recipients are filtered out of the SMTP relay step (non-suppressed recipients on the same send still reach the relay normally), and the configuration set's event destinations receive a synthetic `Bounce` or `Complaint` event alongside the always-emitted `Send` event. The `SendEmail` API response (`200` + `MessageId`), the stored `SentEmail` visible at `GET /_aws/ses`, and the published event's `mail.destination` all retain the original recipient list — matching the AWS contract that the message is "accepted, just not sent" for suppressed addresses.+Import jobs read their source from Floci's own S3 emulation, so the object has to be uploaded there rather than to real S3; the CSV and newline-delimited JSON record formats are AWS's own ([suppression list](https://docs.aws.amazon.com/ses/latest/dg/sending-email-suppression-list.html#sending-email-suppression-list-manual-add-bulk), [contact list](https://docs.aws.amazon.com/ses/latest/dg/sending-email-list-management.html#configuring-list-management-bulk-import)). A job still running when the emulator restarts is marked `FAILED` on the next start. Not emulated: `FailureInfo.FailedRecordsS3Url` (Floci writes no failed-record file), the per-file record limits, the same-region bucket rule, and `ListImportJobs` pagination, which returns every job in one page.+`SendEmail` honors `ListManagementOptions` (`ContactListName`, optional `TopicName`). When present, each recipient is matched against the named contact list and suppressed as a `Bounce` when opted out — reusing the same relay-exclusion and Bounce-event path as suppression-list filtering, matching AWS ("SES will issue a bounce event for a message that is sent to an unsubscribed contact"). A contact is opted out when `UnsubscribeAll` is set; with a `TopicName`, an explicit `OPT_OUT` preference for that topic (or, absent an explicit preference, the topic's `DefaultSubscriptionStatus` being `OPT_OUT`) suppresses; without a `TopicName`, only `UnsubscribeAll` contacts are suppressed. A recipient that is not yet a contact is created on the list automatically (as on AWS) and then evaluated. Referencing a contact list that does not exist fails the send with `NotFoundException`. The topic-default fallback at send time is an intentional deviation — it is not documented by AWS and mirrors the effective-status model AWS uses for `ListContacts`.For a **single-recipient** list-managed send, Floci injects a functional unsubscribe link (matching AWS, which only does this for one recipient): the `{{amazonSESUnsubscribeUrl}}` body placeholder is replaced (up to twice) and a `List-Unsubscribe` header plus `List-Unsubscribe-Post: List-Unsubscribe=One-Click` are added (applied to the relayed message and shown under `Headers` at `GET /_aws/ses`; a caller-supplied `List-Unsubscribe` is overridden, matching AWS). The `{{amazonSESUnsubscribeUrl}}` placeholder is also preserved through template rendering so a templated body can carry it. On a send without `ListManagementOptions`, the placeholder is left in the body verbatim (neither replaced nor stripped) — this specific behavior is Floci's choice and is not verified against real SES. Unlike AWS's opaque hosted URL, the link points at Floci's own `/_aws/ses/unsubscribe?region=…&contactList=…&address=…&topic=…` endpoint. `GET` (a browser click) only renders a confirmation page and changes nothing — matching AWS's landing-page behavior and avoiding the RFC 8058 hazard where a client or bot that prefetches the link would silently unsubscribe the contact — while `POST` (the one-click request the confirmation form submits) applies the opt-out (a topic → `OPT_OUT` for that topic; no topic → `UnsubscribeAll`), auto-creating the contact if needed. Two deviations from AWS here: the link is carried as readable query parameters rather than an opaque token (so, like the rest of Floci, the endpoint is unauthenticated and only safe on a trusted dev/test network — a `POST` can opt out any contact), and only the one-click URL entry is emitted — its scheme follows Floci's base URL (`http` unless TLS is enabled) — whereas AWS also includes a `mailto:` entry, which Floci can't service since it has no inbound mail endpoint. Raw (MIME) sends do not get link injection yet.diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.javaindex 3d4f49f184..0d55ab6dc2 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java@@ -39,6 +39,7 @@ import io.github.hectorvent.floci.services.ses.SesContactController;import io.github.hectorvent.floci.services.ses.SesCvetController;import io.github.hectorvent.floci.services.ses.SesDedicatedIpController;import io.github.hectorvent.floci.services.ses.SesIdentityController;+import io.github.hectorvent.floci.services.ses.SesImportJobController;import io.github.hectorvent.floci.services.ses.SesSendController;import io.github.hectorvent.floci.services.ses.SesSuppressionController;import io.github.hectorvent.floci.services.ses.SesTagController;@@ -301,7 +302,7 @@ public class ResolvedServiceCatalog {SesDedicatedIpController.class, SesIdentityController.class,SesSendController.class, SesSuppressionController.class,SesTagController.class, SesTemplateController.class,- SesTenantController.class)),+ SesImportJobController.class, SesTenantController.class)),descriptor("es", "opensearch", config.services().opensearch().enabled(), true,"opensearch", storageMode(config.storage().services().opensearch().mode(), config.storage().mode()),config.storage().services().opensearch().flushIntervalMs(), null, ServiceProtocol.REST_JSON,diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesContactService.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesContactService.javaindex 87855be41b..897e1827fa 100644--- a/src/main/java/io/github/hectorvent/floci/services/ses/SesContactService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesContactService.java@@ -386,6 +386,32 @@ public class SesContactService {return existing;}+ /**+ * Replaces a contact outright, as a contact-list import job's {@code PUT} action does. Probed+ * against real AWS: an imported record is the whole contact, so preferences, attributes, and+ * the unsubscribe flag it omits are cleared rather than merged, unlike+ * {@link #updateContact}, whose topic preferences merge by name.+ */+ public Contact replaceContact(String listName, String emailAddress, List<TopicPreference> topicPreferences,+ Boolean unsubscribeAll, String attributesData, String region) {+ validateContactInput(listName, emailAddress, topicPreferences, region);+ String key = contactKey(region, listName, emailAddress);+ Contact existing;+ // Same lock as create/update: a concurrent deleteContactList must not slip between the+ // validation and the write.+ synchronized (contactMutationLock) {+ getContactList(listName, region);+ existing = contactStore.get(key).orElseThrow(() -> contactNotFound(emailAddress));+ existing.setTopicPreferences(topicPreferences);+ existing.setUnsubscribeAll(unsubscribeAll != null && unsubscribeAll);+ existing.setAttributesData(attributesData);+ existing.setLastUpdatedTimestamp(Instant.now(clock));+ contactStore.put(key, existing);+ }+ LOG.infov("Replaced SES contact {0} in list {1}", emailAddress, listName);+ return existing;+ }+public void deleteContact(String listName, String emailAddress, String region) {validateEmailAddress(emailAddress);String key = contactKey(region, listName, emailAddress);diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobController.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobController.javanew file mode 100644index 0000000000..a0e12938e7--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobController.java@@ -0,0 +1,177 @@+package io.github.hectorvent.floci.services.ses;++import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.databind.DeserializationFeature;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.ObjectReader;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.ses.model.ImportJob;+import jakarta.inject.Inject;+import jakarta.ws.rs.Consumes;+import jakarta.ws.rs.GET;+import jakarta.ws.rs.POST;+import jakarta.ws.rs.Path;+import jakarta.ws.rs.PathParam;+import jakarta.ws.rs.Produces;+import jakarta.ws.rs.core.Context;+import jakarta.ws.rs.core.HttpHeaders;+import jakarta.ws.rs.core.MediaType;+import jakarta.ws.rs.core.Response;+import org.jboss.logging.Logger;++import java.util.List;++import static io.github.hectorvent.floci.services.ses.SesV2Json.intMemberOrAbsent;+import static io.github.hectorvent.floci.services.ses.SesV2Json.readRequiredStringField;+import static io.github.hectorvent.floci.services.ses.SesV2Json.remapV1Exception;+import static io.github.hectorvent.floci.services.ses.SesV2Json.requireJsonObject;+import static io.github.hectorvent.floci.services.ses.SesV2Json.requireObjectOrAbsent;+import static io.github.hectorvent.floci.services.ses.SesV2Json.stringMemberOrAbsent;++/**+ * SES V2 import job endpoints ({@code /v2/email/import-jobs}). Every operation is a single-domain+ * call on {@link SesImportJobService}; the JSON shape of {@code ImportDestination} /+ * {@code ImportDataSource} is parsed and rendered here.+ */+@Path("/v2/email")+@Produces(MediaType.APPLICATION_JSON)+@Consumes(MediaType.APPLICATION_JSON)+public class SesImportJobController {++ private static final Logger LOG = Logger.getLogger(SesImportJobController.class);++ private final SesImportJobService importJobService;+ private final RegionResolver regionResolver;+ private final ObjectMapper objectMapper;+ // FAIL_ON_TRAILING_TOKENS so "{...} garbage" is rejected instead of creating a job from the+ // leading object, as the JSON protocol controllers do.+ private final ObjectReader strictReader;++ @Inject+ public SesImportJobController(SesImportJobService importJobService, RegionResolver regionResolver,+ ObjectMapper objectMapper) {+ this.importJobService = importJobService;+ this.regionResolver = regionResolver;+ this.objectMapper = objectMapper;+ this.strictReader = objectMapper.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);+ }++ @POST+ @Path("/import-jobs")+ public Response createImportJob(@Context HttpHeaders headers, String body) {+ String region = regionResolver.resolveRegion(headers);+ try {+ if (body == null || body.isBlank()) {+ throw new AwsException("BadRequestException", "Request body is required.", 400);+ }+ JsonNode request = strictReader.readTree(body);+ requireJsonObject(request);+ JsonNode source = requireObjectOrAbsent(request, "ImportDataSource");+ if (!source.isObject()) {+ throw new AwsException("BadRequestException", "ImportDataSource is required.", 400);+ }+ String s3Url = readRequiredStringField(source, "S3Url");+ String dataFormat = readRequiredStringField(source, "DataFormat");+ JsonNode destination = requireObjectOrAbsent(request, "ImportDestination");+ if (!destination.isObject()) {+ throw new AwsException("BadRequestException", "ImportDestination is required.", 400);+ }+ JsonNode suppression = requireObjectOrAbsent(destination, "SuppressionListDestination");+ JsonNode contactList = requireObjectOrAbsent(destination, "ContactListDestination");+ if (suppression.isObject() == contactList.isObject()) {+ throw new AwsException("BadRequestException",+ "ImportDestination must contain exactly one of SuppressionListDestination "+ + "or ContactListDestination.", 400);+ }+ String accountId = regionResolver.getAccountId();+ ImportJob job;+ if (suppression.isObject()) {+ String action = readRequiredStringField(suppression, "SuppressionListImportAction");+ job = importJobService.createSuppressionListImportJob(region, accountId, s3Url, dataFormat, action);+ } else {+ String listName = readRequiredStringField(contactList, "ContactListName");+ String action = readRequiredStringField(contactList, "ContactListImportAction");+ job = importJobService.createContactListImportJob(region, accountId, s3Url, dataFormat,+ listName, action);+ }+ LOG.infov("SES V2 CreateImportJob: {0}", job.getJobId());+ ObjectNode result = objectMapper.createObjectNode();+ result.put("JobId", job.getJobId());+ return Response.ok(result).build();+ } catch (AwsException e) {+ throw remapV1Exception(e);+ } catch (JsonProcessingException e) {+ throw new AwsException("BadRequestException", e.getMessage(), 400);+ }+ }++ @GET+ @Path("/import-jobs/{jobId}")+ public Response getImportJob(@Context HttpHeaders headers, @PathParam("jobId") String jobId) {+ String region = regionResolver.resolveRegion(headers);+ ImportJob job = importJobService.getImportJob(region, jobId);+ ObjectNode result = objectMapper.createObjectNode();+ writeSummary(result, job);+ ObjectNode source = result.putObject("ImportDataSource");+ source.put("S3Url", job.getS3Url());+ source.put("DataFormat", job.getDataFormat());+ if (job.getCompletedTimestamp() != null) {+ result.put("CompletedTimestamp", job.getCompletedTimestamp().toEpochMilli() / 1000.0);+ }+ if (job.getErrorMessage() != null) {+ result.putObject("FailureInfo").put("ErrorMessage", job.getErrorMessage());+ }+ return Response.ok(result).build();+ }++ @POST+ @Path("/import-jobs/list")+ public Response listImportJobs(@Context HttpHeaders headers, String body) {+ String region = regionResolver.resolveRegion(headers);+ try {+ JsonNode request = (body == null || body.isBlank())+ ? objectMapper.createObjectNode()+ : strictReader.readTree(body);+ requireJsonObject(request);+ String destinationType = stringMemberOrAbsent(request, "ImportDestinationType");+ Integer pageSize = intMemberOrAbsent(request, "PageSize");+ String nextToken = stringMemberOrAbsent(request, "NextToken");+ List<ImportJob> jobs = importJobService.listImportJobs(region, destinationType, pageSize, nextToken);+ ObjectNode result = objectMapper.createObjectNode();+ ArrayNode items = result.putArray("ImportJobs");+ for (ImportJob job : jobs) {+ writeSummary(items.addObject(), job);+ }+ // AWS renders NextToken as an explicit null on the last (here: only) page.+ result.putNull("NextToken");+ return Response.ok(result).build();+ } catch (AwsException e) {+ throw remapV1Exception(e);+ } catch (JsonProcessingException e) {+ throw new AwsException("BadRequestException", e.getMessage(), 400);+ }+ }++ // The ImportJobSummary members, shared by GetImportJob (which adds the source, completion time+ // and failure info on top).+ private static void writeSummary(ObjectNode node, ImportJob job) {+ node.put("JobId", job.getJobId());+ ObjectNode destination = node.putObject("ImportDestination");+ if (ImportJob.DESTINATION_SUPPRESSION_LIST.equals(job.getDestinationType())) {+ destination.putObject("SuppressionListDestination")+ .put("SuppressionListImportAction", job.getImportAction());+ } else {+ ObjectNode contactList = destination.putObject("ContactListDestination");+ contactList.put("ContactListName", job.getContactListName());+ contactList.put("ContactListImportAction", job.getImportAction());+ }+ node.put("JobStatus", job.getJobStatus());+ node.put("CreatedTimestamp", job.getCreatedTimestamp().toEpochMilli() / 1000.0);+ node.put("ProcessedRecordsCount", job.getProcessedRecordsCount());+ node.put("FailedRecordsCount", job.getFailedRecordsCount());+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobService.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobService.javanew file mode 100644index 0000000000..e8700a23e5--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesImportJobService.java@@ -0,0 +1,474 @@+package io.github.hectorvent.floci.services.ses;++import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.core.common.Resettable;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.ses.imports.ImportRecord;+import io.github.hectorvent.floci.services.ses.imports.RecordReader;+import io.github.hectorvent.floci.services.ses.imports.RecordReaderFactory;+import io.github.hectorvent.floci.services.ses.model.ImportJob;+import io.quarkus.runtime.StartupEvent;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.event.Observes;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.io.BufferedReader;+import java.io.IOException;+import java.io.InputStream;+import java.io.InputStreamReader;+import java.nio.charset.StandardCharsets;+import java.time.Clock;+import java.time.Instant;+import java.util.Comparator;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.UUID;+import java.util.concurrent.Executor;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++/**+ * SES V2 import jobs: bulk-loads a CSV or newline-delimited JSON object from S3 into the account+ * suppression list or a contact list. A job is stored as CREATED and returned immediately; a+ * worker then flips it to PROCESSING and applies the records one by one through+ * {@link SesSuppressionService} / {@link SesContactService}, so the imported data lives in those+ * services' stores and only the job record lives here. A record that fails validation is counted+ * in {@code FailedRecordsCount} and the job still completes; the job itself fails when the source+ * cannot be read or the contact list is gone.+ */+@ApplicationScoped+public class SesImportJobService implements Resettable {++ private static final Logger LOG = Logger.getLogger(SesImportJobService.class);++ static final int MAX_CONCURRENT_JOBS = 20;+ // AWS reports an import's counters as it runs, so the worker republishes them as it goes.+ // Every record would mean serializing the job per row, so progress lands in batches.+ static final int PROGRESS_RECORD_INTERVAL = 100;+ static final String RESTART_FAILURE_MESSAGE = "The emulator restarted before the import finished";+ private static final long RESET_ADMISSION_WAIT_MILLIS = 30_000L;+ // The drain only ever waits for the record being applied right now plus the read of the next+ // one, which is a store get/put and a line off a local file, so this deadline is a safety net+ // that a healthy import never reaches. 5s is what every other worker wait in Floci uses+ // (WalStorage, HybridStorage, CloudWatchLogsMetricFilterService, CodePipelineService). Raising+ // it is the wrong direction: a reset walks every Resettable service in turn, so per-service+ // deadlines add up.+ static final long RESET_DRAIN_TIMEOUT_MILLIS = 5_000L;++ private static final Pattern S3_URL = Pattern.compile("^s3://([^/]+)/(.*?([^/]+)/?)$");+ private static final Set<String> DATA_FORMATS = Set.of(ImportJob.FORMAT_CSV, ImportJob.FORMAT_JSON);+ private static final Set<String> IMPORT_ACTIONS = Set.of(ImportJob.ACTION_PUT, ImportJob.ACTION_DELETE);+ private static final Set<String> DESTINATION_TYPES =+ Set.of(ImportJob.DESTINATION_SUPPRESSION_LIST, ImportJob.DESTINATION_CONTACT_LIST);++ private final StorageBackend<String, ImportJob> importJobStore;+ private final S3Service s3Service;+ private final SesSuppressionService suppressionService;+ private final SesContactService contactService;+ private final ObjectMapper objectMapper;+ private final RecordReaderFactory readerFactory;+ private final Clock clock;+ private final Executor workers;+ private final long resetDrainTimeoutMillis;+ // Admission lock: serializes the concurrent-job count against the store write (so two creates+ // cannot both pass the limit check) and against reset, which flips {@code resetting} under it.+ private final Object createLock = new Object();+ private boolean resetting;+ // Reset support: beforeReset bumps the generation so every running worker stops before its+ // next write, then waits on workerMonitor for the workers still inside one. That wait is+ // bounded (see RESET_DRAIN_TIMEOUT_MILLIS), so a worker wedged past the deadline can still+ // land the record it is applying right now; it is logged when that happens.+ private final AtomicInteger generation = new AtomicInteger();+ private final Object workerMonitor = new Object();+ private int activeWorkers;++ @Inject+ public SesImportJobService(StorageFactory storageFactory, S3Service s3Service,+ SesSuppressionService suppressionService, SesContactService contactService,+ ObjectMapper objectMapper, Clock clock) {+ this(storageFactory.create("ses", "ses-import-jobs.json",+ new TypeReference<Map<String, ImportJob>>() {}),+ s3Service, suppressionService, contactService, objectMapper, clock,+ task -> Thread.ofVirtual().start(task));+ }++ SesImportJobService(StorageBackend<String, ImportJob> importJobStore, S3Service s3Service,+ SesSuppressionService suppressionService, SesContactService contactService,+ ObjectMapper objectMapper, Clock clock, Executor workers) {+ this(importJobStore, s3Service, suppressionService, contactService, objectMapper, clock, workers,+ RESET_DRAIN_TIMEOUT_MILLIS);+ }++ SesImportJobService(StorageBackend<String, ImportJob> importJobStore, S3Service s3Service,+ SesSuppressionService suppressionService, SesContactService contactService,+ ObjectMapper objectMapper, Clock clock, Executor workers,+ long resetDrainTimeoutMillis) {+ this.resetDrainTimeoutMillis = resetDrainTimeoutMillis;+ this.importJobStore = importJobStore;+ this.s3Service = s3Service;+ this.suppressionService = suppressionService;+ this.contactService = contactService;+ this.objectMapper = objectMapper;+ this.readerFactory = new RecordReaderFactory(objectMapper);+ this.clock = clock;+ this.workers = workers;+ recoverInterruptedJobs();+ }++ public ImportJob createSuppressionListImportJob(String region, String accountId, String s3Url,+ String dataFormat, String importAction) {+ ImportJob job = newJob(region, s3Url, dataFormat, importAction);+ job.setDestinationType(ImportJob.DESTINATION_SUPPRESSION_LIST);+ return start(job, accountId);+ }++ public ImportJob createContactListImportJob(String region, String accountId, String s3Url,+ String dataFormat, String contactListName,+ String importAction) {+ ImportJob job = newJob(region, s3Url, dataFormat, importAction);+ job.setDestinationType(ImportJob.DESTINATION_CONTACT_LIST);+ job.setContactListName(contactListName);+ requireContactList(job);+ return start(job, accountId);+ }++ /**+ * Probed against real AWS: an import job checks the contact list when it is created, answering+ * NotFoundException even though the model lists only BadRequest / LimitExceeded /+ * TooManyRequests for CreateImportJob. This path also words it differently from the contact-list+ * APIs ("ContactList <name> doesn't exist" rather than "List with name: <name> doesn't exist."),+ * so the message is rewritten here and the contact service keeps its own.+ */+ private void requireContactList(ImportJob job) {+ try {+ contactService.getContactList(job.getContactListName(), job.getRegion());+ } catch (AwsException e) {+ if (!"NotFoundException".equals(e.getErrorCode())) {+ throw e;+ }+ throw new AwsException("NotFoundException",+ "ContactList <" + job.getContactListName() + "> doesn't exist", 404);+ }+ }++ private ImportJob newJob(String region, String s3Url, String dataFormat, String importAction) {+ if (s3Url == null || !S3_URL.matcher(s3Url).matches()) {+ throw validationError("importDataSource.s3Url",+ "Member must satisfy regular expression pattern: " + S3_URL.pattern());+ }+ if (dataFormat == null || !DATA_FORMATS.contains(dataFormat)) {+ throw validationError("importDataSource.dataFormat",+ "Member must satisfy enum value set: [CSV, JSON]");+ }+ if (importAction == null || !IMPORT_ACTIONS.contains(importAction)) {+ throw validationError("importDestination", "Member must satisfy enum value set: [DELETE, PUT]");+ }+ ImportJob job = new ImportJob();+ job.setJobId(UUID.randomUUID().toString());+ job.setRegion(region);+ job.setS3Url(s3Url);+ job.setDataFormat(dataFormat);+ job.setImportAction(importAction);+ job.setJobStatus(ImportJob.STATUS_CREATED);+ job.setCreatedTimestamp(Instant.now(clock));+ return job;+ }++ private ImportJob start(ImportJob job, String accountId) {+ // The worker gets its own copy so the record serialized into the Create response never+ // changes underneath the handler, and runs as the request account so its store writes+ // land in that account's partition.+ ImportJob worker = objectMapper.convertValue(job, ImportJob.class);+ synchronized (createLock) {+ awaitResetFinished();+ long active = importJobStore.scan(k -> k.startsWith(keyPrefix(job.getRegion()))).stream()+ .filter(j -> !j.isTerminal())+ .count();+ if (active >= MAX_CONCURRENT_JOBS) {+ throw new AwsException("LimitExceededException",+ "The maximum number of concurrent import jobs (" + MAX_CONCURRENT_JOBS+ + ") has been reached.", 400);+ }+ publish(key(job.getRegion(), job.getJobId()), job);+ // Persist, capture the generation and dispatch under the same lock beforeReset takes,+ // so a job is either fully admitted before a reset (and then abandoned by its stale+ // generation) or admitted only after the reset has finished.+ int startedIn = generation.get();+ workers.execute(() -> RequestScopes.runAs(accountId, () -> runImport(worker, startedIn)));+ }+ LOG.infov("Created SES import job {0} in region {1}: {2} {3} from {4}", job.getJobId(),+ job.getRegion(), job.getImportAction(), job.getDestinationType(), job.getS3Url());+ return job;+ }++ // Holds createLock. A create that arrives while a reset is in progress waits for afterReset+ // rather than persisting a job that the wipe would then race.+ private void awaitResetFinished() {+ long deadline = System.currentTimeMillis() + RESET_ADMISSION_WAIT_MILLIS;+ while (resetting) {+ long remaining = deadline - System.currentTimeMillis();+ if (remaining <= 0) {+ throw new AwsException("TooManyRequestsException",+ "Import jobs are not accepted while the emulator is resetting.", 429);+ }+ try {+ createLock.wait(remaining);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AwsException("TooManyRequestsException",+ "Import jobs are not accepted while the emulator is resetting.", 429);+ }+ }+ }++ private void runImport(ImportJob job, int startedIn) {+ synchronized (workerMonitor) {+ activeWorkers++;+ }+ try {+ runImportInGeneration(job, startedIn);+ } finally {+ synchronized (workerMonitor) {+ activeWorkers--;+ workerMonitor.notifyAll();+ }+ }+ }++ private void runImportInGeneration(ImportJob job, int startedIn) {+ String storeKey = key(job.getRegion(), job.getJobId());+ if (generation.get() != startedIn) {+ return;+ }+ job.setJobStatus(ImportJob.STATUS_PROCESSING);+ publish(storeKey, job);+ try {+ Matcher url = S3_URL.matcher(job.getS3Url());+ if (!url.matches()) {+ throw new AwsException("BadRequestException", "Invalid S3 URL: " + job.getS3Url(), 400);+ }+ if (ImportJob.DESTINATION_CONTACT_LIST.equals(job.getDestinationType())) {+ requireContactList(job);+ }+ // Streamed and applied one record at a time, so an import never holds more than the+ // current record in memory whatever the object's size.+ try (InputStream raw = s3Service.openObjectStream(url.group(1), url.group(2), null);+ BufferedReader source = new BufferedReader(new InputStreamReader(raw, StandardCharsets.UTF_8))) {+ applyRecords(job, storeKey, startedIn, readerFactory.forJob(job, source));+ }+ job.setJobStatus(ImportJob.STATUS_COMPLETED);+ } catch (AwsException e) {+ job.setJobStatus(ImportJob.STATUS_FAILED);+ job.setErrorMessage(e.getMessage());+ } catch (IOException | RuntimeException e) {+ LOG.errorv(e, "SES import job {0} failed", job.getJobId());+ job.setJobStatus(ImportJob.STATUS_FAILED);+ job.setErrorMessage(e.getMessage());+ }+ job.setCompletedTimestamp(Instant.now(clock));+ if (generation.get() != startedIn) {+ LOG.infov("SES import job {0} abandoned by an emulator reset", job.getJobId());+ return;+ }+ publish(storeKey, job);+ LOG.infov("SES import job {0} {1}: processed={2}, failed={3}", job.getJobId(), job.getJobStatus(),+ job.getProcessedRecordsCount(), job.getFailedRecordsCount());+ }++ // The worker goes on mutating its own ImportJob after each write, so every store write publishes+ // a copy: a backend that hands back the stored instance must never let a poll observe a+ // half-applied transition, such as COMPLETED before the completion timestamp is set.+ private void publish(String storeKey, ImportJob job) {+ importJobStore.put(storeKey, objectMapper.convertValue(job, ImportJob.class));+ }++ private void applyRecords(ImportJob job, String storeKey, int startedIn,+ RecordReader reader)+ throws IOException {+ for (ImportRecord record = reader.next(); record != null; record = reader.next()) {+ if (generation.get() != startedIn) {+ LOG.infov("SES import job {0} abandoned by an emulator reset", job.getJobId());+ return;+ }+ job.setProcessedRecordsCount(job.getProcessedRecordsCount() + 1);+ applyRecord(job, record);+ if (job.getProcessedRecordsCount() % PROGRESS_RECORD_INTERVAL == 0) {+ publish(storeKey, job);+ }+ }+ }++ private void applyRecord(ImportJob job, ImportRecord record) {+ if (record.error() != null) {+ failRecord(job, record.line(), record.error());+ return;+ }+ try {+ apply(job, record);+ } catch (AwsException e) {+ // A NotFound for a contact-list job is a missing contact (a failed record) unless the+ // list itself has gone, which fails the whole job like the pre-run check.+ if (ImportJob.DESTINATION_CONTACT_LIST.equals(job.getDestinationType())+ && "NotFoundException".equals(e.getErrorCode())) {+ requireContactList(job);+ }+ failRecord(job, record.line(), e.getMessage());+ }+ }++ private static void failRecord(ImportJob job, int line, String reason) {+ job.setFailedRecordsCount(job.getFailedRecordsCount() + 1);+ LOG.debugv("SES import job {0} skipped record at line {1}: {2}", job.getJobId(), line, reason);+ }++ private void apply(ImportJob job, ImportRecord record) {+ boolean put = ImportJob.ACTION_PUT.equals(job.getImportAction());+ if (ImportJob.DESTINATION_SUPPRESSION_LIST.equals(job.getDestinationType())) {+ if (put) {+ suppressionService.putSuppressedDestination(job.getRegion(), record.emailAddress(), record.reason());+ } else {+ suppressionService.deleteSuppressedDestination(job.getRegion(), record.emailAddress());+ }+ return;+ }+ String list = job.getContactListName();+ if (!put) {+ contactService.deleteContact(list, record.emailAddress(), job.getRegion());+ return;+ }+ // PUT over an existing contact replaces it outright rather than merging: probed against real+ // AWS, an import record that omits a topic, the attributes, or the unsubscribe flag clears+ // them. That is not UpdateContact's contract, so it has its own service path.+ try {+ contactService.createContact(list, record.emailAddress(), record.topicPreferences(),+ record.unsubscribeAll(), record.attributesData(), job.getRegion());+ } catch (AwsException e) {+ if (!"AlreadyExistsException".equals(e.getErrorCode())) {+ throw e;+ }+ contactService.replaceContact(list, record.emailAddress(), record.topicPreferences(),+ record.unsubscribeAll(), record.attributesData(), job.getRegion());+ }+ }++ public ImportJob getImportJob(String region, String jobId) {+ return importJobStore.get(key(region, jobId))+ .orElseThrow(() -> new AwsException("NotFoundException",+ "Import job " + jobId + " does not exist.", 404));+ }++ public List<ImportJob> listImportJobs(String region, String destinationType, Integer pageSize,+ String nextToken) {+ if (destinationType != null && !DESTINATION_TYPES.contains(destinationType)) {+ throw validationError("importDestinationType",+ "Member must satisfy enum value set: [SUPPRESSION_LIST, CONTACT_LIST]");+ }+ SesTenantService.validateListPaging(pageSize, nextToken);+ return importJobStore.scan(k -> k.startsWith(keyPrefix(region))).stream()+ .filter(j -> destinationType == null || destinationType.equals(j.getDestinationType()))+ .sorted(Comparator.comparing(ImportJob::getCreatedTimestamp)+ .thenComparing(ImportJob::getJobId))+ .toList();+ }++ // ──────────────────────────── Lifecycle ────────────────────────────++ // Recovery happens in the constructor, and a normal-scoped bean is only built on first use, so+ // without this the jobs left non-terminal by a restart would not be failed until an import+ // endpoint was called. Observing startup makes "marked FAILED on the next start" true of the+ // boot itself, which is what the docs promise.+ void onStart(@Observes StartupEvent ignored) {+ LOG.debugv("SES import job service ready");+ }++ @Override+ public void beforeReset() {+ synchronized (createLock) {+ resetting = true;+ generation.incrementAndGet();+ }+ // Drain, but bounded: a reset has to return even when a worker is wedged in a slow read.+ // Every worker re-checks the generation before its next write, so the wait normally covers+ // only the record being applied right now. If the deadline passes anyway that one record+ // can still land after the wipe, which the warning below is there to explain.+ boolean interrupted = false;+ long deadline = System.currentTimeMillis() + resetDrainTimeoutMillis;+ synchronized (workerMonitor) {+ while (activeWorkers > 0) {+ long remaining = deadline - System.currentTimeMillis();+ if (remaining <= 0) {+ LOG.warnv("{0} SES import worker(s) still running after {1}ms; a record in flight may "+ + "write after the reset", activeWorkers, resetDrainTimeoutMillis);+ break;+ }+ try {+ workerMonitor.wait(remaining);+ } catch (InterruptedException e) {+ // Keep draining to the deadline; the interrupt is re-asserted on the way out.+ interrupted = true;+ }+ }+ }+ if (interrupted) {+ Thread.currentThread().interrupt();+ }+ }++ @Override+ public void afterReset() {+ synchronized (createLock) {+ resetting = false;+ createLock.notifyAll();+ }+ }++ @Override+ public void clear() {+ // The job store is wiped by StorageFactory; the imported rows belong to the suppression and+ // contact services, which clear their own stores.+ }+++ // A job interrupted by a restart would otherwise stay CREATED/PROCESSING forever, since no+ // worker survives the process.+ private void recoverInterruptedJobs() {+ if (!(importJobStore instanceof AccountAwareStorageBackend<ImportJob> jobs)) {+ return;+ }+ Instant now = Instant.now(clock);+ for (AccountAwareStorageBackend.AccountEntry<ImportJob> entry : jobs.scanAllAccountEntries(k -> true)) {+ ImportJob job = entry.value();+ if (job.isTerminal()) {+ continue;+ }+ job.setJobStatus(ImportJob.STATUS_FAILED);+ job.setErrorMessage(RESTART_FAILURE_MESSAGE);+ job.setCompletedTimestamp(now);+ jobs.putForAccount(entry.accountId(), entry.key(), job);+ LOG.warnv("SES import job {0} was interrupted by a restart and is now FAILED", job.getJobId());+ }+ }++ private static AwsException validationError(String path, String constraint) {+ return new AwsException("BadRequestException",+ "1 validation error detected: Value at '" + path + "' failed to satisfy constraint: "+ + constraint, 400);+ }++ static String key(String region, String jobId) {+ return keyPrefix(region) + jobId;+ }++ private static String keyPrefix(String region) {+ return "importJob::" + region + "::";+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReader.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReader.javanew file mode 100644index 0000000000..293d7644a5--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReader.java@@ -0,0 +1,261 @@+package io.github.hectorvent.floci.services.ses.imports;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.ses.model.ImportJob;+import io.github.hectorvent.floci.services.ses.model.TopicPreference;++import java.io.IOException;+import java.io.Reader;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++/**+ * Reads RFC 4180 records one at a time: a record ends at an unquoted line break (a quoted+ * attributesData may span lines), the first record of a contact-list file is the header, and+ * a row whose width or quoting is wrong is a failed record. Records are numbered by position.+ *+ * <p>Deliberately not {@link io.github.hectorvent.floci.core.common.CsvParser}, which parses a+ * whole document held as a {@code String} and is permissive by design (a quote inside an+ * unquoted field is literal, an unterminated quoted field simply ends at EOF, and+ * {@code parseAll} applies the backslash escapes S3 Select, Athena, and the Step Functions+ * ItemReader expect). An import reads its source straight off an S3 stream so a large file is+ * never materialized, and it has to report malformed quoting rather than absorb it, since a bad+ * row has to land in {@code FailedRecordsCount} or fail the job. Nothing else in Floci streams+ * CSV out of S3 yet: DynamoDB's {@code ImportTable} sidesteps it by rejecting the CSV+ * {@code InputFormat} outright, so this is the first of its kind, and the place to grow a+ * shared streaming reader from if that changes.+ */+final class CsvRecordReader implements RecordReader {++ private static final String TOPIC_PREFERENCE_COLUMN_PREFIX = "topicPreferences.";+ private final Reader source;+ private final ImportJob job;+ private final boolean contactList;+ private int pending = -1;+ private boolean bomChecked;+ private int recordNumber;+ private Map<String, Integer> columns;++ CsvRecordReader(Reader source, ImportJob job) {+ this.source = source;+ this.job = job;+ this.contactList = ImportJob.DESTINATION_CONTACT_LIST.equals(job.getDestinationType());+ }++ @Override+ public ImportRecord next() throws IOException {+ String raw;+ while ((raw = nextRawRecord()) != null) {+ recordNumber++;+ if (raw.isBlank()) {+ continue;+ }+ List<String> fields;+ try {+ fields = splitCsvLine(raw);+ } catch (InvalidRecordException e) {+ if (contactList && columns == null) {+ throw new AwsException("BadRequestException", "Malformed CSV header: " + e.getMessage(), 400);+ }+ return ImportRecord.invalid(recordNumber, e.getMessage());+ }+ if (contactList && columns == null) {+ columns = readContactHeader(fields);+ continue;+ }+ if (!contactList) {+ // The documented formats are exactly "email,reason" for PUT and "email" for DELETE.+ int expected = ImportJob.ACTION_PUT.equals(job.getImportAction()) ? 2 : 1;+ if (fields.size() != expected) {+ return ImportRecord.invalid(recordNumber,+ "Expected " + expected + " column(s) but found " + fields.size() + ".");+ }+ return new ImportRecord(recordNumber, fields.get(0), expected == 2 ? fields.get(1) : null,+ null, null, null, null);+ }+ if (fields.size() != columns.size()) {+ return ImportRecord.invalid(recordNumber,+ "Expected " + columns.size() + " column(s) but found " + fields.size() + ".");+ }+ return contactRecordFromCsv(recordNumber, fields, columns);+ }+ if (contactList && columns == null) {+ throw new AwsException("BadRequestException",+ "The CSV file must start with a header line naming the emailAddress column.", 400);+ }+ return null;+ }++ // Only a quote at the start of a field opens a quoted section (a bare quote elsewhere is+ // left for splitCsvLine to reject as a malformed row), so one bad row cannot swallow the+ // rest of the file; a doubled quote inside a quoted field stays inside it.+ private String nextRawRecord() throws IOException {+ int start = read();+ if (start < 0) {+ return null;+ }+ pushBack(start);+ StringBuilder current = new StringBuilder();+ boolean quoted = false;+ boolean fieldStart = true;+ int next;+ while ((next = read()) >= 0) {+ char c = (char) next;+ if (quoted) {+ if (c == '"') {+ int peek = read();+ if (peek == '"') {+ current.append(c);+ } else {+ quoted = false;+ pushBack(peek);+ }+ }+ current.append(c);+ } else if (c == '\n' || c == '\r') {+ if (c == '\r') {+ int peek = read();+ if (peek != '\n') {+ pushBack(peek);+ }+ }+ return current.toString();+ } else {+ if (c == '"' && fieldStart) {+ quoted = true;+ }+ fieldStart = c == ',';+ current.append(c);+ }+ }+ if (quoted) {+ throw new AwsException("BadRequestException", "Malformed CSV: unterminated quoted field.", 400);+ }+ return current.toString();+ }++ private int read() throws IOException {+ int c;+ if (pending >= 0) {+ c = pending;+ pending = -1;+ } else {+ c = source.read();+ }+ if (!bomChecked) {+ bomChecked = true;+ if (c == '\uFEFF') {+ return read();+ }+ }+ return c;+ }++ private void pushBack(int c) {+ if (c >= 0) {+ pending = c;+ }+ }++ private static Map<String, Integer> readContactHeader(List<String> header) {+ Map<String, Integer> columns = new LinkedHashMap<>();+ for (int i = 0; i < header.size(); i++) {+ columns.put(header.get(i).trim(), i);+ }+ if (!columns.containsKey("emailAddress")) {+ throw new AwsException("BadRequestException",+ "The CSV header line must contain an emailAddress column.", 400);+ }+ return columns;+ }++ private static ImportRecord contactRecordFromCsv(int line, List<String> fields, Map<String, Integer> columns) {+ String email = column(fields, columns, "emailAddress");+ Boolean unsubscribeAll = null;+ String unsubscribeText = column(fields, columns, "unsubscribeAll");+ if (unsubscribeText != null && !unsubscribeText.isEmpty()) {+ if (unsubscribeText.equalsIgnoreCase("true")) {+ unsubscribeAll = true;+ } else if (unsubscribeText.equalsIgnoreCase("false")) {+ unsubscribeAll = false;+ } else {+ return ImportRecord.invalid(line, "unsubscribeAll must be true or false.");+ }+ }+ String attributesData = column(fields, columns, "attributesData");+ if (attributesData != null && attributesData.isEmpty()) {+ attributesData = null;+ }+ List<TopicPreference> prefs = null;+ for (Map.Entry<String, Integer> entry : columns.entrySet()) {+ if (!entry.getKey().startsWith(TOPIC_PREFERENCE_COLUMN_PREFIX)) {+ continue;+ }+ String status = fields.get(entry.getValue()).trim();+ if (status.isEmpty()) {+ continue;+ }+ if (prefs == null) {+ prefs = new ArrayList<>();+ }+ prefs.add(new TopicPreference(entry.getKey().substring(TOPIC_PREFERENCE_COLUMN_PREFIX.length()),+ status));+ }+ return new ImportRecord(line, email, null, unsubscribeAll, attributesData, prefs, null);+ }++ private static String column(List<String> fields, Map<String, Integer> columns, String name) {+ Integer index = columns.get(name);+ return index == null ? null : fields.get(index);+ }++ // RFC 4180 field splitting: a quoted field may contain commas and line breaks, and a doubled+ // quote inside a quoted field is a literal quote. A quote that does not open a field is data,+ // not an error: the developer guide's own contact row writes attributesData unquoted as+ // {"Name": "John"}, and core/common/CsvParser keeps such a quote literal as well. Anything but+ // a delimiter after a closing quote is still malformed, since that row cannot be read back.+ static List<String> splitCsvLine(String line) {+ List<String> fields = new ArrayList<>();+ StringBuilder current = new StringBuilder();+ boolean quoted = false;+ boolean afterClosingQuote = false;+ for (int i = 0; i < line.length(); i++) {+ char c = line.charAt(i);+ if (quoted) {+ if (c == '"') {+ if (i + 1 < line.length() && line.charAt(i + 1) == '"') {+ current.append('"');+ i++;+ } else {+ quoted = false;+ afterClosingQuote = true;+ }+ } else {+ current.append(c);+ }+ } else if (afterClosingQuote) {+ if (c != ',') {+ throw new InvalidRecordException("Malformed CSV: unexpected character after a closing quote.");+ }+ fields.add(current.toString());+ current.setLength(0);+ afterClosingQuote = false;+ } else if (c == '"' && current.isEmpty()) {+ quoted = true;+ } else if (c == ',') {+ fields.add(current.toString());+ current.setLength(0);+ } else {+ current.append(c);+ }+ }+ if (quoted) {+ throw new InvalidRecordException("Malformed CSV: unterminated quoted field.");+ }+ fields.add(current.toString());+ return fields;+ }++}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/ImportRecord.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/ImportRecord.javanew file mode 100644index 0000000000..70c50e7bdb--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/ImportRecord.java@@ -0,0 +1,18 @@+package io.github.hectorvent.floci.services.ses.imports;++import io.github.hectorvent.floci.services.ses.model.TopicPreference;++import java.util.List;++/**+ * One parsed source record. {@code error} is set when the record could not be turned into an+ * import (it still counts as processed and failed, and the job carries on), mirroring how AWS+ * reports a partially failed import as COMPLETED with a FailedRecordsCount.+ */+public record ImportRecord(int line, String emailAddress, String reason, Boolean unsubscribeAll,+ String attributesData, List<TopicPreference> topicPreferences, String error) {++ public static ImportRecord invalid(int line, String error) {+ return new ImportRecord(line, null, null, null, null, null, error);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/InvalidRecordException.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/InvalidRecordException.javanew file mode 100644index 0000000000..ac239cb8d6--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/InvalidRecordException.java@@ -0,0 +1,13 @@+package io.github.hectorvent.floci.services.ses.imports;+++/**+ * A source record that cannot be turned into an import. Caught by the reader that raised it and+ * reported as a failed record, so one bad row never fails the whole job.+ */+final class InvalidRecordException extends RuntimeException {++ InvalidRecordException(String message) {+ super(message);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReader.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReader.javanew file mode 100644index 0000000000..ef8d152d15--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReader.java@@ -0,0 +1,102 @@+package io.github.hectorvent.floci.services.ses.imports;++import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectReader;+import io.github.hectorvent.floci.services.ses.model.ImportJob;+import io.github.hectorvent.floci.services.ses.model.TopicPreference;++import java.io.BufferedReader;+import java.io.IOException;+import java.util.ArrayList;+import java.util.List;++/** Reads newline-delimited JSON: one complete object per line. */+final class JsonLineReader implements RecordReader {++ private static final String BOM = "\uFEFF";+ private final BufferedReader source;+ private final ImportJob job;+ private final ObjectReader lineReader;+ private int lineNumber;++ JsonLineReader(BufferedReader source, ImportJob job, ObjectReader lineReader) {+ this.source = source;+ this.job = job;+ this.lineReader = lineReader;+ }++ @Override+ public ImportRecord next() throws IOException {+ String read;+ while ((read = source.readLine()) != null) {+ lineNumber++;+ String line = (lineNumber == 1 ? stripBom(read) : read).trim();+ if (line.isEmpty()) {+ continue;+ }+ JsonNode node;+ try {+ node = lineReader.readTree(line);+ } catch (JsonProcessingException e) {+ return ImportRecord.invalid(lineNumber, "Malformed JSON: " + e.getOriginalMessage());+ }+ try {+ return jsonRecord(lineNumber, node, job);+ } catch (InvalidRecordException e) {+ return ImportRecord.invalid(lineNumber, e.getMessage());+ }+ }+ return null;+ }++ private static ImportRecord jsonRecord(int line, JsonNode node, ImportJob job) {+ if (!node.isObject()) {+ throw new InvalidRecordException("Each line must be a JSON object.");+ }+ String email = textMember(node, "emailAddress");+ if (ImportJob.DESTINATION_SUPPRESSION_LIST.equals(job.getDestinationType())) {+ return new ImportRecord(line, email, textMember(node, "reason"), null, null, null, null);+ }+ Boolean unsubscribeAll = null;+ JsonNode unsubscribeNode = node.get("unsubscribeAll");+ if (unsubscribeNode != null && !unsubscribeNode.isNull()) {+ if (!unsubscribeNode.isBoolean()) {+ throw new InvalidRecordException("unsubscribeAll must be a boolean.");+ }+ unsubscribeAll = unsubscribeNode.booleanValue();+ }+ List<TopicPreference> prefs = null;+ JsonNode prefsNode = node.get("topicPreferences");+ if (prefsNode != null && !prefsNode.isNull()) {+ if (!prefsNode.isArray()) {+ throw new InvalidRecordException("topicPreferences must be an array.");+ }+ prefs = new ArrayList<>();+ for (JsonNode p : prefsNode) {+ if (!p.isObject()) {+ throw new InvalidRecordException("Each topicPreferences element must be a JSON object.");+ }+ prefs.add(new TopicPreference(textMember(p, "topicName"), textMember(p, "subscriptionStatus")));+ }+ }+ return new ImportRecord(line, email, null, unsubscribeAll, textMember(node, "attributesData"), prefs, null);+ }++ // A present member of the wrong JSON type is an invalid record, not coerced text, matching the+ // SerializationException the CreateContact / PutSuppressedDestination bodies get.+ private static String textMember(JsonNode node, String field) {+ JsonNode member = node.get(field);+ if (member == null || member.isNull()) {+ return null;+ }+ if (!member.isTextual()) {+ throw new InvalidRecordException(field + " must be a JSON string.");+ }+ return member.textValue();+ }++ private static String stripBom(String text) {+ return text.startsWith(BOM) ? text.substring(BOM.length()) : text;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReader.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReader.javanew file mode 100644index 0000000000..da239be0d6--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReader.java@@ -0,0 +1,13 @@+package io.github.hectorvent.floci.services.ses.imports;++import java.io.IOException;++/**+ * Yields an import job's source records lazily, one at a time, so a large object is never held in+ * memory. {@code null} ends the source; a problem with the file as a whole rather than with one+ * record is an {@code AwsException} that fails the job.+ */+public interface RecordReader {++ ImportRecord next() throws IOException;+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReaderFactory.java b/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReaderFactory.javanew file mode 100644index 0000000000..c5a6ba2f22--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/imports/RecordReaderFactory.java@@ -0,0 +1,36 @@+package io.github.hectorvent.floci.services.ses.imports;++import com.fasterxml.jackson.databind.DeserializationFeature;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.ObjectReader;++import io.github.hectorvent.floci.services.ses.model.ImportJob;++import java.io.BufferedReader;++/**+ * Builds the reader for an import job's DataFormat. Takes the mapper rather than a ready-made+ * {@link ObjectReader} on purpose: reading a JSON line strictly is a requirement of the format, not+ * a caller's preference, so the setting lives here instead of leaking to whoever wires this up,+ * where a plain reader would compile and quietly accept malformed lines.+ */+public final class RecordReaderFactory {++ /**+ * FAIL_ON_TRAILING_TOKENS so a line like {@code {"emailAddress":"x"} garbage} is a malformed+ * record rather than a silently accepted object, as the JSON protocol controllers read their+ * bodies. Built once: an import reads thousands of lines through it.+ */+ private final ObjectReader lineReader;++ public RecordReaderFactory(ObjectMapper objectMapper) {+ this.lineReader = objectMapper.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);+ }++ /** CSV is parsed without Jackson, so only the JSON reader is handed the line reader. */+ public RecordReader forJob(ImportJob job, BufferedReader source) {+ return ImportJob.FORMAT_JSON.equals(job.getDataFormat())+ ? new JsonLineReader(source, job, lineReader)+ : new CsvRecordReader(source, job);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/model/ImportJob.java b/src/main/java/io/github/hectorvent/floci/services/ses/model/ImportJob.javanew file mode 100644index 0000000000..ffb6b6ad74--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ses/model/ImportJob.java@@ -0,0 +1,92 @@+package io.github.hectorvent.floci.services.ses.model;++import com.fasterxml.jackson.annotation.JsonIgnore;+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import com.fasterxml.jackson.annotation.JsonInclude;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;++/**+ * A SES V2 import job: the S3 source, the suppression-list or contact-list destination, and the+ * job's progress. The destination is flattened (type + action + optional contact list name) since+ * a job has exactly one; the controller rebuilds the AWS {@code ImportDestination} shape from it.+ */+@RegisterForReflection+@JsonIgnoreProperties(ignoreUnknown = true)+@JsonInclude(JsonInclude.Include.NON_NULL)+public class ImportJob {++ public static final String DESTINATION_SUPPRESSION_LIST = "SUPPRESSION_LIST";+ public static final String DESTINATION_CONTACT_LIST = "CONTACT_LIST";+ public static final String ACTION_PUT = "PUT";+ public static final String ACTION_DELETE = "DELETE";+ public static final String FORMAT_CSV = "CSV";+ public static final String FORMAT_JSON = "JSON";+ public static final String STATUS_CREATED = "CREATED";+ public static final String STATUS_PROCESSING = "PROCESSING";+ public static final String STATUS_COMPLETED = "COMPLETED";+ public static final String STATUS_FAILED = "FAILED";++ private String jobId;+ private String region;+ private String destinationType;+ private String importAction;+ private String contactListName;+ private String s3Url;+ private String dataFormat;+ private String jobStatus;+ private Instant createdTimestamp;+ private Instant completedTimestamp;+ private long processedRecordsCount;+ private long failedRecordsCount;+ private String errorMessage;++ public ImportJob() {}++ public String getJobId() { return jobId; }+ public void setJobId(String jobId) { this.jobId = jobId; }++ public String getRegion() { return region; }+ public void setRegion(String region) { this.region = region; }++ public String getDestinationType() { return destinationType; }+ public void setDestinationType(String destinationType) { this.destinationType = destinationType; }++ public String getImportAction() { return importAction; }+ public void setImportAction(String importAction) { this.importAction = importAction; }++ public String getContactListName() { return contactListName; }+ public void setContactListName(String contactListName) { this.contactListName = contactListName; }++ public String getS3Url() { return s3Url; }+ public void setS3Url(String s3Url) { this.s3Url = s3Url; }++ public String getDataFormat() { return dataFormat; }+ public void setDataFormat(String dataFormat) { this.dataFormat = dataFormat; }++ public String getJobStatus() { return jobStatus; }+ public void setJobStatus(String jobStatus) { this.jobStatus = jobStatus; }++ public Instant getCreatedTimestamp() { return createdTimestamp; }+ public void setCreatedTimestamp(Instant createdTimestamp) { this.createdTimestamp = createdTimestamp; }++ public Instant getCompletedTimestamp() { return completedTimestamp; }+ public void setCompletedTimestamp(Instant completedTimestamp) { this.completedTimestamp = completedTimestamp; }++ public long getProcessedRecordsCount() { return processedRecordsCount; }+ public void setProcessedRecordsCount(long processedRecordsCount) {+ this.processedRecordsCount = processedRecordsCount;+ }++ public long getFailedRecordsCount() { return failedRecordsCount; }+ public void setFailedRecordsCount(long failedRecordsCount) { this.failedRecordsCount = failedRecordsCount; }++ public String getErrorMessage() { return errorMessage; }+ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }++ @JsonIgnore+ public boolean isTerminal() {+ return STATUS_COMPLETED.equals(jobStatus) || STATUS_FAILED.equals(jobStatus);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobServiceTest.javanew file mode 100644index 0000000000..2934073fe2--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobServiceTest.java@@ -0,0 +1,656 @@+package io.github.hectorvent.floci.services.ses;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.ses.model.Contact;+import io.github.hectorvent.floci.services.ses.model.ImportJob;+import io.github.hectorvent.floci.services.ses.model.Topic;+import io.github.hectorvent.floci.services.ses.model.TopicPreference;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayInputStream;+import java.nio.charset.StandardCharsets;+import java.time.Clock;+import java.time.Instant;+import java.time.ZoneOffset;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.Executor;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.stream.Collectors;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * Unit test for the import-job domain: request validation, the CSV / newline-delimited JSON record+ * formats, per-record failure counting versus whole-job failure, and the concurrency limit. The+ * worker executor runs inline so a job is terminal by the time create returns; S3 is a mock, the+ * suppression and contact services are real ones over in-memory stores so the imported rows can be+ * read back.+ */+class SesImportJobServiceTest {++ private static final String REGION = "eu-west-3";+ private static final String ACCOUNT = "000000000000";+ private static final String BUCKET = "imports";+ private static final String LIST = "newsletter";++ private S3Service s3Service;+ private SesSuppressionService suppressionService;+ private SesContactService contactService;+ private InMemoryStorage<String, ImportJob> jobStore;+ private SesImportJobService service;++ @BeforeEach+ void setUp() {+ s3Service = mock(S3Service.class);+ Clock clock = Clock.fixed(Instant.parse("2026-09-18T00:00:00Z"), ZoneOffset.UTC);+ suppressionService = new SesSuppressionService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ new InMemoryStorage<>());+ contactService = new SesContactService(new InMemoryStorage<>(), new InMemoryStorage<>(), clock);+ contactService.createContactList(LIST, null,+ List.of(new Topic("Sports", "Sports", "OPT_OUT", null),+ new Topic("Cycling", "Cycling", "OPT_IN", null)),+ null, REGION);+ jobStore = new InMemoryStorage<>();+ service = newService(Runnable::run);+ }++ private SesImportJobService newService(Executor workers) {+ return new SesImportJobService(jobStore, s3Service, suppressionService, contactService,+ new ObjectMapper().findAndRegisterModules(), Clock.systemUTC(), workers);+ }++ private void stubObject(String key, String content) {+ when(s3Service.openObjectStream(BUCKET, key, null)).thenAnswer(+ invocation -> new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)));+ }++ private static String url(String key) {+ return "s3://" + BUCKET + "/" + key;+ }++ // ──────────────────────────── Suppression list ────────────────────────────++ @Test+ void suppressionCsvPut_addsEveryRowAndCompletes() {+ stubObject("add.csv", "[email protected],BOUNCE\[email protected],COMPLAINT\n");++ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("add.csv"), "CSV", "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());+ assertEquals(2, stored.getProcessedRecordsCount());+ assertEquals(0, stored.getFailedRecordsCount());+ assertNotNull(stored.getCompletedTimestamp());+ assertNull(stored.getErrorMessage());+ assertEquals("BOUNCE", suppressionService.getSuppressedDestination(REGION, "[email protected]").getReason());+ assertEquals("COMPLAINT", suppressionService.getSuppressedDestination(REGION, "[email protected]").getReason());+ }++ @Test+ void suppressionJsonPut_countsInvalidRowsAsFailedButCompletes() {+ stubObject("add.json", """+ {"emailAddress":"[email protected]","reason":"BOUNCE"}++ {"emailAddress":"[email protected]","reason":"SPAM"}+ not json at all+ {"emailAddress":"[email protected]","reason":"COMPLAINT"}+ """);++ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("add.json"), "JSON", "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());+ assertEquals(4, stored.getProcessedRecordsCount());+ assertEquals(2, stored.getFailedRecordsCount());+ assertEquals(2, suppressionService.listSuppressedDestinations(REGION, null).size());+ }++ @Test+ void suppressionCsvDelete_removesListedAddresses_missingOnesFail() {+ suppressionService.putSuppressedDestination(REGION, "[email protected]", "BOUNCE");+ stubObject("remove.csv", "[email protected]\[email protected]\n");++ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("remove.csv"), "CSV", "DELETE");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());+ assertEquals(2, stored.getProcessedRecordsCount());+ assertEquals(1, stored.getFailedRecordsCount());+ assertTrue(suppressionService.listSuppressedDestinations(REGION, null).isEmpty());+ }++ // ──────────────────────────── Contact list ────────────────────────────++ @Test+ void contactCsvPut_readsHeaderTopicColumnsAndQuotedAttributes() {+ stubObject("contacts.csv", """+ emailAddress,unsubscribeAll,attributesData,topicPreferences.Sports,topicPreferences.Cycling+ [email protected],false,"{""Name"":""Alice"",""City"":""Paris""}",OPT_IN,OPT_OUT+ [email protected],true,,OPT_OUT,+ """);++ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());+ assertEquals(2, stored.getProcessedRecordsCount());+ assertEquals(0, stored.getFailedRecordsCount());+ Contact alice = contactService.getContact(LIST, "[email protected]", REGION).contact();+ assertEquals("{\"Name\":\"Alice\",\"City\":\"Paris\"}", alice.getAttributesData());+ assertEquals(Map.of("Sports", "OPT_IN", "Cycling", "OPT_OUT"), preferences(alice));+ Contact bob = contactService.getContact(LIST, "[email protected]", REGION).contact();+ assertTrue(bob.isUnsubscribeAll());+ assertNull(bob.getAttributesData());+ assertEquals(Map.of("Sports", "OPT_OUT"), preferences(bob));+ }++ @Test+ void contactJsonPut_overridesExistingContact() {+ contactService.createContact(LIST, "[email protected]",+ List.of(new TopicPreference("Sports", "OPT_OUT")), false, "{\"old\":true}", REGION);+ stubObject("contacts.json", """+ {"emailAddress":"[email protected]","unsubscribeAll":true,"attributesData":"{\\"new\\":true}",+ "topicPreferences":[{"topicName":"Cycling","subscriptionStatus":"OPT_IN"}]}+ """.replace("\n", ""));++ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.json"), "JSON", LIST, "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());+ assertEquals(0, stored.getFailedRecordsCount());+ Contact alice = contactService.getContact(LIST, "[email protected]", REGION).contact();+ assertTrue(alice.isUnsubscribeAll());+ assertEquals("{\"new\":true}", alice.getAttributesData());+ // Probed against real AWS: a PUT import replaces the contact outright, so the Sports+ // preference this record omits is cleared rather than merged.+ assertEquals(Map.of("Cycling", "OPT_IN"), preferences(alice));++ // A record carrying only the address clears the rest for the same reason.+ stubObject("bare.json", "{\"emailAddress\":\"[email protected]\"}\n");+ service.createContactListImportJob(REGION, ACCOUNT, url("bare.json"), "JSON", LIST, "PUT");++ Contact cleared = contactService.getContact(LIST, "[email protected]", REGION).contact();+ assertEquals(Map.of(), preferences(cleared));+ assertFalse(cleared.isUnsubscribeAll());+ assertNull(cleared.getAttributesData());+ }++ @Test+ void contactCsv_unknownTopicAndBadBooleanAreFailedRecords() {+ stubObject("contacts.csv", """+ emailAddress,unsubscribeAll,topicPreferences.Chess+ [email protected],false,OPT_IN+ [email protected],maybe,+ """);++ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());+ assertEquals(3, stored.getProcessedRecordsCount());+ assertEquals(2, stored.getFailedRecordsCount());+ assertEquals(1, contactService.listContacts(LIST, REGION).contacts().size());+ }++ @Test+ void contactJsonDelete_removesContacts() {+ contactService.createContact(LIST, "[email protected]", null, false, null, REGION);+ stubObject("remove.json", "{\"emailAddress\":\"[email protected]\"}\n{\"emailAddress\":\"[email protected]\"}\n");++ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("remove.json"), "JSON", LIST, "DELETE");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());+ assertEquals(1, stored.getFailedRecordsCount());+ assertTrue(contactService.listContacts(LIST, REGION).contacts().isEmpty());+ }++ @Test+ void contactCsv_withoutEmailAddressHeader_failsTheJob() {+ stubObject("contacts.csv", "email,unsubscribeAll\[email protected],false\n");++ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_FAILED, stored.getJobStatus());+ assertTrue(stored.getErrorMessage().contains("emailAddress"));+ assertNotNull(stored.getCompletedTimestamp());+ }++ // ──────────────────────────── Whole-job failure ────────────────────────────++ @Test+ void missingObject_failsTheJobWithTheS3Message() {+ when(s3Service.openObjectStream(eq(BUCKET), anyString(), eq(null))).thenThrow(+ new AwsException("NoSuchKey", "The specified key does not exist.", 404));++ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("missing.csv"), "CSV", "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_FAILED, stored.getJobStatus());+ assertEquals("The specified key does not exist.", stored.getErrorMessage());+ assertEquals(0, stored.getProcessedRecordsCount());+ }++ @Test+ void contactListDeletedBeforeTheWorkerRuns_failsTheJob() {+ stubObject("contacts.csv", "emailAddress\[email protected]\n");+ // Hold the worker so the list can be deleted between create and run.+ List<Runnable> queued = new ArrayList<>();+ SesImportJobService deferred = newService(queued::add);++ ImportJob job = deferred.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");+ assertEquals(ImportJob.STATUS_CREATED, deferred.getImportJob(REGION, job.getJobId()).getJobStatus());+ contactService.deleteContactList(LIST, REGION);+ queued.forEach(Runnable::run);++ ImportJob stored = deferred.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_FAILED, stored.getJobStatus());+ assertNotNull(stored.getErrorMessage());+ }++ // ──────────────────────────── Validation ────────────────────────────++ @Test+ void create_rejectsBadUrlFormatAndAction() {+ assertBadRequest(() -> service.createSuppressionListImportJob(REGION, ACCOUNT, "https://x/y", "CSV", "PUT"));+ assertBadRequest(() -> service.createSuppressionListImportJob(REGION, ACCOUNT, "s3://bucket", "CSV", "PUT"));+ assertBadRequest(() -> service.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "XML", "PUT"));+ assertBadRequest(() -> service.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "UPSERT"));+ assertTrue(service.listImportJobs(REGION, null, null, null).isEmpty());+ }++ @Test+ void create_unknownContactList_isNotFound() {+ AwsException e = assertThrows(AwsException.class,+ () -> service.createContactListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "missing", "PUT"));+ assertEquals("NotFoundException", e.getErrorCode());+ // Probed against real AWS: this path words it differently from the contact-list APIs.+ assertEquals("ContactList <missing> doesn't exist", e.getMessage());+ assertTrue(service.listImportJobs(REGION, null, null, null).isEmpty());+ }++ @Test+ void create_rejectsMoreThanTwentyInProgressJobs() {+ stubObject("a.csv", "[email protected],BOUNCE\n");+ SesImportJobService parked = newService(task -> { });+ for (int i = 0; i < SesImportJobService.MAX_CONCURRENT_JOBS; i++) {+ parked.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT");+ }++ AwsException e = assertThrows(AwsException.class,+ () -> parked.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT"));+ assertEquals("LimitExceededException", e.getErrorCode());+ // Jobs in another region do not count against this region's limit.+ assertEquals(ImportJob.STATUS_CREATED,+ parked.createSuppressionListImportJob("us-east-1", ACCOUNT, url("a.csv"), "CSV", "PUT").getJobStatus());+ }++ @Test+ void getImportJob_unknownId_isNotFound() {+ AwsException e = assertThrows(AwsException.class, () -> service.getImportJob(REGION, "nope"));+ assertEquals("NotFoundException", e.getErrorCode());+ assertEquals(404, e.getHttpStatus());+ }++ @Test+ void listImportJobs_filtersByDestinationTypeAndRejectsBadPaging() {+ stubObject("a.csv", "[email protected],BOUNCE\n");+ stubObject("c.csv", "emailAddress\[email protected]\n");+ service.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT");+ service.createContactListImportJob(REGION, ACCOUNT, url("c.csv"), "CSV", LIST, "PUT");++ assertEquals(2, service.listImportJobs(REGION, null, null, null).size());+ assertEquals(1, service.listImportJobs(REGION, "CONTACT_LIST", 10, null).size());+ assertEquals(1, service.listImportJobs(REGION, "SUPPRESSION_LIST", null, null).size());+ assertTrue(service.listImportJobs("us-east-1", null, null, null).isEmpty());+ assertBadRequest(() -> service.listImportJobs(REGION, "EXPORT", null, null));+ assertBadRequest(() -> service.listImportJobs(REGION, null, 0, null));+ assertBadRequest(() -> service.listImportJobs(REGION, null, null, "token"));+ }++ @Test+ void jsonLine_withTrailingTokensOrWrongMemberTypes_isAFailedRecord() {+ stubObject("contacts.json", """+ {"emailAddress":"[email protected]"} garbage+ {"emailAddress":"[email protected]","attributesData":123}+ {"emailAddress":"[email protected]","topicPreferences":[{"topicName":"Sports","subscriptionStatus":true}]}+ {"emailAddress":"[email protected]","topicPreferences":["Sports"]}+ {"emailAddress":"[email protected]"}+ """);++ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.json"), "JSON", LIST, "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());+ assertEquals(5, stored.getProcessedRecordsCount());+ assertEquals(4, stored.getFailedRecordsCount());+ assertEquals(1, contactService.listContacts(LIST, REGION).contacts().size());+ }++ @Test+ void contactCsv_quotedFieldMayContainLineBreaks() {+ stubObject("contacts.csv", "emailAddress,attributesData\r\n"+ + "[email protected],\"{\r\n \"\"Name\"\": \"\"Alice\"\"\r\n}\"\r\n"+ + "[email protected],\r\n");++ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());+ assertEquals(2, stored.getProcessedRecordsCount());+ assertEquals(0, stored.getFailedRecordsCount());+ assertEquals("{\r\n \"Name\": \"Alice\"\r\n}",+ contactService.getContact(LIST, "[email protected]", REGION).contact().getAttributesData());+ }++ @Test+ void create_isSerializedAgainstTheConcurrencyLimit() throws InterruptedException {+ stubObject("a.csv", "[email protected],BOUNCE\n");+ SesImportJobService parked = newService(task -> { });+ int attempts = SesImportJobService.MAX_CONCURRENT_JOBS * 2;+ CountDownLatch start = new CountDownLatch(1);+ AtomicInteger rejected = new AtomicInteger();+ List<Thread> threads = new ArrayList<>();+ for (int i = 0; i < attempts; i++) {+ Thread t = new Thread(() -> {+ try {+ start.await();+ parked.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT");+ } catch (AwsException e) {+ rejected.incrementAndGet();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ });+ threads.add(t);+ t.start();+ }+ start.countDown();+ for (Thread t : threads) {+ t.join();+ }++ assertEquals(SesImportJobService.MAX_CONCURRENT_JOBS, rejected.get());+ assertEquals(SesImportJobService.MAX_CONCURRENT_JOBS, parked.listImportJobs(REGION, null, null, null).size());+ }++ @Test+ void beforeReset_stopsAQueuedWorkerBeforeItWrites() {+ stubObject("a.csv", "[email protected],BOUNCE\n");+ List<Runnable> queued = new ArrayList<>();+ SesImportJobService deferred = newService(queued::add);+ ImportJob job = deferred.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT");++ deferred.beforeReset();+ queued.forEach(Runnable::run);+ deferred.afterReset();++ assertEquals(ImportJob.STATUS_CREATED, deferred.getImportJob(REGION, job.getJobId()).getJobStatus());+ assertTrue(suppressionService.listSuppressedDestinations(REGION, null).isEmpty());+ }++ @Test+ void beforeReset_waitsForARunningWorkerAndBlocksCreatesUntilAfterReset() throws Exception {+ CountDownLatch s3Blocked = new CountDownLatch(1);+ CountDownLatch releaseS3 = new CountDownLatch(1);+ when(s3Service.openObjectStream(BUCKET, "slow.csv", null)).thenAnswer(invocation -> {+ s3Blocked.countDown();+ releaseS3.await();+ return new ByteArrayInputStream("[email protected],BOUNCE\n".getBytes(StandardCharsets.UTF_8));+ });+ stubObject("a.csv", "[email protected],BOUNCE\n");+ SesImportJobService threaded = newService(task -> new Thread(task).start());+ ImportJob slow = threaded.createSuppressionListImportJob(REGION, ACCOUNT, url("slow.csv"), "CSV", "PUT");+ assertTrue(s3Blocked.await(5, TimeUnit.SECONDS));++ Thread reset = new Thread(threaded::beforeReset);+ reset.start();+ reset.join(300);+ assertTrue(reset.isAlive(), "beforeReset must wait for the running worker");+ // The create has to race the drain, not follow it: a job admitted between beforeReset and+ // the storage wipe is exactly what the admission gate exists to stop.+ AtomicInteger created = new AtomicInteger();+ Thread create = new Thread(() -> {+ threaded.createSuppressionListImportJob(REGION, ACCOUNT, url("a.csv"), "CSV", "PUT");+ created.incrementAndGet();+ });+ create.start();+ create.join(300);+ assertEquals(0, created.get(), "a create arriving while the drain runs waits");++ releaseS3.countDown();+ reset.join(5_000);+ assertFalse(reset.isAlive(), "beforeReset returns once the worker has left");+ create.join(300);+ assertEquals(0, created.get(), "and keeps waiting until the reset finishes");++ threaded.afterReset();+ create.join(5_000);+ assertEquals(1, created.get());+ // The slow worker was abandoned: no write after the reset started.+ assertEquals(ImportJob.STATUS_PROCESSING, threaded.getImportJob(REGION, slow.getJobId()).getJobStatus());+ assertTrue(suppressionService.findSuppressedDestination(REGION, "[email protected]").isEmpty());+ }++ @Test+ void beforeReset_givesUpOnAWedgedWorkerInsteadOfHangingTheReset() throws Exception {+ CountDownLatch blocked = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ when(s3Service.openObjectStream(BUCKET, "wedged.csv", null)).thenAnswer(invocation -> {+ blocked.countDown();+ release.await();+ return new ByteArrayInputStream("[email protected],BOUNCE\n".getBytes(StandardCharsets.UTF_8));+ });+ SesImportJobService threaded = new SesImportJobService(jobStore, s3Service, suppressionService,+ contactService, new ObjectMapper().findAndRegisterModules(), Clock.systemUTC(),+ task -> new Thread(task).start(), 200);+ threaded.createSuppressionListImportJob(REGION, ACCOUNT, url("wedged.csv"), "CSV", "PUT");+ assertTrue(blocked.await(5, TimeUnit.SECONDS));++ // The worker never finishes, so the drain has to time out rather than block the reset.+ long started = System.currentTimeMillis();+ threaded.beforeReset();+ long waited = System.currentTimeMillis() - started;++ assertTrue(waited >= 200, "the drain should wait for its deadline, waited " + waited + "ms");+ assertTrue(waited < 5_000, "beforeReset must return on the deadline, waited " + waited + "ms");+ release.countDown();+ threaded.afterReset();+ }++ @Test+ void storedJobIsASnapshot_soAnEarlierPollIsNeverMutatedUnderTheReader() throws Exception {+ CountDownLatch blocked = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ when(s3Service.openObjectStream(BUCKET, "slow.csv", null)).thenAnswer(invocation -> {+ blocked.countDown();+ release.await();+ return new ByteArrayInputStream("[email protected],BOUNCE\n".getBytes(StandardCharsets.UTF_8));+ });+ SesImportJobService threaded = newService(task -> new Thread(task).start());+ ImportJob job = threaded.createSuppressionListImportJob(REGION, ACCOUNT, url("slow.csv"), "CSV", "PUT");+ assertTrue(blocked.await(5, TimeUnit.SECONDS));++ ImportJob midRun = threaded.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_PROCESSING, midRun.getJobStatus());+ release.countDown();+ ImportJob finished = awaitTerminal(threaded, job.getJobId());++ assertEquals(ImportJob.STATUS_COMPLETED, finished.getJobStatus());+ assertNotNull(finished.getCompletedTimestamp());+ assertEquals(1, finished.getProcessedRecordsCount());+ // The worker keeps mutating its own copy, so the record the earlier poll returned is frozen:+ // a poll can never catch a transition half applied (COMPLETED with no CompletedTimestamp).+ assertEquals(ImportJob.STATUS_PROCESSING, midRun.getJobStatus());+ assertEquals(0, midRun.getProcessedRecordsCount());+ assertNull(midRun.getCompletedTimestamp());+ }++ private static ImportJob awaitTerminal(SesImportJobService service, String jobId) throws InterruptedException {+ for (int i = 0; i < 250; i++) {+ ImportJob job = service.getImportJob(REGION, jobId);+ if (job.isTerminal()) {+ return job;+ }+ Thread.sleep(20);+ }+ throw new AssertionError("import job " + jobId + " never reached a terminal status");+ }++ @Test+ void longImportRepublishesItsCountersAsItRuns() {+ int records = SesImportJobService.PROGRESS_RECORD_INTERVAL * 2 + 5;+ StringBuilder csv = new StringBuilder();+ for (int i = 0; i < records; i++) {+ csv.append("user").append(i).append("@example.com,BOUNCE\n");+ }+ stubObject("bulk.csv", csv.toString());+ List<Long> publishedCounts = new ArrayList<>();+ StorageBackend<String, ImportJob> recording = new InMemoryStorage<>() {+ @Override+ public void put(String key, ImportJob value) {+ publishedCounts.add(value.getProcessedRecordsCount());+ super.put(key, value);+ }+ };+ SesImportJobService service = new SesImportJobService(recording, s3Service, suppressionService,+ contactService, new ObjectMapper().findAndRegisterModules(), Clock.systemUTC(), Runnable::run);++ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("bulk.csv"), "CSV", "PUT");++ assertEquals(records, service.getImportJob(REGION, job.getJobId()).getProcessedRecordsCount());+ // A poll partway through sees real progress rather than 0 until the job ends.+ assertEquals(List.of(0L, 0L, 100L, 200L, (long) records), publishedCounts);+ }++ @Test+ void suppressionCsv_wrongColumnCountIsAFailedRecord() {+ stubObject("mixed.csv", "[email protected],BOUNCE,extra\[email protected]\[email protected],COMPLAINT\n");++ ImportJob job = service.createSuppressionListImportJob(REGION, ACCOUNT, url("mixed.csv"), "CSV", "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(3, stored.getProcessedRecordsCount());+ assertEquals(2, stored.getFailedRecordsCount());+ assertEquals(1, suppressionService.listSuppressedDestinations(REGION, null).size());+ }++ @Test+ void csv_unterminatedQuote_failsTheJob() {+ stubObject("broken.csv", "emailAddress,attributesData\[email protected],\"{unterminated\n");++ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("broken.csv"), "CSV", LIST, "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_FAILED, stored.getJobStatus());+ assertTrue(stored.getErrorMessage().contains("unterminated"));+ assertTrue(contactService.listContacts(LIST, REGION).contacts().isEmpty());+ }++ @Test+ void contactListDeletedDuringTheRun_failsTheJob() {+ stubObject("contacts.csv", "emailAddress\[email protected]\[email protected]\n");+ SesContactService deletingService = mock(SesContactService.class);+ when(deletingService.getContactList(LIST, REGION)).thenReturn(contactService.getContactList(LIST, REGION));+ when(deletingService.createContact(eq(LIST), anyString(), any(),+ any(), any(), eq(REGION)))+ .thenAnswer(invocation -> {+ when(deletingService.getContactList(LIST, REGION))+ .thenThrow(new AwsException("NotFoundException", "List " + LIST + " not found.", 404));+ throw new AwsException("NotFoundException", "List " + LIST + " not found.", 404);+ });+ SesImportJobService racing = new SesImportJobService(jobStore, s3Service, suppressionService,+ deletingService, new ObjectMapper().findAndRegisterModules(), Clock.systemUTC(), Runnable::run);++ ImportJob job = racing.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");++ ImportJob stored = racing.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_FAILED, stored.getJobStatus());+ assertEquals("ContactList <" + LIST + "> doesn't exist", stored.getErrorMessage());+ assertEquals(1, stored.getProcessedRecordsCount());+ }++ @Test+ void contactCsv_wrongRowWidthOrBadQuotingIsAFailedRecord() {+ stubObject("contacts.csv", """+ emailAddress,attributesData+ [email protected],"{}",extra+ [email protected],"x"junk+ [email protected],ab"c+ """);++ ImportJob job = service.createContactListImportJob(REGION, ACCOUNT, url("contacts.csv"), "CSV", LIST, "PUT");++ ImportJob stored = service.getImportJob(REGION, job.getJobId());+ assertEquals(ImportJob.STATUS_COMPLETED, stored.getJobStatus());+ assertEquals(4, stored.getProcessedRecordsCount());+ // The wrong-width row and the one with data after a closing quote fail; carol's bare quote+ // is data, as the developer guide's own unquoted attributesData shows.+ assertEquals(2, stored.getFailedRecordsCount());+ assertEquals(2, contactService.listContacts(LIST, REGION).contacts().size());+ }++ @Test+ void recoverInterruptedJobs_failsNonTerminalJobsLeftInAnAccountAwareStore() {+ AccountAwareStorageBackend<ImportJob> persisted =+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, ACCOUNT);+ ImportJob stuck = new ImportJob();+ stuck.setJobId("stuck");+ stuck.setRegion(REGION);+ stuck.setDestinationType(ImportJob.DESTINATION_SUPPRESSION_LIST);+ stuck.setImportAction("PUT");+ stuck.setJobStatus(ImportJob.STATUS_PROCESSING);+ stuck.setCreatedTimestamp(Instant.now());+ persisted.putForAccount(ACCOUNT, SesImportJobService.key(REGION, "stuck"), stuck);+ ImportJob done = new ImportJob();+ done.setJobId("done");+ done.setRegion(REGION);+ done.setDestinationType(ImportJob.DESTINATION_SUPPRESSION_LIST);+ done.setImportAction("PUT");+ done.setJobStatus(ImportJob.STATUS_COMPLETED);+ done.setCreatedTimestamp(Instant.now());+ persisted.putForAccount(ACCOUNT, SesImportJobService.key(REGION, "done"), done);++ SesImportJobService restarted = new SesImportJobService(persisted, s3Service, suppressionService,+ contactService, new ObjectMapper().findAndRegisterModules(), Clock.systemUTC(), Runnable::run);++ ImportJob recovered = restarted.getImportJob(REGION, "stuck");+ assertEquals(ImportJob.STATUS_FAILED, recovered.getJobStatus());+ assertEquals(SesImportJobService.RESTART_FAILURE_MESSAGE, recovered.getErrorMessage());+ assertNotNull(recovered.getCompletedTimestamp());+ assertEquals(ImportJob.STATUS_COMPLETED, restarted.getImportJob(REGION, "done").getJobStatus());+ }++ private static void assertBadRequest(Runnable call) {+ AwsException e = assertThrows(AwsException.class, call::run);+ assertEquals("BadRequestException", e.getErrorCode());+ }++ private static Map<String, String> preferences(Contact contact) {+ return contact.getTopicPreferences().stream()+ .collect(Collectors.toMap(TopicPreference::getTopicName, TopicPreference::getSubscriptionStatus));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobV2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobV2IntegrationTest.javanew file mode 100644index 0000000000..0cee1ac41b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesImportJobV2IntegrationTest.java@@ -0,0 +1,309 @@+package io.github.hectorvent.floci.services.ses;++import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import java.util.function.Supplier;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasItems;+import static org.hamcrest.Matchers.hasSize;+import static org.hamcrest.Matchers.not;+import static org.hamcrest.Matchers.notNullValue;+import static org.hamcrest.Matchers.nullValue;+import static org.junit.jupiter.api.Assertions.assertEquals;++/**+ * Integration tests for the SES V2 import-job endpoints: {@code POST /v2/email/import-jobs}+ * (CreateImportJob), {@code GET /v2/email/import-jobs/{JobId}} (GetImportJob) and+ * {@code POST /v2/email/import-jobs/list} (ListImportJobs), end to end through the local S3+ * emulation into the suppression list and a contact list. The job runs on a background worker as+ * on AWS, so the tests poll GetImportJob until it is terminal. Uses its own region so the+ * suppression list and the one-per-region contact list do not collide with other SES tests.+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class SesImportJobV2IntegrationTest {++ private static final String AUTH =+ "AWS4-HMAC-SHA256 Credential=AKID/20260101/eu-west-3/ses/aws4_request";+ private static final String BUCKET = "ses-import-job-it";+ private static final String LIST = "import-job-list";++ private static String suppressionJobId;+ private static String contactJobId;++ @Test+ @Order(0)+ void setupBucketAndContactList() {+ given().when().put("/" + BUCKET).then().statusCode(200);+ given().contentType("application/json").header("Authorization", AUTH)+ .body("""+ {"ContactListName":"%s","Topics":[+ {"TopicName":"Sports","DisplayName":"Sports","DefaultSubscriptionStatus":"OPT_OUT"},+ {"TopicName":"Cycling","DisplayName":"Cycling","DefaultSubscriptionStatus":"OPT_IN"}]}+ """.formatted(LIST))+ .when().post("/v2/email/contact-lists").then().statusCode(200);+ }++ @Test+ @Order(1)+ void createImportJob_suppressionCsv_runsToCompletedAndFillsTheList() {+ putObject("suppress.csv", "[email protected],BOUNCE\[email protected],COMPLAINT\[email protected],SPAM\n");++ suppressionJobId = given().contentType("application/json").header("Authorization", AUTH)+ .body("""+ {"ImportDataSource":{"S3Url":"s3://%s/suppress.csv","DataFormat":"CSV"},+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"}}}+ """.formatted(BUCKET))+ .when().post("/v2/email/import-jobs").then().statusCode(200)+ .body("JobId", notNullValue())+ .extract().jsonPath().getString("JobId");++ assertEquals("COMPLETED", pollUntilTerminal(suppressionJobId));+ given().header("Authorization", AUTH)+ .when().get("/v2/email/import-jobs/" + suppressionJobId).then().statusCode(200)+ .body("JobId", equalTo(suppressionJobId))+ .body("JobStatus", equalTo("COMPLETED"))+ .body("ImportDataSource.S3Url", equalTo("s3://" + BUCKET + "/suppress.csv"))+ .body("ImportDataSource.DataFormat", equalTo("CSV"))+ .body("ImportDestination.SuppressionListDestination.SuppressionListImportAction", equalTo("PUT"))+ .body("ImportDestination.ContactListDestination", nullValue())+ .body("ProcessedRecordsCount", equalTo(3))+ .body("FailedRecordsCount", equalTo(1))+ .body("CreatedTimestamp", notNullValue())+ .body("CompletedTimestamp", notNullValue())+ .body("FailureInfo", nullValue());++ given().header("Authorization", AUTH)+ .when().get("/v2/email/suppression/addresses").then().statusCode(200)+ .body("SuppressedDestinationSummaries.EmailAddress",+ hasItems("[email protected]", "[email protected]"))+ .body("SuppressedDestinationSummaries", hasSize(2));+ }++ @Test+ @Order(2)+ void createImportJob_contactJson_addsContactsWithPreferences() {+ putObject("contacts.json", """+ {"emailAddress":"[email protected]","unsubscribeAll":false,"attributesData":"{\\"Name\\":\\"Carol\\"}","topicPreferences":[{"topicName":"Sports","subscriptionStatus":"OPT_IN"}]}+ {"emailAddress":"[email protected]","unsubscribeAll":true}+ """);++ contactJobId = given().contentType("application/json").header("Authorization", AUTH)+ .body("""+ {"ImportDataSource":{"S3Url":"s3://%s/contacts.json","DataFormat":"JSON"},+ "ImportDestination":{"ContactListDestination":{"ContactListName":"%s","ContactListImportAction":"PUT"}}}+ """.formatted(BUCKET, LIST))+ .when().post("/v2/email/import-jobs").then().statusCode(200)+ .extract().jsonPath().getString("JobId");++ assertEquals("COMPLETED", pollUntilTerminal(contactJobId));+ given().header("Authorization", AUTH)+ .when().get("/v2/email/import-jobs/" + contactJobId).then().statusCode(200)+ .body("ImportDestination.ContactListDestination.ContactListName", equalTo(LIST))+ .body("ImportDestination.ContactListDestination.ContactListImportAction", equalTo("PUT"))+ .body("ProcessedRecordsCount", equalTo(2))+ .body("FailedRecordsCount", equalTo(0));++ given().header("Authorization", AUTH)+ .when().get("/v2/email/contact-lists/" + LIST + "/contacts/[email protected]").then().statusCode(200)+ .body("AttributesData", equalTo("{\"Name\":\"Carol\"}"))+ .body("TopicPreferences[0].TopicName", equalTo("Sports"))+ .body("TopicPreferences[0].SubscriptionStatus", equalTo("OPT_IN"));+ given().header("Authorization", AUTH)+ .when().get("/v2/email/contact-lists/" + LIST + "/contacts/[email protected]").then().statusCode(200)+ .body("UnsubscribeAll", equalTo(true));+ }++ @Test+ @Order(3)+ void listImportJobs_returnsSummariesAndFiltersByDestinationType() {+ given().contentType("application/json").header("Authorization", AUTH)+ .when().post("/v2/email/import-jobs/list").then().statusCode(200)+ .body("ImportJobs.JobId", hasItems(suppressionJobId, contactJobId))+ .body("ImportJobs.find { it.JobId == '" + contactJobId + "' }.JobStatus", equalTo("COMPLETED"))+ .body("ImportJobs.find { it.JobId == '" + contactJobId + "' }.ImportDataSource", nullValue())+ .body("NextToken", nullValue());++ given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"ImportDestinationType\":\"SUPPRESSION_LIST\",\"PageSize\":10}")+ .when().post("/v2/email/import-jobs/list").then().statusCode(200)+ .body("ImportJobs.JobId", hasItems(suppressionJobId))+ .body("ImportJobs.JobId", not(hasItems(contactJobId)));++ given().contentType("application/json").header("Authorization", AUTH)+ .body("{\"ImportDestinationType\":\"EXPORT\"}")+ .when().post("/v2/email/import-jobs/list").then().statusCode(400)+ .body("__type", equalTo("BadRequestException"));+ }++ @Test+ @Order(4)+ void createImportJob_missingObject_failsTheJob() {+ String jobId = given().contentType("application/json").header("Authorization", AUTH)+ .body("""+ {"ImportDataSource":{"S3Url":"s3://%s/does-not-exist.csv","DataFormat":"CSV"},+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"DELETE"}}}+ """.formatted(BUCKET))+ .when().post("/v2/email/import-jobs").then().statusCode(200)+ .extract().jsonPath().getString("JobId");++ assertEquals("FAILED", pollUntilTerminal(jobId));+ given().header("Authorization", AUTH)+ .when().get("/v2/email/import-jobs/" + jobId).then().statusCode(200)+ .body("FailureInfo.ErrorMessage", containsString("does not exist"))+ .body("ProcessedRecordsCount", equalTo(0));+ }++ @Test+ @Order(5)+ void createImportJob_validationErrors() {+ String badFormat = """+ {"ImportDataSource":{"S3Url":"s3://%s/x.csv","DataFormat":"XML"},+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"}}}+ """.formatted(BUCKET);+ String bothDestinations = """+ {"ImportDataSource":{"S3Url":"s3://%s/x.csv","DataFormat":"CSV"},+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"},+ "ContactListDestination":{"ContactListName":"%s","ContactListImportAction":"PUT"}}}+ """.formatted(BUCKET, LIST);+ String noDataSource = """+ {"ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"}}}+ """;+ String unknownList = """+ {"ImportDataSource":{"S3Url":"s3://%s/x.csv","DataFormat":"CSV"},+ "ImportDestination":{"ContactListDestination":{"ContactListName":"nope","ContactListImportAction":"PUT"}}}+ """.formatted(BUCKET);++ String trailingTokens = """+ {"ImportDataSource":{"S3Url":"s3://%s/x.csv","DataFormat":"CSV"},+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"}}} garbage+ """.formatted(BUCKET);++ for (String body : new String[] {badFormat, bothDestinations, noDataSource, trailingTokens}) {+ given().contentType("application/json").header("Authorization", AUTH).body(body)+ .when().post("/v2/email/import-jobs").then().statusCode(400)+ .body("__type", equalTo("BadRequestException"));+ }+ given().contentType("application/json").header("Authorization", AUTH)+ .when().post("/v2/email/import-jobs").then().statusCode(400)+ .body("__type", equalTo("BadRequestException"));+ given().contentType("application/json").header("Authorization", AUTH).body(unknownList)+ .when().post("/v2/email/import-jobs").then().statusCode(404)+ .body("__type", equalTo("NotFoundException"));+ }++ @Test+ @Order(6)+ void getImportJob_unknownId_isNotFound() {+ given().header("Authorization", AUTH)+ .when().get("/v2/email/import-jobs/00000000-0000-0000-0000-000000000000").then().statusCode(404)+ .body("__type", equalTo("NotFoundException"));+ }++ @Test+ @Order(7)+ void importJobsAndTheirRecordsAreScopedToTheCallingAccount() {+ // A 12-digit access key id in the credential becomes the account (AccountContextFilter), so+ // these two headers are two accounts hitting the same region, neither of them the default+ // account the rest of this class runs as.+ String authA = accountAuth("111111111111", "ses");+ String authB = accountAuth("222222222222", "ses");+ // The credential's service scope routes the request, so the S3 calls need an s3-scoped one.+ String s3AuthA = accountAuth("111111111111", "s3");+ String bucketA = BUCKET + "-a";++ given().header("Authorization", s3AuthA).when().put("/" + bucketA).then().statusCode(200);+ given().header("Authorization", s3AuthA).contentType("text/plain").body("[email protected],BOUNCE\n")+ .when().put("/" + bucketA + "/suppress.csv").then().statusCode(200);++ String jobA = given().contentType("application/json").header("Authorization", authA)+ .body("""+ {"ImportDataSource":{"S3Url":"s3://%s/suppress.csv","DataFormat":"CSV"},+ "ImportDestination":{"SuppressionListDestination":{"SuppressionListImportAction":"PUT"}}}+ """.formatted(bucketA))+ .when().post("/v2/email/import-jobs").then().statusCode(200)+ .extract().jsonPath().getString("JobId");++ // Reaching COMPLETED under account A also proves the worker's terminal write landed in A's+ // partition: RequestScopes.runAs carries the account onto the worker thread, and without it+ // the write would go to the default account and this poll would never leave PROCESSING.+ assertEquals("COMPLETED", pollUntilTerminal(jobA, authA));++ given().header("Authorization", authB)+ .when().get("/v2/email/import-jobs/" + jobA).then().statusCode(404)+ .body("__type", equalTo("NotFoundException"));+ given().contentType("application/json").header("Authorization", authB)+ .when().post("/v2/email/import-jobs/list").then().statusCode(200)+ .body("ImportJobs.JobId", not(hasItems(jobA)));+ given().contentType("application/json").header("Authorization", authA)+ .when().post("/v2/email/import-jobs/list").then().statusCode(200)+ .body("ImportJobs.JobId", hasItems(jobA));++ // The imported rows are account-scoped too, not just the job record.+ given().header("Authorization", authA)+ .when().get("/v2/email/suppression/addresses").then().statusCode(200)+ .body("SuppressedDestinationSummaries.EmailAddress", hasItems("[email protected]"));+ given().header("Authorization", authB)+ .when().get("/v2/email/suppression/addresses").then().statusCode(200)+ .body("SuppressedDestinationSummaries.EmailAddress", not(hasItems("[email protected]")));++ given().header("Authorization", authA)+ .when().delete("/v2/email/suppression/addresses/[email protected]").then().statusCode(200);+ given().header("Authorization", s3AuthA).when().delete("/" + bucketA + "/suppress.csv").then().statusCode(204);+ given().header("Authorization", s3AuthA).when().delete("/" + bucketA).then().statusCode(204);+ }++ private static String accountAuth(String accountId, String service) {+ return "AWS4-HMAC-SHA256 Credential=" + accountId + "/20260101/eu-west-3/" + service + "/aws4_request";+ }++ @Test+ @Order(8)+ void cleanup() {+ given().header("Authorization", AUTH)+ .when().delete("/v2/email/contact-lists/" + LIST).then().statusCode(200);+ for (String address : new String[] {"[email protected]", "[email protected]"}) {+ given().header("Authorization", AUTH)+ .when().delete("/v2/email/suppression/addresses/" + address).then().statusCode(200);+ }+ }++ private static void putObject(String key, String content) {+ given().contentType("text/plain").body(content)+ .when().put("/" + BUCKET + "/" + key).then().statusCode(200);+ }++ private static String pollUntilTerminal(String jobId) {+ return pollUntilTerminal(jobId, AUTH);+ }++ private static String pollUntilTerminal(String jobId, String auth) {+ Supplier<Response> call = () -> given().header("Authorization", auth)+ .when().get("/v2/email/import-jobs/" + jobId);+ long deadline = System.currentTimeMillis() + 10_000;+ while (System.currentTimeMillis() < deadline) {+ String status = call.get().then().statusCode(200).extract().jsonPath().getString("JobStatus");+ if ("COMPLETED".equals(status) || "FAILED".equals(status)) {+ return status;+ }+ sleep(100);+ }+ return "TIMEOUT";+ }++ private static void sleep(long millis) {+ try {+ Thread.sleep(millis);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReaderTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReaderTest.javanew file mode 100644index 0000000000..260a052f6e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ses/imports/CsvRecordReaderTest.java@@ -0,0 +1,196 @@+package io.github.hectorvent.floci.services.ses.imports;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.ses.model.ImportJob;+import io.github.hectorvent.floci.services.ses.model.TopicPreference;+import org.junit.jupiter.api.Test;++import java.io.BufferedReader;+import java.io.IOException;+import java.io.StringReader;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.stream.Collectors;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Unit test for the CSV reader: where a record ends, how a field is split, and which malformed+ * input fails one record versus the whole file. Reading is driven through the factory, the way the+ * import worker reaches it.+ */+class CsvRecordReaderTest {++ private static ImportJob job(String destinationType, String action, String contactList) {+ ImportJob job = new ImportJob();+ job.setDataFormat(ImportJob.FORMAT_CSV);+ job.setDestinationType(destinationType);+ job.setImportAction(action);+ job.setContactListName(contactList);+ return job;+ }++ private static ImportJob suppression(String action) {+ return job(ImportJob.DESTINATION_SUPPRESSION_LIST, action, null);+ }++ private static ImportJob contacts() {+ return job(ImportJob.DESTINATION_CONTACT_LIST, ImportJob.ACTION_PUT, "newsletter");+ }++ private static RecordReader reader(ImportJob job, String content) {+ return new RecordReaderFactory(new ObjectMapper())+ .forJob(job, new BufferedReader(new StringReader(content)));+ }++ private static List<ImportRecord> readAll(ImportJob job, String content) throws IOException {+ RecordReader reader = reader(job, content);+ List<ImportRecord> records = new ArrayList<>();+ for (ImportRecord record = reader.next(); record != null; record = reader.next()) {+ records.add(record);+ }+ return records;+ }++ private static Map<String, String> preferences(ImportRecord record) {+ return record.topicPreferences().stream()+ .collect(Collectors.toMap(TopicPreference::getTopicName, TopicPreference::getSubscriptionStatus));+ }++ @Test+ void suppressionRows_carryTheAddressReasonAndLineNumber() throws IOException {+ List<ImportRecord> records = readAll(suppression(ImportJob.ACTION_PUT),+ "[email protected],BOUNCE\[email protected],COMPLAINT\n");++ assertEquals(2, records.size());+ assertEquals("[email protected]", records.get(0).emailAddress());+ assertEquals("BOUNCE", records.get(0).reason());+ assertEquals(1, records.get(0).line());+ assertEquals(2, records.get(1).line());+ }++ @Test+ void deleteRowsCarryNoReason_andAnExtraColumnFailsTheRecord() throws IOException {+ List<ImportRecord> records = readAll(suppression(ImportJob.ACTION_DELETE),+ "[email protected]\[email protected],BOUNCE\n");++ assertNull(records.get(0).reason());+ assertNull(records.get(0).error());+ assertTrue(records.get(1).error().contains("1 column(s) but found 2"));+ }++ @Test+ void quotedFieldMaySpanLines_andTheNextRecordKeepsCounting() throws IOException {+ List<ImportRecord> records = readAll(contacts(), "emailAddress,attributesData\r\n"+ + "[email protected],\"{\r\n \"\"Name\"\": \"\"Alice\"\"\r\n}\"\r\n"+ + "[email protected],\r\n");++ assertEquals(2, records.size());+ assertEquals("{\r\n \"Name\": \"Alice\"\r\n}", records.get(0).attributesData());+ // The header is record 1 and the multi-line row is record 2, so the row after it is 3:+ // records are numbered by position, not by physical line.+ assertEquals(2, records.get(0).line());+ assertEquals(3, records.get(1).line());+ assertEquals("[email protected]", records.get(1).emailAddress());+ }++ @Test+ void headerTopicColumnsBecomePreferences_andBlankCellsAreLeftOut() throws IOException {+ List<ImportRecord> records = readAll(contacts(),+ "emailAddress,unsubscribeAll,topicPreferences.Sports,topicPreferences.Cycling\n"+ + "[email protected],true,OPT_IN,\n");++ ImportRecord alice = records.get(0);+ assertEquals(Boolean.TRUE, alice.unsubscribeAll());+ assertEquals(Map.of("Sports", "OPT_IN"), preferences(alice));+ }++ @Test+ void aBomAndBlankLinesDoNotBecomeRecords() throws IOException {+ List<ImportRecord> records = readAll(suppression(ImportJob.ACTION_DELETE),+ "[email protected]\n\n\[email protected]\n");++ assertEquals(2, records.size());+ assertEquals("[email protected]", records.get(0).emailAddress());+ // Blank lines are skipped but still counted, so the reported line matches the file.+ assertEquals(4, records.get(1).line());+ }++ @Test+ void theDeveloperGuidesOwnContactRowIsAccepted() throws IOException {+ // Straight out of the SES list-management guide: attributesData is written unquoted, so the+ // quotes inside it are data. Rejecting them would refuse the documented format.+ List<ImportRecord> records = readAll(contacts(),+ "emailAddress,unsubscribeAll,attributesData,topicPreferences.Sports,topicPreferences.Cycling\n"+ + "[email protected],false,{\"Name\": \"John\"},OPT_IN,OPT_OUT\n"+ + "[email protected],true,,OPT_OUT,OPT_OUT\n");++ assertEquals(2, records.size());+ assertNull(records.get(0).error());+ assertEquals("{\"Name\": \"John\"}", records.get(0).attributesData());+ assertEquals(Map.of("Sports", "OPT_IN", "Cycling", "OPT_OUT"), preferences(records.get(0)));+ assertEquals(Boolean.TRUE, records.get(1).unsubscribeAll());+ }++ @Test+ void garbageAfterAClosingQuoteFailsOneRecordAndTheReaderCarriesOn() throws IOException {+ List<ImportRecord> records = readAll(contacts(), "emailAddress,attributesData\n"+ + "[email protected],\"x\"junk\n"+ + "[email protected],ab\"c\n"+ + "[email protected],fine\n");++ // A field that opened a quoted run has to end at the delimiter; anything else cannot be+ // read back unambiguously.+ assertTrue(records.get(0).error().contains("closing quote"));+ // A quote that never opened a run is data, so this row is fine.+ assertNull(records.get(1).error());+ assertEquals("ab\"c", records.get(1).attributesData());+ assertEquals("[email protected]", records.get(2).emailAddress());+ assertNull(records.get(2).error());+ }++ @Test+ void anUnterminatedQuoteFailsTheWholeFile() {+ RecordReader reader = reader(contacts(), "emailAddress,attributesData\[email protected],\"{oops\n");++ AwsException e = assertThrows(AwsException.class, () -> {+ for (ImportRecord record = reader.next(); record != null; record = reader.next()) {+ // drain until the scanner reaches the end of the unterminated field+ }+ });+ assertEquals("BadRequestException", e.getErrorCode());+ assertTrue(e.getMessage().contains("unterminated"));+ }++ @Test+ void aContactFileWithoutAnEmailAddressHeaderFailsTheWholeFile() {+ RecordReader reader = reader(contacts(), "email,unsubscribeAll\[email protected],false\n");++ AwsException e = assertThrows(AwsException.class, reader::next);+ assertEquals("BadRequestException", e.getErrorCode());+ assertTrue(e.getMessage().contains("emailAddress"));+ }++ @Test+ void anEmptyContactFileFailsForTheMissingHeader() {+ AwsException e = assertThrows(AwsException.class, () -> reader(contacts(), "").next());+ assertTrue(e.getMessage().contains("header line"));+ }++ @Test+ void anEmptySuppressionFileYieldsNoRecords() throws IOException {+ assertTrue(readAll(suppression(ImportJob.ACTION_PUT), "").isEmpty());+ }++ @Test+ void splitCsvLine_handlesQuotesCommasAndEscapedQuotes() {+ assertEquals(List.of("a", "b,c", "say \"hi\"", ""),+ CsvRecordReader.splitCsvLine("a,\"b,c\",\"say \"\"hi\"\"\","));+ assertEquals(List.of(""), CsvRecordReader.splitCsvLine(""));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReaderTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReaderTest.javanew file mode 100644index 0000000000..27bf2a997d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ses/imports/JsonLineReaderTest.java@@ -0,0 +1,126 @@+package io.github.hectorvent.floci.services.ses.imports;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.services.ses.model.ImportJob;+import io.github.hectorvent.floci.services.ses.model.TopicPreference;+import org.junit.jupiter.api.Test;++import java.io.BufferedReader;+import java.io.IOException;+import java.io.StringReader;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.stream.Collectors;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Unit test for the newline-delimited JSON reader: one object per line, and which malformed lines+ * become failed records. Reading is driven through the factory, the way the import worker reaches+ * it, so the strict parse the factory configures is exercised too.+ */+class JsonLineReaderTest {++ private static ImportJob job(String destinationType) {+ ImportJob job = new ImportJob();+ job.setDataFormat(ImportJob.FORMAT_JSON);+ job.setDestinationType(destinationType);+ job.setImportAction(ImportJob.ACTION_PUT);+ job.setContactListName("newsletter");+ return job;+ }++ private static List<ImportRecord> readAll(ImportJob job, String content) throws IOException {+ RecordReader reader = new RecordReaderFactory(new ObjectMapper())+ .forJob(job, new BufferedReader(new StringReader(content)));+ List<ImportRecord> records = new ArrayList<>();+ for (ImportRecord record = reader.next(); record != null; record = reader.next()) {+ records.add(record);+ }+ return records;+ }++ private static Map<String, String> preferences(ImportRecord record) {+ return record.topicPreferences().stream()+ .collect(Collectors.toMap(TopicPreference::getTopicName, TopicPreference::getSubscriptionStatus));+ }++ @Test+ void suppressionLinesCarryTheAddressAndReason() throws IOException {+ List<ImportRecord> records = readAll(job(ImportJob.DESTINATION_SUPPRESSION_LIST), """+ {"emailAddress":"[email protected]","reason":"BOUNCE"}+ {"emailAddress":"[email protected]"}+ """);++ assertEquals(2, records.size());+ assertEquals("[email protected]", records.get(0).emailAddress());+ assertEquals("BOUNCE", records.get(0).reason());+ assertNull(records.get(1).reason());+ }++ @Test+ void contactLinesCarryPreferencesAttributesAndTheUnsubscribeFlag() throws IOException {+ List<ImportRecord> records = readAll(job(ImportJob.DESTINATION_CONTACT_LIST), """+ {"emailAddress":"[email protected]","unsubscribeAll":false,"attributesData":"{\\"Name\\":\\"Alice\\"}",\+ "topicPreferences":[{"topicName":"Sports","subscriptionStatus":"OPT_IN"},\+ {"topicName":"Cycling","subscriptionStatus":"OPT_OUT"}]}+ """);++ ImportRecord alice = records.get(0);+ assertEquals(Boolean.FALSE, alice.unsubscribeAll());+ assertEquals("{\"Name\":\"Alice\"}", alice.attributesData());+ assertEquals(Map.of("Sports", "OPT_IN", "Cycling", "OPT_OUT"), preferences(alice));+ }++ @Test+ void aBomAndBlankLinesDoNotBecomeRecords() throws IOException {+ List<ImportRecord> records = readAll(job(ImportJob.DESTINATION_SUPPRESSION_LIST),+ "{\"emailAddress\":\"[email protected]\"}\n\n \n{\"emailAddress\":\"[email protected]\"}\n");++ assertEquals(2, records.size());+ assertEquals("[email protected]", records.get(0).emailAddress());+ // Blank lines are skipped but still counted, so the reported line matches the file.+ assertEquals(4, records.get(1).line());+ }++ @Test+ void malformedJsonAndTrailingTokensAreFailedRecords() throws IOException {+ List<ImportRecord> records = readAll(job(ImportJob.DESTINATION_SUPPRESSION_LIST), """+ not json at all+ {"emailAddress":"[email protected]"} garbage+ ["emailAddress"]+ {"emailAddress":"[email protected]"}+ """);++ assertTrue(records.get(0).error().contains("Malformed JSON"));+ // The factory configures FAIL_ON_TRAILING_TOKENS, so the leading object is not accepted.+ assertTrue(records.get(1).error().contains("Malformed JSON"));+ assertTrue(records.get(2).error().contains("JSON object"));+ assertNull(records.get(3).error());+ }++ @Test+ void membersOfTheWrongTypeAreFailedRecordsRatherThanCoerced() throws IOException {+ List<ImportRecord> records = readAll(job(ImportJob.DESTINATION_CONTACT_LIST), """+ {"emailAddress":"[email protected]","attributesData":123}+ {"emailAddress":"[email protected]","unsubscribeAll":"yes"}+ {"emailAddress":"[email protected]","topicPreferences":"Sports"}+ {"emailAddress":"[email protected]","topicPreferences":["Sports"]}+ {"emailAddress":"[email protected]","topicPreferences":[{"topicName":"Sports","subscriptionStatus":true}]}+ """);++ assertTrue(records.get(0).error().contains("attributesData"));+ assertTrue(records.get(1).error().contains("unsubscribeAll"));+ assertTrue(records.get(2).error().contains("topicPreferences"));+ assertTrue(records.get(3).error().contains("JSON object"));+ assertTrue(records.get(4).error().contains("subscriptionStatus"));+ }++ @Test+ void anEmptySourceYieldsNoRecords() throws IOException {+ assertTrue(readAll(job(ImportJob.DESTINATION_SUPPRESSION_LIST), "").isEmpty());+ }+}