Light, fluffy, and always free - The AWS Local Emulator alternative
feat(emr-serverless): start, get, list and cancel job runs (#6202)
Priamo Germosen1f1e5db1bc7d
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.javanew file mode 100644index 0000000000..de1e6ad277--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java@@ -0,0 +1,149 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.services.emrserverless.EmrServerlessClient;+import software.amazon.awssdk.services.emrserverless.model.Application;+import software.amazon.awssdk.services.emrserverless.model.ApplicationState;+import software.amazon.awssdk.services.emrserverless.model.ConflictException;+import software.amazon.awssdk.services.emrserverless.model.JobRun;+import software.amazon.awssdk.services.emrserverless.model.JobRunMode;+import software.amazon.awssdk.services.emrserverless.model.JobRunState;+import software.amazon.awssdk.services.emrserverless.model.JobRunSummary;+import software.amazon.awssdk.services.emrserverless.model.StartJobRunResponse;+import software.amazon.awssdk.services.emrserverless.model.ValidationException;++import java.time.Instant;+import java.time.temporal.ChronoUnit;+import java.util.List;+import java.util.Map;+import java.util.UUID;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;++@DisplayName("EMR Serverless job runs")+class EmrServerlessJobRunTest {++ private static final String ROLE = "arn:aws:iam::000000000000:role/emr-serverless-job";++ private static EmrServerlessClient emr;+ private static String applicationId;++ @BeforeAll+ static void setup() {+ emr = TestFixtures.emrServerlessClient();+ applicationId = emr.createApplication(r -> r+ .name(TestFixtures.uniqueName("jobs-app"))+ .releaseLabel("emr-7.5.0")+ .type("SPARK")+ .clientToken(UUID.randomUUID().toString())).applicationId();+ }++ @AfterAll+ static void cleanup() {+ if (emr == null) {+ return;+ }+ try {+ emr.stopApplication(r -> r.applicationId(applicationId));+ emr.deleteApplication(r -> r.applicationId(applicationId));+ }+ catch (Exception ignored) {+ // Cleanup is best effort: the application is unique to this run and the emulator is disposable.+ }+ emr.close();+ }++ private static StartJobRunResponse startSparkRun(JobRunMode mode) {+ return emr.startJobRun(r -> r+ .applicationId(applicationId)+ .executionRoleArn(ROLE)+ .name("nightly")+ .mode(mode)+ .tags(Map.of("team", "data"))+ .jobDriver(d -> d.sparkSubmit(s -> s+ .entryPoint("s3://bucket/jobs/app.jar")+ .entryPointArguments("s3://bucket/in")+ .sparkSubmitParameters("--class example.Main"))));+ }++ @Test+ @DisplayName("StartJobRun auto-starts the application and GetJobRun reports a finished run")+ void startAndGet() {+ Instant before = Instant.now().minus(1, ChronoUnit.DAYS);+ StartJobRunResponse started = startSparkRun(JobRunMode.BATCH);+ Instant after = Instant.now().plus(1, ChronoUnit.DAYS);++ assertThat(started.applicationId()).isEqualTo(applicationId);+ assertThat(started.arn()).endsWith("/applications/" + applicationId + "/jobruns/" + started.jobRunId());+ Application application = emr.getApplication(r -> r.applicationId(applicationId)).application();+ assertThat(application.state()).isEqualTo(ApplicationState.STARTED);+ // Epoch seconds on the wire; milliseconds would read as a year past 50,000.+ assertThat(application.createdAt()).isBetween(before, after);+ assertThat(application.updatedAt()).isBetween(application.createdAt(), after);++ JobRun run = emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(started.jobRunId())).jobRun();+ assertThat(run.state()).isEqualTo(JobRunState.SUCCESS);+ assertThat(run.executionRole()).isEqualTo(ROLE);+ assertThat(run.releaseLabel()).isEqualTo("emr-7.5.0");+ assertThat(run.createdBy()).startsWith("arn:aws:");+ assertThat(run.createdAt()).isBetween(before, after);+ assertThat(run.jobDriver().sparkSubmit().entryPoint()).isEqualTo("s3://bucket/jobs/app.jar");+ assertThat(run.tags()).containsEntry("team", "data");+ assertThat(run.executionTimeoutMinutes()).isEqualTo(720L);+ assertThat(run.mode()).isEqualTo(JobRunMode.BATCH);+ }++ @Test+ @DisplayName("A repeated client token returns the original run; reused with another request it conflicts")+ void clientTokenIdempotency() {+ String token = UUID.randomUUID().toString();+ String first = emr.startJobRun(r -> r.applicationId(applicationId).clientToken(token)+ .executionRoleArn(ROLE).jobDriver(d -> d.sparkSubmit(s -> s.entryPoint("s3://bucket/a.jar"))))+ .jobRunId();+ String again = emr.startJobRun(r -> r.applicationId(applicationId).clientToken(token)+ .executionRoleArn(ROLE).jobDriver(d -> d.sparkSubmit(s -> s.entryPoint("s3://bucket/a.jar"))))+ .jobRunId();+ assertThat(again).isEqualTo(first);++ assertThatThrownBy(() -> emr.startJobRun(r -> r.applicationId(applicationId).clientToken(token)+ .executionRoleArn(ROLE).jobDriver(d -> d.sparkSubmit(s -> s.entryPoint("s3://bucket/b.jar")))))+ .isInstanceOf(ConflictException.class);+ }++ @Test+ @DisplayName("CancelJobRun cancels a streaming run and refuses a finished one")+ void cancel() {+ String streaming = startSparkRun(JobRunMode.STREAMING).jobRunId();+ assertThat(emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(streaming)).jobRun().state())+ .isEqualTo(JobRunState.RUNNING);++ emr.cancelJobRun(r -> r.applicationId(applicationId).jobRunId(streaming));+ assertThat(emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(streaming)).jobRun().state())+ .isEqualTo(JobRunState.CANCELLED);++ assertThatThrownBy(() -> emr.cancelJobRun(r -> r.applicationId(applicationId).jobRunId(streaming)))+ .isInstanceOf(ValidationException.class)+ .hasMessageContaining("Job run is not in a cancellable state");+ }++ @Test+ @DisplayName("ListJobRuns returns summaries filtered by state")+ void list() {+ String streaming = startSparkRun(JobRunMode.STREAMING).jobRunId();+ try {+ List<JobRunSummary> running = emr.listJobRuns(r -> r.applicationId(applicationId)+ .states(JobRunState.RUNNING)).jobRuns();+ assertThat(running).extracting(JobRunSummary::id).contains(streaming);+ assertThat(running).allSatisfy(summary -> assertThat(summary.state()).isEqualTo(JobRunState.RUNNING));+ assertThat(emr.listJobRunsPaginator(r -> r.applicationId(applicationId)).jobRuns())+ .extracting(JobRunSummary::applicationId).containsOnly(applicationId);+ }+ finally {+ emr.cancelJobRun(r -> r.applicationId(applicationId).jobRunId(streaming));+ }+ }+}diff --git a/docs/services/emr-serverless.md b/docs/services/emr-serverless.mdindex c6d72375ae..cdeda3875e 100644--- a/docs/services/emr-serverless.md+++ b/docs/services/emr-serverless.md@@ -1,6 +1,6 @@# EMR Serverless-EMR Serverless is an AWS service that provides serverless data analytics environments. Floci provides an emulator for the management-plane API (REST JSON) which covers the basic lifecycle of an Application.+EMR Serverless is an AWS service that provides serverless data analytics environments. Floci provides an emulator for the management-plane API (REST JSON) which covers the lifecycle of an Application and of the job runs submitted to it.## Configuration@@ -8,6 +8,7 @@ EMR Serverless is an AWS service that provides serverless data analytics environ|-----|-------------|---------|| `floci.emrserverless.enabled` | Whether the EMR Serverless API is enabled | `true` || `floci.emrserverless.port` | The port the service is exposed on | `4566` |+| `floci.services.emrserverless.job-run-duration-seconds` | How long a `BATCH` job run stays `RUNNING` before it succeeds; `0` makes it `SUCCESS` as soon as it starts | `0` |## Endpoints@@ -20,6 +21,10 @@ The emulator implements the standard AWS `emr-serverless` service endpoints:* `DELETE /applications/{applicationId}` (DeleteApplication)* `POST /applications/{applicationId}/start` (StartApplication)* `POST /applications/{applicationId}/stop` (StopApplication)+* `POST /applications/{applicationId}/jobruns` (StartJobRun)+* `GET /applications/{applicationId}/jobruns` (ListJobRuns)+* `GET /applications/{applicationId}/jobruns/{jobRunId}` (GetJobRun)+* `DELETE /applications/{applicationId}/jobruns/{jobRunId}` (CancelJobRun)* `POST /tags/{resourceArn}` (TagResource)* `DELETE /tags/{resourceArn}?tagKeys=...` (UntagResource)* `GET /tags/{resourceArn}` (ListTagsForResource)@@ -36,10 +41,39 @@ An application created in one Region is not returned by list, get, or tag requesreport that state so lifecycle waiters can observe it, while an unfiltered `ListApplications`omits the terminated application. A terminated application cannot be started or stopped again.+## Job runs++Job runs are metadata only: nothing executes. A run follows the job run states and `GetJobRun`'s+response shape, and settles when it is read.++* **Lifecycle**: a run is `RUNNING` from the moment it is submitted. With the default+ `job-run-duration-seconds` of `0`, a `BATCH` run is `SUCCESS` as soon as it starts; a positive+ duration keeps it `RUNNING` that long. A run whose duration would pass its+ `executionTimeoutMinutes` (720 when not given, as AWS reports it) is `CANCELLED` at the timeout.+ A `STREAMING` run stays `RUNNING` until it is cancelled.+* **Auto-start**: a run submitted to a `CREATED` or `STOPPED` application starts the application,+ as `autoStartConfiguration.enabled` defaults to `true`. With auto-start turned off, the run is+ refused with `ValidationException` until the application is started. A run on a terminated+ application is refused.+* **Idempotency**: `clientToken` is the idempotency token. Repeating a request with the same token+ returns the original run; reusing the token with a different request answers `ConflictException`+ (HTTP 409) "Request did not match the original".+* **CancelJobRun** moves a run that has not finished to `CANCELLED`. A finished run answers+ `ValidationException` "Job run is not in a cancellable state".+* **ListJobRuns** returns the newest runs first, filtered by `states`, `createdAtAfter`,+ `createdAtBefore` and `mode`, at most 50 per page. With several `states`, the runs are grouped by+ state in the order the filter names them. An unknown application has no runs. A `nextToken`+ resumes only the listing it came from, with the same application and filters.+* `createdBy` is the caller's ARN, as `GetCallerIdentity` reports it. `releaseLabel` and+ `networkConfiguration` come from the application.+## Limitations and Differences from AWS-* **Jobs not implemented**: Floci supports the provisioning of the `Application` management plane (which satisfies tools like Terraform's `aws_emrserverless_application` resource). Executing actual Spark or Hive jobs against these applications via `StartJobRun` is not currently implemented.+* **Jobs do not execute**: `StartJobRun` records a job run and moves it through its states, but no Spark or Hive job runs.+* **Unsourced errors**: AWS documents neither the error for a run submitted to an application whose auto-start is off nor the one for a run on a terminated application; Floci answers both with `ValidationException`. A run needs a `jobDriver`, as every `JobRun` carries one, and `jobDriver` is a union, so a run without a driver, or with one that sets both `sparkSubmit` and `hive` or neither, is refused with `ValidationException` too, in Floci's own wording.+* **StopApplication with running jobs**: AWS requires running jobs to finish or be cancelled before an application stops. Floci does not enforce it; with the default duration of `0` no `BATCH` run is ever running.+* **Not emulated**: job run attempts and retries, queuing (`QUEUED`), `totalResourceUtilization` and `billedResourceUtilization`, and `shutdownGracePeriodInSeconds` (accepted and ignored).* **Instant Start/Stop**: Floci marks the application `STARTED` or `STOPPED` immediately without provisioning actual compute capacity in the background.* **Data Plane**: Floci does not implement the data-plane API or any execution environments.-* **Job run tags**: job runs are not emulated, so a job run ARN (`.../jobruns/<id>`) answers- `ResourceNotFoundException` on the tagging APIs.+* **Job run tags**: the tagging APIs do not accept job run ARNs (`.../jobruns/<id>`) yet; they answer+ `ResourceNotFoundException`. Tags given to `StartJobRun` are stored and returned by `GetJobRun`.diff --git a/docs/services/index.md b/docs/services/index.mdindex 851d1db247..1181b191f7 100644--- a/docs/services/index.md+++ b/docs/services/index.md@@ -80,7 +80,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac| [Redshift Data API](redshift-data.md) | `POST /` + `X-Amz-Target: RedshiftData.*` | JSON 1.1 | 11 || [Redshift Serverless](redshift-serverless.md) | `POST /` + `X-Amz-Target: RedshiftServerless.*` | JSON 1.1 | 55 || [EMR](emr.md) | `POST /` + `X-Amz-Target: ElasticMapReduce.*` | JSON 1.1 | 40 |-| [EMR Serverless](emr-serverless.md) | `/applications/*`, `/tags/*` | REST JSON | 10 |+| [EMR Serverless](emr-serverless.md) | `/applications/*`, `/tags/*` | REST JSON | 14 || [Data Firehose](firehose.md) | `POST /` + `X-Amz-Target: Firehose_20150804.*` | JSON 1.1 | 12 || [ECS](ecs.md) | `POST /` + `X-Amz-Target: AmazonEC2ContainerServiceV20141113.*` | JSON 1.1 | 59 || [EFS](efs.md) | `/2015-02-01/...` | REST JSON | 26 |diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 48f74cb775..0a6e1ed0e4 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1085,6 +1085,10 @@ public interface EmulatorConfig {interface EmrServerlessServiceConfig {@WithDefault("true")boolean enabled();++ /** How long a BATCH job run stays RUNNING before it succeeds; 0 = a run succeeds as soon as it starts. */+ @WithDefault("0")+ int jobRunDurationSeconds();}interface Route53ResolverServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.javaindex b09b7e76e9..063cf0e8e4 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java@@ -4,13 +4,20 @@ import io.github.hectorvent.floci.core.common.PaginatedResult;import io.github.hectorvent.floci.core.common.Pagination;import io.github.hectorvent.floci.services.emrserverless.model.Application;import io.github.hectorvent.floci.services.emrserverless.model.ApplicationSummary;+import io.github.hectorvent.floci.services.emrserverless.model.CancelJobRunResponse;import io.github.hectorvent.floci.services.emrserverless.model.CreateApplicationRequest;import io.github.hectorvent.floci.services.emrserverless.model.CreateApplicationResponse;import io.github.hectorvent.floci.services.emrserverless.model.DeleteApplicationResponse;import io.github.hectorvent.floci.services.emrserverless.model.GetApplicationResponse;+import io.github.hectorvent.floci.services.emrserverless.model.GetJobRunResponse;+import io.github.hectorvent.floci.services.emrserverless.model.JobRun;+import io.github.hectorvent.floci.services.emrserverless.model.JobRunSummary;import io.github.hectorvent.floci.services.emrserverless.model.ListApplicationsRequest;import io.github.hectorvent.floci.services.emrserverless.model.ListApplicationsResponse;+import io.github.hectorvent.floci.services.emrserverless.model.ListJobRunsResponse;import io.github.hectorvent.floci.services.emrserverless.model.StartApplicationResponse;+import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunRequest;+import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunResponse;import io.github.hectorvent.floci.services.emrserverless.model.StopApplicationResponse;import io.github.hectorvent.floci.services.emrserverless.model.UpdateApplicationRequest;import io.github.hectorvent.floci.services.emrserverless.model.UpdateApplicationResponse;@@ -36,10 +43,12 @@ import java.util.List;public class EmrServerlessController {private final EmrServerlessService service;+ private final EmrServerlessJobRunService jobRuns;@Inject- public EmrServerlessController(EmrServerlessService service) {+ public EmrServerlessController(EmrServerlessService service, EmrServerlessJobRunService jobRuns) {this.service = service;+ this.jobRuns = jobRuns;}@POST@@ -116,4 +125,55 @@ public class EmrServerlessController {StopApplicationResponse response = new StopApplicationResponse();return response;}++ @POST+ @Path("/{applicationId}/jobruns")+ public StartJobRunResponse startJobRun(@PathParam("applicationId") String applicationId,+ StartJobRunRequest request) {+ JobRun run = jobRuns.startJobRun(applicationId, request);+ StartJobRunResponse response = new StartJobRunResponse();+ response.setApplicationId(run.getApplicationId());+ response.setJobRunId(run.getJobRunId());+ response.setArn(run.getArn());+ return response;+ }++ @GET+ @Path("/{applicationId}/jobruns")+ public ListJobRunsResponse listJobRuns(@PathParam("applicationId") String applicationId,+ @QueryParam("states") List<String> states,+ @QueryParam("createdAtAfter") String createdAtAfter,+ @QueryParam("createdAtBefore") String createdAtBefore,+ @QueryParam("mode") String mode,+ @QueryParam("maxResults") String maxResults,+ @QueryParam("nextToken") String nextToken) {+ PaginatedResult<JobRunSummary> page = jobRuns.listJobRuns(applicationId, states,+ EmrServerlessJobRunService.parseTimestamp(createdAtAfter, "createdAtAfter"),+ EmrServerlessJobRunService.parseTimestamp(createdAtBefore, "createdAtBefore"),+ mode, Pagination.parseMaxResults(maxResults, "ValidationException"), nextToken);+ ListJobRunsResponse response = new ListJobRunsResponse();+ response.setJobRuns(page.items());+ response.setNextToken(page.nextToken());+ return response;+ }++ @GET+ @Path("/{applicationId}/jobruns/{jobRunId}")+ public GetJobRunResponse getJobRun(@PathParam("applicationId") String applicationId,+ @PathParam("jobRunId") String jobRunId) {+ GetJobRunResponse response = new GetJobRunResponse();+ response.setJobRun(jobRuns.getJobRun(applicationId, jobRunId));+ return response;+ }++ @DELETE+ @Path("/{applicationId}/jobruns/{jobRunId}")+ public CancelJobRunResponse cancelJobRun(@PathParam("applicationId") String applicationId,+ @PathParam("jobRunId") String jobRunId) {+ JobRun run = jobRuns.cancelJobRun(applicationId, jobRunId);+ CancelJobRunResponse response = new CancelJobRunResponse();+ response.setApplicationId(run.getApplicationId());+ response.setJobRunId(run.getJobRunId());+ return response;+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.javanew file mode 100644index 0000000000..15cc34739a--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java@@ -0,0 +1,416 @@+package io.github.hectorvent.floci.services.emrserverless;++import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.PaginatedResult;+import io.github.hectorvent.floci.core.common.Pagination;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.RequestContext;+import io.github.hectorvent.floci.core.common.SmithyValidationErrors;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.emrserverless.model.Application;+import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;+import io.github.hectorvent.floci.services.emrserverless.model.JobRun;+import io.github.hectorvent.floci.services.emrserverless.model.JobRunRecord;+import io.github.hectorvent.floci.services.emrserverless.model.JobRunSummary;+import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunRequest;+import io.github.hectorvent.floci.services.iam.IamService;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.time.Clock;+import java.time.Duration;+import java.time.Instant;+import java.time.OffsetDateTime;+import java.time.format.DateTimeParseException;+import java.time.temporal.ChronoUnit;+import java.util.ArrayList;+import java.util.HashMap;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.regex.Pattern;++/**+ * EMR Serverless job runs. Nothing executes: a run follows the job run states and GetJobRun's+ * shape, and settles lazily when read. Times are kept to the millisecond, as AWS reports them. With the default {@code job-run-duration-seconds} of 0 a+ * BATCH run is SUCCESS as soon as it starts; a positive duration keeps it RUNNING that long, which+ * makes CancelJobRun and the execution timeout observable. A STREAMING run, which EMR Serverless+ * "expects to run for a long time", stays RUNNING until it is cancelled.+ *+ * <p>Every public method is synchronized: admitting a run (look up its client token, then write+ * it), settling one and cancelling one each read and then write a run, and must not interleave.+ */+@ApplicationScoped+public class EmrServerlessJobRunService {++ private static final Logger LOG = Logger.getLogger(EmrServerlessJobRunService.class);++ static final String STATE_RUNNING = "RUNNING";+ static final String STATE_SUCCESS = "SUCCESS";+ static final String STATE_CANCELLED = "CANCELLED";+ private static final String MODE_BATCH = "BATCH";+ private static final String MODE_STREAMING = "STREAMING";++ private static final Set<String> TERMINAL_STATES = Set.of(STATE_SUCCESS, "FAILED", STATE_CANCELLED);+ private static final List<String> MODES = List.of(MODE_BATCH, MODE_STREAMING);+ // JobRun.executionTimeoutMinutes: "If you didn't specify a timeout, this value defaults to 720 minutes."+ private static final long DEFAULT_EXECUTION_TIMEOUT_MINUTES = 720;+ // The model's Duration, which bounds executionTimeoutMinutes.+ private static final long MAX_EXECUTION_TIMEOUT_MINUTES = 1_000_000;+ private static final int MAX_LIST_RESULTS = 50;+ // IAMRoleArn's pattern, as the model and AWS's validation message spell it.+ private static final String ROLE_ARN_PATTERN =+ "arn:(aws[a-zA-Z0-9-]*):iam::([0-9]{12}):(role((\\u002F)|(\\u002F[\\u0021-\\u007F]+\\u002F))[\\w+=,.@-]+)";+ private static final Pattern ROLE_ARN = Pattern.compile(ROLE_ARN_PATTERN);++ private final AccountAwareStorageBackend<JobRunRecord> store;+ private final EmrServerlessService applications;+ private final IamService iamService;+ private final RegionResolver regionResolver;+ private final RequestContext requestContext;+ private final ObjectMapper objectMapper;+ private final int runDurationSeconds;+ private final Clock clock;++ @Inject+ public EmrServerlessJobRunService(StorageFactory storageFactory, EmrServerlessService applications,+ IamService iamService, RegionResolver regionResolver,+ RequestContext requestContext, ObjectMapper objectMapper,+ EmulatorConfig config, Clock clock) {+ this(storageFactory.create("emrserverless", "emr-serverless-job-runs.json", new TypeReference<>() {}),+ applications, iamService, regionResolver, requestContext, objectMapper,+ config.services().emrserverless().jobRunDurationSeconds(), clock);+ }++ EmrServerlessJobRunService(AccountAwareStorageBackend<JobRunRecord> store, EmrServerlessService applications,+ IamService iamService, RegionResolver regionResolver, RequestContext requestContext,+ ObjectMapper objectMapper, int runDurationSeconds, Clock clock) {+ if (runDurationSeconds < 0) {+ throw new IllegalArgumentException(+ "floci.services.emrserverless.job-run-duration-seconds must not be negative: " + runDurationSeconds);+ }+ this.store = store;+ this.applications = applications;+ this.iamService = iamService;+ this.regionResolver = regionResolver;+ this.requestContext = requestContext;+ this.objectMapper = objectMapper;+ this.runDurationSeconds = runDurationSeconds;+ this.clock = clock;+ }++ /**+ * Starts a run, or returns the run an earlier request with the same client token started. The+ * token is StartJobRun's idempotency token; reused with a different request it answers+ * {@code ConflictException} "Request did not match the original", as AWS does.+ */+ public synchronized JobRun startJobRun(String applicationId, StartJobRunRequest request) {+ StartJobRunRequest body = request != null ? request : new StartJobRunRequest();+ validate(body);+ applications.getApplication(applicationId);+ JsonNode original = requestOf(applicationId, body);+ JobRunRecord existing = runWithClientToken(body.getClientToken());+ if (existing != null) {+ if (!original.equals(existing.getRequest())) {+ throw new AwsException("ConflictException", "Request did not match the original", 409);+ }+ return settle(existing).getJobRun();+ }+ // Before admission, which may auto-start the application: a refused run leaves it as it was.+ if (body.getTags() != null) {+ EmrServerlessService.validateTags(body.getTags());+ }+ JobRunRecord record = applications.admitJobRun(applicationId, app -> newRun(app, body, original));+ LOG.infov("Started EMR Serverless job run {0} on application {1}", record.getJobRun().getJobRunId(),+ applicationId);+ return settle(record).getJobRun();+ }++ public synchronized JobRun getJobRun(String applicationId, String jobRunId) {+ applications.getApplication(applicationId);+ return settle(findRun(applicationId, jobRunId)).getJobRun();+ }++ /**+ * The application's runs, newest first. ListJobRuns models no ResourceNotFoundException, so an+ * unknown application has no runs rather than an error. With several {@code states}, "the+ * resulting list will be grouped by the state", in the order the filter names them. A run+ * started without a mode is a BATCH run. A {@code nextToken} resumes only the listing it came+ * from: its cursor holds a state group, which means nothing under another application or filter.+ */+ public synchronized PaginatedResult<JobRunSummary> listJobRuns(String applicationId, List<String> states,+ Instant createdAtAfter, Instant createdAtBefore,+ String mode, Integer maxResults, String nextToken) {+ if (maxResults != null) {+ String violation = outOfRange("maxResults", maxResults, 1, MAX_LIST_RESULTS);+ if (violation != null) {+ throw SmithyValidationErrors.validationException(violation);+ }+ }+ List<JobRunRecord> matching = new ArrayList<>();+ Map<String, String> cursors = new HashMap<>();+ for (JobRunRecord record : store.scan(key -> true)) {+ JobRun run = record.getJobRun();+ if (!applicationId.equals(run.getApplicationId()) || !inRequestRegion(run)) {+ continue;+ }+ settle(record);+ if ((states != null && !states.isEmpty() && !states.contains(run.getState()))+ || (createdAtAfter != null && !run.getCreatedAt().isAfter(createdAtAfter))+ || (createdAtBefore != null && !run.getCreatedAt().isBefore(createdAtBefore))+ || (mode != null && !mode.equals(run.getMode() != null ? run.getMode() : MODE_BATCH))) {+ continue;+ }+ int group = states == null || states.isEmpty() ? 0 : states.indexOf(run.getState());+ cursors.put(run.getJobRunId(), String.format("%02d%020d%s", group,+ Long.MAX_VALUE - run.getCreatedAt().toEpochMilli(), run.getJobRunId()));+ matching.add(record);+ }+ String namespace = String.join("|", "ListJobRuns", applicationId,+ String.valueOf(states != null ? states : List.of()), String.valueOf(createdAtAfter),+ String.valueOf(createdAtBefore), String.valueOf(mode));+ PaginatedResult<JobRunRecord> page = Pagination.paginate(matching,+ record -> cursors.get(record.getJobRun().getJobRunId()),+ maxResults != null ? maxResults : MAX_LIST_RESULTS, nextToken, namespace,+ token -> new AwsException("ValidationException", "Invalid nextToken.", 400));+ return new PaginatedResult<>(page.items().stream().map(EmrServerlessJobRunService::toSummary).toList(),+ page.nextToken());+ }++ /** Cancels a run that has not finished. A finished run is refused, with AWS's wording. */+ public synchronized JobRun cancelJobRun(String applicationId, String jobRunId) {+ applications.getApplication(applicationId);+ JobRunRecord record = settle(findRun(applicationId, jobRunId));+ if (isTerminal(record.getJobRun())) {+ throw new AwsException("ValidationException", "Job run is not in a cancellable state", 400);+ }+ finish(record, STATE_CANCELLED, clock.instant().truncatedTo(ChronoUnit.MILLIS));+ LOG.infov("Cancelled EMR Serverless job run {0} on application {1}", jobRunId, applicationId);+ return record.getJobRun();+ }++ /**+ * A ListJobRuns time filter. Query-string timestamps are ISO 8601 in restJson1; epoch seconds+ * are accepted too.+ */+ static Instant parseTimestamp(String value, String member) {+ if (value == null || value.isBlank()) {+ return null;+ }+ try {+ return OffsetDateTime.parse(value).toInstant();+ } catch (DateTimeParseException notIso) {+ try {+ return Instant.ofEpochMilli(Math.round(Double.parseDouble(value) * 1000));+ } catch (NumberFormatException notEpoch) {+ throw new AwsException("ValidationException", member + " must be a timestamp.", 400);+ }+ }+ }++ private JobRunRecord newRun(Application app, StartJobRunRequest request, JsonNode original) {+ AwsArnUtils.Arn applicationArn = AwsArnUtils.parse(app.getArn());+ String jobRunId = EmrServerlessService.newId();+ Instant now = clock.instant().truncatedTo(ChronoUnit.MILLIS);++ JobRun run = new JobRun();+ run.setApplicationId(app.getApplicationId());+ run.setJobRunId(jobRunId);+ run.setName(request.getName());+ run.setArn(AwsArnUtils.Arn.of("emr-serverless", applicationArn.region(), applicationArn.accountId(),+ applicationArn.resource() + "/jobruns/" + jobRunId).toString());+ run.setCreatedBy(createdBy(applicationArn.accountId()));+ run.setCreatedAt(now);+ run.setUpdatedAt(now);+ run.setExecutionRole(request.getExecutionRoleArn());+ run.setExecutionIamPolicy(request.getExecutionIamPolicy());+ run.setState(STATE_RUNNING);+ run.setStateDetails("");+ run.setReleaseLabel(app.getReleaseLabel());+ run.setConfigurationOverrides(request.getConfigurationOverrides());+ run.setJobDriver(request.getJobDriver());+ run.setTags(request.getTags() != null ? new LinkedHashMap<>(request.getTags()) : new LinkedHashMap<>());+ run.setNetworkConfiguration(app.getNetworkConfiguration());+ run.setExecutionTimeoutMinutes(request.getExecutionTimeoutMinutes() != null+ ? request.getExecutionTimeoutMinutes() : DEFAULT_EXECUTION_TIMEOUT_MINUTES);+ run.setMode(request.getMode());+ run.setRetryPolicy(request.getRetryPolicy());+ run.setStartedAt(now);++ JobRunRecord record = new JobRunRecord();+ record.setJobRun(run);+ record.setClientToken(request.getClientToken());+ record.setRequest(original);+ record.setApplicationType(app.getType());+ store.put(jobRunId, record);+ return record;+ }++ /**+ * Finishes a BATCH run that has been RUNNING for the configured duration, or cancels it at its+ * execution timeout when that comes first: "If the job run exceeds this duration, Amazon EMR+ * Serverless cancels it automatically."+ */+ private JobRunRecord settle(JobRunRecord record) {+ JobRun run = record.getJobRun();+ if (isTerminal(run) || MODE_STREAMING.equals(run.getMode())) {+ return record;+ }+ long timeoutSeconds = run.getExecutionTimeoutMinutes() == null || run.getExecutionTimeoutMinutes() == 0+ ? Long.MAX_VALUE : run.getExecutionTimeoutMinutes() * 60;+ if (runDurationSeconds > timeoutSeconds) {+ Instant timedOutAt = run.getStartedAt().plusSeconds(timeoutSeconds);+ if (!clock.instant().isBefore(timedOutAt)) {+ finish(record, STATE_CANCELLED, timedOutAt);+ }+ return record;+ }+ Instant finishedAt = run.getStartedAt().plusSeconds(runDurationSeconds);+ if (!clock.instant().isBefore(finishedAt)) {+ finish(record, STATE_SUCCESS, finishedAt);+ }+ return record;+ }++ private void finish(JobRunRecord record, String state, Instant endedAt) {+ JobRun run = record.getJobRun();+ run.setState(state);+ run.setEndedAt(endedAt);+ run.setUpdatedAt(endedAt);+ run.setTotalExecutionDurationSeconds((int) Duration.between(run.getStartedAt(), endedAt).toSeconds());+ store.put(run.getJobRunId(), record);+ }++ private JobRunRecord findRun(String applicationId, String jobRunId) {+ return store.get(jobRunId)+ .filter(record -> applicationId.equals(record.getJobRun().getApplicationId()))+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",+ "Job run " + jobRunId + " not found", 404));+ }++ private JobRunRecord runWithClientToken(String clientToken) {+ for (JobRunRecord record : store.scan(key -> true)) {+ if (clientToken.equals(record.getClientToken()) && inRequestRegion(record.getJobRun())) {+ return record;+ }+ }+ return null;+ }++ /**+ * What a retry must repeat: the application and every member of the body but the token itself.+ * Read back from its JSON text, so that its numbers compare equal to those of a stored request+ * once storage has written and read it.+ */+ private JsonNode requestOf(String applicationId, StartJobRunRequest request) {+ ObjectNode node;+ try {+ node = (ObjectNode) objectMapper.readTree(objectMapper.writeValueAsString(request));+ } catch (JsonProcessingException e) {+ throw new IllegalStateException("Cannot serialize a StartJobRun request", e);+ }+ node.remove("clientToken");+ node.put("applicationId", applicationId);+ return node;+ }++ // The caller, as GetCallerIdentity reports it; the account root for a key IAM does not know.+ private String createdBy(String accountId) {+ String accessKeyId = requestContext != null ? requestContext.getAccessKeyId() : null;+ return iamService.resolveCallerArn(accessKeyId)+ .orElseGet(() -> regionResolver.buildGlobalArn("iam", accountId, "root"));+ }++ private boolean inRequestRegion(JobRun run) {+ return regionResolver.getRegion().equals(AwsArnUtils.parse(run.getArn()).region());+ }++ private static boolean isTerminal(JobRun run) {+ return TERMINAL_STATES.contains(run.getState());+ }++ private static void validate(StartJobRunRequest request) {+ List<String> violations = new ArrayList<>();+ if (request.getClientToken() == null) {+ violations.add(notNull("clientToken"));+ }+ if (request.getExecutionRoleArn() == null) {+ violations.add(notNull("executionRoleArn"));+ } else if (!ROLE_ARN.matcher(request.getExecutionRoleArn()).matches()) {+ violations.add("Value at 'executionRoleArn' failed to satisfy constraint: "+ + "Member must satisfy regular expression pattern: " + ROLE_ARN_PATTERN);+ }+ if (request.getMode() != null && !MODES.contains(request.getMode())) {+ violations.add("Value '" + request.getMode() + "' at 'mode' failed to satisfy constraint: "+ + "Member must satisfy enum value set: " + MODES);+ }+ if (request.getExecutionTimeoutMinutes() != null) {+ String violation = outOfRange("executionTimeoutMinutes", request.getExecutionTimeoutMinutes(), 0,+ MAX_EXECUTION_TIMEOUT_MINUTES);+ if (violation != null) {+ violations.add(violation);+ }+ }+ JobDriver driver = request.getJobDriver();+ if (driver != null && driver.getSparkSubmit() != null && driver.getSparkSubmit().getEntryPoint() == null) {+ violations.add(notNull("jobDriver.sparkSubmit.entryPoint"));+ }+ if (driver != null && driver.getHive() != null && driver.getHive().getQuery() == null) {+ violations.add(notNull("jobDriver.hive.query"));+ }+ if (!violations.isEmpty()) {+ throw SmithyValidationErrors.validationException(violations);+ }+ // StartJobRun's shape leaves jobDriver optional, but GetJobRun's JobRun requires one, so a run is+ // refused without it. JobDriver is a union, "so only one of the following members can be+ // specified". No source gives AWS's wording for a run without a driver, or with both.+ if (driver == null || (driver.getSparkSubmit() == null) == (driver.getHive() == null)) {+ throw new AwsException("ValidationException", "jobDriver must contain exactly one union member", 400);+ }+ }++ private static String notNull(String member) {+ return "Value null at '" + member + "' failed to satisfy constraint: Member must not be null";+ }++ private static String outOfRange(String member, long value, long min, long max) {+ if (value < min) {+ return "Value '" + value + "' at '" + member + "' failed to satisfy constraint: "+ + "Member must have value greater than or equal to " + min;+ }+ if (value > max) {+ return "Value '" + value + "' at '" + member + "' failed to satisfy constraint: "+ + "Member must have value less than or equal to " + max;+ }+ return null;+ }++ private static JobRunSummary toSummary(JobRunRecord record) {+ JobRun run = record.getJobRun();+ JobRunSummary summary = new JobRunSummary();+ summary.setApplicationId(run.getApplicationId());+ summary.setId(run.getJobRunId());+ summary.setName(run.getName());+ summary.setMode(run.getMode());+ summary.setArn(run.getArn());+ summary.setCreatedBy(run.getCreatedBy());+ summary.setCreatedAt(run.getCreatedAt());+ summary.setUpdatedAt(run.getUpdatedAt());+ summary.setExecutionRole(run.getExecutionRole());+ summary.setState(run.getState());+ summary.setStateDetails(run.getStateDetails());+ summary.setReleaseLabel(run.getReleaseLabel());+ summary.setType(record.getApplicationType());+ return summary;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.javaindex 0d1f4acd25..4d34a99467 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java@@ -18,12 +18,15 @@ import io.github.hectorvent.floci.services.emrserverless.model.UpdateApplicationimport jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import java.time.Clock;import java.time.Instant;+import java.time.temporal.ChronoUnit;import java.util.Date;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.UUID;+import java.util.function.Function;import java.util.regex.Matcher;import java.util.regex.Pattern;import java.util.stream.Collectors;@@ -42,15 +45,17 @@ public class EmrServerlessService {private final EmulatorConfig config;private final AccountAwareStorageBackend<Application> storage;private final RegionResolver regionResolver;+ private final Clock clock;@InjectRequestContext requestContext;@Injectpublic EmrServerlessService(EmulatorConfig config, StorageFactory storageFactory,- RegionResolver regionResolver) {+ RegionResolver regionResolver, Clock clock) {this.config = config;this.regionResolver = regionResolver;+ this.clock = clock;this.storage = storageFactory.create("emrserverless", "emr-serverless-applications.json",new TypeReference<Map<String, Application>>() {});}@@ -75,9 +80,9 @@ public class EmrServerlessService {}}- String id = generateId();+ String id = newId();String arn = buildArn(id);- long now = System.currentTimeMillis();+ Instant createdAt = now();Application app = new Application();app.setApplicationId(id);@@ -88,8 +93,8 @@ public class EmrServerlessService {app.setType(request.getType());app.setState("CREATED");app.setStateDetails("");- app.setCreatedAt(now);- app.setUpdatedAt(now);+ app.setCreatedAt(createdAt);+ app.setUpdatedAt(createdAt);if (request.getTags() != null) {validateTags(request.getTags());}@@ -188,7 +193,7 @@ public class EmrServerlessService {app.setJobLevelCostAllocationConfiguration(request.getJobLevelCostAllocationConfiguration());}- app.setUpdatedAt(System.currentTimeMillis());+ app.setUpdatedAt(now());storage.put(applicationId, app);return app;}@@ -200,7 +205,7 @@ public class EmrServerlessService {throw new AwsException("ValidationException", "Application must be in a stopped or created state in order to be deleted.", 400);}app.setState("TERMINATED");- app.setUpdatedAt(System.currentTimeMillis());+ app.setUpdatedAt(now());storage.put(applicationId, app);}@@ -214,7 +219,7 @@ public class EmrServerlessService {throw new AwsException("ValidationException", "Application is terminated.", 400);}app.setState("STARTED");- app.setUpdatedAt(System.currentTimeMillis());+ app.setUpdatedAt(now());storage.put(applicationId, app);}@@ -228,10 +233,40 @@ public class EmrServerlessService {throw new AwsException("ValidationException", "Application is terminated.", 400);}app.setState("STOPPED");- app.setUpdatedAt(System.currentTimeMillis());+ app.setUpdatedAt(now());storage.put(applicationId, app);}+ /**+ * Admits a job run to the application and returns what {@code admit} builds from it. An+ * application "only accepts jobs when it's in" STARTED, and auto-start, on unless the+ * application turns it off, starts a CREATED or STOPPED one "on job submission" (EMR Serverless+ * user guide, "Application states" and "Default application behavior"). This service's lock is+ * held throughout, so the application cannot be stopped or deleted while the run is admitted.+ */+ public synchronized <T> T admitJobRun(String applicationId, Function<Application, T> admit) {+ Application app = getApplication(applicationId);+ String state = app.getState();+ if ("TERMINATED".equals(state)) {+ throw new AwsException("ValidationException", "Application is terminated.", 400);+ }+ if (!"STARTED".equals(state)) {+ if (!autoStartEnabled(app)) {+ throw new AwsException("ValidationException", "Application " + applicationId+ + " must be started before a job run is submitted, as its auto-start is disabled.", 400);+ }+ startApplication(applicationId);+ app = getApplication(applicationId);+ }+ return admit.apply(app);+ }++ // AutoStartConfig.enabled "Defaults to true".+ private static boolean autoStartEnabled(Application app) {+ return app.getAutoStartConfiguration() == null+ || !Boolean.FALSE.equals(app.getAutoStartConfiguration().getEnabled());+ }+// ---- Tags ---------------------------------------------------------------------------------public synchronized Map<String, String> listTags(String arn) {@@ -276,8 +311,8 @@ public class EmrServerlessService {/*** The application an EMR Serverless ARN names:* {@code arn:<partition>:emr-serverless:<region>:<account>:/applications/<id>}, the form- * CreateApplication returns. A job run ARN ({@code .../jobruns/<id>}) is refused, as job runs are- * not emulated.+ * CreateApplication returns. A job run ARN ({@code .../jobruns/<id>}) is refused, as job run tags+ * are not emulated yet.*/private Application applicationFor(String arn) {AwsArnUtils.Arn parsed;@@ -303,7 +338,7 @@ public class EmrServerlessService {}/** At most 50 tags, each key and value within TagKey and TagValue and free of the reserved prefix. */- private static void validateTags(Map<String, String> tags) {+ static void validateTags(Map<String, String> tags) {if (tags.size() > MAX_TAGS) {throw new AwsException("ValidationException","A resource can have at most " + MAX_TAGS + " tags.", 400);@@ -334,7 +369,8 @@ public class EmrServerlessService {return text.regionMatches(true, 0, RESERVED_PREFIX, 0, RESERVED_PREFIX.length());}- private String generateId() {+ /** An application or job run id: 16 lowercase hexadecimal characters, within the model's [0-9a-z]+. */+ static String newId() {return UUID.randomUUID().toString().replace("-", "").substring(0, 16);}@@ -356,6 +392,10 @@ public class EmrServerlessService {}}+ private Instant now() {+ return clock.instant().truncatedTo(ChronoUnit.MILLIS);+ }+private ApplicationSummary toSummary(Application app) {ApplicationSummary summary = new ApplicationSummary();summary.setId(app.getApplicationId());diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/Application.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/Application.javaindex 378c800518..a27d78cd55 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/Application.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/Application.java@@ -1,13 +1,22 @@package io.github.hectorvent.floci.services.emrserverless.model;+import com.fasterxml.jackson.annotation.JsonFormat;import io.quarkus.runtime.annotations.RegisterForReflection;+import java.time.Instant;import java.util.Date;import java.util.List;import java.util.Map;+/** GetApplication's application. Timestamps go out as epoch seconds, the restJson1 default. */@RegisterForReflectionpublic class Application {++ // Applications persisted before their timestamps were epoch seconds hold epoch milliseconds, and+ // those read back as seconds. 1e11 seconds is the year 5138, while 1e11 milliseconds is 1973, before+ // any application, so a value at or past it was written as milliseconds.+ private static final long LEGACY_MILLIS_FLOOR = 100_000_000_000L;+private String applicationId;private String arn;private String name;@@ -16,8 +25,10 @@ public class Application {private String state;private String stateDetails;private String clientToken;- private Long createdAt;- private Long updatedAt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant createdAt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant updatedAt;private Map<String, String> tags;private String architecture;private Map<String, InitialCapacityConfig> initialCapacity;@@ -51,10 +62,10 @@ public class Application {public void setStateDetails(String stateDetails) { this.stateDetails = stateDetails; }public String getClientToken() { return clientToken; }public void setClientToken(String clientToken) { this.clientToken = clientToken; }- public Long getCreatedAt() { return createdAt; }- public void setCreatedAt(Long createdAt) { this.createdAt = createdAt; }- public Long getUpdatedAt() { return updatedAt; }- public void setUpdatedAt(Long updatedAt) { this.updatedAt = updatedAt; }+ public Instant getCreatedAt() { return createdAt; }+ public void setCreatedAt(Instant createdAt) { this.createdAt = fromLegacyMillis(createdAt); }+ public Instant getUpdatedAt() { return updatedAt; }+ public void setUpdatedAt(Instant updatedAt) { this.updatedAt = fromLegacyMillis(updatedAt); }public Map<String, String> getTags() { return tags; }public void setTags(Map<String, String> tags) { this.tags = tags; }public String getArchitecture() { return architecture; }@@ -87,4 +98,9 @@ public class Application {public void setIdentityCenterConfiguration(IdentityCenterConfiguration identityCenterConfiguration) { this.identityCenterConfiguration = identityCenterConfiguration; }public JobLevelCostAllocationConfiguration getJobLevelCostAllocationConfiguration() { return jobLevelCostAllocationConfiguration; }public void setJobLevelCostAllocationConfiguration(JobLevelCostAllocationConfiguration jobLevelCostAllocationConfiguration) { this.jobLevelCostAllocationConfiguration = jobLevelCostAllocationConfiguration; }++ private static Instant fromLegacyMillis(Instant value) {+ return value != null && value.getEpochSecond() >= LEGACY_MILLIS_FLOOR+ ? Instant.ofEpochMilli(value.getEpochSecond()) : value;+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ApplicationSummary.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ApplicationSummary.javaindex 8d993d3fda..55c3f601c6 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ApplicationSummary.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ApplicationSummary.java@@ -1,19 +1,24 @@package io.github.hectorvent.floci.services.emrserverless.model;+import com.fasterxml.jackson.annotation.JsonFormat;import io.quarkus.runtime.annotations.RegisterForReflection;+import java.time.Instant;+@RegisterForReflectionpublic class ApplicationSummary {private String architecture;private String arn;- private Long createdAt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant createdAt;private String id;private String name;private String releaseLabel;private String state;private String stateDetails;private String type;- private Long updatedAt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant updatedAt;public ApplicationSummary() {}@@ -33,11 +38,11 @@ public class ApplicationSummary {this.arn = arn;}- public Long getCreatedAt() {+ public Instant getCreatedAt() {return createdAt;}- public void setCreatedAt(Long createdAt) {+ public void setCreatedAt(Instant createdAt) {this.createdAt = createdAt;}@@ -89,11 +94,11 @@ public class ApplicationSummary {this.type = type;}- public Long getUpdatedAt() {+ public Instant getUpdatedAt() {return updatedAt;}- public void setUpdatedAt(Long updatedAt) {+ public void setUpdatedAt(Instant updatedAt) {this.updatedAt = updatedAt;}}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/CancelJobRunResponse.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/CancelJobRunResponse.javanew file mode 100644index 0000000000..0a3be4592d--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/CancelJobRunResponse.java@@ -0,0 +1,14 @@+package io.github.hectorvent.floci.services.emrserverless.model;++import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+public class CancelJobRunResponse {+ private String applicationId;+ private String jobRunId;++ public String getApplicationId() { return applicationId; }+ public void setApplicationId(String applicationId) { this.applicationId = applicationId; }+ public String getJobRunId() { return jobRunId; }+ public void setJobRunId(String jobRunId) { this.jobRunId = jobRunId; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/GetJobRunResponse.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/GetJobRunResponse.javanew file mode 100644index 0000000000..1da266e6db--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/GetJobRunResponse.java@@ -0,0 +1,11 @@+package io.github.hectorvent.floci.services.emrserverless.model;++import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+public class GetJobRunResponse {+ private JobRun jobRun;++ public JobRun getJobRun() { return jobRun; }+ public void setJobRun(JobRun jobRun) { this.jobRun = jobRun; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobDriver.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobDriver.javanew file mode 100644index 0000000000..a770ce4b3e--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobDriver.java@@ -0,0 +1,49 @@+package io.github.hectorvent.floci.services.emrserverless.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.List;++/** What a job run executes: a Spark submission or a Hive query. */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class JobDriver {+ private SparkSubmit sparkSubmit;+ private Hive hive;++ public SparkSubmit getSparkSubmit() { return sparkSubmit; }+ public void setSparkSubmit(SparkSubmit sparkSubmit) { this.sparkSubmit = sparkSubmit; }+ public Hive getHive() { return hive; }+ public void setHive(Hive hive) { this.hive = hive; }++ @RegisterForReflection+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class SparkSubmit {+ private String entryPoint;+ private List<String> entryPointArguments;+ private String sparkSubmitParameters;++ public String getEntryPoint() { return entryPoint; }+ public void setEntryPoint(String entryPoint) { this.entryPoint = entryPoint; }+ public List<String> getEntryPointArguments() { return entryPointArguments; }+ public void setEntryPointArguments(List<String> entryPointArguments) { this.entryPointArguments = entryPointArguments; }+ public String getSparkSubmitParameters() { return sparkSubmitParameters; }+ public void setSparkSubmitParameters(String sparkSubmitParameters) { this.sparkSubmitParameters = sparkSubmitParameters; }+ }++ @RegisterForReflection+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class Hive {+ private String query;+ private String initQueryFile;+ private String parameters;++ public String getQuery() { return query; }+ public void setQuery(String query) { this.query = query; }+ public String getInitQueryFile() { return initQueryFile; }+ public void setInitQueryFile(String initQueryFile) { this.initQueryFile = initQueryFile; }+ public String getParameters() { return parameters; }+ public void setParameters(String parameters) { this.parameters = parameters; }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.javanew file mode 100644index 0000000000..44b3b65838--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java@@ -0,0 +1,85 @@+package io.github.hectorvent.floci.services.emrserverless.model;++import com.fasterxml.jackson.annotation.JsonFormat;+import com.fasterxml.jackson.annotation.JsonInclude;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;+import java.util.Map;++/** GetJobRun's jobRun. Timestamps go out as epoch seconds, the restJson1 default. */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class JobRun {+ private String applicationId;+ private String jobRunId;+ private String name;+ private String arn;+ private String createdBy;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant createdAt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant updatedAt;+ private String executionRole;+ private Map<String, Object> executionIamPolicy;+ private String state;+ private String stateDetails;+ private String releaseLabel;+ private Map<String, Object> configurationOverrides;+ private JobDriver jobDriver;+ private Map<String, String> tags;+ private NetworkConfiguration networkConfiguration;+ private Integer totalExecutionDurationSeconds;+ private Long executionTimeoutMinutes;+ private String mode;+ private Map<String, Object> retryPolicy;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant startedAt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant endedAt;++ public String getApplicationId() { return applicationId; }+ public void setApplicationId(String applicationId) { this.applicationId = applicationId; }+ public String getJobRunId() { return jobRunId; }+ public void setJobRunId(String jobRunId) { this.jobRunId = jobRunId; }+ public String getName() { return name; }+ public void setName(String name) { this.name = name; }+ public String getArn() { return arn; }+ public void setArn(String arn) { this.arn = arn; }+ public String getCreatedBy() { return createdBy; }+ public void setCreatedBy(String createdBy) { this.createdBy = createdBy; }+ public Instant getCreatedAt() { return createdAt; }+ public void setCreatedAt(Instant createdAt) { this.createdAt = createdAt; }+ public Instant getUpdatedAt() { return updatedAt; }+ public void setUpdatedAt(Instant updatedAt) { this.updatedAt = updatedAt; }+ public String getExecutionRole() { return executionRole; }+ public void setExecutionRole(String executionRole) { this.executionRole = executionRole; }+ public Map<String, Object> getExecutionIamPolicy() { return executionIamPolicy; }+ public void setExecutionIamPolicy(Map<String, Object> executionIamPolicy) { this.executionIamPolicy = executionIamPolicy; }+ public String getState() { return state; }+ public void setState(String state) { this.state = state; }+ public String getStateDetails() { return stateDetails; }+ public void setStateDetails(String stateDetails) { this.stateDetails = stateDetails; }+ public String getReleaseLabel() { return releaseLabel; }+ public void setReleaseLabel(String releaseLabel) { this.releaseLabel = releaseLabel; }+ public Map<String, Object> getConfigurationOverrides() { return configurationOverrides; }+ public void setConfigurationOverrides(Map<String, Object> configurationOverrides) { this.configurationOverrides = configurationOverrides; }+ public JobDriver getJobDriver() { return jobDriver; }+ public void setJobDriver(JobDriver jobDriver) { this.jobDriver = jobDriver; }+ public Map<String, String> getTags() { return tags; }+ public void setTags(Map<String, String> tags) { this.tags = tags; }+ public NetworkConfiguration getNetworkConfiguration() { return networkConfiguration; }+ public void setNetworkConfiguration(NetworkConfiguration networkConfiguration) { this.networkConfiguration = networkConfiguration; }+ public Integer getTotalExecutionDurationSeconds() { return totalExecutionDurationSeconds; }+ public void setTotalExecutionDurationSeconds(Integer totalExecutionDurationSeconds) { this.totalExecutionDurationSeconds = totalExecutionDurationSeconds; }+ public Long getExecutionTimeoutMinutes() { return executionTimeoutMinutes; }+ public void setExecutionTimeoutMinutes(Long executionTimeoutMinutes) { this.executionTimeoutMinutes = executionTimeoutMinutes; }+ public String getMode() { return mode; }+ public void setMode(String mode) { this.mode = mode; }+ public Map<String, Object> getRetryPolicy() { return retryPolicy; }+ public void setRetryPolicy(Map<String, Object> retryPolicy) { this.retryPolicy = retryPolicy; }+ public Instant getStartedAt() { return startedAt; }+ public void setStartedAt(Instant startedAt) { this.startedAt = startedAt; }+ public Instant getEndedAt() { return endedAt; }+ public void setEndedAt(Instant endedAt) { this.endedAt = endedAt; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunRecord.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunRecord.javanew file mode 100644index 0000000000..e55acce436--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunRecord.java@@ -0,0 +1,26 @@+package io.github.hectorvent.floci.services.emrserverless.model;++import com.fasterxml.jackson.databind.JsonNode;+import io.quarkus.runtime.annotations.RegisterForReflection;++/**+ * A stored job run: the run as GetJobRun reports it, plus what only Floci needs. The client token+ * and the request it came with decide whether a repeated StartJobRun is a retry or a conflict; the+ * application type fills ListJobRuns' {@code type}.+ */+@RegisterForReflection+public class JobRunRecord {+ private JobRun jobRun;+ private String clientToken;+ private JsonNode request;+ private String applicationType;++ public JobRun getJobRun() { return jobRun; }+ public void setJobRun(JobRun jobRun) { this.jobRun = jobRun; }+ public String getClientToken() { return clientToken; }+ public void setClientToken(String clientToken) { this.clientToken = clientToken; }+ public JsonNode getRequest() { return request; }+ public void setRequest(JsonNode request) { this.request = request; }+ public String getApplicationType() { return applicationType; }+ public void setApplicationType(String applicationType) { this.applicationType = applicationType; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.javanew file mode 100644index 0000000000..bf54a96730--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java@@ -0,0 +1,55 @@+package io.github.hectorvent.floci.services.emrserverless.model;++import com.fasterxml.jackson.annotation.JsonFormat;+import com.fasterxml.jackson.annotation.JsonInclude;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;++/** One entry of ListJobRuns. Unlike JobRun, the run's id is {@code id}. */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class JobRunSummary {+ private String applicationId;+ private String id;+ private String name;+ private String mode;+ private String arn;+ private String createdBy;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant createdAt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant updatedAt;+ private String executionRole;+ private String state;+ private String stateDetails;+ private String releaseLabel;+ private String type;++ public String getApplicationId() { return applicationId; }+ public void setApplicationId(String applicationId) { this.applicationId = applicationId; }+ public String getId() { return id; }+ public void setId(String id) { this.id = id; }+ public String getName() { return name; }+ public void setName(String name) { this.name = name; }+ public String getMode() { return mode; }+ public void setMode(String mode) { this.mode = mode; }+ public String getArn() { return arn; }+ public void setArn(String arn) { this.arn = arn; }+ public String getCreatedBy() { return createdBy; }+ public void setCreatedBy(String createdBy) { this.createdBy = createdBy; }+ public Instant getCreatedAt() { return createdAt; }+ public void setCreatedAt(Instant createdAt) { this.createdAt = createdAt; }+ public Instant getUpdatedAt() { return updatedAt; }+ public void setUpdatedAt(Instant updatedAt) { this.updatedAt = updatedAt; }+ public String getExecutionRole() { return executionRole; }+ public void setExecutionRole(String executionRole) { this.executionRole = executionRole; }+ public String getState() { return state; }+ public void setState(String state) { this.state = state; }+ public String getStateDetails() { return stateDetails; }+ public void setStateDetails(String stateDetails) { this.stateDetails = stateDetails; }+ public String getReleaseLabel() { return releaseLabel; }+ public void setReleaseLabel(String releaseLabel) { this.releaseLabel = releaseLabel; }+ public String getType() { return type; }+ public void setType(String type) { this.type = type; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunsResponse.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunsResponse.javanew file mode 100644index 0000000000..a62fe22ef7--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunsResponse.java@@ -0,0 +1,18 @@+package io.github.hectorvent.floci.services.emrserverless.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.List;++@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class ListJobRunsResponse {+ private List<JobRunSummary> jobRuns;+ private String nextToken;++ public List<JobRunSummary> getJobRuns() { return jobRuns; }+ public void setJobRuns(List<JobRunSummary> jobRuns) { this.jobRuns = jobRuns; }+ public String getNextToken() { return nextToken; }+ public void setNextToken(String nextToken) { this.nextToken = nextToken; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunRequest.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunRequest.javanew file mode 100644index 0000000000..c091afc95a--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunRequest.java@@ -0,0 +1,46 @@+package io.github.hectorvent.floci.services.emrserverless.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.Map;++/**+ * StartJobRun's body. configurationOverrides, executionIamPolicy and retryPolicy are kept as sent+ * and returned by GetJobRun unchanged.+ */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class StartJobRunRequest {+ private String clientToken;+ private String executionRoleArn;+ private Map<String, Object> executionIamPolicy;+ private JobDriver jobDriver;+ private Map<String, Object> configurationOverrides;+ private Map<String, String> tags;+ private Long executionTimeoutMinutes;+ private String name;+ private String mode;+ private Map<String, Object> retryPolicy;++ public String getClientToken() { return clientToken; }+ public void setClientToken(String clientToken) { this.clientToken = clientToken; }+ public String getExecutionRoleArn() { return executionRoleArn; }+ public void setExecutionRoleArn(String executionRoleArn) { this.executionRoleArn = executionRoleArn; }+ public Map<String, Object> getExecutionIamPolicy() { return executionIamPolicy; }+ public void setExecutionIamPolicy(Map<String, Object> executionIamPolicy) { this.executionIamPolicy = executionIamPolicy; }+ public JobDriver getJobDriver() { return jobDriver; }+ public void setJobDriver(JobDriver jobDriver) { this.jobDriver = jobDriver; }+ public Map<String, Object> getConfigurationOverrides() { return configurationOverrides; }+ public void setConfigurationOverrides(Map<String, Object> configurationOverrides) { this.configurationOverrides = configurationOverrides; }+ public Map<String, String> getTags() { return tags; }+ public void setTags(Map<String, String> tags) { this.tags = tags; }+ public Long getExecutionTimeoutMinutes() { return executionTimeoutMinutes; }+ public void setExecutionTimeoutMinutes(Long executionTimeoutMinutes) { this.executionTimeoutMinutes = executionTimeoutMinutes; }+ public String getName() { return name; }+ public void setName(String name) { this.name = name; }+ public String getMode() { return mode; }+ public void setMode(String mode) { this.mode = mode; }+ public Map<String, Object> getRetryPolicy() { return retryPolicy; }+ public void setRetryPolicy(Map<String, Object> retryPolicy) { this.retryPolicy = retryPolicy; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunResponse.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunResponse.javanew file mode 100644index 0000000000..0d06a8f1bb--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunResponse.java@@ -0,0 +1,17 @@+package io.github.hectorvent.floci.services.emrserverless.model;++import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+public class StartJobRunResponse {+ private String applicationId;+ private String jobRunId;+ private String arn;++ public String getApplicationId() { return applicationId; }+ public void setApplicationId(String applicationId) { this.applicationId = applicationId; }+ public String getJobRunId() { return jobRunId; }+ public void setJobRunId(String jobRunId) { this.jobRunId = jobRunId; }+ public String getArn() { return arn; }+ public void setArn(String arn) { this.arn = arn; }+}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 544ce54464..2b7cb321ce 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -954,6 +954,7 @@ floci:enabled: trueemrserverless:enabled: true+ job-run-duration-seconds: 0 # 0 = BATCH job runs succeed as soon as they startssoadmin:enabled: truessooidc:diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.javaindex ab99865c6c..f2c2ebbf6d 100644--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java@@ -1,32 +1,46 @@package io.github.hectorvent.floci.services.emrserverless;+import io.github.hectorvent.floci.testing.EmrServerlessRequests;+import io.github.hectorvent.floci.testing.MutableClock;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;+import io.restassured.config.JsonConfig;+import io.restassured.config.RestAssuredConfig;+import io.restassured.path.json.config.JsonPathConfig;+import io.restassured.specification.RequestSpecification;+import jakarta.inject.Inject;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.MethodOrderer;import org.junit.jupiter.api.Order;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.TestMethodOrder;-import static io.restassured.RestAssured.given;+import java.math.BigDecimal;+import java.time.Duration;+import java.util.UUID;+import static org.hamcrest.Matchers.*;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTest@TestMethodOrder(MethodOrderer.OrderAnnotation.class)public class EmrServerlessIntegrationTest {- private static final String JSON_CONTENT_TYPE = "application/json";+ private static final RestAssuredConfig DECIMAL_NUMBERS = RestAssuredConfig.config()+ .jsonConfig(JsonConfig.jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.BIG_DECIMAL));private static String applicationId;+ @Inject+ MutableClock clock;+@BeforeAllstatic void configureRestAssured() {RestAssuredJsonUtils.configureAwsContentTypes();}- private io.restassured.specification.RequestSpecification givenReq() {- return given()- .header("Authorization", "AWS4-HMAC-SHA256 Credential=TEST/20260819/us-east-1/emr-serverless/aws4_request, SignedHeaders=host;x-amz-date, Signature=test")- .contentType(JSON_CONTENT_TYPE);+ private RequestSpecification givenReq() {+ return EmrServerlessRequests.emrServerless("us-east-1");}@Test@@ -286,4 +300,42 @@ public class EmrServerlessIntegrationTest {.statusCode(200).body("application.state", equalTo("TERMINATED"));}++ @Test+ @Order(9)+ void applicationTimestampsAreEpochSecondsWithMilliseconds() {+ // A Region of its own, so ListApplications holds this application and no other test's.+ String region = "ap-southeast-4";+ clock.advance(Duration.ofMillis(250));+ BigDecimal before = BigDecimal.valueOf(clock.instant().toEpochMilli(), 3);+ String id = EmrServerlessRequests.emrServerless(region)+ .body("""+ {"name": "stamped-app", "releaseLabel": "emr-6.6.0", "type": "SPARK", "clientToken": "%s"}+ """.formatted(UUID.randomUUID()))+ .when()+ .post("/applications")+ .then()+ .statusCode(200)+ .extract().path("applicationId");++ BigDecimal createdAt = EmrServerlessRequests.emrServerless(region).config(DECIMAL_NUMBERS)+ .when()+ .get("/applications/" + id)+ .then()+ .statusCode(200)+ .extract().path("application.createdAt");+ BigDecimal updatedAt = EmrServerlessRequests.emrServerless(region).config(DECIMAL_NUMBERS)+ .get("/applications/" + id)+ .then().statusCode(200).extract().path("application.updatedAt");+ BigDecimal listed = EmrServerlessRequests.emrServerless(region).config(DECIMAL_NUMBERS)+ .get("/applications")+ .then().statusCode(200).extract().path("applications[0].createdAt");++ // The test clock moves a millisecond on every read, so the application was created a little later.+ assertTrue(createdAt.compareTo(before) > 0 && createdAt.compareTo(before.add(BigDecimal.ONE)) < 0,+ "createdAt " + createdAt + " is not just after " + before);+ assertTrue(createdAt.stripTrailingZeros().scale() <= 3, "createdAt " + createdAt + " is finer than milliseconds");+ assertEquals(0, createdAt.compareTo(updatedAt), "updatedAt " + updatedAt + " differs from createdAt " + createdAt);+ assertEquals(0, createdAt.compareTo(listed), "ListApplications reports createdAt " + listed);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.javanew file mode 100644index 0000000000..e5146d22d4--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java@@ -0,0 +1,485 @@+package io.github.hectorvent.floci.services.emrserverless;++import io.github.hectorvent.floci.testing.EmrServerlessRequests;+import io.github.hectorvent.floci.testing.MutableClock;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.config.JsonConfig;+import io.restassured.config.RestAssuredConfig;+import io.restassured.path.json.config.JsonPathConfig;+import io.restassured.response.Response;+import io.restassured.specification.RequestSpecification;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.math.BigDecimal;+import java.time.Duration;+import java.time.Instant;+import java.util.List;+import java.util.UUID;++import static org.hamcrest.Matchers.*;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++@QuarkusTest+class EmrServerlessJobRunIntegrationTest {++ private static final String ROLE = "arn:aws:iam::000000000000:role/emr-serverless-job";+ private static final RestAssuredConfig DECIMAL_NUMBERS = RestAssuredConfig.config()+ .jsonConfig(JsonConfig.jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.BIG_DECIMAL));++ @Inject+ MutableClock clock;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ private static RequestSpecification givenReq() {+ return EmrServerlessRequests.emrServerless("us-east-1");+ }++ private static String createApplication(String extra) {+ return givenReq()+ .body("""+ {"name": "jobs-%s", "releaseLabel": "emr-7.5.0", "type": "SPARK", "clientToken": "%s"%s}+ """.formatted(UUID.randomUUID(), UUID.randomUUID(), extra))+ .when().post("/applications")+ .then().statusCode(200)+ .extract().path("applicationId");+ }++ private static String sparkRun(String clientToken, String extra) {+ return """+ {"clientToken": "%s", "executionRoleArn": "%s",+ "jobDriver": {"sparkSubmit": {"entryPoint": "s3://bucket/jobs/app.jar",+ "entryPointArguments": ["s3://bucket/in"],+ "sparkSubmitParameters": "--class example.Main"}}%s}+ """.formatted(clientToken, ROLE, extra);+ }++ private static Response startJobRun(String applicationId, String body) {+ return givenReq().body(body).when().post("/applications/" + applicationId + "/jobruns");+ }++ private static String startedRunId(String applicationId, String body) {+ return startJobRun(applicationId, body).then().statusCode(200).extract().path("jobRunId");+ }++ @Test+ void startJobRunAutoStartsTheApplicationAndSucceedsByDefault() {+ String applicationId = createApplication("");+ givenReq().when().get("/applications/" + applicationId)+ .then().statusCode(200).body("application.state", equalTo("CREATED"));++ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ", \"name\": \"nightly\""))+ .then().statusCode(200)+ .body("applicationId", equalTo(applicationId))+ .body("jobRunId", matchesPattern("[0-9a-z]{16}"))+ .body("arn", matchesPattern("arn:aws:emr-serverless:us-east-1:000000000000:/applications/"+ + applicationId + "/jobruns/[0-9a-z]{16}"));++ givenReq().when().get("/applications/" + applicationId)+ .then().statusCode(200).body("application.state", equalTo("STARTED"));+ }++ @Test+ void getJobRunReportsTheRunAsGetJobRunShapesIt() {+ String applicationId = createApplication("");+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(),+ ", \"name\": \"nightly\", \"tags\": {\"team\": \"data\"}, \"mode\": \"BATCH\""));++ givenReq().when().get("/applications/" + applicationId + "/jobruns/" + jobRunId)+ .then().statusCode(200)+ .body("jobRun.applicationId", equalTo(applicationId))+ .body("jobRun.jobRunId", equalTo(jobRunId))+ .body("jobRun.name", equalTo("nightly"))+ .body("jobRun.arn", endsWith("/applications/" + applicationId + "/jobruns/" + jobRunId))+ .body("jobRun.createdBy", equalTo("arn:aws:iam::000000000000:root"))+ .body("jobRun.createdAt", instanceOf(Number.class))+ .body("jobRun.updatedAt", instanceOf(Number.class))+ .body("jobRun.executionRole", equalTo(ROLE))+ .body("jobRun.state", equalTo("SUCCESS"))+ .body("jobRun.stateDetails", equalTo(""))+ .body("jobRun.releaseLabel", equalTo("emr-7.5.0"))+ .body("jobRun.jobDriver.sparkSubmit.entryPoint", equalTo("s3://bucket/jobs/app.jar"))+ .body("jobRun.jobDriver.sparkSubmit.entryPointArguments", contains("s3://bucket/in"))+ .body("jobRun.jobDriver.sparkSubmit.sparkSubmitParameters", equalTo("--class example.Main"))+ .body("jobRun.tags.team", equalTo("data"))+ .body("jobRun.mode", equalTo("BATCH"))+ .body("jobRun.executionTimeoutMinutes", equalTo(720))+ .body("jobRun.totalExecutionDurationSeconds", equalTo(0))+ .body("jobRun.startedAt", instanceOf(Number.class))+ .body("jobRun.endedAt", instanceOf(Number.class))+ .body("jobRun", not(hasKey("hive")));+ }++ @Test+ void timestampsAreEpochSecondsWithMilliseconds() {+ String applicationId = createApplication("");+ clock.advance(Duration.ofMillis(250));+ Instant submittedAt = clock.instant();+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));++ BigDecimal createdAt = givenReq().config(DECIMAL_NUMBERS)+ .when().get("/applications/" + applicationId + "/jobruns/" + jobRunId)+ .then().statusCode(200).extract().path("jobRun.createdAt");+ // The test clock moves a millisecond on every read, so the run was created a little later.+ BigDecimal submitted = BigDecimal.valueOf(submittedAt.toEpochMilli(), 3);+ assertTrue(createdAt.compareTo(submitted) >= 0 && createdAt.compareTo(submitted.add(BigDecimal.ONE)) < 0,+ "createdAt " + createdAt + " is not just after " + submitted);+ assertTrue(createdAt.stripTrailingZeros().scale() <= 3, "createdAt " + createdAt + " is finer than milliseconds");+ }++ @Test+ void runWithoutTagsReportsAnEmptyTagMap() {+ String applicationId = createApplication("");+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));++ givenReq().when().get("/applications/" + applicationId + "/jobruns/" + jobRunId)+ .then().statusCode(200)+ .body("jobRun.tags", anEmptyMap())+ .body("jobRun", not(hasKey("mode")));+ }++ @Test+ void repeatedClientTokenReturnsTheOriginalRun() {+ String applicationId = createApplication("");+ String body = sparkRun(UUID.randomUUID().toString(), ", \"tags\": {\"a\": \"1\", \"b\": \"2\"}");+ String first = startedRunId(applicationId, body);++ assertEquals(first, startedRunId(applicationId, body));+ }++ @Test+ void reusedClientTokenWithAnotherRequestIsAConflict() {+ String applicationId = createApplication("");+ String token = UUID.randomUUID().toString();+ startedRunId(applicationId, sparkRun(token, ", \"name\": \"first\""));++ startJobRun(applicationId, sparkRun(token, ", \"name\": \"second\""))+ .then().statusCode(409)+ .body("message", equalTo("Request did not match the original"))+ .body("__type", equalTo("ConflictException"));+ }++ @Test+ void newClientTokenStartsAnotherRun() {+ String applicationId = createApplication("");+ String first = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));++ assertNotEquals(first, startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), "")));+ }++ @Test+ void missingRequiredMembersAreAValidationException() {+ String applicationId = createApplication("");++ startJobRun(applicationId, "{\"jobDriver\": {\"sparkSubmit\": {}}}")+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("3 validation errors detected: "+ + "Value null at 'clientToken' failed to satisfy constraint: Member must not be null; "+ + "Value null at 'executionRoleArn' failed to satisfy constraint: Member must not be null; "+ + "Value null at 'jobDriver.sparkSubmit.entryPoint' failed to satisfy constraint: "+ + "Member must not be null"));+ }++ @Test+ void executionRoleOutsideTheRolePatternIsAValidationException() {+ String applicationId = createApplication("");++ startJobRun(applicationId, """+ {"clientToken": "t-1", "executionRoleArn": "arn:aws:iam::000000000000:user/someone"}+ """)+ .then().statusCode(400)+ .body("message", equalTo("1 validation error detected: Value at 'executionRoleArn' failed to "+ + "satisfy constraint: Member must satisfy regular expression pattern: "+ + "arn:(aws[a-zA-Z0-9-]*):iam::([0-9]{12}):(role((" + backslashU("002F") + ")|("+ + backslashU("002F") + "[" + backslashU("0021") + "-" + backslashU("007F") + "]+"+ + backslashU("002F") + "))[" + (char) 92 + "w+=,.@-]+)"));+ }++ // The pattern is quoted with its escapes as text, so the expected message spells them out.+ private static String backslashU(String hex) {+ return (char) 92 + "u" + hex;+ }++ @Test+ void unknownModeIsAValidationException() {+ String applicationId = createApplication("");++ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ", \"mode\": \"INTERACTIVE\""))+ .then().statusCode(400)+ .body("message", equalTo("1 validation error detected: Value 'INTERACTIVE' at 'mode' failed to "+ + "satisfy constraint: Member must satisfy enum value set: [BATCH, STREAMING]"));+ }++ @Test+ void executionTimeoutOutsideItsRangeIsAValidationException() {+ String applicationId = createApplication("");++ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ", \"executionTimeoutMinutes\": -1"))+ .then().statusCode(400)+ .body("message", equalTo("1 validation error detected: Value '-1' at 'executionTimeoutMinutes' "+ + "failed to satisfy constraint: Member must have value greater than or equal to 0"));+ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ", \"executionTimeoutMinutes\": 1000001"))+ .then().statusCode(400)+ .body("message", equalTo("1 validation error detected: Value '1000001' at 'executionTimeoutMinutes' "+ + "failed to satisfy constraint: Member must have value less than or equal to 1000000"));+ }++ @Test+ void aRunNeedsExactlyOneJobDriver() {+ String applicationId = createApplication("");+ String missing = """+ {"clientToken": "%s", "executionRoleArn": "%s"}+ """.formatted(UUID.randomUUID(), ROLE);+ String both = """+ {"clientToken": "%s", "executionRoleArn": "%s",+ "jobDriver": {"sparkSubmit": {"entryPoint": "s3://bucket/jobs/app.jar"}, "hive": {"query": "SELECT 1"}}}+ """.formatted(UUID.randomUUID(), ROLE);+ String neither = """+ {"clientToken": "%s", "executionRoleArn": "%s", "jobDriver": {}}+ """.formatted(UUID.randomUUID(), ROLE);++ for (String body : List.of(missing, both, neither)) {+ startJobRun(applicationId, body)+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("jobDriver must contain exactly one union member"));+ }+ givenReq().when().get("/applications/" + applicationId)+ .then().body("application.state", equalTo("CREATED"));+ }++ @Test+ void refusedTagsLeaveTheApplicationAsItWas() {+ String applicationId = createApplication("");++ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ", \"tags\": {\"aws:team\": \"data\"}"))+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"));+ givenReq().when().get("/applications/" + applicationId)+ .then().body("application.state", equalTo("CREATED"));+ givenReq().when().get("/applications/" + applicationId + "/jobruns")+ .then().body("jobRuns", empty());+ }++ @Test+ void runOnAnUnknownApplicationIsResourceNotFound() {+ startJobRun("00missing0000000", sparkRun(UUID.randomUUID().toString(), ""))+ .then().statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"));+ }++ @Test+ void unknownJobRunIsResourceNotFound() {+ String applicationId = createApplication("");++ givenReq().when().get("/applications/" + applicationId + "/jobruns/00missing0000000")+ .then().statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"));+ givenReq().when().delete("/applications/" + applicationId + "/jobruns/00missing0000000")+ .then().statusCode(404);+ }++ @Test+ void jobRunOfAnotherApplicationIsResourceNotFound() {+ String owner = createApplication("");+ String other = createApplication("");+ String jobRunId = startedRunId(owner, sparkRun(UUID.randomUUID().toString(), ""));++ givenReq().when().get("/applications/" + other + "/jobruns/" + jobRunId)+ .then().statusCode(404);+ }++ @Test+ void disabledAutoStartRefusesARunUntilTheApplicationStarts() {+ String applicationId = createApplication(", \"autoStartConfiguration\": {\"enabled\": false}");++ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ""))+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"));+ givenReq().when().get("/applications/" + applicationId)+ .then().body("application.state", equalTo("CREATED"));++ givenReq().when().post("/applications/" + applicationId + "/start").then().statusCode(200);+ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), "")).then().statusCode(200);+ }++ @Test+ void runOnATerminatedApplicationIsRefused() {+ String applicationId = createApplication("");+ givenReq().when().delete("/applications/" + applicationId).then().statusCode(200);++ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ""))+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"));+ }++ @Test+ void cancellingAFinishedRunIsAValidationException() {+ String applicationId = createApplication("");+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));++ givenReq().when().delete("/applications/" + applicationId + "/jobruns/" + jobRunId)+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("Job run is not in a cancellable state"));+ }++ @Test+ void streamingRunKeepsRunningUntilCancelled() {+ String applicationId = createApplication("");+ String jobRunId = startedRunId(applicationId,+ sparkRun(UUID.randomUUID().toString(), ", \"mode\": \"STREAMING\""));+ String path = "/applications/" + applicationId + "/jobruns/" + jobRunId;++ givenReq().when().get(path).then().statusCode(200)+ .body("jobRun.state", equalTo("RUNNING"))+ .body("jobRun", not(hasKey("endedAt")))+ .body("jobRun", not(hasKey("totalExecutionDurationSeconds")));++ givenReq().when().delete(path).then().statusCode(200)+ .body("applicationId", equalTo(applicationId))+ .body("jobRunId", equalTo(jobRunId));++ givenReq().when().get(path).then().statusCode(200)+ .body("jobRun.state", equalTo("CANCELLED"))+ .body("jobRun.endedAt", instanceOf(Number.class));+ }++ @Test+ void listJobRunsReturnsTheNewestFirst() {+ String applicationId = createApplication("");+ String older = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));+ clock.advance(Duration.ofSeconds(5));+ String newer = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));++ givenReq().when().get("/applications/" + applicationId + "/jobruns")+ .then().statusCode(200)+ .body("jobRuns.id", contains(newer, older))+ .body("jobRuns[0].applicationId", equalTo(applicationId))+ .body("jobRuns[0].state", equalTo("SUCCESS"))+ .body("jobRuns[0].type", equalTo("SPARK"))+ .body("jobRuns[0].releaseLabel", equalTo("emr-7.5.0"))+ .body("jobRuns[0].createdBy", equalTo("arn:aws:iam::000000000000:root"))+ .body("jobRuns[0]", not(hasKey("jobRunId")))+ .body("jobRuns[0]", not(hasKey("jobDriver")));+ }++ @Test+ void listJobRunsPagesWithNextToken() {+ String applicationId = createApplication("");+ String older = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));+ clock.advance(Duration.ofSeconds(5));+ String newer = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));++ String nextToken = givenReq().queryParam("maxResults", 1)+ .when().get("/applications/" + applicationId + "/jobruns")+ .then().statusCode(200)+ .body("jobRuns.id", contains(newer))+ .body("nextToken", notNullValue())+ .extract().path("nextToken");++ givenReq().queryParam("maxResults", 1).queryParam("nextToken", nextToken)+ .when().get("/applications/" + applicationId + "/jobruns")+ .then().statusCode(200)+ .body("jobRuns.id", contains(older))+ .body("$", not(hasKey("nextToken")));+ }++ @Test+ void listJobRunsRefusesAPageLargerThanFifty() {+ String applicationId = createApplication("");++ givenReq().queryParam("maxResults", 51)+ .when().get("/applications/" + applicationId + "/jobruns")+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("1 validation error detected: Value '51' at 'maxResults' failed to "+ + "satisfy constraint: Member must have value less than or equal to 50"));+ }++ @Test+ void listJobRunsRefusesATokenFromAnotherListing() {+ String applicationId = createApplication("");+ String other = createApplication("");+ startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));+ clock.advance(Duration.ofSeconds(5));+ startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));+ String nextToken = givenReq().queryParam("maxResults", 1)+ .when().get("/applications/" + applicationId + "/jobruns")+ .then().statusCode(200)+ .extract().path("nextToken");++ givenReq().queryParam("nextToken", nextToken)+ .when().get("/applications/" + other + "/jobruns")+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"));+ givenReq().queryParam("nextToken", nextToken).queryParam("states", "SUCCESS")+ .when().get("/applications/" + applicationId + "/jobruns")+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"));+ }++ @Test+ void listJobRunsFiltersOnStateModeAndCreationTime() {+ String applicationId = createApplication("");+ String batch = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));+ clock.advance(Duration.ofSeconds(10));+ Instant between = clock.instant();+ clock.advance(Duration.ofSeconds(10));+ String streaming = startedRunId(applicationId,+ sparkRun(UUID.randomUUID().toString(), ", \"mode\": \"STREAMING\""));+ String base = "/applications/" + applicationId + "/jobruns";++ givenReq().queryParam("states", "RUNNING").when().get(base)+ .then().statusCode(200).body("jobRuns.id", contains(streaming));+ givenReq().queryParam("mode", "STREAMING").when().get(base)+ .then().statusCode(200).body("jobRuns.id", contains(streaming));+ givenReq().queryParam("mode", "BATCH").when().get(base)+ .then().statusCode(200).body("jobRuns.id", contains(batch));+ givenReq().queryParam("createdAtBefore", between.toString()).when().get(base)+ .then().statusCode(200).body("jobRuns.id", contains(batch));+ givenReq().queryParam("createdAtAfter", between.toString()).when().get(base)+ .then().statusCode(200).body("jobRuns.id", contains(streaming));+ }++ @Test+ void listJobRunsGroupsSeveralStatesInTheFilterOrder() {+ String applicationId = createApplication("");+ String streaming = startedRunId(applicationId,+ sparkRun(UUID.randomUUID().toString(), ", \"mode\": \"STREAMING\""));+ clock.advance(Duration.ofSeconds(5));+ String batch = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));++ givenReq().queryParam("states", List.of("RUNNING", "SUCCESS"))+ .when().get("/applications/" + applicationId + "/jobruns")+ .then().statusCode(200)+ .body("jobRuns.id", contains(streaming, batch));+ }++ @Test+ void listJobRunsOfAnUnknownApplicationIsEmpty() {+ givenReq().when().get("/applications/00missing0000000/jobruns")+ .then().statusCode(200)+ .body("jobRuns", empty());+ }++ @Test+ void jobRunsStayInTheirRegion() {+ String applicationId = createApplication("");+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));++ EmrServerlessRequests.emrServerless("eu-west-1").when().get("/applications/" + applicationId + "/jobruns/" + jobRunId)+ .then().statusCode(404);+ EmrServerlessRequests.emrServerless("eu-west-1").when().get("/applications/" + applicationId + "/jobruns")+ .then().statusCode(200).body("jobRuns", empty());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.javanew file mode 100644index 0000000000..f3b4cd5520--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java@@ -0,0 +1,189 @@+package io.github.hectorvent.floci.services.emrserverless;++import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.services.emrserverless.model.CreateApplicationRequest;+import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;+import io.github.hectorvent.floci.services.emrserverless.model.JobRun;+import io.github.hectorvent.floci.services.emrserverless.model.JobRunRecord;+import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunRequest;+import io.github.hectorvent.floci.services.iam.IamService;+import io.github.hectorvent.floci.testing.InMemoryStorageFactory;+import io.github.hectorvent.floci.testing.MutableClock;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.time.Instant;+import java.util.Map;+import java.util.Optional;+import java.util.UUID;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++class EmrServerlessJobRunServiceTest {++ private static final String REGION = "us-east-1";+ private static final String ACCOUNT_ID = "000000000000";+ private static final String ROLE = "arn:aws:iam::000000000000:role/emr-serverless-job";++ private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());+ private EmrServerlessService applications;+ private IamService iamService;+ private AccountAwareStorageBackend<JobRunRecord> store;+ private MutableClock clock;+ private String applicationId;++ @BeforeEach+ void setUp() {+ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.defaultAccountId()).thenReturn(ACCOUNT_ID);+ clock = new MutableClock();+ applications = new EmrServerlessService(config, new InMemoryStorageFactory(ACCOUNT_ID),+ new RegionResolver(REGION, ACCOUNT_ID), clock);+ iamService = mock(IamService.class);+ when(iamService.resolveCallerArn(any())).thenReturn(Optional.empty());+ store = AccountAwareStorageBackend.inMemory(ACCOUNT_ID);++ CreateApplicationRequest create = new CreateApplicationRequest();+ create.setName("jobs");+ create.setReleaseLabel("emr-7.5.0");+ create.setType("SPARK");+ create.setClientToken(UUID.randomUUID().toString());+ applicationId = applications.createApplication(create).getApplicationId();+ }++ private EmrServerlessJobRunService service(int runDurationSeconds) {+ return new EmrServerlessJobRunService(store, applications, iamService, new RegionResolver(REGION, ACCOUNT_ID),+ null, objectMapper, runDurationSeconds, clock);+ }++ private static StartJobRunRequest request(String mode, Long executionTimeoutMinutes) {+ JobDriver.SparkSubmit sparkSubmit = new JobDriver.SparkSubmit();+ sparkSubmit.setEntryPoint("s3://bucket/jobs/app.jar");+ JobDriver driver = new JobDriver();+ driver.setSparkSubmit(sparkSubmit);+ StartJobRunRequest request = new StartJobRunRequest();+ request.setClientToken(UUID.randomUUID().toString());+ request.setExecutionRoleArn(ROLE);+ request.setJobDriver(driver);+ request.setMode(mode);+ request.setExecutionTimeoutMinutes(executionTimeoutMinutes);+ return request;+ }++ @Test+ void runSucceedsAsSoonAsItStartsByDefault() {+ JobRun run = service(0).startJobRun(applicationId, request(null, null));++ assertEquals("SUCCESS", run.getState());+ assertEquals(run.getCreatedAt(), run.getStartedAt());+ assertEquals(run.getCreatedAt(), run.getEndedAt());+ assertEquals(run.getCreatedAt(), run.getUpdatedAt());+ assertEquals(0, run.getTotalExecutionDurationSeconds());+ }++ @Test+ void positiveDurationKeepsTheRunRunningUntilItHasPassed() {+ EmrServerlessJobRunService service = service(30);+ JobRun started = service.startJobRun(applicationId, request(null, null));+ String jobRunId = started.getJobRunId();+ Instant submittedAt = started.getCreatedAt();++ clock.advance(Duration.ofSeconds(29));+ JobRun running = service.getJobRun(applicationId, jobRunId);+ assertEquals("RUNNING", running.getState());+ assertNull(running.getEndedAt());+ assertNull(running.getTotalExecutionDurationSeconds());++ clock.advance(Duration.ofSeconds(1));+ JobRun finished = service.getJobRun(applicationId, jobRunId);+ assertEquals("SUCCESS", finished.getState());+ assertEquals(submittedAt.plusSeconds(30), finished.getEndedAt());+ assertEquals(submittedAt.plusSeconds(30), finished.getUpdatedAt());+ assertEquals(30, finished.getTotalExecutionDurationSeconds());+ }++ @Test+ void cancellingARunningRunEndsItCancelled() {+ EmrServerlessJobRunService service = service(60);+ String jobRunId = service.startJobRun(applicationId, request(null, null)).getJobRunId();+ clock.advance(Duration.ofSeconds(10));++ JobRun cancelled = service.cancelJobRun(applicationId, jobRunId);++ assertEquals("CANCELLED", cancelled.getState());+ assertEquals(10, cancelled.getTotalExecutionDurationSeconds());+ clock.advance(Duration.ofMinutes(5));+ assertEquals("CANCELLED", service.getJobRun(applicationId, jobRunId).getState());+ }++ @Test+ void runIsCancelledAtItsExecutionTimeoutWhenItWouldRunLonger() {+ EmrServerlessJobRunService service = service(600);+ JobRun started = service.startJobRun(applicationId, request(null, 5L));+ String jobRunId = started.getJobRunId();+ Instant submittedAt = started.getCreatedAt();++ clock.advance(Duration.ofMinutes(5).minusSeconds(1));+ assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId).getState());++ clock.advance(Duration.ofSeconds(1));+ JobRun timedOut = service.getJobRun(applicationId, jobRunId);+ assertEquals("CANCELLED", timedOut.getState());+ assertEquals(submittedAt.plus(Duration.ofMinutes(5)), timedOut.getEndedAt());+ }++ @Test+ void streamingRunIgnoresTheRunDuration() {+ EmrServerlessJobRunService service = service(0);+ String jobRunId = service.startJobRun(applicationId, request("STREAMING", null)).getJobRunId();++ clock.advance(Duration.ofDays(2));++ assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId).getState());+ }++ @Test+ void retryIsRecognisedAfterTheStoredRunWasWrittenAndReadBack() throws Exception {+ EmrServerlessJobRunService service = service(0);+ StartJobRunRequest request = request("BATCH", 15L);+ request.setTags(Map.of("team", "data"));+ String jobRunId = service.startJobRun(applicationId, request).getJobRunId();++ // As persistent storage does: the record goes out as JSON and comes back.+ JobRunRecord stored = store.get(jobRunId).orElseThrow();+ store.put(jobRunId, objectMapper.readValue(objectMapper.writeValueAsString(stored), JobRunRecord.class));++ assertEquals(jobRunId, service.startJobRun(applicationId, request).getJobRunId());+ request.setExecutionTimeoutMinutes(30L);+ AwsException conflict = assertThrows(AwsException.class, () -> service.startJobRun(applicationId, request));+ assertEquals("ConflictException", conflict.getErrorCode());+ assertEquals(409, conflict.getHttpStatus());+ }++ @Test+ void negativeRunDurationIsRefused() {+ assertThrows(IllegalArgumentException.class, () -> service(-1));+ }++ @Test+ void parseTimestampReadsIso8601AndEpochSeconds() {+ assertEquals(Instant.parse("2026-10-21T12:00:00Z"),+ EmrServerlessJobRunService.parseTimestamp("2026-10-21T12:00:00Z", "createdAtAfter"));+ assertEquals(Instant.parse("2026-10-21T12:00:00.500Z"),+ EmrServerlessJobRunService.parseTimestamp("1792584000.5", "createdAtAfter"));+ AwsException invalid = assertThrows(AwsException.class,+ () -> EmrServerlessJobRunService.parseTimestamp("yesterday", "createdAtAfter"));+ assertEquals("ValidationException", invalid.getErrorCode());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessServiceTest.javanew file mode 100644index 0000000000..66b9c7984e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessServiceTest.java@@ -0,0 +1,91 @@+package io.github.hectorvent.floci.services.emrserverless;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.PersistentStorage;+import io.github.hectorvent.floci.services.emrserverless.model.Application;+import io.github.hectorvent.floci.services.emrserverless.model.CreateApplicationRequest;+import io.github.hectorvent.floci.testing.InMemoryStorageFactory;+import io.github.hectorvent.floci.testing.MutableClock;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.io.IOException;+import java.nio.file.Files;+import java.nio.file.Path;+import java.time.Duration;+import java.time.Instant;+import java.util.Map;+import java.util.UUID;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++class EmrServerlessServiceTest {++ private static final String REGION = "us-east-1";+ private static final String ACCOUNT_ID = "000000000000";+ private static final TypeReference<Map<String, Application>> APPLICATIONS = new TypeReference<>() {};++ @TempDir+ Path tempDir;++ @Test+ void applicationIsStampedFromTheClockToTheMillisecond() {+ MutableClock clock = new MutableClock();+ clock.advance(Duration.ofNanos(250_400_000));+ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.defaultAccountId()).thenReturn(ACCOUNT_ID);+ EmrServerlessService service = new EmrServerlessService(config, new InMemoryStorageFactory(ACCOUNT_ID),+ new RegionResolver(REGION, ACCOUNT_ID), clock);+ CreateApplicationRequest create = new CreateApplicationRequest();+ create.setReleaseLabel("emr-7.5.0");+ create.setType("SPARK");+ create.setClientToken(UUID.randomUUID().toString());++ Application created = service.createApplication(create);+ assertEquals(Instant.parse("2026-01-01T00:00:00.250Z"), created.getCreatedAt());+ assertEquals(created.getCreatedAt(), created.getUpdatedAt());++ // The test clock moves a millisecond on every read.+ clock.advance(Duration.ofMinutes(5));+ service.startApplication(created.getApplicationId());+ Application started = service.getApplication(created.getApplicationId());+ assertEquals(Instant.parse("2026-01-01T00:00:00.250Z"), started.getCreatedAt());+ assertEquals(Instant.parse("2026-01-01T00:05:00.251Z"), started.getUpdatedAt());+ }++ @Test+ void applicationPersistedWithMillisecondTimestampsReadsBackAtTheSameInstant() throws IOException {+ // As Floci stored applications before their timestamps were epoch seconds.+ Path file = tempDir.resolve("emr-serverless-applications.json");+ Files.writeString(file, """+ {"00abc": {"applicationId": "00abc", "createdAt": 1792584000123, "updatedAt": 1792584060456}}+ """);+ PersistentStorage<String, Application> storage = new PersistentStorage<>(file, APPLICATIONS);+ storage.load();++ Application app = storage.get("00abc").orElseThrow();+ assertEquals(Instant.parse("2026-10-21T12:00:00.123Z"), app.getCreatedAt());+ assertEquals(Instant.parse("2026-10-21T12:01:00.456Z"), app.getUpdatedAt());+ }++ @Test+ void applicationTimestampsSurviveAReload() {+ Path file = tempDir.resolve("emr-serverless-applications.json");+ Application app = new Application();+ app.setApplicationId("00abc");+ app.setCreatedAt(Instant.parse("2026-10-21T12:00:00.123Z"));+ app.setUpdatedAt(Instant.parse("2026-10-21T12:01:00.456Z"));+ new PersistentStorage<>(file, APPLICATIONS).put("00abc", app);++ PersistentStorage<String, Application> reader = new PersistentStorage<>(file, APPLICATIONS);+ reader.load();++ Application reloaded = reader.get("00abc").orElseThrow();+ assertEquals(app.getCreatedAt(), reloaded.getCreatedAt());+ assertEquals(app.getUpdatedAt(), reloaded.getUpdatedAt());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java b/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.javanew file mode 100644index 0000000000..29cf8cb33f--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java@@ -0,0 +1,17 @@+package io.github.hectorvent.floci.testing;++import io.restassured.specification.RequestSpecification;++import static io.restassured.RestAssured.given;++/** One EMR Serverless REST JSON request, signed for a named Region. */+public final class EmrServerlessRequests {+ private EmrServerlessRequests() {}++ public static RequestSpecification emrServerless(String region) {+ return given()+ .header("Authorization", "AWS4-HMAC-SHA256 Credential=TEST/20260819/" + region+ + "/emr-serverless/aws4_request, SignedHeaders=host;x-amz-date, Signature=test")+ .contentType("application/json");+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex dca4a32d2d..3a19e2ba5e 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -496,6 +496,7 @@ floci:enabled: trueemrserverless:enabled: true+ job-run-duration-seconds: 0 # 0 = BATCH job runs succeed as soon as they startssoadmin:enabled: truessooidc: