Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(emr-serverless): start, get, list and cancel job runs (#6202)

Priamo Germosen1f1e5db1bc7d
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java
new file mode 100644
index 0000000000..de1e6ad277
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java
@@ -0,0 +1,149 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.services.emrserverless.EmrServerlessClient;
+import software.amazon.awssdk.services.emrserverless.model.Application;
+import software.amazon.awssdk.services.emrserverless.model.ApplicationState;
+import software.amazon.awssdk.services.emrserverless.model.ConflictException;
+import software.amazon.awssdk.services.emrserverless.model.JobRun;
+import software.amazon.awssdk.services.emrserverless.model.JobRunMode;
+import software.amazon.awssdk.services.emrserverless.model.JobRunState;
+import software.amazon.awssdk.services.emrserverless.model.JobRunSummary;
+import software.amazon.awssdk.services.emrserverless.model.StartJobRunResponse;
+import software.amazon.awssdk.services.emrserverless.model.ValidationException;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+@DisplayName("EMR Serverless job runs")
+class EmrServerlessJobRunTest {
+
+ private static final String ROLE = "arn:aws:iam::000000000000:role/emr-serverless-job";
+
+ private static EmrServerlessClient emr;
+ private static String applicationId;
+
+ @BeforeAll
+ static void setup() {
+ emr = TestFixtures.emrServerlessClient();
+ applicationId = emr.createApplication(r -> r
+ .name(TestFixtures.uniqueName("jobs-app"))
+ .releaseLabel("emr-7.5.0")
+ .type("SPARK")
+ .clientToken(UUID.randomUUID().toString())).applicationId();
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (emr == null) {
+ return;
+ }
+ try {
+ emr.stopApplication(r -> r.applicationId(applicationId));
+ emr.deleteApplication(r -> r.applicationId(applicationId));
+ }
+ catch (Exception ignored) {
+ // Cleanup is best effort: the application is unique to this run and the emulator is disposable.
+ }
+ emr.close();
+ }
+
+ private static StartJobRunResponse startSparkRun(JobRunMode mode) {
+ return emr.startJobRun(r -> r
+ .applicationId(applicationId)
+ .executionRoleArn(ROLE)
+ .name("nightly")
+ .mode(mode)
+ .tags(Map.of("team", "data"))
+ .jobDriver(d -> d.sparkSubmit(s -> s
+ .entryPoint("s3://bucket/jobs/app.jar")
+ .entryPointArguments("s3://bucket/in")
+ .sparkSubmitParameters("--class example.Main"))));
+ }
+
+ @Test
+ @DisplayName("StartJobRun auto-starts the application and GetJobRun reports a finished run")
+ void startAndGet() {
+ Instant before = Instant.now().minus(1, ChronoUnit.DAYS);
+ StartJobRunResponse started = startSparkRun(JobRunMode.BATCH);
+ Instant after = Instant.now().plus(1, ChronoUnit.DAYS);
+
+ assertThat(started.applicationId()).isEqualTo(applicationId);
+ assertThat(started.arn()).endsWith("/applications/" + applicationId + "/jobruns/" + started.jobRunId());
+ Application application = emr.getApplication(r -> r.applicationId(applicationId)).application();
+ assertThat(application.state()).isEqualTo(ApplicationState.STARTED);
+ // Epoch seconds on the wire; milliseconds would read as a year past 50,000.
+ assertThat(application.createdAt()).isBetween(before, after);
+ assertThat(application.updatedAt()).isBetween(application.createdAt(), after);
+
+ JobRun run = emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(started.jobRunId())).jobRun();
+ assertThat(run.state()).isEqualTo(JobRunState.SUCCESS);
+ assertThat(run.executionRole()).isEqualTo(ROLE);
+ assertThat(run.releaseLabel()).isEqualTo("emr-7.5.0");
+ assertThat(run.createdBy()).startsWith("arn:aws:");
+ assertThat(run.createdAt()).isBetween(before, after);
+ assertThat(run.jobDriver().sparkSubmit().entryPoint()).isEqualTo("s3://bucket/jobs/app.jar");
+ assertThat(run.tags()).containsEntry("team", "data");
+ assertThat(run.executionTimeoutMinutes()).isEqualTo(720L);
+ assertThat(run.mode()).isEqualTo(JobRunMode.BATCH);
+ }
+
+ @Test
+ @DisplayName("A repeated client token returns the original run; reused with another request it conflicts")
+ void clientTokenIdempotency() {
+ String token = UUID.randomUUID().toString();
+ String first = emr.startJobRun(r -> r.applicationId(applicationId).clientToken(token)
+ .executionRoleArn(ROLE).jobDriver(d -> d.sparkSubmit(s -> s.entryPoint("s3://bucket/a.jar"))))
+ .jobRunId();
+ String again = emr.startJobRun(r -> r.applicationId(applicationId).clientToken(token)
+ .executionRoleArn(ROLE).jobDriver(d -> d.sparkSubmit(s -> s.entryPoint("s3://bucket/a.jar"))))
+ .jobRunId();
+ assertThat(again).isEqualTo(first);
+
+ assertThatThrownBy(() -> emr.startJobRun(r -> r.applicationId(applicationId).clientToken(token)
+ .executionRoleArn(ROLE).jobDriver(d -> d.sparkSubmit(s -> s.entryPoint("s3://bucket/b.jar")))))
+ .isInstanceOf(ConflictException.class);
+ }
+
+ @Test
+ @DisplayName("CancelJobRun cancels a streaming run and refuses a finished one")
+ void cancel() {
+ String streaming = startSparkRun(JobRunMode.STREAMING).jobRunId();
+ assertThat(emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(streaming)).jobRun().state())
+ .isEqualTo(JobRunState.RUNNING);
+
+ emr.cancelJobRun(r -> r.applicationId(applicationId).jobRunId(streaming));
+ assertThat(emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(streaming)).jobRun().state())
+ .isEqualTo(JobRunState.CANCELLED);
+
+ assertThatThrownBy(() -> emr.cancelJobRun(r -> r.applicationId(applicationId).jobRunId(streaming)))
+ .isInstanceOf(ValidationException.class)
+ .hasMessageContaining("Job run is not in a cancellable state");
+ }
+
+ @Test
+ @DisplayName("ListJobRuns returns summaries filtered by state")
+ void list() {
+ String streaming = startSparkRun(JobRunMode.STREAMING).jobRunId();
+ try {
+ List<JobRunSummary> running = emr.listJobRuns(r -> r.applicationId(applicationId)
+ .states(JobRunState.RUNNING)).jobRuns();
+ assertThat(running).extracting(JobRunSummary::id).contains(streaming);
+ assertThat(running).allSatisfy(summary -> assertThat(summary.state()).isEqualTo(JobRunState.RUNNING));
+ assertThat(emr.listJobRunsPaginator(r -> r.applicationId(applicationId)).jobRuns())
+ .extracting(JobRunSummary::applicationId).containsOnly(applicationId);
+ }
+ finally {
+ emr.cancelJobRun(r -> r.applicationId(applicationId).jobRunId(streaming));
+ }
+ }
+}
diff --git a/docs/services/emr-serverless.md b/docs/services/emr-serverless.md
index c6d72375ae..cdeda3875e 100644
--- a/docs/services/emr-serverless.md
+++ b/docs/services/emr-serverless.md
@@ -1,6 +1,6 @@
# EMR Serverless
-EMR Serverless is an AWS service that provides serverless data analytics environments. Floci provides an emulator for the management-plane API (REST JSON) which covers the basic lifecycle of an Application.
+EMR Serverless is an AWS service that provides serverless data analytics environments. Floci provides an emulator for the management-plane API (REST JSON) which covers the lifecycle of an Application and of the job runs submitted to it.
## Configuration
@@ -8,6 +8,7 @@ EMR Serverless is an AWS service that provides serverless data analytics environ
|-----|-------------|---------|
| `floci.emrserverless.enabled` | Whether the EMR Serverless API is enabled | `true` |
| `floci.emrserverless.port` | The port the service is exposed on | `4566` |
+| `floci.services.emrserverless.job-run-duration-seconds` | How long a `BATCH` job run stays `RUNNING` before it succeeds; `0` makes it `SUCCESS` as soon as it starts | `0` |
## Endpoints
@@ -20,6 +21,10 @@ The emulator implements the standard AWS `emr-serverless` service endpoints:
* `DELETE /applications/{applicationId}` (DeleteApplication)
* `POST /applications/{applicationId}/start` (StartApplication)
* `POST /applications/{applicationId}/stop` (StopApplication)
+* `POST /applications/{applicationId}/jobruns` (StartJobRun)
+* `GET /applications/{applicationId}/jobruns` (ListJobRuns)
+* `GET /applications/{applicationId}/jobruns/{jobRunId}` (GetJobRun)
+* `DELETE /applications/{applicationId}/jobruns/{jobRunId}` (CancelJobRun)
* `POST /tags/{resourceArn}` (TagResource)
* `DELETE /tags/{resourceArn}?tagKeys=...` (UntagResource)
* `GET /tags/{resourceArn}` (ListTagsForResource)
@@ -36,10 +41,39 @@ An application created in one Region is not returned by list, get, or tag reques
report that state so lifecycle waiters can observe it, while an unfiltered `ListApplications`
omits the terminated application. A terminated application cannot be started or stopped again.
+## Job runs
+
+Job runs are metadata only: nothing executes. A run follows the job run states and `GetJobRun`'s
+response shape, and settles when it is read.
+
+* **Lifecycle**: a run is `RUNNING` from the moment it is submitted. With the default
+ `job-run-duration-seconds` of `0`, a `BATCH` run is `SUCCESS` as soon as it starts; a positive
+ duration keeps it `RUNNING` that long. A run whose duration would pass its
+ `executionTimeoutMinutes` (720 when not given, as AWS reports it) is `CANCELLED` at the timeout.
+ A `STREAMING` run stays `RUNNING` until it is cancelled.
+* **Auto-start**: a run submitted to a `CREATED` or `STOPPED` application starts the application,
+ as `autoStartConfiguration.enabled` defaults to `true`. With auto-start turned off, the run is
+ refused with `ValidationException` until the application is started. A run on a terminated
+ application is refused.
+* **Idempotency**: `clientToken` is the idempotency token. Repeating a request with the same token
+ returns the original run; reusing the token with a different request answers `ConflictException`
+ (HTTP 409) "Request did not match the original".
+* **CancelJobRun** moves a run that has not finished to `CANCELLED`. A finished run answers
+ `ValidationException` "Job run is not in a cancellable state".
+* **ListJobRuns** returns the newest runs first, filtered by `states`, `createdAtAfter`,
+ `createdAtBefore` and `mode`, at most 50 per page. With several `states`, the runs are grouped by
+ state in the order the filter names them. An unknown application has no runs. A `nextToken`
+ resumes only the listing it came from, with the same application and filters.
+* `createdBy` is the caller's ARN, as `GetCallerIdentity` reports it. `releaseLabel` and
+ `networkConfiguration` come from the application.
+
## Limitations and Differences from AWS
-* **Jobs not implemented**: Floci supports the provisioning of the `Application` management plane (which satisfies tools like Terraform's `aws_emrserverless_application` resource). Executing actual Spark or Hive jobs against these applications via `StartJobRun` is not currently implemented.
+* **Jobs do not execute**: `StartJobRun` records a job run and moves it through its states, but no Spark or Hive job runs.
+* **Unsourced errors**: AWS documents neither the error for a run submitted to an application whose auto-start is off nor the one for a run on a terminated application; Floci answers both with `ValidationException`. A run needs a `jobDriver`, as every `JobRun` carries one, and `jobDriver` is a union, so a run without a driver, or with one that sets both `sparkSubmit` and `hive` or neither, is refused with `ValidationException` too, in Floci's own wording.
+* **StopApplication with running jobs**: AWS requires running jobs to finish or be cancelled before an application stops. Floci does not enforce it; with the default duration of `0` no `BATCH` run is ever running.
+* **Not emulated**: job run attempts and retries, queuing (`QUEUED`), `totalResourceUtilization` and `billedResourceUtilization`, and `shutdownGracePeriodInSeconds` (accepted and ignored).
* **Instant Start/Stop**: Floci marks the application `STARTED` or `STOPPED` immediately without provisioning actual compute capacity in the background.
* **Data Plane**: Floci does not implement the data-plane API or any execution environments.
-* **Job run tags**: job runs are not emulated, so a job run ARN (`.../jobruns/<id>`) answers
- `ResourceNotFoundException` on the tagging APIs.
+* **Job run tags**: the tagging APIs do not accept job run ARNs (`.../jobruns/<id>`) yet; they answer
+ `ResourceNotFoundException`. Tags given to `StartJobRun` are stored and returned by `GetJobRun`.
diff --git a/docs/services/index.md b/docs/services/index.md
index 851d1db247..1181b191f7 100644
--- a/docs/services/index.md
+++ b/docs/services/index.md
@@ -80,7 +80,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac
| [Redshift Data API](redshift-data.md) | `POST /` + `X-Amz-Target: RedshiftData.*` | JSON 1.1 | 11 |
| [Redshift Serverless](redshift-serverless.md) | `POST /` + `X-Amz-Target: RedshiftServerless.*` | JSON 1.1 | 55 |
| [EMR](emr.md) | `POST /` + `X-Amz-Target: ElasticMapReduce.*` | JSON 1.1 | 40 |
-| [EMR Serverless](emr-serverless.md) | `/applications/*`, `/tags/*` | REST JSON | 10 |
+| [EMR Serverless](emr-serverless.md) | `/applications/*`, `/tags/*` | REST JSON | 14 |
| [Data Firehose](firehose.md) | `POST /` + `X-Amz-Target: Firehose_20150804.*` | JSON 1.1 | 12 |
| [ECS](ecs.md) | `POST /` + `X-Amz-Target: AmazonEC2ContainerServiceV20141113.*` | JSON 1.1 | 59 |
| [EFS](efs.md) | `/2015-02-01/...` | REST JSON | 26 |
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 48f74cb775..0a6e1ed0e4 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1085,6 +1085,10 @@ public interface EmulatorConfig {
interface EmrServerlessServiceConfig {
@WithDefault("true")
boolean enabled();
+
+ /** How long a BATCH job run stays RUNNING before it succeeds; 0 = a run succeeds as soon as it starts. */
+ @WithDefault("0")
+ int jobRunDurationSeconds();
}
interface Route53ResolverServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java
index b09b7e76e9..063cf0e8e4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java
@@ -4,13 +4,20 @@ import io.github.hectorvent.floci.core.common.PaginatedResult;
import io.github.hectorvent.floci.core.common.Pagination;
import io.github.hectorvent.floci.services.emrserverless.model.Application;
import io.github.hectorvent.floci.services.emrserverless.model.ApplicationSummary;
+import io.github.hectorvent.floci.services.emrserverless.model.CancelJobRunResponse;
import io.github.hectorvent.floci.services.emrserverless.model.CreateApplicationRequest;
import io.github.hectorvent.floci.services.emrserverless.model.CreateApplicationResponse;
import io.github.hectorvent.floci.services.emrserverless.model.DeleteApplicationResponse;
import io.github.hectorvent.floci.services.emrserverless.model.GetApplicationResponse;
+import io.github.hectorvent.floci.services.emrserverless.model.GetJobRunResponse;
+import io.github.hectorvent.floci.services.emrserverless.model.JobRun;
+import io.github.hectorvent.floci.services.emrserverless.model.JobRunSummary;
import io.github.hectorvent.floci.services.emrserverless.model.ListApplicationsRequest;
import io.github.hectorvent.floci.services.emrserverless.model.ListApplicationsResponse;
+import io.github.hectorvent.floci.services.emrserverless.model.ListJobRunsResponse;
import io.github.hectorvent.floci.services.emrserverless.model.StartApplicationResponse;
+import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunRequest;
+import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunResponse;
import io.github.hectorvent.floci.services.emrserverless.model.StopApplicationResponse;
import io.github.hectorvent.floci.services.emrserverless.model.UpdateApplicationRequest;
import io.github.hectorvent.floci.services.emrserverless.model.UpdateApplicationResponse;
@@ -36,10 +43,12 @@ import java.util.List;
public class EmrServerlessController {
private final EmrServerlessService service;
+ private final EmrServerlessJobRunService jobRuns;
@Inject
- public EmrServerlessController(EmrServerlessService service) {
+ public EmrServerlessController(EmrServerlessService service, EmrServerlessJobRunService jobRuns) {
this.service = service;
+ this.jobRuns = jobRuns;
}
@POST
@@ -116,4 +125,55 @@ public class EmrServerlessController {
StopApplicationResponse response = new StopApplicationResponse();
return response;
}
+
+ @POST
+ @Path("/{applicationId}/jobruns")
+ public StartJobRunResponse startJobRun(@PathParam("applicationId") String applicationId,
+ StartJobRunRequest request) {
+ JobRun run = jobRuns.startJobRun(applicationId, request);
+ StartJobRunResponse response = new StartJobRunResponse();
+ response.setApplicationId(run.getApplicationId());
+ response.setJobRunId(run.getJobRunId());
+ response.setArn(run.getArn());
+ return response;
+ }
+
+ @GET
+ @Path("/{applicationId}/jobruns")
+ public ListJobRunsResponse listJobRuns(@PathParam("applicationId") String applicationId,
+ @QueryParam("states") List<String> states,
+ @QueryParam("createdAtAfter") String createdAtAfter,
+ @QueryParam("createdAtBefore") String createdAtBefore,
+ @QueryParam("mode") String mode,
+ @QueryParam("maxResults") String maxResults,
+ @QueryParam("nextToken") String nextToken) {
+ PaginatedResult<JobRunSummary> page = jobRuns.listJobRuns(applicationId, states,
+ EmrServerlessJobRunService.parseTimestamp(createdAtAfter, "createdAtAfter"),
+ EmrServerlessJobRunService.parseTimestamp(createdAtBefore, "createdAtBefore"),
+ mode, Pagination.parseMaxResults(maxResults, "ValidationException"), nextToken);
+ ListJobRunsResponse response = new ListJobRunsResponse();
+ response.setJobRuns(page.items());
+ response.setNextToken(page.nextToken());
+ return response;
+ }
+
+ @GET
+ @Path("/{applicationId}/jobruns/{jobRunId}")
+ public GetJobRunResponse getJobRun(@PathParam("applicationId") String applicationId,
+ @PathParam("jobRunId") String jobRunId) {
+ GetJobRunResponse response = new GetJobRunResponse();
+ response.setJobRun(jobRuns.getJobRun(applicationId, jobRunId));
+ return response;
+ }
+
+ @DELETE
+ @Path("/{applicationId}/jobruns/{jobRunId}")
+ public CancelJobRunResponse cancelJobRun(@PathParam("applicationId") String applicationId,
+ @PathParam("jobRunId") String jobRunId) {
+ JobRun run = jobRuns.cancelJobRun(applicationId, jobRunId);
+ CancelJobRunResponse response = new CancelJobRunResponse();
+ response.setApplicationId(run.getApplicationId());
+ response.setJobRunId(run.getJobRunId());
+ return response;
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java
new file mode 100644
index 0000000000..15cc34739a
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java
@@ -0,0 +1,416 @@
+package io.github.hectorvent.floci.services.emrserverless;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.PaginatedResult;
+import io.github.hectorvent.floci.core.common.Pagination;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.RequestContext;
+import io.github.hectorvent.floci.core.common.SmithyValidationErrors;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.emrserverless.model.Application;
+import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;
+import io.github.hectorvent.floci.services.emrserverless.model.JobRun;
+import io.github.hectorvent.floci.services.emrserverless.model.JobRunRecord;
+import io.github.hectorvent.floci.services.emrserverless.model.JobRunSummary;
+import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunRequest;
+import io.github.hectorvent.floci.services.iam.IamService;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * EMR Serverless job runs. Nothing executes: a run follows the job run states and GetJobRun's
+ * shape, and settles lazily when read. Times are kept to the millisecond, as AWS reports them. With the default {@code job-run-duration-seconds} of 0 a
+ * BATCH run is SUCCESS as soon as it starts; a positive duration keeps it RUNNING that long, which
+ * makes CancelJobRun and the execution timeout observable. A STREAMING run, which EMR Serverless
+ * "expects to run for a long time", stays RUNNING until it is cancelled.
+ *
+ * <p>Every public method is synchronized: admitting a run (look up its client token, then write
+ * it), settling one and cancelling one each read and then write a run, and must not interleave.
+ */
+@ApplicationScoped
+public class EmrServerlessJobRunService {
+
+ private static final Logger LOG = Logger.getLogger(EmrServerlessJobRunService.class);
+
+ static final String STATE_RUNNING = "RUNNING";
+ static final String STATE_SUCCESS = "SUCCESS";
+ static final String STATE_CANCELLED = "CANCELLED";
+ private static final String MODE_BATCH = "BATCH";
+ private static final String MODE_STREAMING = "STREAMING";
+
+ private static final Set<String> TERMINAL_STATES = Set.of(STATE_SUCCESS, "FAILED", STATE_CANCELLED);
+ private static final List<String> MODES = List.of(MODE_BATCH, MODE_STREAMING);
+ // JobRun.executionTimeoutMinutes: "If you didn't specify a timeout, this value defaults to 720 minutes."
+ private static final long DEFAULT_EXECUTION_TIMEOUT_MINUTES = 720;
+ // The model's Duration, which bounds executionTimeoutMinutes.
+ private static final long MAX_EXECUTION_TIMEOUT_MINUTES = 1_000_000;
+ private static final int MAX_LIST_RESULTS = 50;
+ // IAMRoleArn's pattern, as the model and AWS's validation message spell it.
+ private static final String ROLE_ARN_PATTERN =
+ "arn:(aws[a-zA-Z0-9-]*):iam::([0-9]{12}):(role((\\u002F)|(\\u002F[\\u0021-\\u007F]+\\u002F))[\\w+=,.@-]+)";
+ private static final Pattern ROLE_ARN = Pattern.compile(ROLE_ARN_PATTERN);
+
+ private final AccountAwareStorageBackend<JobRunRecord> store;
+ private final EmrServerlessService applications;
+ private final IamService iamService;
+ private final RegionResolver regionResolver;
+ private final RequestContext requestContext;
+ private final ObjectMapper objectMapper;
+ private final int runDurationSeconds;
+ private final Clock clock;
+
+ @Inject
+ public EmrServerlessJobRunService(StorageFactory storageFactory, EmrServerlessService applications,
+ IamService iamService, RegionResolver regionResolver,
+ RequestContext requestContext, ObjectMapper objectMapper,
+ EmulatorConfig config, Clock clock) {
+ this(storageFactory.create("emrserverless", "emr-serverless-job-runs.json", new TypeReference<>() {}),
+ applications, iamService, regionResolver, requestContext, objectMapper,
+ config.services().emrserverless().jobRunDurationSeconds(), clock);
+ }
+
+ EmrServerlessJobRunService(AccountAwareStorageBackend<JobRunRecord> store, EmrServerlessService applications,
+ IamService iamService, RegionResolver regionResolver, RequestContext requestContext,
+ ObjectMapper objectMapper, int runDurationSeconds, Clock clock) {
+ if (runDurationSeconds < 0) {
+ throw new IllegalArgumentException(
+ "floci.services.emrserverless.job-run-duration-seconds must not be negative: " + runDurationSeconds);
+ }
+ this.store = store;
+ this.applications = applications;
+ this.iamService = iamService;
+ this.regionResolver = regionResolver;
+ this.requestContext = requestContext;
+ this.objectMapper = objectMapper;
+ this.runDurationSeconds = runDurationSeconds;
+ this.clock = clock;
+ }
+
+ /**
+ * Starts a run, or returns the run an earlier request with the same client token started. The
+ * token is StartJobRun's idempotency token; reused with a different request it answers
+ * {@code ConflictException} "Request did not match the original", as AWS does.
+ */
+ public synchronized JobRun startJobRun(String applicationId, StartJobRunRequest request) {
+ StartJobRunRequest body = request != null ? request : new StartJobRunRequest();
+ validate(body);
+ applications.getApplication(applicationId);
+ JsonNode original = requestOf(applicationId, body);
+ JobRunRecord existing = runWithClientToken(body.getClientToken());
+ if (existing != null) {
+ if (!original.equals(existing.getRequest())) {
+ throw new AwsException("ConflictException", "Request did not match the original", 409);
+ }
+ return settle(existing).getJobRun();
+ }
+ // Before admission, which may auto-start the application: a refused run leaves it as it was.
+ if (body.getTags() != null) {
+ EmrServerlessService.validateTags(body.getTags());
+ }
+ JobRunRecord record = applications.admitJobRun(applicationId, app -> newRun(app, body, original));
+ LOG.infov("Started EMR Serverless job run {0} on application {1}", record.getJobRun().getJobRunId(),
+ applicationId);
+ return settle(record).getJobRun();
+ }
+
+ public synchronized JobRun getJobRun(String applicationId, String jobRunId) {
+ applications.getApplication(applicationId);
+ return settle(findRun(applicationId, jobRunId)).getJobRun();
+ }
+
+ /**
+ * The application's runs, newest first. ListJobRuns models no ResourceNotFoundException, so an
+ * unknown application has no runs rather than an error. With several {@code states}, "the
+ * resulting list will be grouped by the state", in the order the filter names them. A run
+ * started without a mode is a BATCH run. A {@code nextToken} resumes only the listing it came
+ * from: its cursor holds a state group, which means nothing under another application or filter.
+ */
+ public synchronized PaginatedResult<JobRunSummary> listJobRuns(String applicationId, List<String> states,
+ Instant createdAtAfter, Instant createdAtBefore,
+ String mode, Integer maxResults, String nextToken) {
+ if (maxResults != null) {
+ String violation = outOfRange("maxResults", maxResults, 1, MAX_LIST_RESULTS);
+ if (violation != null) {
+ throw SmithyValidationErrors.validationException(violation);
+ }
+ }
+ List<JobRunRecord> matching = new ArrayList<>();
+ Map<String, String> cursors = new HashMap<>();
+ for (JobRunRecord record : store.scan(key -> true)) {
+ JobRun run = record.getJobRun();
+ if (!applicationId.equals(run.getApplicationId()) || !inRequestRegion(run)) {
+ continue;
+ }
+ settle(record);
+ if ((states != null && !states.isEmpty() && !states.contains(run.getState()))
+ || (createdAtAfter != null && !run.getCreatedAt().isAfter(createdAtAfter))
+ || (createdAtBefore != null && !run.getCreatedAt().isBefore(createdAtBefore))
+ || (mode != null && !mode.equals(run.getMode() != null ? run.getMode() : MODE_BATCH))) {
+ continue;
+ }
+ int group = states == null || states.isEmpty() ? 0 : states.indexOf(run.getState());
+ cursors.put(run.getJobRunId(), String.format("%02d%020d%s", group,
+ Long.MAX_VALUE - run.getCreatedAt().toEpochMilli(), run.getJobRunId()));
+ matching.add(record);
+ }
+ String namespace = String.join("|", "ListJobRuns", applicationId,
+ String.valueOf(states != null ? states : List.of()), String.valueOf(createdAtAfter),
+ String.valueOf(createdAtBefore), String.valueOf(mode));
+ PaginatedResult<JobRunRecord> page = Pagination.paginate(matching,
+ record -> cursors.get(record.getJobRun().getJobRunId()),
+ maxResults != null ? maxResults : MAX_LIST_RESULTS, nextToken, namespace,
+ token -> new AwsException("ValidationException", "Invalid nextToken.", 400));
+ return new PaginatedResult<>(page.items().stream().map(EmrServerlessJobRunService::toSummary).toList(),
+ page.nextToken());
+ }
+
+ /** Cancels a run that has not finished. A finished run is refused, with AWS's wording. */
+ public synchronized JobRun cancelJobRun(String applicationId, String jobRunId) {
+ applications.getApplication(applicationId);
+ JobRunRecord record = settle(findRun(applicationId, jobRunId));
+ if (isTerminal(record.getJobRun())) {
+ throw new AwsException("ValidationException", "Job run is not in a cancellable state", 400);
+ }
+ finish(record, STATE_CANCELLED, clock.instant().truncatedTo(ChronoUnit.MILLIS));
+ LOG.infov("Cancelled EMR Serverless job run {0} on application {1}", jobRunId, applicationId);
+ return record.getJobRun();
+ }
+
+ /**
+ * A ListJobRuns time filter. Query-string timestamps are ISO 8601 in restJson1; epoch seconds
+ * are accepted too.
+ */
+ static Instant parseTimestamp(String value, String member) {
+ if (value == null || value.isBlank()) {
+ return null;
+ }
+ try {
+ return OffsetDateTime.parse(value).toInstant();
+ } catch (DateTimeParseException notIso) {
+ try {
+ return Instant.ofEpochMilli(Math.round(Double.parseDouble(value) * 1000));
+ } catch (NumberFormatException notEpoch) {
+ throw new AwsException("ValidationException", member + " must be a timestamp.", 400);
+ }
+ }
+ }
+
+ private JobRunRecord newRun(Application app, StartJobRunRequest request, JsonNode original) {
+ AwsArnUtils.Arn applicationArn = AwsArnUtils.parse(app.getArn());
+ String jobRunId = EmrServerlessService.newId();
+ Instant now = clock.instant().truncatedTo(ChronoUnit.MILLIS);
+
+ JobRun run = new JobRun();
+ run.setApplicationId(app.getApplicationId());
+ run.setJobRunId(jobRunId);
+ run.setName(request.getName());
+ run.setArn(AwsArnUtils.Arn.of("emr-serverless", applicationArn.region(), applicationArn.accountId(),
+ applicationArn.resource() + "/jobruns/" + jobRunId).toString());
+ run.setCreatedBy(createdBy(applicationArn.accountId()));
+ run.setCreatedAt(now);
+ run.setUpdatedAt(now);
+ run.setExecutionRole(request.getExecutionRoleArn());
+ run.setExecutionIamPolicy(request.getExecutionIamPolicy());
+ run.setState(STATE_RUNNING);
+ run.setStateDetails("");
+ run.setReleaseLabel(app.getReleaseLabel());
+ run.setConfigurationOverrides(request.getConfigurationOverrides());
+ run.setJobDriver(request.getJobDriver());
+ run.setTags(request.getTags() != null ? new LinkedHashMap<>(request.getTags()) : new LinkedHashMap<>());
+ run.setNetworkConfiguration(app.getNetworkConfiguration());
+ run.setExecutionTimeoutMinutes(request.getExecutionTimeoutMinutes() != null
+ ? request.getExecutionTimeoutMinutes() : DEFAULT_EXECUTION_TIMEOUT_MINUTES);
+ run.setMode(request.getMode());
+ run.setRetryPolicy(request.getRetryPolicy());
+ run.setStartedAt(now);
+
+ JobRunRecord record = new JobRunRecord();
+ record.setJobRun(run);
+ record.setClientToken(request.getClientToken());
+ record.setRequest(original);
+ record.setApplicationType(app.getType());
+ store.put(jobRunId, record);
+ return record;
+ }
+
+ /**
+ * Finishes a BATCH run that has been RUNNING for the configured duration, or cancels it at its
+ * execution timeout when that comes first: "If the job run exceeds this duration, Amazon EMR
+ * Serverless cancels it automatically."
+ */
+ private JobRunRecord settle(JobRunRecord record) {
+ JobRun run = record.getJobRun();
+ if (isTerminal(run) || MODE_STREAMING.equals(run.getMode())) {
+ return record;
+ }
+ long timeoutSeconds = run.getExecutionTimeoutMinutes() == null || run.getExecutionTimeoutMinutes() == 0
+ ? Long.MAX_VALUE : run.getExecutionTimeoutMinutes() * 60;
+ if (runDurationSeconds > timeoutSeconds) {
+ Instant timedOutAt = run.getStartedAt().plusSeconds(timeoutSeconds);
+ if (!clock.instant().isBefore(timedOutAt)) {
+ finish(record, STATE_CANCELLED, timedOutAt);
+ }
+ return record;
+ }
+ Instant finishedAt = run.getStartedAt().plusSeconds(runDurationSeconds);
+ if (!clock.instant().isBefore(finishedAt)) {
+ finish(record, STATE_SUCCESS, finishedAt);
+ }
+ return record;
+ }
+
+ private void finish(JobRunRecord record, String state, Instant endedAt) {
+ JobRun run = record.getJobRun();
+ run.setState(state);
+ run.setEndedAt(endedAt);
+ run.setUpdatedAt(endedAt);
+ run.setTotalExecutionDurationSeconds((int) Duration.between(run.getStartedAt(), endedAt).toSeconds());
+ store.put(run.getJobRunId(), record);
+ }
+
+ private JobRunRecord findRun(String applicationId, String jobRunId) {
+ return store.get(jobRunId)
+ .filter(record -> applicationId.equals(record.getJobRun().getApplicationId()))
+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",
+ "Job run " + jobRunId + " not found", 404));
+ }
+
+ private JobRunRecord runWithClientToken(String clientToken) {
+ for (JobRunRecord record : store.scan(key -> true)) {
+ if (clientToken.equals(record.getClientToken()) && inRequestRegion(record.getJobRun())) {
+ return record;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * What a retry must repeat: the application and every member of the body but the token itself.
+ * Read back from its JSON text, so that its numbers compare equal to those of a stored request
+ * once storage has written and read it.
+ */
+ private JsonNode requestOf(String applicationId, StartJobRunRequest request) {
+ ObjectNode node;
+ try {
+ node = (ObjectNode) objectMapper.readTree(objectMapper.writeValueAsString(request));
+ } catch (JsonProcessingException e) {
+ throw new IllegalStateException("Cannot serialize a StartJobRun request", e);
+ }
+ node.remove("clientToken");
+ node.put("applicationId", applicationId);
+ return node;
+ }
+
+ // The caller, as GetCallerIdentity reports it; the account root for a key IAM does not know.
+ private String createdBy(String accountId) {
+ String accessKeyId = requestContext != null ? requestContext.getAccessKeyId() : null;
+ return iamService.resolveCallerArn(accessKeyId)
+ .orElseGet(() -> regionResolver.buildGlobalArn("iam", accountId, "root"));
+ }
+
+ private boolean inRequestRegion(JobRun run) {
+ return regionResolver.getRegion().equals(AwsArnUtils.parse(run.getArn()).region());
+ }
+
+ private static boolean isTerminal(JobRun run) {
+ return TERMINAL_STATES.contains(run.getState());
+ }
+
+ private static void validate(StartJobRunRequest request) {
+ List<String> violations = new ArrayList<>();
+ if (request.getClientToken() == null) {
+ violations.add(notNull("clientToken"));
+ }
+ if (request.getExecutionRoleArn() == null) {
+ violations.add(notNull("executionRoleArn"));
+ } else if (!ROLE_ARN.matcher(request.getExecutionRoleArn()).matches()) {
+ violations.add("Value at 'executionRoleArn' failed to satisfy constraint: "
+ + "Member must satisfy regular expression pattern: " + ROLE_ARN_PATTERN);
+ }
+ if (request.getMode() != null && !MODES.contains(request.getMode())) {
+ violations.add("Value '" + request.getMode() + "' at 'mode' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: " + MODES);
+ }
+ if (request.getExecutionTimeoutMinutes() != null) {
+ String violation = outOfRange("executionTimeoutMinutes", request.getExecutionTimeoutMinutes(), 0,
+ MAX_EXECUTION_TIMEOUT_MINUTES);
+ if (violation != null) {
+ violations.add(violation);
+ }
+ }
+ JobDriver driver = request.getJobDriver();
+ if (driver != null && driver.getSparkSubmit() != null && driver.getSparkSubmit().getEntryPoint() == null) {
+ violations.add(notNull("jobDriver.sparkSubmit.entryPoint"));
+ }
+ if (driver != null && driver.getHive() != null && driver.getHive().getQuery() == null) {
+ violations.add(notNull("jobDriver.hive.query"));
+ }
+ if (!violations.isEmpty()) {
+ throw SmithyValidationErrors.validationException(violations);
+ }
+ // StartJobRun's shape leaves jobDriver optional, but GetJobRun's JobRun requires one, so a run is
+ // refused without it. JobDriver is a union, "so only one of the following members can be
+ // specified". No source gives AWS's wording for a run without a driver, or with both.
+ if (driver == null || (driver.getSparkSubmit() == null) == (driver.getHive() == null)) {
+ throw new AwsException("ValidationException", "jobDriver must contain exactly one union member", 400);
+ }
+ }
+
+ private static String notNull(String member) {
+ return "Value null at '" + member + "' failed to satisfy constraint: Member must not be null";
+ }
+
+ private static String outOfRange(String member, long value, long min, long max) {
+ if (value < min) {
+ return "Value '" + value + "' at '" + member + "' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to " + min;
+ }
+ if (value > max) {
+ return "Value '" + value + "' at '" + member + "' failed to satisfy constraint: "
+ + "Member must have value less than or equal to " + max;
+ }
+ return null;
+ }
+
+ private static JobRunSummary toSummary(JobRunRecord record) {
+ JobRun run = record.getJobRun();
+ JobRunSummary summary = new JobRunSummary();
+ summary.setApplicationId(run.getApplicationId());
+ summary.setId(run.getJobRunId());
+ summary.setName(run.getName());
+ summary.setMode(run.getMode());
+ summary.setArn(run.getArn());
+ summary.setCreatedBy(run.getCreatedBy());
+ summary.setCreatedAt(run.getCreatedAt());
+ summary.setUpdatedAt(run.getUpdatedAt());
+ summary.setExecutionRole(run.getExecutionRole());
+ summary.setState(run.getState());
+ summary.setStateDetails(run.getStateDetails());
+ summary.setReleaseLabel(run.getReleaseLabel());
+ summary.setType(record.getApplicationType());
+ return summary;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java
index 0d1f4acd25..4d34a99467 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java
@@ -18,12 +18,15 @@ import io.github.hectorvent.floci.services.emrserverless.model.UpdateApplication
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
+import java.time.Clock;
import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -42,15 +45,17 @@ public class EmrServerlessService {
private final EmulatorConfig config;
private final AccountAwareStorageBackend<Application> storage;
private final RegionResolver regionResolver;
+ private final Clock clock;
@Inject
RequestContext requestContext;
@Inject
public EmrServerlessService(EmulatorConfig config, StorageFactory storageFactory,
- RegionResolver regionResolver) {
+ RegionResolver regionResolver, Clock clock) {
this.config = config;
this.regionResolver = regionResolver;
+ this.clock = clock;
this.storage = storageFactory.create("emrserverless", "emr-serverless-applications.json",
new TypeReference<Map<String, Application>>() {});
}
@@ -75,9 +80,9 @@ public class EmrServerlessService {
}
}
- String id = generateId();
+ String id = newId();
String arn = buildArn(id);
- long now = System.currentTimeMillis();
+ Instant createdAt = now();
Application app = new Application();
app.setApplicationId(id);
@@ -88,8 +93,8 @@ public class EmrServerlessService {
app.setType(request.getType());
app.setState("CREATED");
app.setStateDetails("");
- app.setCreatedAt(now);
- app.setUpdatedAt(now);
+ app.setCreatedAt(createdAt);
+ app.setUpdatedAt(createdAt);
if (request.getTags() != null) {
validateTags(request.getTags());
}
@@ -188,7 +193,7 @@ public class EmrServerlessService {
app.setJobLevelCostAllocationConfiguration(request.getJobLevelCostAllocationConfiguration());
}
- app.setUpdatedAt(System.currentTimeMillis());
+ app.setUpdatedAt(now());
storage.put(applicationId, app);
return app;
}
@@ -200,7 +205,7 @@ public class EmrServerlessService {
throw new AwsException("ValidationException", "Application must be in a stopped or created state in order to be deleted.", 400);
}
app.setState("TERMINATED");
- app.setUpdatedAt(System.currentTimeMillis());
+ app.setUpdatedAt(now());
storage.put(applicationId, app);
}
@@ -214,7 +219,7 @@ public class EmrServerlessService {
throw new AwsException("ValidationException", "Application is terminated.", 400);
}
app.setState("STARTED");
- app.setUpdatedAt(System.currentTimeMillis());
+ app.setUpdatedAt(now());
storage.put(applicationId, app);
}
@@ -228,10 +233,40 @@ public class EmrServerlessService {
throw new AwsException("ValidationException", "Application is terminated.", 400);
}
app.setState("STOPPED");
- app.setUpdatedAt(System.currentTimeMillis());
+ app.setUpdatedAt(now());
storage.put(applicationId, app);
}
+ /**
+ * Admits a job run to the application and returns what {@code admit} builds from it. An
+ * application "only accepts jobs when it's in" STARTED, and auto-start, on unless the
+ * application turns it off, starts a CREATED or STOPPED one "on job submission" (EMR Serverless
+ * user guide, "Application states" and "Default application behavior"). This service's lock is
+ * held throughout, so the application cannot be stopped or deleted while the run is admitted.
+ */
+ public synchronized <T> T admitJobRun(String applicationId, Function<Application, T> admit) {
+ Application app = getApplication(applicationId);
+ String state = app.getState();
+ if ("TERMINATED".equals(state)) {
+ throw new AwsException("ValidationException", "Application is terminated.", 400);
+ }
+ if (!"STARTED".equals(state)) {
+ if (!autoStartEnabled(app)) {
+ throw new AwsException("ValidationException", "Application " + applicationId
+ + " must be started before a job run is submitted, as its auto-start is disabled.", 400);
+ }
+ startApplication(applicationId);
+ app = getApplication(applicationId);
+ }
+ return admit.apply(app);
+ }
+
+ // AutoStartConfig.enabled "Defaults to true".
+ private static boolean autoStartEnabled(Application app) {
+ return app.getAutoStartConfiguration() == null
+ || !Boolean.FALSE.equals(app.getAutoStartConfiguration().getEnabled());
+ }
+
// ---- Tags ---------------------------------------------------------------------------------
public synchronized Map<String, String> listTags(String arn) {
@@ -276,8 +311,8 @@ public class EmrServerlessService {
/**
* The application an EMR Serverless ARN names:
* {@code arn:<partition>:emr-serverless:<region>:<account>:/applications/<id>}, the form
- * CreateApplication returns. A job run ARN ({@code .../jobruns/<id>}) is refused, as job runs are
- * not emulated.
+ * CreateApplication returns. A job run ARN ({@code .../jobruns/<id>}) is refused, as job run tags
+ * are not emulated yet.
*/
private Application applicationFor(String arn) {
AwsArnUtils.Arn parsed;
@@ -303,7 +338,7 @@ public class EmrServerlessService {
}
/** At most 50 tags, each key and value within TagKey and TagValue and free of the reserved prefix. */
- private static void validateTags(Map<String, String> tags) {
+ static void validateTags(Map<String, String> tags) {
if (tags.size() > MAX_TAGS) {
throw new AwsException("ValidationException",
"A resource can have at most " + MAX_TAGS + " tags.", 400);
@@ -334,7 +369,8 @@ public class EmrServerlessService {
return text.regionMatches(true, 0, RESERVED_PREFIX, 0, RESERVED_PREFIX.length());
}
- private String generateId() {
+ /** An application or job run id: 16 lowercase hexadecimal characters, within the model's [0-9a-z]+. */
+ static String newId() {
return UUID.randomUUID().toString().replace("-", "").substring(0, 16);
}
@@ -356,6 +392,10 @@ public class EmrServerlessService {
}
}
+ private Instant now() {
+ return clock.instant().truncatedTo(ChronoUnit.MILLIS);
+ }
+
private ApplicationSummary toSummary(Application app) {
ApplicationSummary summary = new ApplicationSummary();
summary.setId(app.getApplicationId());
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/Application.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/Application.java
index 378c800518..a27d78cd55 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/Application.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/Application.java
@@ -1,13 +1,22 @@
package io.github.hectorvent.floci.services.emrserverless.model;
+import com.fasterxml.jackson.annotation.JsonFormat;
import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.time.Instant;
import java.util.Date;
import java.util.List;
import java.util.Map;
+/** GetApplication's application. Timestamps go out as epoch seconds, the restJson1 default. */
@RegisterForReflection
public class Application {
+
+ // Applications persisted before their timestamps were epoch seconds hold epoch milliseconds, and
+ // those read back as seconds. 1e11 seconds is the year 5138, while 1e11 milliseconds is 1973, before
+ // any application, so a value at or past it was written as milliseconds.
+ private static final long LEGACY_MILLIS_FLOOR = 100_000_000_000L;
+
private String applicationId;
private String arn;
private String name;
@@ -16,8 +25,10 @@ public class Application {
private String state;
private String stateDetails;
private String clientToken;
- private Long createdAt;
- private Long updatedAt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant createdAt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant updatedAt;
private Map<String, String> tags;
private String architecture;
private Map<String, InitialCapacityConfig> initialCapacity;
@@ -51,10 +62,10 @@ public class Application {
public void setStateDetails(String stateDetails) { this.stateDetails = stateDetails; }
public String getClientToken() { return clientToken; }
public void setClientToken(String clientToken) { this.clientToken = clientToken; }
- public Long getCreatedAt() { return createdAt; }
- public void setCreatedAt(Long createdAt) { this.createdAt = createdAt; }
- public Long getUpdatedAt() { return updatedAt; }
- public void setUpdatedAt(Long updatedAt) { this.updatedAt = updatedAt; }
+ public Instant getCreatedAt() { return createdAt; }
+ public void setCreatedAt(Instant createdAt) { this.createdAt = fromLegacyMillis(createdAt); }
+ public Instant getUpdatedAt() { return updatedAt; }
+ public void setUpdatedAt(Instant updatedAt) { this.updatedAt = fromLegacyMillis(updatedAt); }
public Map<String, String> getTags() { return tags; }
public void setTags(Map<String, String> tags) { this.tags = tags; }
public String getArchitecture() { return architecture; }
@@ -87,4 +98,9 @@ public class Application {
public void setIdentityCenterConfiguration(IdentityCenterConfiguration identityCenterConfiguration) { this.identityCenterConfiguration = identityCenterConfiguration; }
public JobLevelCostAllocationConfiguration getJobLevelCostAllocationConfiguration() { return jobLevelCostAllocationConfiguration; }
public void setJobLevelCostAllocationConfiguration(JobLevelCostAllocationConfiguration jobLevelCostAllocationConfiguration) { this.jobLevelCostAllocationConfiguration = jobLevelCostAllocationConfiguration; }
+
+ private static Instant fromLegacyMillis(Instant value) {
+ return value != null && value.getEpochSecond() >= LEGACY_MILLIS_FLOOR
+ ? Instant.ofEpochMilli(value.getEpochSecond()) : value;
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ApplicationSummary.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ApplicationSummary.java
index 8d993d3fda..55c3f601c6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ApplicationSummary.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ApplicationSummary.java
@@ -1,19 +1,24 @@
package io.github.hectorvent.floci.services.emrserverless.model;
+import com.fasterxml.jackson.annotation.JsonFormat;
import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.time.Instant;
+
@RegisterForReflection
public class ApplicationSummary {
private String architecture;
private String arn;
- private Long createdAt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant createdAt;
private String id;
private String name;
private String releaseLabel;
private String state;
private String stateDetails;
private String type;
- private Long updatedAt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant updatedAt;
public ApplicationSummary() {}
@@ -33,11 +38,11 @@ public class ApplicationSummary {
this.arn = arn;
}
- public Long getCreatedAt() {
+ public Instant getCreatedAt() {
return createdAt;
}
- public void setCreatedAt(Long createdAt) {
+ public void setCreatedAt(Instant createdAt) {
this.createdAt = createdAt;
}
@@ -89,11 +94,11 @@ public class ApplicationSummary {
this.type = type;
}
- public Long getUpdatedAt() {
+ public Instant getUpdatedAt() {
return updatedAt;
}
- public void setUpdatedAt(Long updatedAt) {
+ public void setUpdatedAt(Instant updatedAt) {
this.updatedAt = updatedAt;
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/CancelJobRunResponse.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/CancelJobRunResponse.java
new file mode 100644
index 0000000000..0a3be4592d
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/CancelJobRunResponse.java
@@ -0,0 +1,14 @@
+package io.github.hectorvent.floci.services.emrserverless.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class CancelJobRunResponse {
+ private String applicationId;
+ private String jobRunId;
+
+ public String getApplicationId() { return applicationId; }
+ public void setApplicationId(String applicationId) { this.applicationId = applicationId; }
+ public String getJobRunId() { return jobRunId; }
+ public void setJobRunId(String jobRunId) { this.jobRunId = jobRunId; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/GetJobRunResponse.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/GetJobRunResponse.java
new file mode 100644
index 0000000000..1da266e6db
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/GetJobRunResponse.java
@@ -0,0 +1,11 @@
+package io.github.hectorvent.floci.services.emrserverless.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class GetJobRunResponse {
+ private JobRun jobRun;
+
+ public JobRun getJobRun() { return jobRun; }
+ public void setJobRun(JobRun jobRun) { this.jobRun = jobRun; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobDriver.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobDriver.java
new file mode 100644
index 0000000000..a770ce4b3e
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobDriver.java
@@ -0,0 +1,49 @@
+package io.github.hectorvent.floci.services.emrserverless.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+/** What a job run executes: a Spark submission or a Hive query. */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class JobDriver {
+ private SparkSubmit sparkSubmit;
+ private Hive hive;
+
+ public SparkSubmit getSparkSubmit() { return sparkSubmit; }
+ public void setSparkSubmit(SparkSubmit sparkSubmit) { this.sparkSubmit = sparkSubmit; }
+ public Hive getHive() { return hive; }
+ public void setHive(Hive hive) { this.hive = hive; }
+
+ @RegisterForReflection
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class SparkSubmit {
+ private String entryPoint;
+ private List<String> entryPointArguments;
+ private String sparkSubmitParameters;
+
+ public String getEntryPoint() { return entryPoint; }
+ public void setEntryPoint(String entryPoint) { this.entryPoint = entryPoint; }
+ public List<String> getEntryPointArguments() { return entryPointArguments; }
+ public void setEntryPointArguments(List<String> entryPointArguments) { this.entryPointArguments = entryPointArguments; }
+ public String getSparkSubmitParameters() { return sparkSubmitParameters; }
+ public void setSparkSubmitParameters(String sparkSubmitParameters) { this.sparkSubmitParameters = sparkSubmitParameters; }
+ }
+
+ @RegisterForReflection
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class Hive {
+ private String query;
+ private String initQueryFile;
+ private String parameters;
+
+ public String getQuery() { return query; }
+ public void setQuery(String query) { this.query = query; }
+ public String getInitQueryFile() { return initQueryFile; }
+ public void setInitQueryFile(String initQueryFile) { this.initQueryFile = initQueryFile; }
+ public String getParameters() { return parameters; }
+ public void setParameters(String parameters) { this.parameters = parameters; }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java
new file mode 100644
index 0000000000..44b3b65838
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java
@@ -0,0 +1,85 @@
+package io.github.hectorvent.floci.services.emrserverless.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.Map;
+
+/** GetJobRun's jobRun. Timestamps go out as epoch seconds, the restJson1 default. */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class JobRun {
+ private String applicationId;
+ private String jobRunId;
+ private String name;
+ private String arn;
+ private String createdBy;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant createdAt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant updatedAt;
+ private String executionRole;
+ private Map<String, Object> executionIamPolicy;
+ private String state;
+ private String stateDetails;
+ private String releaseLabel;
+ private Map<String, Object> configurationOverrides;
+ private JobDriver jobDriver;
+ private Map<String, String> tags;
+ private NetworkConfiguration networkConfiguration;
+ private Integer totalExecutionDurationSeconds;
+ private Long executionTimeoutMinutes;
+ private String mode;
+ private Map<String, Object> retryPolicy;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant startedAt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant endedAt;
+
+ public String getApplicationId() { return applicationId; }
+ public void setApplicationId(String applicationId) { this.applicationId = applicationId; }
+ public String getJobRunId() { return jobRunId; }
+ public void setJobRunId(String jobRunId) { this.jobRunId = jobRunId; }
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+ public String getArn() { return arn; }
+ public void setArn(String arn) { this.arn = arn; }
+ public String getCreatedBy() { return createdBy; }
+ public void setCreatedBy(String createdBy) { this.createdBy = createdBy; }
+ public Instant getCreatedAt() { return createdAt; }
+ public void setCreatedAt(Instant createdAt) { this.createdAt = createdAt; }
+ public Instant getUpdatedAt() { return updatedAt; }
+ public void setUpdatedAt(Instant updatedAt) { this.updatedAt = updatedAt; }
+ public String getExecutionRole() { return executionRole; }
+ public void setExecutionRole(String executionRole) { this.executionRole = executionRole; }
+ public Map<String, Object> getExecutionIamPolicy() { return executionIamPolicy; }
+ public void setExecutionIamPolicy(Map<String, Object> executionIamPolicy) { this.executionIamPolicy = executionIamPolicy; }
+ public String getState() { return state; }
+ public void setState(String state) { this.state = state; }
+ public String getStateDetails() { return stateDetails; }
+ public void setStateDetails(String stateDetails) { this.stateDetails = stateDetails; }
+ public String getReleaseLabel() { return releaseLabel; }
+ public void setReleaseLabel(String releaseLabel) { this.releaseLabel = releaseLabel; }
+ public Map<String, Object> getConfigurationOverrides() { return configurationOverrides; }
+ public void setConfigurationOverrides(Map<String, Object> configurationOverrides) { this.configurationOverrides = configurationOverrides; }
+ public JobDriver getJobDriver() { return jobDriver; }
+ public void setJobDriver(JobDriver jobDriver) { this.jobDriver = jobDriver; }
+ public Map<String, String> getTags() { return tags; }
+ public void setTags(Map<String, String> tags) { this.tags = tags; }
+ public NetworkConfiguration getNetworkConfiguration() { return networkConfiguration; }
+ public void setNetworkConfiguration(NetworkConfiguration networkConfiguration) { this.networkConfiguration = networkConfiguration; }
+ public Integer getTotalExecutionDurationSeconds() { return totalExecutionDurationSeconds; }
+ public void setTotalExecutionDurationSeconds(Integer totalExecutionDurationSeconds) { this.totalExecutionDurationSeconds = totalExecutionDurationSeconds; }
+ public Long getExecutionTimeoutMinutes() { return executionTimeoutMinutes; }
+ public void setExecutionTimeoutMinutes(Long executionTimeoutMinutes) { this.executionTimeoutMinutes = executionTimeoutMinutes; }
+ public String getMode() { return mode; }
+ public void setMode(String mode) { this.mode = mode; }
+ public Map<String, Object> getRetryPolicy() { return retryPolicy; }
+ public void setRetryPolicy(Map<String, Object> retryPolicy) { this.retryPolicy = retryPolicy; }
+ public Instant getStartedAt() { return startedAt; }
+ public void setStartedAt(Instant startedAt) { this.startedAt = startedAt; }
+ public Instant getEndedAt() { return endedAt; }
+ public void setEndedAt(Instant endedAt) { this.endedAt = endedAt; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunRecord.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunRecord.java
new file mode 100644
index 0000000000..e55acce436
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunRecord.java
@@ -0,0 +1,26 @@
+package io.github.hectorvent.floci.services.emrserverless.model;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+/**
+ * A stored job run: the run as GetJobRun reports it, plus what only Floci needs. The client token
+ * and the request it came with decide whether a repeated StartJobRun is a retry or a conflict; the
+ * application type fills ListJobRuns' {@code type}.
+ */
+@RegisterForReflection
+public class JobRunRecord {
+ private JobRun jobRun;
+ private String clientToken;
+ private JsonNode request;
+ private String applicationType;
+
+ public JobRun getJobRun() { return jobRun; }
+ public void setJobRun(JobRun jobRun) { this.jobRun = jobRun; }
+ public String getClientToken() { return clientToken; }
+ public void setClientToken(String clientToken) { this.clientToken = clientToken; }
+ public JsonNode getRequest() { return request; }
+ public void setRequest(JsonNode request) { this.request = request; }
+ public String getApplicationType() { return applicationType; }
+ public void setApplicationType(String applicationType) { this.applicationType = applicationType; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java
new file mode 100644
index 0000000000..bf54a96730
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java
@@ -0,0 +1,55 @@
+package io.github.hectorvent.floci.services.emrserverless.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+
+/** One entry of ListJobRuns. Unlike JobRun, the run's id is {@code id}. */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class JobRunSummary {
+ private String applicationId;
+ private String id;
+ private String name;
+ private String mode;
+ private String arn;
+ private String createdBy;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant createdAt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant updatedAt;
+ private String executionRole;
+ private String state;
+ private String stateDetails;
+ private String releaseLabel;
+ private String type;
+
+ public String getApplicationId() { return applicationId; }
+ public void setApplicationId(String applicationId) { this.applicationId = applicationId; }
+ public String getId() { return id; }
+ public void setId(String id) { this.id = id; }
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+ public String getMode() { return mode; }
+ public void setMode(String mode) { this.mode = mode; }
+ public String getArn() { return arn; }
+ public void setArn(String arn) { this.arn = arn; }
+ public String getCreatedBy() { return createdBy; }
+ public void setCreatedBy(String createdBy) { this.createdBy = createdBy; }
+ public Instant getCreatedAt() { return createdAt; }
+ public void setCreatedAt(Instant createdAt) { this.createdAt = createdAt; }
+ public Instant getUpdatedAt() { return updatedAt; }
+ public void setUpdatedAt(Instant updatedAt) { this.updatedAt = updatedAt; }
+ public String getExecutionRole() { return executionRole; }
+ public void setExecutionRole(String executionRole) { this.executionRole = executionRole; }
+ public String getState() { return state; }
+ public void setState(String state) { this.state = state; }
+ public String getStateDetails() { return stateDetails; }
+ public void setStateDetails(String stateDetails) { this.stateDetails = stateDetails; }
+ public String getReleaseLabel() { return releaseLabel; }
+ public void setReleaseLabel(String releaseLabel) { this.releaseLabel = releaseLabel; }
+ public String getType() { return type; }
+ public void setType(String type) { this.type = type; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunsResponse.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunsResponse.java
new file mode 100644
index 0000000000..a62fe22ef7
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunsResponse.java
@@ -0,0 +1,18 @@
+package io.github.hectorvent.floci.services.emrserverless.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ListJobRunsResponse {
+ private List<JobRunSummary> jobRuns;
+ private String nextToken;
+
+ public List<JobRunSummary> getJobRuns() { return jobRuns; }
+ public void setJobRuns(List<JobRunSummary> jobRuns) { this.jobRuns = jobRuns; }
+ public String getNextToken() { return nextToken; }
+ public void setNextToken(String nextToken) { this.nextToken = nextToken; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunRequest.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunRequest.java
new file mode 100644
index 0000000000..c091afc95a
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunRequest.java
@@ -0,0 +1,46 @@
+package io.github.hectorvent.floci.services.emrserverless.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.Map;
+
+/**
+ * StartJobRun's body. configurationOverrides, executionIamPolicy and retryPolicy are kept as sent
+ * and returned by GetJobRun unchanged.
+ */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class StartJobRunRequest {
+ private String clientToken;
+ private String executionRoleArn;
+ private Map<String, Object> executionIamPolicy;
+ private JobDriver jobDriver;
+ private Map<String, Object> configurationOverrides;
+ private Map<String, String> tags;
+ private Long executionTimeoutMinutes;
+ private String name;
+ private String mode;
+ private Map<String, Object> retryPolicy;
+
+ public String getClientToken() { return clientToken; }
+ public void setClientToken(String clientToken) { this.clientToken = clientToken; }
+ public String getExecutionRoleArn() { return executionRoleArn; }
+ public void setExecutionRoleArn(String executionRoleArn) { this.executionRoleArn = executionRoleArn; }
+ public Map<String, Object> getExecutionIamPolicy() { return executionIamPolicy; }
+ public void setExecutionIamPolicy(Map<String, Object> executionIamPolicy) { this.executionIamPolicy = executionIamPolicy; }
+ public JobDriver getJobDriver() { return jobDriver; }
+ public void setJobDriver(JobDriver jobDriver) { this.jobDriver = jobDriver; }
+ public Map<String, Object> getConfigurationOverrides() { return configurationOverrides; }
+ public void setConfigurationOverrides(Map<String, Object> configurationOverrides) { this.configurationOverrides = configurationOverrides; }
+ public Map<String, String> getTags() { return tags; }
+ public void setTags(Map<String, String> tags) { this.tags = tags; }
+ public Long getExecutionTimeoutMinutes() { return executionTimeoutMinutes; }
+ public void setExecutionTimeoutMinutes(Long executionTimeoutMinutes) { this.executionTimeoutMinutes = executionTimeoutMinutes; }
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+ public String getMode() { return mode; }
+ public void setMode(String mode) { this.mode = mode; }
+ public Map<String, Object> getRetryPolicy() { return retryPolicy; }
+ public void setRetryPolicy(Map<String, Object> retryPolicy) { this.retryPolicy = retryPolicy; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunResponse.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunResponse.java
new file mode 100644
index 0000000000..0d06a8f1bb
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/StartJobRunResponse.java
@@ -0,0 +1,17 @@
+package io.github.hectorvent.floci.services.emrserverless.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public class StartJobRunResponse {
+ private String applicationId;
+ private String jobRunId;
+ private String arn;
+
+ public String getApplicationId() { return applicationId; }
+ public void setApplicationId(String applicationId) { this.applicationId = applicationId; }
+ public String getJobRunId() { return jobRunId; }
+ public void setJobRunId(String jobRunId) { this.jobRunId = jobRunId; }
+ public String getArn() { return arn; }
+ public void setArn(String arn) { this.arn = arn; }
+}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 544ce54464..2b7cb321ce 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -954,6 +954,7 @@ floci:
enabled: true
emrserverless:
enabled: true
+ job-run-duration-seconds: 0 # 0 = BATCH job runs succeed as soon as they start
ssoadmin:
enabled: true
ssooidc:
diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java
index ab99865c6c..f2c2ebbf6d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java
@@ -1,32 +1,46 @@
package io.github.hectorvent.floci.services.emrserverless;
+import io.github.hectorvent.floci.testing.EmrServerlessRequests;
+import io.github.hectorvent.floci.testing.MutableClock;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.config.JsonConfig;
+import io.restassured.config.RestAssuredConfig;
+import io.restassured.path.json.config.JsonPathConfig;
+import io.restassured.specification.RequestSpecification;
+import jakarta.inject.Inject;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
-import static io.restassured.RestAssured.given;
+import java.math.BigDecimal;
+import java.time.Duration;
+import java.util.UUID;
+
import static org.hamcrest.Matchers.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
@QuarkusTest
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class EmrServerlessIntegrationTest {
- private static final String JSON_CONTENT_TYPE = "application/json";
+ private static final RestAssuredConfig DECIMAL_NUMBERS = RestAssuredConfig.config()
+ .jsonConfig(JsonConfig.jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.BIG_DECIMAL));
private static String applicationId;
+ @Inject
+ MutableClock clock;
+
@BeforeAll
static void configureRestAssured() {
RestAssuredJsonUtils.configureAwsContentTypes();
}
- private io.restassured.specification.RequestSpecification givenReq() {
- return given()
- .header("Authorization", "AWS4-HMAC-SHA256 Credential=TEST/20260819/us-east-1/emr-serverless/aws4_request, SignedHeaders=host;x-amz-date, Signature=test")
- .contentType(JSON_CONTENT_TYPE);
+ private RequestSpecification givenReq() {
+ return EmrServerlessRequests.emrServerless("us-east-1");
}
@Test
@@ -286,4 +300,42 @@ public class EmrServerlessIntegrationTest {
.statusCode(200)
.body("application.state", equalTo("TERMINATED"));
}
+
+ @Test
+ @Order(9)
+ void applicationTimestampsAreEpochSecondsWithMilliseconds() {
+ // A Region of its own, so ListApplications holds this application and no other test's.
+ String region = "ap-southeast-4";
+ clock.advance(Duration.ofMillis(250));
+ BigDecimal before = BigDecimal.valueOf(clock.instant().toEpochMilli(), 3);
+ String id = EmrServerlessRequests.emrServerless(region)
+ .body("""
+ {"name": "stamped-app", "releaseLabel": "emr-6.6.0", "type": "SPARK", "clientToken": "%s"}
+ """.formatted(UUID.randomUUID()))
+ .when()
+ .post("/applications")
+ .then()
+ .statusCode(200)
+ .extract().path("applicationId");
+
+ BigDecimal createdAt = EmrServerlessRequests.emrServerless(region).config(DECIMAL_NUMBERS)
+ .when()
+ .get("/applications/" + id)
+ .then()
+ .statusCode(200)
+ .extract().path("application.createdAt");
+ BigDecimal updatedAt = EmrServerlessRequests.emrServerless(region).config(DECIMAL_NUMBERS)
+ .get("/applications/" + id)
+ .then().statusCode(200).extract().path("application.updatedAt");
+ BigDecimal listed = EmrServerlessRequests.emrServerless(region).config(DECIMAL_NUMBERS)
+ .get("/applications")
+ .then().statusCode(200).extract().path("applications[0].createdAt");
+
+ // The test clock moves a millisecond on every read, so the application was created a little later.
+ assertTrue(createdAt.compareTo(before) > 0 && createdAt.compareTo(before.add(BigDecimal.ONE)) < 0,
+ "createdAt " + createdAt + " is not just after " + before);
+ assertTrue(createdAt.stripTrailingZeros().scale() <= 3, "createdAt " + createdAt + " is finer than milliseconds");
+ assertEquals(0, createdAt.compareTo(updatedAt), "updatedAt " + updatedAt + " differs from createdAt " + createdAt);
+ assertEquals(0, createdAt.compareTo(listed), "ListApplications reports createdAt " + listed);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java
new file mode 100644
index 0000000000..e5146d22d4
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java
@@ -0,0 +1,485 @@
+package io.github.hectorvent.floci.services.emrserverless;
+
+import io.github.hectorvent.floci.testing.EmrServerlessRequests;
+import io.github.hectorvent.floci.testing.MutableClock;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.config.JsonConfig;
+import io.restassured.config.RestAssuredConfig;
+import io.restassured.path.json.config.JsonPathConfig;
+import io.restassured.response.Response;
+import io.restassured.specification.RequestSpecification;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.math.BigDecimal;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.UUID;
+
+import static org.hamcrest.Matchers.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@QuarkusTest
+class EmrServerlessJobRunIntegrationTest {
+
+ private static final String ROLE = "arn:aws:iam::000000000000:role/emr-serverless-job";
+ private static final RestAssuredConfig DECIMAL_NUMBERS = RestAssuredConfig.config()
+ .jsonConfig(JsonConfig.jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.BIG_DECIMAL));
+
+ @Inject
+ MutableClock clock;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ private static RequestSpecification givenReq() {
+ return EmrServerlessRequests.emrServerless("us-east-1");
+ }
+
+ private static String createApplication(String extra) {
+ return givenReq()
+ .body("""
+ {"name": "jobs-%s", "releaseLabel": "emr-7.5.0", "type": "SPARK", "clientToken": "%s"%s}
+ """.formatted(UUID.randomUUID(), UUID.randomUUID(), extra))
+ .when().post("/applications")
+ .then().statusCode(200)
+ .extract().path("applicationId");
+ }
+
+ private static String sparkRun(String clientToken, String extra) {
+ return """
+ {"clientToken": "%s", "executionRoleArn": "%s",
+ "jobDriver": {"sparkSubmit": {"entryPoint": "s3://bucket/jobs/app.jar",
+ "entryPointArguments": ["s3://bucket/in"],
+ "sparkSubmitParameters": "--class example.Main"}}%s}
+ """.formatted(clientToken, ROLE, extra);
+ }
+
+ private static Response startJobRun(String applicationId, String body) {
+ return givenReq().body(body).when().post("/applications/" + applicationId + "/jobruns");
+ }
+
+ private static String startedRunId(String applicationId, String body) {
+ return startJobRun(applicationId, body).then().statusCode(200).extract().path("jobRunId");
+ }
+
+ @Test
+ void startJobRunAutoStartsTheApplicationAndSucceedsByDefault() {
+ String applicationId = createApplication("");
+ givenReq().when().get("/applications/" + applicationId)
+ .then().statusCode(200).body("application.state", equalTo("CREATED"));
+
+ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ", \"name\": \"nightly\""))
+ .then().statusCode(200)
+ .body("applicationId", equalTo(applicationId))
+ .body("jobRunId", matchesPattern("[0-9a-z]{16}"))
+ .body("arn", matchesPattern("arn:aws:emr-serverless:us-east-1:000000000000:/applications/"
+ + applicationId + "/jobruns/[0-9a-z]{16}"));
+
+ givenReq().when().get("/applications/" + applicationId)
+ .then().statusCode(200).body("application.state", equalTo("STARTED"));
+ }
+
+ @Test
+ void getJobRunReportsTheRunAsGetJobRunShapesIt() {
+ String applicationId = createApplication("");
+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(),
+ ", \"name\": \"nightly\", \"tags\": {\"team\": \"data\"}, \"mode\": \"BATCH\""));
+
+ givenReq().when().get("/applications/" + applicationId + "/jobruns/" + jobRunId)
+ .then().statusCode(200)
+ .body("jobRun.applicationId", equalTo(applicationId))
+ .body("jobRun.jobRunId", equalTo(jobRunId))
+ .body("jobRun.name", equalTo("nightly"))
+ .body("jobRun.arn", endsWith("/applications/" + applicationId + "/jobruns/" + jobRunId))
+ .body("jobRun.createdBy", equalTo("arn:aws:iam::000000000000:root"))
+ .body("jobRun.createdAt", instanceOf(Number.class))
+ .body("jobRun.updatedAt", instanceOf(Number.class))
+ .body("jobRun.executionRole", equalTo(ROLE))
+ .body("jobRun.state", equalTo("SUCCESS"))
+ .body("jobRun.stateDetails", equalTo(""))
+ .body("jobRun.releaseLabel", equalTo("emr-7.5.0"))
+ .body("jobRun.jobDriver.sparkSubmit.entryPoint", equalTo("s3://bucket/jobs/app.jar"))
+ .body("jobRun.jobDriver.sparkSubmit.entryPointArguments", contains("s3://bucket/in"))
+ .body("jobRun.jobDriver.sparkSubmit.sparkSubmitParameters", equalTo("--class example.Main"))
+ .body("jobRun.tags.team", equalTo("data"))
+ .body("jobRun.mode", equalTo("BATCH"))
+ .body("jobRun.executionTimeoutMinutes", equalTo(720))
+ .body("jobRun.totalExecutionDurationSeconds", equalTo(0))
+ .body("jobRun.startedAt", instanceOf(Number.class))
+ .body("jobRun.endedAt", instanceOf(Number.class))
+ .body("jobRun", not(hasKey("hive")));
+ }
+
+ @Test
+ void timestampsAreEpochSecondsWithMilliseconds() {
+ String applicationId = createApplication("");
+ clock.advance(Duration.ofMillis(250));
+ Instant submittedAt = clock.instant();
+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+
+ BigDecimal createdAt = givenReq().config(DECIMAL_NUMBERS)
+ .when().get("/applications/" + applicationId + "/jobruns/" + jobRunId)
+ .then().statusCode(200).extract().path("jobRun.createdAt");
+ // The test clock moves a millisecond on every read, so the run was created a little later.
+ BigDecimal submitted = BigDecimal.valueOf(submittedAt.toEpochMilli(), 3);
+ assertTrue(createdAt.compareTo(submitted) >= 0 && createdAt.compareTo(submitted.add(BigDecimal.ONE)) < 0,
+ "createdAt " + createdAt + " is not just after " + submitted);
+ assertTrue(createdAt.stripTrailingZeros().scale() <= 3, "createdAt " + createdAt + " is finer than milliseconds");
+ }
+
+ @Test
+ void runWithoutTagsReportsAnEmptyTagMap() {
+ String applicationId = createApplication("");
+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+
+ givenReq().when().get("/applications/" + applicationId + "/jobruns/" + jobRunId)
+ .then().statusCode(200)
+ .body("jobRun.tags", anEmptyMap())
+ .body("jobRun", not(hasKey("mode")));
+ }
+
+ @Test
+ void repeatedClientTokenReturnsTheOriginalRun() {
+ String applicationId = createApplication("");
+ String body = sparkRun(UUID.randomUUID().toString(), ", \"tags\": {\"a\": \"1\", \"b\": \"2\"}");
+ String first = startedRunId(applicationId, body);
+
+ assertEquals(first, startedRunId(applicationId, body));
+ }
+
+ @Test
+ void reusedClientTokenWithAnotherRequestIsAConflict() {
+ String applicationId = createApplication("");
+ String token = UUID.randomUUID().toString();
+ startedRunId(applicationId, sparkRun(token, ", \"name\": \"first\""));
+
+ startJobRun(applicationId, sparkRun(token, ", \"name\": \"second\""))
+ .then().statusCode(409)
+ .body("message", equalTo("Request did not match the original"))
+ .body("__type", equalTo("ConflictException"));
+ }
+
+ @Test
+ void newClientTokenStartsAnotherRun() {
+ String applicationId = createApplication("");
+ String first = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+
+ assertNotEquals(first, startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), "")));
+ }
+
+ @Test
+ void missingRequiredMembersAreAValidationException() {
+ String applicationId = createApplication("");
+
+ startJobRun(applicationId, "{\"jobDriver\": {\"sparkSubmit\": {}}}")
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo("3 validation errors detected: "
+ + "Value null at 'clientToken' failed to satisfy constraint: Member must not be null; "
+ + "Value null at 'executionRoleArn' failed to satisfy constraint: Member must not be null; "
+ + "Value null at 'jobDriver.sparkSubmit.entryPoint' failed to satisfy constraint: "
+ + "Member must not be null"));
+ }
+
+ @Test
+ void executionRoleOutsideTheRolePatternIsAValidationException() {
+ String applicationId = createApplication("");
+
+ startJobRun(applicationId, """
+ {"clientToken": "t-1", "executionRoleArn": "arn:aws:iam::000000000000:user/someone"}
+ """)
+ .then().statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value at 'executionRoleArn' failed to "
+ + "satisfy constraint: Member must satisfy regular expression pattern: "
+ + "arn:(aws[a-zA-Z0-9-]*):iam::([0-9]{12}):(role((" + backslashU("002F") + ")|("
+ + backslashU("002F") + "[" + backslashU("0021") + "-" + backslashU("007F") + "]+"
+ + backslashU("002F") + "))[" + (char) 92 + "w+=,.@-]+)"));
+ }
+
+ // The pattern is quoted with its escapes as text, so the expected message spells them out.
+ private static String backslashU(String hex) {
+ return (char) 92 + "u" + hex;
+ }
+
+ @Test
+ void unknownModeIsAValidationException() {
+ String applicationId = createApplication("");
+
+ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ", \"mode\": \"INTERACTIVE\""))
+ .then().statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value 'INTERACTIVE' at 'mode' failed to "
+ + "satisfy constraint: Member must satisfy enum value set: [BATCH, STREAMING]"));
+ }
+
+ @Test
+ void executionTimeoutOutsideItsRangeIsAValidationException() {
+ String applicationId = createApplication("");
+
+ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ", \"executionTimeoutMinutes\": -1"))
+ .then().statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value '-1' at 'executionTimeoutMinutes' "
+ + "failed to satisfy constraint: Member must have value greater than or equal to 0"));
+ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ", \"executionTimeoutMinutes\": 1000001"))
+ .then().statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value '1000001' at 'executionTimeoutMinutes' "
+ + "failed to satisfy constraint: Member must have value less than or equal to 1000000"));
+ }
+
+ @Test
+ void aRunNeedsExactlyOneJobDriver() {
+ String applicationId = createApplication("");
+ String missing = """
+ {"clientToken": "%s", "executionRoleArn": "%s"}
+ """.formatted(UUID.randomUUID(), ROLE);
+ String both = """
+ {"clientToken": "%s", "executionRoleArn": "%s",
+ "jobDriver": {"sparkSubmit": {"entryPoint": "s3://bucket/jobs/app.jar"}, "hive": {"query": "SELECT 1"}}}
+ """.formatted(UUID.randomUUID(), ROLE);
+ String neither = """
+ {"clientToken": "%s", "executionRoleArn": "%s", "jobDriver": {}}
+ """.formatted(UUID.randomUUID(), ROLE);
+
+ for (String body : List.of(missing, both, neither)) {
+ startJobRun(applicationId, body)
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo("jobDriver must contain exactly one union member"));
+ }
+ givenReq().when().get("/applications/" + applicationId)
+ .then().body("application.state", equalTo("CREATED"));
+ }
+
+ @Test
+ void refusedTagsLeaveTheApplicationAsItWas() {
+ String applicationId = createApplication("");
+
+ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ", \"tags\": {\"aws:team\": \"data\"}"))
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ givenReq().when().get("/applications/" + applicationId)
+ .then().body("application.state", equalTo("CREATED"));
+ givenReq().when().get("/applications/" + applicationId + "/jobruns")
+ .then().body("jobRuns", empty());
+ }
+
+ @Test
+ void runOnAnUnknownApplicationIsResourceNotFound() {
+ startJobRun("00missing0000000", sparkRun(UUID.randomUUID().toString(), ""))
+ .then().statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ }
+
+ @Test
+ void unknownJobRunIsResourceNotFound() {
+ String applicationId = createApplication("");
+
+ givenReq().when().get("/applications/" + applicationId + "/jobruns/00missing0000000")
+ .then().statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ givenReq().when().delete("/applications/" + applicationId + "/jobruns/00missing0000000")
+ .then().statusCode(404);
+ }
+
+ @Test
+ void jobRunOfAnotherApplicationIsResourceNotFound() {
+ String owner = createApplication("");
+ String other = createApplication("");
+ String jobRunId = startedRunId(owner, sparkRun(UUID.randomUUID().toString(), ""));
+
+ givenReq().when().get("/applications/" + other + "/jobruns/" + jobRunId)
+ .then().statusCode(404);
+ }
+
+ @Test
+ void disabledAutoStartRefusesARunUntilTheApplicationStarts() {
+ String applicationId = createApplication(", \"autoStartConfiguration\": {\"enabled\": false}");
+
+ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ""))
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ givenReq().when().get("/applications/" + applicationId)
+ .then().body("application.state", equalTo("CREATED"));
+
+ givenReq().when().post("/applications/" + applicationId + "/start").then().statusCode(200);
+ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), "")).then().statusCode(200);
+ }
+
+ @Test
+ void runOnATerminatedApplicationIsRefused() {
+ String applicationId = createApplication("");
+ givenReq().when().delete("/applications/" + applicationId).then().statusCode(200);
+
+ startJobRun(applicationId, sparkRun(UUID.randomUUID().toString(), ""))
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ }
+
+ @Test
+ void cancellingAFinishedRunIsAValidationException() {
+ String applicationId = createApplication("");
+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+
+ givenReq().when().delete("/applications/" + applicationId + "/jobruns/" + jobRunId)
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo("Job run is not in a cancellable state"));
+ }
+
+ @Test
+ void streamingRunKeepsRunningUntilCancelled() {
+ String applicationId = createApplication("");
+ String jobRunId = startedRunId(applicationId,
+ sparkRun(UUID.randomUUID().toString(), ", \"mode\": \"STREAMING\""));
+ String path = "/applications/" + applicationId + "/jobruns/" + jobRunId;
+
+ givenReq().when().get(path).then().statusCode(200)
+ .body("jobRun.state", equalTo("RUNNING"))
+ .body("jobRun", not(hasKey("endedAt")))
+ .body("jobRun", not(hasKey("totalExecutionDurationSeconds")));
+
+ givenReq().when().delete(path).then().statusCode(200)
+ .body("applicationId", equalTo(applicationId))
+ .body("jobRunId", equalTo(jobRunId));
+
+ givenReq().when().get(path).then().statusCode(200)
+ .body("jobRun.state", equalTo("CANCELLED"))
+ .body("jobRun.endedAt", instanceOf(Number.class));
+ }
+
+ @Test
+ void listJobRunsReturnsTheNewestFirst() {
+ String applicationId = createApplication("");
+ String older = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+ clock.advance(Duration.ofSeconds(5));
+ String newer = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+
+ givenReq().when().get("/applications/" + applicationId + "/jobruns")
+ .then().statusCode(200)
+ .body("jobRuns.id", contains(newer, older))
+ .body("jobRuns[0].applicationId", equalTo(applicationId))
+ .body("jobRuns[0].state", equalTo("SUCCESS"))
+ .body("jobRuns[0].type", equalTo("SPARK"))
+ .body("jobRuns[0].releaseLabel", equalTo("emr-7.5.0"))
+ .body("jobRuns[0].createdBy", equalTo("arn:aws:iam::000000000000:root"))
+ .body("jobRuns[0]", not(hasKey("jobRunId")))
+ .body("jobRuns[0]", not(hasKey("jobDriver")));
+ }
+
+ @Test
+ void listJobRunsPagesWithNextToken() {
+ String applicationId = createApplication("");
+ String older = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+ clock.advance(Duration.ofSeconds(5));
+ String newer = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+
+ String nextToken = givenReq().queryParam("maxResults", 1)
+ .when().get("/applications/" + applicationId + "/jobruns")
+ .then().statusCode(200)
+ .body("jobRuns.id", contains(newer))
+ .body("nextToken", notNullValue())
+ .extract().path("nextToken");
+
+ givenReq().queryParam("maxResults", 1).queryParam("nextToken", nextToken)
+ .when().get("/applications/" + applicationId + "/jobruns")
+ .then().statusCode(200)
+ .body("jobRuns.id", contains(older))
+ .body("$", not(hasKey("nextToken")));
+ }
+
+ @Test
+ void listJobRunsRefusesAPageLargerThanFifty() {
+ String applicationId = createApplication("");
+
+ givenReq().queryParam("maxResults", 51)
+ .when().get("/applications/" + applicationId + "/jobruns")
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo("1 validation error detected: Value '51' at 'maxResults' failed to "
+ + "satisfy constraint: Member must have value less than or equal to 50"));
+ }
+
+ @Test
+ void listJobRunsRefusesATokenFromAnotherListing() {
+ String applicationId = createApplication("");
+ String other = createApplication("");
+ startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+ clock.advance(Duration.ofSeconds(5));
+ startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+ String nextToken = givenReq().queryParam("maxResults", 1)
+ .when().get("/applications/" + applicationId + "/jobruns")
+ .then().statusCode(200)
+ .extract().path("nextToken");
+
+ givenReq().queryParam("nextToken", nextToken)
+ .when().get("/applications/" + other + "/jobruns")
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ givenReq().queryParam("nextToken", nextToken).queryParam("states", "SUCCESS")
+ .when().get("/applications/" + applicationId + "/jobruns")
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ }
+
+ @Test
+ void listJobRunsFiltersOnStateModeAndCreationTime() {
+ String applicationId = createApplication("");
+ String batch = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+ clock.advance(Duration.ofSeconds(10));
+ Instant between = clock.instant();
+ clock.advance(Duration.ofSeconds(10));
+ String streaming = startedRunId(applicationId,
+ sparkRun(UUID.randomUUID().toString(), ", \"mode\": \"STREAMING\""));
+ String base = "/applications/" + applicationId + "/jobruns";
+
+ givenReq().queryParam("states", "RUNNING").when().get(base)
+ .then().statusCode(200).body("jobRuns.id", contains(streaming));
+ givenReq().queryParam("mode", "STREAMING").when().get(base)
+ .then().statusCode(200).body("jobRuns.id", contains(streaming));
+ givenReq().queryParam("mode", "BATCH").when().get(base)
+ .then().statusCode(200).body("jobRuns.id", contains(batch));
+ givenReq().queryParam("createdAtBefore", between.toString()).when().get(base)
+ .then().statusCode(200).body("jobRuns.id", contains(batch));
+ givenReq().queryParam("createdAtAfter", between.toString()).when().get(base)
+ .then().statusCode(200).body("jobRuns.id", contains(streaming));
+ }
+
+ @Test
+ void listJobRunsGroupsSeveralStatesInTheFilterOrder() {
+ String applicationId = createApplication("");
+ String streaming = startedRunId(applicationId,
+ sparkRun(UUID.randomUUID().toString(), ", \"mode\": \"STREAMING\""));
+ clock.advance(Duration.ofSeconds(5));
+ String batch = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+
+ givenReq().queryParam("states", List.of("RUNNING", "SUCCESS"))
+ .when().get("/applications/" + applicationId + "/jobruns")
+ .then().statusCode(200)
+ .body("jobRuns.id", contains(streaming, batch));
+ }
+
+ @Test
+ void listJobRunsOfAnUnknownApplicationIsEmpty() {
+ givenReq().when().get("/applications/00missing0000000/jobruns")
+ .then().statusCode(200)
+ .body("jobRuns", empty());
+ }
+
+ @Test
+ void jobRunsStayInTheirRegion() {
+ String applicationId = createApplication("");
+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+
+ EmrServerlessRequests.emrServerless("eu-west-1").when().get("/applications/" + applicationId + "/jobruns/" + jobRunId)
+ .then().statusCode(404);
+ EmrServerlessRequests.emrServerless("eu-west-1").when().get("/applications/" + applicationId + "/jobruns")
+ .then().statusCode(200).body("jobRuns", empty());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java
new file mode 100644
index 0000000000..f3b4cd5520
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java
@@ -0,0 +1,189 @@
+package io.github.hectorvent.floci.services.emrserverless;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.services.emrserverless.model.CreateApplicationRequest;
+import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;
+import io.github.hectorvent.floci.services.emrserverless.model.JobRun;
+import io.github.hectorvent.floci.services.emrserverless.model.JobRunRecord;
+import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunRequest;
+import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.testing.InMemoryStorageFactory;
+import io.github.hectorvent.floci.testing.MutableClock;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class EmrServerlessJobRunServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT_ID = "000000000000";
+ private static final String ROLE = "arn:aws:iam::000000000000:role/emr-serverless-job";
+
+ private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());
+ private EmrServerlessService applications;
+ private IamService iamService;
+ private AccountAwareStorageBackend<JobRunRecord> store;
+ private MutableClock clock;
+ private String applicationId;
+
+ @BeforeEach
+ void setUp() {
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ when(config.defaultAccountId()).thenReturn(ACCOUNT_ID);
+ clock = new MutableClock();
+ applications = new EmrServerlessService(config, new InMemoryStorageFactory(ACCOUNT_ID),
+ new RegionResolver(REGION, ACCOUNT_ID), clock);
+ iamService = mock(IamService.class);
+ when(iamService.resolveCallerArn(any())).thenReturn(Optional.empty());
+ store = AccountAwareStorageBackend.inMemory(ACCOUNT_ID);
+
+ CreateApplicationRequest create = new CreateApplicationRequest();
+ create.setName("jobs");
+ create.setReleaseLabel("emr-7.5.0");
+ create.setType("SPARK");
+ create.setClientToken(UUID.randomUUID().toString());
+ applicationId = applications.createApplication(create).getApplicationId();
+ }
+
+ private EmrServerlessJobRunService service(int runDurationSeconds) {
+ return new EmrServerlessJobRunService(store, applications, iamService, new RegionResolver(REGION, ACCOUNT_ID),
+ null, objectMapper, runDurationSeconds, clock);
+ }
+
+ private static StartJobRunRequest request(String mode, Long executionTimeoutMinutes) {
+ JobDriver.SparkSubmit sparkSubmit = new JobDriver.SparkSubmit();
+ sparkSubmit.setEntryPoint("s3://bucket/jobs/app.jar");
+ JobDriver driver = new JobDriver();
+ driver.setSparkSubmit(sparkSubmit);
+ StartJobRunRequest request = new StartJobRunRequest();
+ request.setClientToken(UUID.randomUUID().toString());
+ request.setExecutionRoleArn(ROLE);
+ request.setJobDriver(driver);
+ request.setMode(mode);
+ request.setExecutionTimeoutMinutes(executionTimeoutMinutes);
+ return request;
+ }
+
+ @Test
+ void runSucceedsAsSoonAsItStartsByDefault() {
+ JobRun run = service(0).startJobRun(applicationId, request(null, null));
+
+ assertEquals("SUCCESS", run.getState());
+ assertEquals(run.getCreatedAt(), run.getStartedAt());
+ assertEquals(run.getCreatedAt(), run.getEndedAt());
+ assertEquals(run.getCreatedAt(), run.getUpdatedAt());
+ assertEquals(0, run.getTotalExecutionDurationSeconds());
+ }
+
+ @Test
+ void positiveDurationKeepsTheRunRunningUntilItHasPassed() {
+ EmrServerlessJobRunService service = service(30);
+ JobRun started = service.startJobRun(applicationId, request(null, null));
+ String jobRunId = started.getJobRunId();
+ Instant submittedAt = started.getCreatedAt();
+
+ clock.advance(Duration.ofSeconds(29));
+ JobRun running = service.getJobRun(applicationId, jobRunId);
+ assertEquals("RUNNING", running.getState());
+ assertNull(running.getEndedAt());
+ assertNull(running.getTotalExecutionDurationSeconds());
+
+ clock.advance(Duration.ofSeconds(1));
+ JobRun finished = service.getJobRun(applicationId, jobRunId);
+ assertEquals("SUCCESS", finished.getState());
+ assertEquals(submittedAt.plusSeconds(30), finished.getEndedAt());
+ assertEquals(submittedAt.plusSeconds(30), finished.getUpdatedAt());
+ assertEquals(30, finished.getTotalExecutionDurationSeconds());
+ }
+
+ @Test
+ void cancellingARunningRunEndsItCancelled() {
+ EmrServerlessJobRunService service = service(60);
+ String jobRunId = service.startJobRun(applicationId, request(null, null)).getJobRunId();
+ clock.advance(Duration.ofSeconds(10));
+
+ JobRun cancelled = service.cancelJobRun(applicationId, jobRunId);
+
+ assertEquals("CANCELLED", cancelled.getState());
+ assertEquals(10, cancelled.getTotalExecutionDurationSeconds());
+ clock.advance(Duration.ofMinutes(5));
+ assertEquals("CANCELLED", service.getJobRun(applicationId, jobRunId).getState());
+ }
+
+ @Test
+ void runIsCancelledAtItsExecutionTimeoutWhenItWouldRunLonger() {
+ EmrServerlessJobRunService service = service(600);
+ JobRun started = service.startJobRun(applicationId, request(null, 5L));
+ String jobRunId = started.getJobRunId();
+ Instant submittedAt = started.getCreatedAt();
+
+ clock.advance(Duration.ofMinutes(5).minusSeconds(1));
+ assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId).getState());
+
+ clock.advance(Duration.ofSeconds(1));
+ JobRun timedOut = service.getJobRun(applicationId, jobRunId);
+ assertEquals("CANCELLED", timedOut.getState());
+ assertEquals(submittedAt.plus(Duration.ofMinutes(5)), timedOut.getEndedAt());
+ }
+
+ @Test
+ void streamingRunIgnoresTheRunDuration() {
+ EmrServerlessJobRunService service = service(0);
+ String jobRunId = service.startJobRun(applicationId, request("STREAMING", null)).getJobRunId();
+
+ clock.advance(Duration.ofDays(2));
+
+ assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId).getState());
+ }
+
+ @Test
+ void retryIsRecognisedAfterTheStoredRunWasWrittenAndReadBack() throws Exception {
+ EmrServerlessJobRunService service = service(0);
+ StartJobRunRequest request = request("BATCH", 15L);
+ request.setTags(Map.of("team", "data"));
+ String jobRunId = service.startJobRun(applicationId, request).getJobRunId();
+
+ // As persistent storage does: the record goes out as JSON and comes back.
+ JobRunRecord stored = store.get(jobRunId).orElseThrow();
+ store.put(jobRunId, objectMapper.readValue(objectMapper.writeValueAsString(stored), JobRunRecord.class));
+
+ assertEquals(jobRunId, service.startJobRun(applicationId, request).getJobRunId());
+ request.setExecutionTimeoutMinutes(30L);
+ AwsException conflict = assertThrows(AwsException.class, () -> service.startJobRun(applicationId, request));
+ assertEquals("ConflictException", conflict.getErrorCode());
+ assertEquals(409, conflict.getHttpStatus());
+ }
+
+ @Test
+ void negativeRunDurationIsRefused() {
+ assertThrows(IllegalArgumentException.class, () -> service(-1));
+ }
+
+ @Test
+ void parseTimestampReadsIso8601AndEpochSeconds() {
+ assertEquals(Instant.parse("2026-10-21T12:00:00Z"),
+ EmrServerlessJobRunService.parseTimestamp("2026-10-21T12:00:00Z", "createdAtAfter"));
+ assertEquals(Instant.parse("2026-10-21T12:00:00.500Z"),
+ EmrServerlessJobRunService.parseTimestamp("1792584000.5", "createdAtAfter"));
+ AwsException invalid = assertThrows(AwsException.class,
+ () -> EmrServerlessJobRunService.parseTimestamp("yesterday", "createdAtAfter"));
+ assertEquals("ValidationException", invalid.getErrorCode());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessServiceTest.java
new file mode 100644
index 0000000000..66b9c7984e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessServiceTest.java
@@ -0,0 +1,91 @@
+package io.github.hectorvent.floci.services.emrserverless;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.PersistentStorage;
+import io.github.hectorvent.floci.services.emrserverless.model.Application;
+import io.github.hectorvent.floci.services.emrserverless.model.CreateApplicationRequest;
+import io.github.hectorvent.floci.testing.InMemoryStorageFactory;
+import io.github.hectorvent.floci.testing.MutableClock;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class EmrServerlessServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT_ID = "000000000000";
+ private static final TypeReference<Map<String, Application>> APPLICATIONS = new TypeReference<>() {};
+
+ @TempDir
+ Path tempDir;
+
+ @Test
+ void applicationIsStampedFromTheClockToTheMillisecond() {
+ MutableClock clock = new MutableClock();
+ clock.advance(Duration.ofNanos(250_400_000));
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ when(config.defaultAccountId()).thenReturn(ACCOUNT_ID);
+ EmrServerlessService service = new EmrServerlessService(config, new InMemoryStorageFactory(ACCOUNT_ID),
+ new RegionResolver(REGION, ACCOUNT_ID), clock);
+ CreateApplicationRequest create = new CreateApplicationRequest();
+ create.setReleaseLabel("emr-7.5.0");
+ create.setType("SPARK");
+ create.setClientToken(UUID.randomUUID().toString());
+
+ Application created = service.createApplication(create);
+ assertEquals(Instant.parse("2026-01-01T00:00:00.250Z"), created.getCreatedAt());
+ assertEquals(created.getCreatedAt(), created.getUpdatedAt());
+
+ // The test clock moves a millisecond on every read.
+ clock.advance(Duration.ofMinutes(5));
+ service.startApplication(created.getApplicationId());
+ Application started = service.getApplication(created.getApplicationId());
+ assertEquals(Instant.parse("2026-01-01T00:00:00.250Z"), started.getCreatedAt());
+ assertEquals(Instant.parse("2026-01-01T00:05:00.251Z"), started.getUpdatedAt());
+ }
+
+ @Test
+ void applicationPersistedWithMillisecondTimestampsReadsBackAtTheSameInstant() throws IOException {
+ // As Floci stored applications before their timestamps were epoch seconds.
+ Path file = tempDir.resolve("emr-serverless-applications.json");
+ Files.writeString(file, """
+ {"00abc": {"applicationId": "00abc", "createdAt": 1792584000123, "updatedAt": 1792584060456}}
+ """);
+ PersistentStorage<String, Application> storage = new PersistentStorage<>(file, APPLICATIONS);
+ storage.load();
+
+ Application app = storage.get("00abc").orElseThrow();
+ assertEquals(Instant.parse("2026-10-21T12:00:00.123Z"), app.getCreatedAt());
+ assertEquals(Instant.parse("2026-10-21T12:01:00.456Z"), app.getUpdatedAt());
+ }
+
+ @Test
+ void applicationTimestampsSurviveAReload() {
+ Path file = tempDir.resolve("emr-serverless-applications.json");
+ Application app = new Application();
+ app.setApplicationId("00abc");
+ app.setCreatedAt(Instant.parse("2026-10-21T12:00:00.123Z"));
+ app.setUpdatedAt(Instant.parse("2026-10-21T12:01:00.456Z"));
+ new PersistentStorage<>(file, APPLICATIONS).put("00abc", app);
+
+ PersistentStorage<String, Application> reader = new PersistentStorage<>(file, APPLICATIONS);
+ reader.load();
+
+ Application reloaded = reader.get("00abc").orElseThrow();
+ assertEquals(app.getCreatedAt(), reloaded.getCreatedAt());
+ assertEquals(app.getUpdatedAt(), reloaded.getUpdatedAt());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java b/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java
new file mode 100644
index 0000000000..29cf8cb33f
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java
@@ -0,0 +1,17 @@
+package io.github.hectorvent.floci.testing;
+
+import io.restassured.specification.RequestSpecification;
+
+import static io.restassured.RestAssured.given;
+
+/** One EMR Serverless REST JSON request, signed for a named Region. */
+public final class EmrServerlessRequests {
+ private EmrServerlessRequests() {}
+
+ public static RequestSpecification emrServerless(String region) {
+ return given()
+ .header("Authorization", "AWS4-HMAC-SHA256 Credential=TEST/20260819/" + region
+ + "/emr-serverless/aws4_request, SignedHeaders=host;x-amz-date, Signature=test")
+ .contentType("application/json");
+ }
+}
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index dca4a32d2d..3a19e2ba5e 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -496,6 +496,7 @@ floci:
enabled: true
emrserverless:
enabled: true
+ job-run-duration-seconds: 0 # 0 = BATCH job runs succeed as soon as they start
ssoadmin:
enabled: true
ssooidc: