Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(emr-serverless): tag job runs and report their attempts (#6297)

Priamo Germosen63d577a245c4
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java
index de1e6ad277..56c1ab4575 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java
@@ -9,9 +9,11 @@ import software.amazon.awssdk.services.emrserverless.model.Application;
import software.amazon.awssdk.services.emrserverless.model.ApplicationState;
import software.amazon.awssdk.services.emrserverless.model.ConflictException;
import software.amazon.awssdk.services.emrserverless.model.JobRun;
+import software.amazon.awssdk.services.emrserverless.model.JobRunAttemptSummary;
import software.amazon.awssdk.services.emrserverless.model.JobRunMode;
import software.amazon.awssdk.services.emrserverless.model.JobRunState;
import software.amazon.awssdk.services.emrserverless.model.JobRunSummary;
+import software.amazon.awssdk.services.emrserverless.model.ResourceNotFoundException;
import software.amazon.awssdk.services.emrserverless.model.StartJobRunResponse;
import software.amazon.awssdk.services.emrserverless.model.ValidationException;
@@ -146,4 +148,43 @@ class EmrServerlessJobRunTest {
emr.cancelJobRun(r -> r.applicationId(applicationId).jobRunId(streaming));
}
}
+
+ @Test
+ @DisplayName("A run is its one attempt: GetJobRun and ListJobRunAttempts report attempt 1")
+ void attempts() {
+ String jobRunId = startSparkRun(JobRunMode.BATCH).jobRunId();
+
+ JobRun run = emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(jobRunId).attempt(1)).jobRun();
+ assertThat(run.attempt()).isEqualTo(1);
+ assertThat(run.attemptCreatedAt()).isEqualTo(run.createdAt());
+ assertThat(run.attemptUpdatedAt()).isEqualTo(run.updatedAt());
+
+ List<JobRunAttemptSummary> attempts = emr.listJobRunAttempts(r -> r
+ .applicationId(applicationId).jobRunId(jobRunId)).jobRunAttempts();
+ assertThat(attempts).singleElement().satisfies(attempt -> {
+ assertThat(attempt.id()).isEqualTo(jobRunId);
+ assertThat(attempt.attempt()).isEqualTo(1);
+ assertThat(attempt.jobCreatedAt()).isEqualTo(run.createdAt());
+ assertThat(attempt.state()).isEqualTo(JobRunState.SUCCESS);
+ assertThat(attempt.mode()).isEqualTo(JobRunMode.BATCH);
+ });
+
+ assertThatThrownBy(() -> emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(jobRunId).attempt(2)))
+ .isInstanceOf(ResourceNotFoundException.class);
+ }
+
+ @Test
+ @DisplayName("A run's ARN takes tags, which GetJobRun reports")
+ void jobRunTags() {
+ StartJobRunResponse started = startSparkRun(JobRunMode.BATCH);
+
+ emr.tagResource(r -> r.resourceArn(started.arn()).tags(Map.of("env", "dev")));
+ assertThat(emr.listTagsForResource(r -> r.resourceArn(started.arn())).tags())
+ .containsEntry("team", "data")
+ .containsEntry("env", "dev");
+
+ emr.untagResource(r -> r.resourceArn(started.arn()).tagKeys("env"));
+ assertThat(emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(started.jobRunId())).jobRun().tags())
+ .containsOnlyKeys("team");
+ }
}
diff --git a/docs/services/emr-serverless.md b/docs/services/emr-serverless.md
index cdeda3875e..eda94667ca 100644
--- a/docs/services/emr-serverless.md
+++ b/docs/services/emr-serverless.md
@@ -25,17 +25,26 @@ The emulator implements the standard AWS `emr-serverless` service endpoints:
* `GET /applications/{applicationId}/jobruns` (ListJobRuns)
* `GET /applications/{applicationId}/jobruns/{jobRunId}` (GetJobRun)
* `DELETE /applications/{applicationId}/jobruns/{jobRunId}` (CancelJobRun)
+* `GET /applications/{applicationId}/jobruns/{jobRunId}/attempts` (ListJobRunAttempts)
* `POST /tags/{resourceArn}` (TagResource)
* `DELETE /tags/{resourceArn}?tagKeys=...` (UntagResource)
* `GET /tags/{resourceArn}` (ListTagsForResource)
-The tagging operations work on application ARNs, as `CreateApplication` returns them
-(`arn:aws:emr-serverless:<region>:<account>:/applications/<id>`). They share the tags `CreateApplication`
-stores, so `GetApplication` always reports the current set. Following the EMR Serverless user guide, a
-resource holds at most 50 tags, and `aws:` in any case is reserved as the prefix of a tag key or value.
+The tagging operations work on application and job run ARNs, as `CreateApplication` and `StartJobRun`
+return them (`arn:aws:emr-serverless:<region>:<account>:/applications/<id>` and
+`.../applications/<id>/jobruns/<id>`). They share the tags those calls store, so `GetApplication` and
+`GetJobRun` always report the current set. Following the EMR Serverless user guide, a resource holds
+at most 50 tags, and `aws:` in any case is reserved as the prefix of a tag key or value. A session ARN
+(`.../sessions/<id>`) answers `ResourceNotFoundException`, as sessions are not emulated. Once an
+application is deleted, its ARN and its runs' ARNs answer `ResourceNotFoundException` too, while
+`GetApplication` and `GetJobRun` still report them.
-Applications, creation client tokens, and application tags are scoped to the request's AWS Region.
-An application created in one Region is not returned by list, get, or tag requests signed for another.
+Applications, job runs, creation client tokens, and tags are scoped to the request's AWS Region. An
+application or run created in one Region is not returned by list, get, or tag requests signed for
+another.
+
+`ListApplications` refuses a `states` filter naming a state that does not exist, or more than 7, and a
+`maxResults` outside 1 to 50, with the `ValidationException` AWS's validation layer answers.
`DeleteApplication` leaves the application in `TERMINATED` state. `GetApplication` continues to
report that state so lifecycle waiters can observe it, while an unfiltered `ListApplications`
@@ -58,22 +67,29 @@ response shape, and settles when it is read.
* **Idempotency**: `clientToken` is the idempotency token. Repeating a request with the same token
returns the original run; reusing the token with a different request answers `ConflictException`
(HTTP 409) "Request did not match the original".
+* **Execution timeout**: `executionTimeoutMinutes` is 0 to 1,000,000, 0 meaning no timeout. From
+ release `emr-7.11.0`, a `BATCH` run's timeout must be 1 to 10080 minutes (7 days), as the model and
+ the 7.11.0 release notes say; a run outside that is refused with `ValidationException`.
* **CancelJobRun** moves a run that has not finished to `CANCELLED`. A finished run answers
`ValidationException` "Job run is not in a cancellable state".
* **ListJobRuns** returns the newest runs first, filtered by `states`, `createdAtAfter`,
`createdAtBefore` and `mode`, at most 50 per page. With several `states`, the runs are grouped by
- state in the order the filter names them. An unknown application has no runs. A `nextToken`
+ state in the order the filter names them. A state or mode that does not exist, or more than 8
+ states, is refused with `ValidationException`. An unknown application has no runs. A `nextToken`
resumes only the listing it came from, with the same application and filters.
+* **Attempts**: nothing runs, so no run fails and none is retried: every run is a single attempt,
+ number 1. `GetJobRun`, `ListJobRuns` and `ListJobRunAttempts` report it, with the run's own
+ creation and update times. `GetJobRun` with `attempt` 1, or without one, returns the run; an
+ `attempt` of 2 or more answers `ResourceNotFoundException`, and one below 1 or not an integer
+ `ValidationException`.
* `createdBy` is the caller's ARN, as `GetCallerIdentity` reports it. `releaseLabel` and
`networkConfiguration` come from the application.
## Limitations and Differences from AWS
* **Jobs do not execute**: `StartJobRun` records a job run and moves it through its states, but no Spark or Hive job runs.
-* **Unsourced errors**: AWS documents neither the error for a run submitted to an application whose auto-start is off nor the one for a run on a terminated application; Floci answers both with `ValidationException`. A run needs a `jobDriver`, as every `JobRun` carries one, and `jobDriver` is a union, so a run without a driver, or with one that sets both `sparkSubmit` and `hive` or neither, is refused with `ValidationException` too, in Floci's own wording.
+* **Unsourced errors**: AWS documents neither the error for a run submitted to an application whose auto-start is off nor the one for a run on a terminated application; Floci answers both with `ValidationException`. The same holds for a `BATCH` timeout outside 1 to 10080 minutes on `emr-7.11.0` or later (`ValidationException`) and for `GetJobRun` naming an attempt a run never made (`ResourceNotFoundException`), both in Floci's own wording. A run needs a `jobDriver`, as every `JobRun` carries one, and `jobDriver` is a union, so a run without a driver, or with one that sets both `sparkSubmit` and `hive` or neither, is refused with `ValidationException` too, in Floci's own wording.
* **StopApplication with running jobs**: AWS requires running jobs to finish or be cancelled before an application stops. Floci does not enforce it; with the default duration of `0` no `BATCH` run is ever running.
-* **Not emulated**: job run attempts and retries, queuing (`QUEUED`), `totalResourceUtilization` and `billedResourceUtilization`, and `shutdownGracePeriodInSeconds` (accepted and ignored).
+* **Not emulated**: retries (`retryPolicy` is stored and returned, but no run fails, so none is retried), queuing (`QUEUED`), `totalResourceUtilization` and `billedResourceUtilization`, and `shutdownGracePeriodInSeconds` (accepted and ignored).
* **Instant Start/Stop**: Floci marks the application `STARTED` or `STOPPED` immediately without provisioning actual compute capacity in the background.
* **Data Plane**: Floci does not implement the data-plane API or any execution environments.
-* **Job run tags**: the tagging APIs do not accept job run ARNs (`.../jobruns/<id>`) yet; they answer
- `ResourceNotFoundException`. Tags given to `StartJobRun` are stored and returned by `GetJobRun`.
diff --git a/docs/services/index.md b/docs/services/index.md
index c5c2d8db28..a5df0a6964 100644
--- a/docs/services/index.md
+++ b/docs/services/index.md
@@ -80,7 +80,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac
| [Redshift Data API](redshift-data.md) | `POST /` + `X-Amz-Target: RedshiftData.*` | JSON 1.1 | 11 |
| [Redshift Serverless](redshift-serverless.md) | `POST /` + `X-Amz-Target: RedshiftServerless.*` | JSON 1.1 | 55 |
| [EMR](emr.md) | `POST /` + `X-Amz-Target: ElasticMapReduce.*` | JSON 1.1 | 40 |
-| [EMR Serverless](emr-serverless.md) | `/applications/*`, `/tags/*` | REST JSON | 14 |
+| [EMR Serverless](emr-serverless.md) | `/applications/*`, `/tags/*` | REST JSON | 15 |
| [Data Firehose](firehose.md) | `POST /` + `X-Amz-Target: Firehose_20150804.*` | JSON 1.1 | 12 |
| [ECS](ecs.md) | `POST /` + `X-Amz-Target: AmazonEC2ContainerServiceV20141113.*` | JSON 1.1 | 59 |
| [EFS](efs.md) | `/2015-02-01/...` | REST JSON | 26 |
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/SmithyValidationErrors.java b/src/main/java/io/github/hectorvent/floci/core/common/SmithyValidationErrors.java
index 7e1a2913ff..bb33c6cf82 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/SmithyValidationErrors.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/SmithyValidationErrors.java
@@ -66,4 +66,32 @@ public final class SmithyValidationErrors {
public static AwsException badRequest(String violation) {
return badRequest(List.of(violation));
}
+
+ /** The violation for a number outside {@code min} to {@code max}, or {@code null} for one inside. */
+ public static String outOfRange(String path, long value, long min, long max) {
+ if (value < min) {
+ return "Value '" + value + "' at '" + path + "' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to " + min;
+ }
+ if (value > max) {
+ return "Value '" + value + "' at '" + path + "' failed to satisfy constraint: "
+ + "Member must have value less than or equal to " + max;
+ }
+ return null;
+ }
+
+ /** The violation for a value outside its enum, listing the allowed values in the model's order. */
+ public static String notInEnum(String value, String path, List<String> allowed) {
+ return "Value '" + value + "' at '" + path + "' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: " + allowed;
+ }
+
+ /**
+ * The violation for a list holding a value outside its members' enum. The list is checked as a
+ * whole, so the whole list is the offending value and the enum constraint is nested.
+ */
+ public static String notAllInEnum(List<String> values, String path, List<String> allowed) {
+ return "Value '" + values + "' at '" + path + "' failed to satisfy constraint: "
+ + "Member must satisfy constraint: [Member must satisfy enum value set: " + allowed + "]";
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java
index 063cf0e8e4..1759cbf5d6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java
@@ -11,9 +11,11 @@ import io.github.hectorvent.floci.services.emrserverless.model.DeleteApplication
import io.github.hectorvent.floci.services.emrserverless.model.GetApplicationResponse;
import io.github.hectorvent.floci.services.emrserverless.model.GetJobRunResponse;
import io.github.hectorvent.floci.services.emrserverless.model.JobRun;
+import io.github.hectorvent.floci.services.emrserverless.model.JobRunAttemptSummary;
import io.github.hectorvent.floci.services.emrserverless.model.JobRunSummary;
import io.github.hectorvent.floci.services.emrserverless.model.ListApplicationsRequest;
import io.github.hectorvent.floci.services.emrserverless.model.ListApplicationsResponse;
+import io.github.hectorvent.floci.services.emrserverless.model.ListJobRunAttemptsResponse;
import io.github.hectorvent.floci.services.emrserverless.model.ListJobRunsResponse;
import io.github.hectorvent.floci.services.emrserverless.model.StartApplicationResponse;
import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunRequest;
@@ -160,9 +162,25 @@ public class EmrServerlessController {
@GET
@Path("/{applicationId}/jobruns/{jobRunId}")
public GetJobRunResponse getJobRun(@PathParam("applicationId") String applicationId,
- @PathParam("jobRunId") String jobRunId) {
+ @PathParam("jobRunId") String jobRunId,
+ @QueryParam("attempt") String attempt) {
GetJobRunResponse response = new GetJobRunResponse();
- response.setJobRun(jobRuns.getJobRun(applicationId, jobRunId));
+ response.setJobRun(jobRuns.getJobRun(applicationId, jobRunId,
+ EmrServerlessJobRunService.parseAttempt(attempt)));
+ return response;
+ }
+
+ @GET
+ @Path("/{applicationId}/jobruns/{jobRunId}/attempts")
+ public ListJobRunAttemptsResponse listJobRunAttempts(@PathParam("applicationId") String applicationId,
+ @PathParam("jobRunId") String jobRunId,
+ @QueryParam("maxResults") String maxResults,
+ @QueryParam("nextToken") String nextToken) {
+ PaginatedResult<JobRunAttemptSummary> page = jobRuns.listJobRunAttempts(applicationId, jobRunId,
+ Pagination.parseMaxResults(maxResults, "ValidationException"), nextToken);
+ ListJobRunAttemptsResponse response = new ListJobRunAttemptsResponse();
+ response.setJobRunAttempts(page.items());
+ response.setNextToken(page.nextToken());
return response;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java
index 15cc34739a..5d1407bfc3 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java
@@ -18,6 +18,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.emrserverless.model.Application;
import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;
import io.github.hectorvent.floci.services.emrserverless.model.JobRun;
+import io.github.hectorvent.floci.services.emrserverless.model.JobRunAttemptSummary;
import io.github.hectorvent.floci.services.emrserverless.model.JobRunRecord;
import io.github.hectorvent.floci.services.emrserverless.model.JobRunSummary;
import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunRequest;
@@ -38,6 +39,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
@@ -62,11 +64,21 @@ public class EmrServerlessJobRunService {
private static final String MODE_STREAMING = "STREAMING";
private static final Set<String> TERMINAL_STATES = Set.of(STATE_SUCCESS, "FAILED", STATE_CANCELLED);
+ private static final List<String> STATES = List.of("SUBMITTED", "PENDING", "SCHEDULED", STATE_RUNNING,
+ STATE_SUCCESS, "FAILED", "CANCELLING", STATE_CANCELLED, "QUEUED");
+ // JobRunStateSet holds at most 8 states.
+ private static final int MAX_FILTER_STATES = 8;
private static final List<String> MODES = List.of(MODE_BATCH, MODE_STREAMING);
// JobRun.executionTimeoutMinutes: "If you didn't specify a timeout, this value defaults to 720 minutes."
private static final long DEFAULT_EXECUTION_TIMEOUT_MINUTES = 720;
// The model's Duration, which bounds executionTimeoutMinutes.
private static final long MAX_EXECUTION_TIMEOUT_MINUTES = 1_000_000;
+ // The EMR Serverless 7.11.0 release notes: for BATCH job runs the maximum "is 7 days from this release
+ // onwards", and the timeout "can no longer be set to 0".
+ private static final long MAX_BATCH_EXECUTION_TIMEOUT_MINUTES = 10_080;
+ private static final Pattern RELEASE_LABEL = Pattern.compile("emr-(\\d{1,6})\\.(\\d{1,6})(?:\\..*)?");
+ // Nothing runs, so no run fails and none is retried: every run is its own one attempt.
+ private static final int ATTEMPT = 1;
private static final int MAX_LIST_RESULTS = 50;
// IAMRoleArn's pattern, as the model and AWS's validation message spell it.
private static final String ROLE_ARN_PATTERN =
@@ -130,15 +142,54 @@ public class EmrServerlessJobRunService {
if (body.getTags() != null) {
EmrServerlessService.validateTags(body.getTags());
}
- JobRunRecord record = applications.admitJobRun(applicationId, app -> newRun(app, body, original));
+ JobRunRecord record = applications.admitJobRun(applicationId, app -> checkBatchTimeout(app, body),
+ app -> newRun(app, body, original));
LOG.infov("Started EMR Serverless job run {0} on application {1}", record.getJobRun().getJobRunId(),
applicationId);
return settle(record).getJobRun();
}
- public synchronized JobRun getJobRun(String applicationId, String jobRunId) {
+ /**
+ * The run, as its {@code attempt} saw it; without one, "the latest attempt". A run has one
+ * attempt, so any other number names nothing. No source gives AWS's answer for an attempt a run
+ * never made; ResourceNotFoundException, in Floci's words, is the modeled error that fits.
+ */
+ public synchronized JobRun getJobRun(String applicationId, String jobRunId, Integer attempt) {
+ if (attempt != null) {
+ String violation = SmithyValidationErrors.outOfRange("attempt", attempt, ATTEMPT, Integer.MAX_VALUE);
+ if (violation != null) {
+ throw SmithyValidationErrors.validationException(violation);
+ }
+ }
+ applications.getApplication(applicationId);
+ JobRunRecord record = findRun(applicationId, jobRunId);
+ if (attempt != null && attempt != ATTEMPT) {
+ throw new AwsException("ResourceNotFoundException",
+ "Job run " + jobRunId + " has no attempt " + attempt, 404);
+ }
+ return settle(record).getJobRun();
+ }
+
+ /** A run's attempts, which are its one attempt. */
+ public synchronized PaginatedResult<JobRunAttemptSummary> listJobRunAttempts(String applicationId,
+ String jobRunId,
+ Integer maxResults,
+ String nextToken) {
+ if (maxResults != null) {
+ String violation = SmithyValidationErrors.outOfRange("maxResults", maxResults, 1, MAX_LIST_RESULTS);
+ if (violation != null) {
+ throw SmithyValidationErrors.validationException(violation);
+ }
+ }
applications.getApplication(applicationId);
- return settle(findRun(applicationId, jobRunId)).getJobRun();
+ JobRunRecord record = settle(findRun(applicationId, jobRunId));
+ PaginatedResult<JobRunRecord> page = Pagination.paginate(List.of(record),
+ only -> String.format("%010d", ATTEMPT), maxResults != null ? maxResults : MAX_LIST_RESULTS,
+ nextToken, String.join("|", "ListJobRunAttempts", applicationId, jobRunId),
+ token -> new AwsException("ValidationException", "Invalid nextToken.", 400));
+ return new PaginatedResult<>(
+ page.items().stream().map(EmrServerlessJobRunService::toAttemptSummary).toList(),
+ page.nextToken());
}
/**
@@ -151,12 +202,20 @@ public class EmrServerlessJobRunService {
public synchronized PaginatedResult<JobRunSummary> listJobRuns(String applicationId, List<String> states,
Instant createdAtAfter, Instant createdAtBefore,
String mode, Integer maxResults, String nextToken) {
+ List<String> violations = new ArrayList<>();
if (maxResults != null) {
- String violation = outOfRange("maxResults", maxResults, 1, MAX_LIST_RESULTS);
+ String violation = SmithyValidationErrors.outOfRange("maxResults", maxResults, 1, MAX_LIST_RESULTS);
if (violation != null) {
- throw SmithyValidationErrors.validationException(violation);
+ violations.add(violation);
}
}
+ violations.addAll(EmrServerlessService.statesViolations(states, STATES, MAX_FILTER_STATES));
+ if (mode != null && !MODES.contains(mode)) {
+ violations.add(SmithyValidationErrors.notInEnum(mode, "mode", MODES));
+ }
+ if (!violations.isEmpty()) {
+ throw SmithyValidationErrors.validationException(violations);
+ }
List<JobRunRecord> matching = new ArrayList<>();
Map<String, String> cursors = new HashMap<>();
for (JobRunRecord record : store.scan(key -> true)) {
@@ -199,6 +258,35 @@ public class EmrServerlessJobRunService {
return record.getJobRun();
}
+ public synchronized Map<String, String> listTags(EmrServerlessService.ResourceArn resource) {
+ JobRun run = runFor(resource).getJobRun();
+ return run.getTags() != null ? new LinkedHashMap<>(run.getTags()) : new LinkedHashMap<>();
+ }
+
+ public synchronized void tagResource(EmrServerlessService.ResourceArn resource, Map<String, String> tags) {
+ JobRunRecord record = runFor(resource);
+ record.getJobRun().setTags(EmrServerlessService.withTags(record.getJobRun().getTags(), tags));
+ store.put(record.getJobRun().getJobRunId(), record);
+ }
+
+ public synchronized void untagResource(EmrServerlessService.ResourceArn resource, List<String> tagKeys) {
+ JobRunRecord record = runFor(resource);
+ record.getJobRun().setTags(EmrServerlessService.withoutTags(record.getJobRun().getTags(), tagKeys));
+ store.put(record.getJobRun().getJobRunId(), record);
+ }
+
+ /** GetJobRun's {@code attempt}, an integer in the query string. */
+ static Integer parseAttempt(String value) {
+ if (value == null || value.isBlank()) {
+ return null;
+ }
+ try {
+ return Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ throw new AwsException("ValidationException", "attempt must be an integer.", 400);
+ }
+ }
+
/**
* A ListJobRuns time filter. Query-string timestamps are ISO 8601 in restJson1; epoch seconds
* are accepted too.
@@ -245,6 +333,9 @@ public class EmrServerlessJobRunService {
? request.getExecutionTimeoutMinutes() : DEFAULT_EXECUTION_TIMEOUT_MINUTES);
run.setMode(request.getMode());
run.setRetryPolicy(request.getRetryPolicy());
+ run.setAttempt(ATTEMPT);
+ run.setAttemptCreatedAt(now);
+ run.setAttemptUpdatedAt(now);
run.setStartedAt(now);
JobRunRecord record = new JobRunRecord();
@@ -287,6 +378,7 @@ public class EmrServerlessJobRunService {
run.setState(state);
run.setEndedAt(endedAt);
run.setUpdatedAt(endedAt);
+ run.setAttemptUpdatedAt(endedAt);
run.setTotalExecutionDurationSeconds((int) Duration.between(run.getStartedAt(), endedAt).toSeconds());
store.put(run.getJobRunId(), record);
}
@@ -298,6 +390,16 @@ public class EmrServerlessJobRunService {
"Job run " + jobRunId + " not found", 404));
}
+ // The run a tagging ARN names. The ARN must be the run's own, so one naming another account names nothing,
+ // and a deleted application's runs name nothing, as the application's own ARN does.
+ private JobRunRecord runFor(EmrServerlessService.ResourceArn resource) {
+ return store.get(resource.jobRunId())
+ .filter(record -> resource.applicationId().equals(record.getJobRun().getApplicationId()))
+ .filter(record -> resource.arn().equals(record.getJobRun().getArn()))
+ .filter(record -> applications.isLive(resource.applicationId()))
+ .orElseThrow(() -> EmrServerlessService.resourceNotFound(resource.arn()));
+ }
+
private JobRunRecord runWithClientToken(String clientToken) {
for (JobRunRecord record : store.scan(key -> true)) {
if (clientToken.equals(record.getClientToken()) && inRequestRegion(record.getJobRun())) {
@@ -351,12 +453,11 @@ public class EmrServerlessJobRunService {
+ "Member must satisfy regular expression pattern: " + ROLE_ARN_PATTERN);
}
if (request.getMode() != null && !MODES.contains(request.getMode())) {
- violations.add("Value '" + request.getMode() + "' at 'mode' failed to satisfy constraint: "
- + "Member must satisfy enum value set: " + MODES);
+ violations.add(SmithyValidationErrors.notInEnum(request.getMode(), "mode", MODES));
}
if (request.getExecutionTimeoutMinutes() != null) {
- String violation = outOfRange("executionTimeoutMinutes", request.getExecutionTimeoutMinutes(), 0,
- MAX_EXECUTION_TIMEOUT_MINUTES);
+ String violation = SmithyValidationErrors.outOfRange("executionTimeoutMinutes",
+ request.getExecutionTimeoutMinutes(), 0, MAX_EXECUTION_TIMEOUT_MINUTES);
if (violation != null) {
violations.add(violation);
}
@@ -383,16 +484,31 @@ public class EmrServerlessJobRunService {
return "Value null at '" + member + "' failed to satisfy constraint: Member must not be null";
}
- private static String outOfRange(String member, long value, long min, long max) {
- if (value < min) {
- return "Value '" + value + "' at '" + member + "' failed to satisfy constraint: "
- + "Member must have value greater than or equal to " + min;
+ /**
+ * From Amazon EMR release 7.11.0 a BATCH run's timeout is 1 to 10080 minutes; an earlier release, or
+ * a STREAMING run, keeps the model's 0 to 1,000,000, 0 meaning none. No source gives AWS's error, so
+ * the wording is Floci's.
+ */
+ private static void checkBatchTimeout(Application app, StartJobRunRequest request) {
+ Long minutes = request.getExecutionTimeoutMinutes();
+ if (minutes == null || MODE_STREAMING.equals(request.getMode())
+ || !boundsBatchTimeouts(app.getReleaseLabel())) {
+ return;
}
- if (value > max) {
- return "Value '" + value + "' at '" + member + "' failed to satisfy constraint: "
- + "Member must have value less than or equal to " + max;
+ if (minutes == 0 || minutes > MAX_BATCH_EXECUTION_TIMEOUT_MINUTES) {
+ throw new AwsException("ValidationException", "executionTimeoutMinutes must be between 1 and "
+ + MAX_BATCH_EXECUTION_TIMEOUT_MINUTES + " for a BATCH job run on release emr-7.11.0 or later.",
+ 400);
}
- return null;
+ }
+
+ private static boolean boundsBatchTimeouts(String releaseLabel) {
+ Matcher matcher = RELEASE_LABEL.matcher(releaseLabel != null ? releaseLabel : "");
+ if (!matcher.matches()) {
+ return false;
+ }
+ int major = Integer.parseInt(matcher.group(1));
+ return major > 7 || (major == 7 && Integer.parseInt(matcher.group(2)) >= 11);
}
private static JobRunSummary toSummary(JobRunRecord record) {
@@ -411,6 +527,30 @@ public class EmrServerlessJobRunService {
summary.setStateDetails(run.getStateDetails());
summary.setReleaseLabel(run.getReleaseLabel());
summary.setType(record.getApplicationType());
+ summary.setAttempt(run.getAttempt());
+ summary.setAttemptCreatedAt(run.getAttemptCreatedAt());
+ summary.setAttemptUpdatedAt(run.getAttemptUpdatedAt());
+ return summary;
+ }
+
+ private static JobRunAttemptSummary toAttemptSummary(JobRunRecord record) {
+ JobRun run = record.getJobRun();
+ JobRunAttemptSummary summary = new JobRunAttemptSummary();
+ summary.setApplicationId(run.getApplicationId());
+ summary.setId(run.getJobRunId());
+ summary.setName(run.getName());
+ summary.setMode(run.getMode());
+ summary.setArn(run.getArn());
+ summary.setCreatedBy(run.getCreatedBy());
+ summary.setJobCreatedAt(run.getCreatedAt());
+ summary.setCreatedAt(run.getCreatedAt());
+ summary.setUpdatedAt(run.getUpdatedAt());
+ summary.setExecutionRole(run.getExecutionRole());
+ summary.setState(run.getState());
+ summary.setStateDetails(run.getStateDetails());
+ summary.setReleaseLabel(run.getReleaseLabel());
+ summary.setType(record.getApplicationType());
+ summary.setAttempt(ATTEMPT);
return summary;
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java
index 4d34a99467..8196392734 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.PaginatedResult;
import io.github.hectorvent.floci.core.common.Pagination;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.RequestContext;
+import io.github.hectorvent.floci.core.common.SmithyValidationErrors;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.emrserverless.model.Application;
@@ -21,11 +22,13 @@ import jakarta.inject.Inject;
import java.time.Clock;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.function.Consumer;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -40,7 +43,17 @@ public class EmrServerlessService {
private static final int MAX_TAGS = 50;
private static final String RESERVED_PREFIX = "aws:";
private static final Pattern TAG_TEXT = Pattern.compile("[A-Za-z0-9 /_.:=+@-]*");
- private static final Pattern RESOURCE = Pattern.compile("/applications/([0-9a-zA-Z]+)(?:/jobruns/([0-9a-zA-Z]+))?");
+ // ResourceArn's resource: an application, or one of its job runs or sessions.
+ private static final Pattern RESOURCE =
+ Pattern.compile("/applications/([0-9a-zA-Z]+)(?:/(jobruns|sessions)/([0-9a-zA-Z]+))?");
+ private static final List<String> APPLICATION_STATES =
+ List.of("CREATING", "CREATED", "STARTING", "STARTED", "STOPPING", "STOPPED", "TERMINATED");
+ // ApplicationStateSet holds at most 7 states; a ListApplications page holds 1 to 50 applications.
+ private static final int MAX_APPLICATION_STATES = 7;
+ private static final int MAX_LIST_RESULTS = 50;
+
+ /** What an EMR Serverless resource ARN names: an application, or one of its job runs. */
+ public record ResourceArn(String arn, String applicationId, String jobRunId) {}
private final EmulatorConfig config;
private final AccountAwareStorageBackend<Application> storage;
@@ -119,6 +132,18 @@ public class EmrServerlessService {
}
public PaginatedResult<ApplicationSummary> listApplications(ListApplicationsRequest request) {
+ List<String> violations = new ArrayList<>();
+ if (request.getMaxResults() != null) {
+ String violation = SmithyValidationErrors.outOfRange("maxResults", request.getMaxResults(), 1,
+ MAX_LIST_RESULTS);
+ if (violation != null) {
+ violations.add(violation);
+ }
+ }
+ violations.addAll(statesViolations(request.getStates(), APPLICATION_STATES, MAX_APPLICATION_STATES));
+ if (!violations.isEmpty()) {
+ throw SmithyValidationErrors.validationException(violations);
+ }
String region = regionResolver.getRegion();
List<Application> all = storage.scan(k -> true).stream()
.filter(app -> inRegion(app, region)).collect(Collectors.toList());
@@ -128,7 +153,7 @@ public class EmrServerlessService {
all = all.stream().filter(app -> !"TERMINATED".equals(app.getState())).collect(Collectors.toList());
}
- PaginatedResult<Application> page = Pagination.paginate(all, Application::getApplicationId, request.getMaxResults(), request.getNextToken(), 50, "ValidationException");
+ PaginatedResult<Application> page = Pagination.paginate(all, Application::getApplicationId, request.getMaxResults(), request.getNextToken(), MAX_LIST_RESULTS, "ValidationException");
return new PaginatedResult<>(
page.items().stream().map(this::toSummary).collect(Collectors.toList()),
@@ -241,15 +266,18 @@ public class EmrServerlessService {
* Admits a job run to the application and returns what {@code admit} builds from it. An
* application "only accepts jobs when it's in" STARTED, and auto-start, on unless the
* application turns it off, starts a CREATED or STOPPED one "on job submission" (EMR Serverless
- * user guide, "Application states" and "Default application behavior"). This service's lock is
- * held throughout, so the application cannot be stopped or deleted while the run is admitted.
+ * user guide, "Application states" and "Default application behavior"). {@code check} refuses a
+ * run against the application before auto-start can change it. This service's lock is held
+ * throughout, so the application cannot be updated, stopped or deleted while the run is admitted.
*/
- public synchronized <T> T admitJobRun(String applicationId, Function<Application, T> admit) {
+ public synchronized <T> T admitJobRun(String applicationId, Consumer<Application> check,
+ Function<Application, T> admit) {
Application app = getApplication(applicationId);
String state = app.getState();
if ("TERMINATED".equals(state)) {
throw new AwsException("ValidationException", "Application is terminated.", 400);
}
+ check.accept(app);
if (!"STARTED".equals(state)) {
if (!autoStartEnabled(app)) {
throw new AwsException("ValidationException", "Application " + applicationId
@@ -269,52 +297,33 @@ public class EmrServerlessService {
// ---- Tags ---------------------------------------------------------------------------------
- public synchronized Map<String, String> listTags(String arn) {
- Application app = applicationFor(arn);
+ public synchronized Map<String, String> listTags(ResourceArn resource) {
+ Application app = applicationFor(resource);
return app.getTags() != null ? new LinkedHashMap<>(app.getTags()) : new LinkedHashMap<>();
}
- /** Adds the tags to the application's, replacing any with the same key. */
- public synchronized void tagResource(String arn, Map<String, String> tags) {
- Application app = applicationFor(arn);
- // An empty map is a valid TagMap (min 0) and changes nothing; the dispatcher already refuses a
- // missing one.
- if (tags == null) {
- throw new AwsException("ValidationException", "tags is required", 400);
- }
- validateTags(tags);
- Map<String, String> merged = app.getTags() != null ? new LinkedHashMap<>(app.getTags()) : new LinkedHashMap<>();
- merged.putAll(tags);
- if (merged.size() > MAX_TAGS) {
- throw new AwsException("ValidationException",
- "A resource can have at most " + MAX_TAGS + " tags.", 400);
- }
- app.setTags(merged);
+ public synchronized void tagResource(ResourceArn resource, Map<String, String> tags) {
+ Application app = applicationFor(resource);
+ app.setTags(withTags(app.getTags(), tags));
storage.put(app.getApplicationId(), app);
}
- public synchronized void untagResource(String arn, List<String> tagKeys) {
- Application app = applicationFor(arn);
- if (tagKeys == null || tagKeys.isEmpty()) {
- throw new AwsException("ValidationException", "tagKeys is required", 400);
- }
- tagKeys.forEach(EmrServerlessService::validateTagKey);
+ public synchronized void untagResource(ResourceArn resource, List<String> tagKeys) {
+ Application app = applicationFor(resource);
+ Map<String, String> remaining = withoutTags(app.getTags(), tagKeys);
if (app.getTags() == null || app.getTags().isEmpty()) {
return;
}
- Map<String, String> remaining = new LinkedHashMap<>(app.getTags());
- tagKeys.forEach(remaining::remove);
app.setTags(remaining);
storage.put(app.getApplicationId(), app);
}
/**
- * The application an EMR Serverless ARN names:
- * {@code arn:<partition>:emr-serverless:<region>:<account>:/applications/<id>}, the form
- * CreateApplication returns. A job run ARN ({@code .../jobruns/<id>}) is refused, as job run tags
- * are not emulated yet.
+ * Parses a tagging ARN: {@code arn:<partition>:emr-serverless:<region>:<account>:/applications/<id>},
+ * the form CreateApplication returns, with {@code /jobruns/<id>} for a job run. An ARN of another
+ * Region names nothing here, and neither does a session's, as sessions are not emulated.
*/
- private Application applicationFor(String arn) {
+ public ResourceArn resourceArn(String arn) {
AwsArnUtils.Arn parsed;
try {
parsed = AwsArnUtils.parse(arn);
@@ -325,16 +334,75 @@ public class EmrServerlessService {
if (!"emr-serverless".equals(parsed.service()) || !matcher.matches()) {
throw new AwsException("ValidationException", "Invalid resourceArn: " + arn, 400);
}
- if (!parsed.region().equals(regionResolver.getRegion())) {
- throw new AwsException("ResourceNotFoundException", "Resource " + arn + " not found", 404);
- }
- if (matcher.group(2) != null) {
- throw new AwsException("ResourceNotFoundException", "Job run " + matcher.group(2) + " not found", 404);
+ if (!parsed.region().equals(regionResolver.getRegion()) || "sessions".equals(matcher.group(2))) {
+ throw resourceNotFound(arn);
}
- return storage.get(matcher.group(1))
- .filter(app -> arn.equals(app.getArn()))
+ return new ResourceArn(arn, matcher.group(1), matcher.group(3));
+ }
+
+ /** Whether the application exists and has not been deleted, which leaves it TERMINATED. */
+ public synchronized boolean isLive(String applicationId) {
+ return storage.get(applicationId).filter(app -> !"TERMINATED".equals(app.getState())).isPresent();
+ }
+
+ static AwsException resourceNotFound(String arn) {
+ return new AwsException("ResourceNotFoundException", "Resource " + arn + " not found", 404);
+ }
+
+ private Application applicationFor(ResourceArn resource) {
+ return storage.get(resource.applicationId())
+ .filter(app -> resource.arn().equals(app.getArn()))
.filter(app -> !"TERMINATED".equals(app.getState()))
- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Resource " + arn + " not found", 404));
+ .orElseThrow(() -> resourceNotFound(resource.arn()));
+ }
+
+ /**
+ * A resource's tags once TagResource adds {@code added}, which replace any with the same key. An
+ * empty map is a valid TagMap (min 0) and changes nothing; the dispatcher already refuses a missing
+ * one.
+ */
+ static Map<String, String> withTags(Map<String, String> current, Map<String, String> added) {
+ if (added == null) {
+ throw new AwsException("ValidationException", "tags is required", 400);
+ }
+ validateTags(added);
+ Map<String, String> merged = current != null ? new LinkedHashMap<>(current) : new LinkedHashMap<>();
+ merged.putAll(added);
+ if (merged.size() > MAX_TAGS) {
+ throw new AwsException("ValidationException",
+ "A resource can have at most " + MAX_TAGS + " tags.", 400);
+ }
+ return merged;
+ }
+
+ /** A resource's tags once UntagResource removes {@code tagKeys}. */
+ static Map<String, String> withoutTags(Map<String, String> current, List<String> tagKeys) {
+ if (tagKeys == null || tagKeys.isEmpty()) {
+ throw new AwsException("ValidationException", "tagKeys is required", 400);
+ }
+ tagKeys.forEach(EmrServerlessService::validateTagKey);
+ Map<String, String> remaining = current != null ? new LinkedHashMap<>(current) : new LinkedHashMap<>();
+ tagKeys.forEach(remaining::remove);
+ return remaining;
+ }
+
+ /**
+ * The violations of a {@code states} filter: at most {@code max} states, each one of
+ * {@code allowed}. A typo would otherwise match nothing and list nothing.
+ */
+ static List<String> statesViolations(List<String> states, List<String> allowed, int max) {
+ List<String> violations = new ArrayList<>();
+ if (states == null) {
+ return violations;
+ }
+ if (states.size() > max) {
+ violations.add("Value '" + states + "' at 'states' failed to satisfy constraint: "
+ + "Member must have length less than or equal to " + max);
+ }
+ if (!allowed.containsAll(states)) {
+ violations.add(SmithyValidationErrors.notAllInEnum(states, "states", allowed));
+ }
+ return violations;
}
/** At most 50 tags, each key and value within TagKey and TagValue and free of the reserved prefix. */
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagHandler.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagHandler.java
index 103805062a..b785678d3a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagHandler.java
@@ -8,18 +8,21 @@ import java.util.List;
import java.util.Map;
/**
- * EMR Serverless on the shared {@code /tags/{resourceArn}} path. The wire shape is the common one (a
- * lowercase {@code tags} map, a {@code tagKeys} query and {@code POST}), except that TagResource and
- * UntagResource answer 200 rather than 204, and payloads are validated strictly.
+ * EMR Serverless on the shared {@code /tags/{resourceArn}} path, for applications and job runs. The
+ * wire shape is the common one (a lowercase {@code tags} map, a {@code tagKeys} query and
+ * {@code POST}), except that TagResource and UntagResource answer 200 rather than 204, and payloads
+ * are validated strictly.
*/
@ApplicationScoped
public class EmrServerlessTagHandler implements TagHandler {
private final EmrServerlessService service;
+ private final EmrServerlessJobRunService jobRuns;
@Inject
- public EmrServerlessTagHandler(EmrServerlessService service) {
+ public EmrServerlessTagHandler(EmrServerlessService service, EmrServerlessJobRunService jobRuns) {
this.service = service;
+ this.jobRuns = jobRuns;
}
@Override
@@ -45,16 +48,27 @@ public class EmrServerlessTagHandler implements TagHandler {
@Override
public Map<String, String> listTags(String region, String arn) {
- return service.listTags(arn);
+ EmrServerlessService.ResourceArn resource = service.resourceArn(arn);
+ return resource.jobRunId() != null ? jobRuns.listTags(resource) : service.listTags(resource);
}
@Override
public void tagResource(String region, String arn, Map<String, String> tags) {
- service.tagResource(arn, tags);
+ EmrServerlessService.ResourceArn resource = service.resourceArn(arn);
+ if (resource.jobRunId() != null) {
+ jobRuns.tagResource(resource, tags);
+ } else {
+ service.tagResource(resource, tags);
+ }
}
@Override
public void untagResource(String region, String arn, List<String> tagKeys) {
- service.untagResource(arn, tagKeys);
+ EmrServerlessService.ResourceArn resource = service.resourceArn(arn);
+ if (resource.jobRunId() != null) {
+ jobRuns.untagResource(resource, tagKeys);
+ } else {
+ service.untagResource(resource, tagKeys);
+ }
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java
index 44b3b65838..bb4be8e3d6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java
@@ -33,6 +33,11 @@ public class JobRun {
private Long executionTimeoutMinutes;
private String mode;
private Map<String, Object> retryPolicy;
+ private Integer attempt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant attemptCreatedAt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant attemptUpdatedAt;
@JsonFormat(shape = JsonFormat.Shape.NUMBER)
private Instant startedAt;
@JsonFormat(shape = JsonFormat.Shape.NUMBER)
@@ -78,6 +83,12 @@ public class JobRun {
public void setMode(String mode) { this.mode = mode; }
public Map<String, Object> getRetryPolicy() { return retryPolicy; }
public void setRetryPolicy(Map<String, Object> retryPolicy) { this.retryPolicy = retryPolicy; }
+ public Integer getAttempt() { return attempt; }
+ public void setAttempt(Integer attempt) { this.attempt = attempt; }
+ public Instant getAttemptCreatedAt() { return attemptCreatedAt; }
+ public void setAttemptCreatedAt(Instant attemptCreatedAt) { this.attemptCreatedAt = attemptCreatedAt; }
+ public Instant getAttemptUpdatedAt() { return attemptUpdatedAt; }
+ public void setAttemptUpdatedAt(Instant attemptUpdatedAt) { this.attemptUpdatedAt = attemptUpdatedAt; }
public Instant getStartedAt() { return startedAt; }
public void setStartedAt(Instant startedAt) { this.startedAt = startedAt; }
public Instant getEndedAt() { return endedAt; }
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunAttemptSummary.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunAttemptSummary.java
new file mode 100644
index 0000000000..907481ed88
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunAttemptSummary.java
@@ -0,0 +1,65 @@
+package io.github.hectorvent.floci.services.emrserverless.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+
+/**
+ * One entry of ListJobRunAttempts. {@code id} is the run's id, {@code jobCreatedAt} when the run was
+ * created, and {@code createdAt} and {@code updatedAt} the attempt's own times.
+ */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class JobRunAttemptSummary {
+ private String applicationId;
+ private String id;
+ private String name;
+ private String mode;
+ private String arn;
+ private String createdBy;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant jobCreatedAt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant createdAt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant updatedAt;
+ private String executionRole;
+ private String state;
+ private String stateDetails;
+ private String releaseLabel;
+ private String type;
+ private Integer attempt;
+
+ public String getApplicationId() { return applicationId; }
+ public void setApplicationId(String applicationId) { this.applicationId = applicationId; }
+ public String getId() { return id; }
+ public void setId(String id) { this.id = id; }
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+ public String getMode() { return mode; }
+ public void setMode(String mode) { this.mode = mode; }
+ public String getArn() { return arn; }
+ public void setArn(String arn) { this.arn = arn; }
+ public String getCreatedBy() { return createdBy; }
+ public void setCreatedBy(String createdBy) { this.createdBy = createdBy; }
+ public Instant getJobCreatedAt() { return jobCreatedAt; }
+ public void setJobCreatedAt(Instant jobCreatedAt) { this.jobCreatedAt = jobCreatedAt; }
+ public Instant getCreatedAt() { return createdAt; }
+ public void setCreatedAt(Instant createdAt) { this.createdAt = createdAt; }
+ public Instant getUpdatedAt() { return updatedAt; }
+ public void setUpdatedAt(Instant updatedAt) { this.updatedAt = updatedAt; }
+ public String getExecutionRole() { return executionRole; }
+ public void setExecutionRole(String executionRole) { this.executionRole = executionRole; }
+ public String getState() { return state; }
+ public void setState(String state) { this.state = state; }
+ public String getStateDetails() { return stateDetails; }
+ public void setStateDetails(String stateDetails) { this.stateDetails = stateDetails; }
+ public String getReleaseLabel() { return releaseLabel; }
+ public void setReleaseLabel(String releaseLabel) { this.releaseLabel = releaseLabel; }
+ public String getType() { return type; }
+ public void setType(String type) { this.type = type; }
+ public Integer getAttempt() { return attempt; }
+ public void setAttempt(Integer attempt) { this.attempt = attempt; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java
index bf54a96730..b9a8fc9ab6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java
@@ -25,6 +25,11 @@ public class JobRunSummary {
private String stateDetails;
private String releaseLabel;
private String type;
+ private Integer attempt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant attemptCreatedAt;
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant attemptUpdatedAt;
public String getApplicationId() { return applicationId; }
public void setApplicationId(String applicationId) { this.applicationId = applicationId; }
@@ -52,4 +57,10 @@ public class JobRunSummary {
public void setReleaseLabel(String releaseLabel) { this.releaseLabel = releaseLabel; }
public String getType() { return type; }
public void setType(String type) { this.type = type; }
+ public Integer getAttempt() { return attempt; }
+ public void setAttempt(Integer attempt) { this.attempt = attempt; }
+ public Instant getAttemptCreatedAt() { return attemptCreatedAt; }
+ public void setAttemptCreatedAt(Instant attemptCreatedAt) { this.attemptCreatedAt = attemptCreatedAt; }
+ public Instant getAttemptUpdatedAt() { return attemptUpdatedAt; }
+ public void setAttemptUpdatedAt(Instant attemptUpdatedAt) { this.attemptUpdatedAt = attemptUpdatedAt; }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunAttemptsResponse.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunAttemptsResponse.java
new file mode 100644
index 0000000000..919a98f27a
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunAttemptsResponse.java
@@ -0,0 +1,18 @@
+package io.github.hectorvent.floci.services.emrserverless.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ListJobRunAttemptsResponse {
+ private List<JobRunAttemptSummary> jobRunAttempts;
+ private String nextToken;
+
+ public List<JobRunAttemptSummary> getJobRunAttempts() { return jobRunAttempts; }
+ public void setJobRunAttempts(List<JobRunAttemptSummary> jobRunAttempts) { this.jobRunAttempts = jobRunAttempts; }
+ public String getNextToken() { return nextToken; }
+ public void setNextToken(String nextToken) { this.nextToken = nextToken; }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java
index f2c2ebbf6d..0a2935ff72 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java
@@ -17,6 +17,7 @@ import org.junit.jupiter.api.TestMethodOrder;
import java.math.BigDecimal;
import java.time.Duration;
+import java.util.List;
import java.util.UUID;
import static org.hamcrest.Matchers.*;
@@ -338,4 +339,34 @@ public class EmrServerlessIntegrationTest {
assertEquals(0, createdAt.compareTo(updatedAt), "updatedAt " + updatedAt + " differs from createdAt " + createdAt);
assertEquals(0, createdAt.compareTo(listed), "ListApplications reports createdAt " + listed);
}
+
+ @Test
+ @Order(10)
+ void listApplicationsRefusesStatesThatDoNotExistAndPagesOutsideOneToFifty() {
+ givenReq()
+ .queryParam("states", "RUNNING")
+ .queryParam("maxResults", 51)
+ .when()
+ .get("/applications")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo("2 validation errors detected: "
+ + "Value '51' at 'maxResults' failed to satisfy constraint: "
+ + "Member must have value less than or equal to 50; "
+ + "Value '[RUNNING]' at 'states' failed to satisfy constraint: Member must satisfy constraint: "
+ + "[Member must satisfy enum value set: "
+ + "[CREATING, CREATED, STARTING, STARTED, STOPPING, STOPPED, TERMINATED]]"));
+
+ givenReq()
+ .queryParam("states", List.of("CREATING", "CREATED", "STARTING", "STARTED", "STOPPING", "STOPPED",
+ "TERMINATED", "CREATED"))
+ .when()
+ .get("/applications")
+ .then()
+ .statusCode(400)
+ .body("message", equalTo("1 validation error detected: "
+ + "Value '[CREATING, CREATED, STARTING, STARTED, STOPPING, STOPPED, TERMINATED, CREATED]' "
+ + "at 'states' failed to satisfy constraint: Member must have length less than or equal to 7"));
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java
index e5146d22d4..0df1c03e33 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java
@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.config.JsonConfig;
import io.restassured.config.RestAssuredConfig;
+import io.restassured.path.json.JsonPath;
import io.restassured.path.json.config.JsonPathConfig;
import io.restassured.response.Response;
import io.restassured.specification.RequestSpecification;
@@ -44,10 +45,14 @@ class EmrServerlessJobRunIntegrationTest {
}
private static String createApplication(String extra) {
+ return createApplicationOn("emr-7.5.0", extra);
+ }
+
+ private static String createApplicationOn(String releaseLabel, String extra) {
return givenReq()
.body("""
- {"name": "jobs-%s", "releaseLabel": "emr-7.5.0", "type": "SPARK", "clientToken": "%s"%s}
- """.formatted(UUID.randomUUID(), UUID.randomUUID(), extra))
+ {"name": "jobs-%s", "releaseLabel": "%s", "type": "SPARK", "clientToken": "%s"%s}
+ """.formatted(UUID.randomUUID(), releaseLabel, UUID.randomUUID(), extra))
.when().post("/applications")
.then().statusCode(200)
.extract().path("applicationId");
@@ -115,6 +120,9 @@ class EmrServerlessJobRunIntegrationTest {
.body("jobRun.totalExecutionDurationSeconds", equalTo(0))
.body("jobRun.startedAt", instanceOf(Number.class))
.body("jobRun.endedAt", instanceOf(Number.class))
+ .body("jobRun.attempt", equalTo(1))
+ .body("jobRun.attemptCreatedAt", instanceOf(Number.class))
+ .body("jobRun.attemptUpdatedAt", instanceOf(Number.class))
.body("jobRun", not(hasKey("hive")));
}
@@ -369,6 +377,9 @@ class EmrServerlessJobRunIntegrationTest {
.body("jobRuns[0].type", equalTo("SPARK"))
.body("jobRuns[0].releaseLabel", equalTo("emr-7.5.0"))
.body("jobRuns[0].createdBy", equalTo("arn:aws:iam::000000000000:root"))
+ .body("jobRuns[0].attempt", equalTo(1))
+ .body("jobRuns[0].attemptCreatedAt", instanceOf(Number.class))
+ .body("jobRuns[0].attemptUpdatedAt", instanceOf(Number.class))
.body("jobRuns[0]", not(hasKey("jobRunId")))
.body("jobRuns[0]", not(hasKey("jobDriver")));
}
@@ -465,6 +476,29 @@ class EmrServerlessJobRunIntegrationTest {
.body("jobRuns.id", contains(streaming, batch));
}
+ @Test
+ void listJobRunsRefusesStatesAndModesThatDoNotExist() {
+ String base = "/applications/" + createApplication("") + "/jobruns";
+
+ givenReq().queryParam("states", List.of("RUNNING", "SUCCEEDED")).queryParam("mode", "batch")
+ .when().get(base)
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo("2 validation errors detected: "
+ + "Value '[RUNNING, SUCCEEDED]' at 'states' failed to satisfy constraint: Member must "
+ + "satisfy constraint: [Member must satisfy enum value set: [SUBMITTED, PENDING, SCHEDULED, "
+ + "RUNNING, SUCCESS, FAILED, CANCELLING, CANCELLED, QUEUED]]; "
+ + "Value 'batch' at 'mode' failed to satisfy constraint: Member must satisfy enum value set: "
+ + "[BATCH, STREAMING]"));
+ givenReq().queryParam("states", List.of("SUBMITTED", "PENDING", "SCHEDULED", "RUNNING", "SUCCESS", "FAILED",
+ "CANCELLING", "CANCELLED", "QUEUED"))
+ .when().get(base)
+ .then().statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value '[SUBMITTED, PENDING, SCHEDULED, "
+ + "RUNNING, SUCCESS, FAILED, CANCELLING, CANCELLED, QUEUED]' at 'states' failed to satisfy "
+ + "constraint: Member must have length less than or equal to 8"));
+ }
+
@Test
void listJobRunsOfAnUnknownApplicationIsEmpty() {
givenReq().when().get("/applications/00missing0000000/jobruns")
@@ -472,6 +506,114 @@ class EmrServerlessJobRunIntegrationTest {
.body("jobRuns", empty());
}
+ @Test
+ void fromRelease711ABatchRunsTimeoutIsOneToTenThousandEightyMinutes() {
+ String current = createApplicationOn("emr-7.11.0", "");
+ for (String timeout : List.of(", \"executionTimeoutMinutes\": 0",
+ ", \"executionTimeoutMinutes\": 10081", ", \"mode\": \"BATCH\", \"executionTimeoutMinutes\": 0")) {
+ startJobRun(current, sparkRun(UUID.randomUUID().toString(), timeout))
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo("executionTimeoutMinutes must be between 1 and 10080 for a BATCH job "
+ + "run on release emr-7.11.0 or later."));
+ }
+ givenReq().when().get("/applications/" + current)
+ .then().body("application.state", equalTo("CREATED"));
+
+ startJobRun(current, sparkRun(UUID.randomUUID().toString(), ", \"executionTimeoutMinutes\": 10080"))
+ .then().statusCode(200);
+ startJobRun(current, sparkRun(UUID.randomUUID().toString(),
+ ", \"mode\": \"STREAMING\", \"executionTimeoutMinutes\": 0"))
+ .then().statusCode(200);
+ String earlier = createApplicationOn("emr-7.10.0", "");
+ for (String timeout : List.of(", \"executionTimeoutMinutes\": 0", ", \"executionTimeoutMinutes\": 1000000")) {
+ startJobRun(earlier, sparkRun(UUID.randomUUID().toString(), timeout)).then().statusCode(200);
+ }
+ }
+
+ @Test
+ void aRunIsItsOneAttempt() {
+ String applicationId = createApplication("");
+ String path = "/applications/" + applicationId + "/jobruns/"
+ + startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+ JsonPath run = givenReq().config(DECIMAL_NUMBERS).when().get(path)
+ .then().statusCode(200).extract().jsonPath();
+
+ assertEquals(1, run.getInt("jobRun.attempt"));
+ assertEquals(run.<BigDecimal>get("jobRun.createdAt"), run.<BigDecimal>get("jobRun.attemptCreatedAt"));
+ assertEquals(run.<BigDecimal>get("jobRun.updatedAt"), run.<BigDecimal>get("jobRun.attemptUpdatedAt"));
+
+ givenReq().queryParam("attempt", 1).when().get(path)
+ .then().statusCode(200).body("jobRun.attempt", equalTo(1));
+ givenReq().queryParam("attempt", 2).when().get(path)
+ .then().statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ givenReq().queryParam("attempt", 0).when().get(path)
+ .then().statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value '0' at 'attempt' failed to satisfy "
+ + "constraint: Member must have value greater than or equal to 1"));
+ givenReq().queryParam("attempt", "first").when().get(path)
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ // An empty value never reaches the parameter: RESTEasy Reactive binds it as null, so it reads as
+ // no attempt, the latest one, as Pagination.parseMaxResults reads an empty maxResults.
+ givenReq().queryParam("attempt", "").when().get(path)
+ .then().statusCode(200).body("jobRun.attempt", equalTo(1));
+ }
+
+ @Test
+ void listJobRunAttemptsReportsTheRunsOneAttempt() {
+ String applicationId = createApplication("");
+ String jobRunId = startedRunId(applicationId,
+ sparkRun(UUID.randomUUID().toString(), ", \"name\": \"nightly\", \"mode\": \"BATCH\""));
+ String path = "/applications/" + applicationId + "/jobruns/" + jobRunId;
+ JsonPath run = givenReq().config(DECIMAL_NUMBERS).when().get(path)
+ .then().statusCode(200).extract().jsonPath();
+
+ JsonPath attempts = givenReq().config(DECIMAL_NUMBERS).when().get(path + "/attempts")
+ .then().statusCode(200)
+ .body("jobRunAttempts", hasSize(1))
+ .body("$", not(hasKey("nextToken")))
+ .extract().jsonPath();
+ assertEquals(applicationId, attempts.getString("jobRunAttempts[0].applicationId"));
+ assertEquals(jobRunId, attempts.getString("jobRunAttempts[0].id"));
+ assertEquals("nightly", attempts.getString("jobRunAttempts[0].name"));
+ assertEquals("BATCH", attempts.getString("jobRunAttempts[0].mode"));
+ assertEquals(run.getString("jobRun.arn"), attempts.getString("jobRunAttempts[0].arn"));
+ assertEquals("arn:aws:iam::000000000000:root", attempts.getString("jobRunAttempts[0].createdBy"));
+ assertEquals(run.<BigDecimal>get("jobRun.createdAt"),
+ attempts.<BigDecimal>get("jobRunAttempts[0].jobCreatedAt"));
+ assertEquals(run.<BigDecimal>get("jobRun.createdAt"), attempts.<BigDecimal>get("jobRunAttempts[0].createdAt"));
+ assertEquals(run.<BigDecimal>get("jobRun.updatedAt"), attempts.<BigDecimal>get("jobRunAttempts[0].updatedAt"));
+ assertEquals(ROLE, attempts.getString("jobRunAttempts[0].executionRole"));
+ assertEquals("SUCCESS", attempts.getString("jobRunAttempts[0].state"));
+ assertEquals("emr-7.5.0", attempts.getString("jobRunAttempts[0].releaseLabel"));
+ assertEquals("SPARK", attempts.getString("jobRunAttempts[0].type"));
+ assertEquals(1, attempts.getInt("jobRunAttempts[0].attempt"));
+ }
+
+ @Test
+ void listJobRunAttemptsRefusesAPageLargerThanFiftyAndAnUnknownRun() {
+ String applicationId = createApplication("");
+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));
+
+ givenReq().queryParam("maxResults", 51)
+ .when().get("/applications/" + applicationId + "/jobruns/" + jobRunId + "/attempts")
+ .then().statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value '51' at 'maxResults' failed to "
+ + "satisfy constraint: Member must have value less than or equal to 50"));
+ givenReq().when().get("/applications/" + applicationId + "/jobruns/00missing0000000/attempts")
+ .then().statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ givenReq().when().get("/applications/00missing0000000/jobruns/" + jobRunId + "/attempts")
+ .then().statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ givenReq().queryParam("nextToken", "bm90LWEtdG9rZW4")
+ .when().get("/applications/" + applicationId + "/jobruns/" + jobRunId + "/attempts")
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ }
+
@Test
void jobRunsStayInTheirRegion() {
String applicationId = createApplication("");
diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java
index f3b4cd5520..32a88d5f96 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java
@@ -100,13 +100,13 @@ class EmrServerlessJobRunServiceTest {
Instant submittedAt = started.getCreatedAt();
clock.advance(Duration.ofSeconds(29));
- JobRun running = service.getJobRun(applicationId, jobRunId);
+ JobRun running = service.getJobRun(applicationId, jobRunId, null);
assertEquals("RUNNING", running.getState());
assertNull(running.getEndedAt());
assertNull(running.getTotalExecutionDurationSeconds());
clock.advance(Duration.ofSeconds(1));
- JobRun finished = service.getJobRun(applicationId, jobRunId);
+ JobRun finished = service.getJobRun(applicationId, jobRunId, null);
assertEquals("SUCCESS", finished.getState());
assertEquals(submittedAt.plusSeconds(30), finished.getEndedAt());
assertEquals(submittedAt.plusSeconds(30), finished.getUpdatedAt());
@@ -124,7 +124,7 @@ class EmrServerlessJobRunServiceTest {
assertEquals("CANCELLED", cancelled.getState());
assertEquals(10, cancelled.getTotalExecutionDurationSeconds());
clock.advance(Duration.ofMinutes(5));
- assertEquals("CANCELLED", service.getJobRun(applicationId, jobRunId).getState());
+ assertEquals("CANCELLED", service.getJobRun(applicationId, jobRunId, null).getState());
}
@Test
@@ -135,14 +135,31 @@ class EmrServerlessJobRunServiceTest {
Instant submittedAt = started.getCreatedAt();
clock.advance(Duration.ofMinutes(5).minusSeconds(1));
- assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId).getState());
+ assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId, null).getState());
clock.advance(Duration.ofSeconds(1));
- JobRun timedOut = service.getJobRun(applicationId, jobRunId);
+ JobRun timedOut = service.getJobRun(applicationId, jobRunId, null);
assertEquals("CANCELLED", timedOut.getState());
assertEquals(submittedAt.plus(Duration.ofMinutes(5)), timedOut.getEndedAt());
}
+ @Test
+ void anAttemptTheRunNeverMadeIsRefusedBeforeTheRunSettles() {
+ EmrServerlessJobRunService service = service(30);
+ JobRun started = service.startJobRun(applicationId, request(null, null));
+ String jobRunId = started.getJobRunId();
+ clock.advance(Duration.ofSeconds(30));
+
+ AwsException noAttempt = assertThrows(AwsException.class,
+ () -> service.getJobRun(applicationId, jobRunId, 2));
+ assertEquals("ResourceNotFoundException", noAttempt.getErrorCode());
+ assertEquals("RUNNING", store.get(jobRunId).orElseThrow().getJobRun().getState());
+
+ JobRun finished = service.getJobRun(applicationId, jobRunId, 1);
+ assertEquals("SUCCESS", finished.getState());
+ assertEquals(started.getCreatedAt().plusSeconds(30), finished.getEndedAt());
+ }
+
@Test
void streamingRunIgnoresTheRunDuration() {
EmrServerlessJobRunService service = service(0);
@@ -150,7 +167,7 @@ class EmrServerlessJobRunServiceTest {
clock.advance(Duration.ofDays(2));
- assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId).getState());
+ assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId, null).getState());
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessRegionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessRegionIntegrationTest.java
index a10fcff731..5e6af0035f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessRegionIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessRegionIntegrationTest.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.emrserverless;
+import io.github.hectorvent.floci.testing.EmrServerlessRequests;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.response.Response;
@@ -7,11 +8,9 @@ import io.restassured.specification.RequestSpecification;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
import java.util.UUID;
-import static io.restassured.RestAssured.given;
+import static io.github.hectorvent.floci.testing.EmrServerlessRequests.tagsPath;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.not;
@@ -26,15 +25,7 @@ class EmrServerlessRegionIntegrationTest {
}
private static RequestSpecification emr(String region) {
- return given()
- .header("Authorization", "AWS4-HMAC-SHA256 Credential=TEST/20261020/" + region
- + "/emr-serverless/aws4_request, SignedHeaders=host;x-amz-date, Signature=test")
- .contentType("application/json")
- .urlEncodingEnabled(false);
- }
-
- private static String tagsPath(String arn) {
- return "/tags/" + URLEncoder.encode(arn, StandardCharsets.UTF_8);
+ return EmrServerlessRequests.emrServerless(region).urlEncodingEnabled(false);
}
@Test
@@ -89,4 +80,29 @@ class EmrServerlessRegionIntegrationTest {
emr("us-east-1").when().get(tagsPath(eastArn))
.then().statusCode(200).body("tags.team", equalTo("data"));
}
+
+ @Test
+ void jobRunTagsStayInTheRunsRegion() {
+ String applicationId = emr("us-east-1").body("""
+ {"name":"regional-runs","releaseLabel":"emr-7.5.0","type":"SPARK","clientToken":"%s"}
+ """.formatted(UUID.randomUUID()))
+ .when().post("/applications")
+ .then().statusCode(200).extract().path("applicationId");
+ String runArn = emr("us-east-1").body("""
+ {"clientToken":"%s","executionRoleArn":"arn:aws:iam::000000000000:role/emr-serverless-job",
+ "jobDriver":{"sparkSubmit":{"entryPoint":"s3://bucket/jobs/app.jar"}},"tags":{"team":"data"}}
+ """.formatted(UUID.randomUUID()))
+ .when().post("/applications/" + applicationId + "/jobruns")
+ .then().statusCode(200).extract().path("arn");
+
+ emr("eu-west-1").when().get(tagsPath(runArn))
+ .then().statusCode(404).body("__type", equalTo("ResourceNotFoundException"));
+ emr("eu-west-1").body("{\"tags\":{\"team\":\"wrong-region\"}}")
+ .when().post(tagsPath(runArn))
+ .then().statusCode(404).body("__type", equalTo("ResourceNotFoundException"));
+ emr("eu-west-1").when().delete(tagsPath(runArn) + "?tagKeys=team")
+ .then().statusCode(404).body("__type", equalTo("ResourceNotFoundException"));
+ emr("us-east-1").when().get(tagsPath(runArn))
+ .then().statusCode(200).body("tags.team", equalTo("data"));
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagIntegrationTest.java
index 1fb8d799dc..4e16cc1881 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagIntegrationTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.emrserverless;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.testing.EmrServerlessRequests;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.path.json.JsonPath;
@@ -21,7 +22,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import static io.restassured.RestAssured.given;
+import static io.github.hectorvent.floci.testing.EmrServerlessRequests.tagsPath;
import static org.hamcrest.Matchers.aMapWithSize;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasKey;
@@ -41,16 +42,28 @@ class EmrServerlessTagIntegrationTest {
}
private static RequestSpecification emr() {
- return given()
- .header("Authorization", "AWS4-HMAC-SHA256 Credential=TEST/20260929/us-east-1/emr-serverless/aws4_request, "
- + "SignedHeaders=host;x-amz-date, Signature=test")
- .contentType("application/json")
- .urlEncodingEnabled(false);
+ return EmrServerlessRequests.emrServerless("us-east-1").urlEncodingEnabled(false);
}
- /** The SDK sends the ARN as one path segment with ':' and '/' percent-encoded. */
- private static String tagsPath(String arn) {
- return "/tags/" + URLEncoder.encode(arn, StandardCharsets.UTF_8);
+ /** A tags map of {@code count} distinct keys, as JSON. */
+ private static String tagMap(int count) {
+ StringBuilder tags = new StringBuilder("{");
+ for (int i = 0; i < count; i++) {
+ tags.append(i == 0 ? "" : ",").append("\"k").append(i).append("\": \"v\"");
+ }
+ return tags.append("}").toString();
+ }
+
+ private static String[] startJobRun(String applicationId, String tagsJson) {
+ JsonPath started = emr()
+ .body("""
+ { "clientToken": "%s", "executionRoleArn": "arn:aws:iam::000000000000:role/emr-serverless-job",
+ "jobDriver": { "sparkSubmit": { "entryPoint": "s3://bucket/jobs/app.jar" } }, "tags": %s }
+ """.formatted(UUID.randomUUID(), tagsJson))
+ .when().post("/applications/" + applicationId + "/jobruns")
+ .then().statusCode(200)
+ .extract().jsonPath();
+ return new String[] {started.getString("jobRunId"), started.getString("arn")};
}
private static String[] createApplication(String tagsJson) {
@@ -138,12 +151,7 @@ class EmrServerlessTagIntegrationTest {
.then().statusCode(400)
.body("__type", equalTo("ValidationException"));
- StringBuilder tooMany = new StringBuilder("{ \"tags\": {");
- for (int i = 0; i < 51; i++) {
- tooMany.append(i == 0 ? "" : ",").append("\"k").append(i).append("\": \"v\"");
- }
- tooMany.append("} }");
- emr().body(tooMany.toString())
+ emr().body("{ \"tags\": " + tagMap(51) + " }")
.when().post(tagsPath(arn))
.then().statusCode(400)
.body("__type", equalTo("ValidationException"));
@@ -170,12 +178,7 @@ class EmrServerlessTagIntegrationTest {
@Test
void createApplicationAppliesTheSameTagRules() {
- StringBuilder tooMany = new StringBuilder("{");
- for (int i = 0; i < 51; i++) {
- tooMany.append(i == 0 ? "" : ",").append("\"k").append(i).append("\": \"v\"");
- }
- tooMany.append("}");
- for (String tags : List.of(tooMany.toString(), "{ \"bad#key\": \"v\" }")) {
+ for (String tags : List.of(tagMap(51), "{ \"bad#key\": \"v\" }")) {
emr().body("""
{ "name": "limits", "releaseLabel": "emr-7.5.0", "type": "SPARK",
"clientToken": "%s", "tags": %s }
@@ -193,13 +196,8 @@ class EmrServerlessTagIntegrationTest {
@Test
void theUserGuideLimitsApplyAndAnEmptyMapIsANoOp() {
String arn = createApplication("{ \"team\": \"data\" }")[1];
- StringBuilder fortyNine = new StringBuilder("{ \"tags\": {");
- for (int i = 0; i < 49; i++) {
- fortyNine.append(i == 0 ? "" : ",").append("\"k").append(i).append("\": \"v\"");
- }
- fortyNine.append("} }");
- emr().body(fortyNine.toString()).when().post(tagsPath(arn)).then().statusCode(200);
+ emr().body("{ \"tags\": " + tagMap(49) + " }").when().post(tagsPath(arn)).then().statusCode(200);
emr().when().get(tagsPath(arn)).then().statusCode(200).body("tags", aMapWithSize(50));
emr().body("{ \"tags\": { \"one-more\": \"v\" } }")
.when().post(tagsPath(arn))
@@ -221,7 +219,7 @@ class EmrServerlessTagIntegrationTest {
/** Tag writes read, change and store the whole application; concurrent ones must not lose each other. */
@Test
void concurrentTagWritesAreAllKept() throws Exception {
- String arn = createApplication("{}")[1];
+ EmrServerlessService.ResourceArn resource = service.resourceArn(createApplication("{}")[1]);
int threads = 16;
CountDownLatch go = new CountDownLatch(1);
ExecutorService pool = Executors.newFixedThreadPool(threads);
@@ -231,7 +229,7 @@ class EmrServerlessTagIntegrationTest {
String key = "k" + i;
futures.add(pool.submit(() -> {
go.await();
- service.tagResource(arn, Map.of(key, "v"));
+ service.tagResource(resource, Map.of(key, "v"));
return null;
}));
}
@@ -243,20 +241,125 @@ class EmrServerlessTagIntegrationTest {
pool.shutdownNow();
}
- assertEquals(threads, service.listTags(arn).size());
+ assertEquals(threads, service.listTags(resource).size());
}
@Test
void anEmptyTagMapChangesNothingButMissingTagsOrKeysAreRefused() {
- String arn = createApplication("{ \"team\": \"data\" }")[1];
+ EmrServerlessService.ResourceArn resource = service.resourceArn(createApplication("{ \"team\": \"data\" }")[1]);
- service.tagResource(arn, Map.of());
- assertEquals(Map.of("team", "data"), service.listTags(arn));
+ service.tagResource(resource, Map.of());
+ assertEquals(Map.of("team", "data"), service.listTags(resource));
assertEquals("ValidationException",
assertThrows(AwsException.class,
- () -> service.tagResource(arn, null)).getErrorCode());
+ () -> service.tagResource(resource, null)).getErrorCode());
assertEquals("ValidationException",
assertThrows(AwsException.class,
- () -> service.untagResource(arn, List.of())).getErrorCode());
+ () -> service.untagResource(resource, List.of())).getErrorCode());
+ }
+
+ @Test
+ void tagsRoundTripOnAJobRunArnAndShowInGetJobRun() {
+ String[] app = createApplication("{ \"team\": \"app\" }");
+ String[] run = startJobRun(app[0], "{ \"team\": \"data\" }");
+ String arn = run[1];
+
+ emr().when().get(tagsPath(arn))
+ .then().statusCode(200)
+ .body("tags", aMapWithSize(1))
+ .body("tags.team", equalTo("data"));
+
+ emr().body("{ \"tags\": { \"env\": \"dev\", \"team\": \"platform\" } }")
+ .when().post(tagsPath(arn))
+ .then().statusCode(200);
+ emr().when().get("/applications/" + app[0] + "/jobruns/" + run[0])
+ .then().statusCode(200)
+ .body("jobRun.state", equalTo("SUCCESS"))
+ .body("jobRun.tags.team", equalTo("platform"))
+ .body("jobRun.tags.env", equalTo("dev"));
+
+ emr().when().delete(tagsPath(arn) + "?tagKeys=env")
+ .then().statusCode(200);
+ emr().when().get(tagsPath(arn))
+ .then().statusCode(200)
+ .body("tags", not(hasKey("env")))
+ .body("tags.team", equalTo("platform"));
+ emr().when().get(tagsPath(app[1]))
+ .then().statusCode(200)
+ .body("tags", aMapWithSize(1))
+ .body("tags.team", equalTo("app"));
+ }
+
+ @Test
+ void aDeletedApplicationsRunsCannotBeTagged() {
+ String applicationId = createApplication("{}")[0];
+ String[] run = startJobRun(applicationId, "{ \"team\": \"data\" }");
+ String path = tagsPath(run[1]);
+
+ emr().when().post("/applications/" + applicationId + "/stop")
+ .then().statusCode(200);
+ emr().when().delete("/applications/" + applicationId)
+ .then().statusCode(200);
+
+ emr().when().get(path)
+ .then().statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"))
+ .body("message", equalTo("Resource " + run[1] + " not found"));
+ emr().body("{ \"tags\": { \"env\": \"dev\" } }")
+ .when().post(path)
+ .then().statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ emr().when().delete(path + "?tagKeys=team")
+ .then().statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+
+ emr().when().get("/applications/" + applicationId + "/jobruns/" + run[0])
+ .then().statusCode(200)
+ .body("jobRun.tags.team", equalTo("data"))
+ .body("jobRun.tags", not(hasKey("env")));
+ }
+
+ @Test
+ void aJobRunArnMustBeTheRunsOwnAndSessionArnsNameNothing() {
+ String[] owner = createApplication("{}");
+ String[] other = createApplication("{}");
+ String[] run = startJobRun(owner[0], "{ \"team\": \"data\" }");
+
+ for (String arn : List.of(other[1] + "/jobruns/" + run[0],
+ run[1].replace(":000000000000:", ":111111111111:"),
+ owner[1] + "/sessions/00abc0000000000")) {
+ emr().when().get(tagsPath(arn))
+ .then().statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ emr().body("{ \"tags\": { \"env\": \"dev\" } }")
+ .when().post(tagsPath(arn))
+ .then().statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ }
+ emr().when().get(tagsPath(run[1]))
+ .then().statusCode(200)
+ .body("tags", aMapWithSize(1));
+ }
+
+ @Test
+ void jobRunsKeepTheSameTagRules() {
+ String applicationId = createApplication("{}")[0];
+ String arn = startJobRun(applicationId, tagMap(50))[1];
+
+ emr().body("{ \"tags\": { \"one-more\": \"v\" } }")
+ .when().post(tagsPath(arn))
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo("A resource can have at most 50 tags."));
+ emr().body("{ \"tags\": { \"aws:owner\": \"x\" } }")
+ .when().post(tagsPath(arn))
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ emr().when().delete(tagsPath(arn) + "?tagKeys=")
+ .then().statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ emr().when().get(tagsPath(arn))
+ .then().statusCode(200)
+ .body("tags", aMapWithSize(50));
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java b/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java
index 29cf8cb33f..26cd7a2374 100644
--- a/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java
+++ b/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java
@@ -2,6 +2,9 @@ package io.github.hectorvent.floci.testing;
import io.restassured.specification.RequestSpecification;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+
import static io.restassured.RestAssured.given;
/** One EMR Serverless REST JSON request, signed for a named Region. */
@@ -14,4 +17,12 @@ public final class EmrServerlessRequests {
+ "/emr-serverless/aws4_request, SignedHeaders=host;x-amz-date, Signature=test")
.contentType("application/json");
}
+
+ /**
+ * The tagging path for an ARN, sent as the SDK sends it: one path segment with ':' and '/'
+ * percent-encoded. Send it with {@code urlEncodingEnabled(false)}, or it is encoded twice.
+ */
+ public static String tagsPath(String arn) {
+ return "/tags/" + URLEncoder.encode(arn, StandardCharsets.UTF_8);
+ }
}