Light, fluffy, and always free - The AWS Local Emulator alternative
feat(emr-serverless): tag job runs and report their attempts (#6297)
Priamo Germosen63d577a245c4
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.javaindex de1e6ad277..56c1ab4575 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrServerlessJobRunTest.java@@ -9,9 +9,11 @@ import software.amazon.awssdk.services.emrserverless.model.Application;import software.amazon.awssdk.services.emrserverless.model.ApplicationState;import software.amazon.awssdk.services.emrserverless.model.ConflictException;import software.amazon.awssdk.services.emrserverless.model.JobRun;+import software.amazon.awssdk.services.emrserverless.model.JobRunAttemptSummary;import software.amazon.awssdk.services.emrserverless.model.JobRunMode;import software.amazon.awssdk.services.emrserverless.model.JobRunState;import software.amazon.awssdk.services.emrserverless.model.JobRunSummary;+import software.amazon.awssdk.services.emrserverless.model.ResourceNotFoundException;import software.amazon.awssdk.services.emrserverless.model.StartJobRunResponse;import software.amazon.awssdk.services.emrserverless.model.ValidationException;@@ -146,4 +148,43 @@ class EmrServerlessJobRunTest {emr.cancelJobRun(r -> r.applicationId(applicationId).jobRunId(streaming));}}++ @Test+ @DisplayName("A run is its one attempt: GetJobRun and ListJobRunAttempts report attempt 1")+ void attempts() {+ String jobRunId = startSparkRun(JobRunMode.BATCH).jobRunId();++ JobRun run = emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(jobRunId).attempt(1)).jobRun();+ assertThat(run.attempt()).isEqualTo(1);+ assertThat(run.attemptCreatedAt()).isEqualTo(run.createdAt());+ assertThat(run.attemptUpdatedAt()).isEqualTo(run.updatedAt());++ List<JobRunAttemptSummary> attempts = emr.listJobRunAttempts(r -> r+ .applicationId(applicationId).jobRunId(jobRunId)).jobRunAttempts();+ assertThat(attempts).singleElement().satisfies(attempt -> {+ assertThat(attempt.id()).isEqualTo(jobRunId);+ assertThat(attempt.attempt()).isEqualTo(1);+ assertThat(attempt.jobCreatedAt()).isEqualTo(run.createdAt());+ assertThat(attempt.state()).isEqualTo(JobRunState.SUCCESS);+ assertThat(attempt.mode()).isEqualTo(JobRunMode.BATCH);+ });++ assertThatThrownBy(() -> emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(jobRunId).attempt(2)))+ .isInstanceOf(ResourceNotFoundException.class);+ }++ @Test+ @DisplayName("A run's ARN takes tags, which GetJobRun reports")+ void jobRunTags() {+ StartJobRunResponse started = startSparkRun(JobRunMode.BATCH);++ emr.tagResource(r -> r.resourceArn(started.arn()).tags(Map.of("env", "dev")));+ assertThat(emr.listTagsForResource(r -> r.resourceArn(started.arn())).tags())+ .containsEntry("team", "data")+ .containsEntry("env", "dev");++ emr.untagResource(r -> r.resourceArn(started.arn()).tagKeys("env"));+ assertThat(emr.getJobRun(r -> r.applicationId(applicationId).jobRunId(started.jobRunId())).jobRun().tags())+ .containsOnlyKeys("team");+ }}diff --git a/docs/services/emr-serverless.md b/docs/services/emr-serverless.mdindex cdeda3875e..eda94667ca 100644--- a/docs/services/emr-serverless.md+++ b/docs/services/emr-serverless.md@@ -25,17 +25,26 @@ The emulator implements the standard AWS `emr-serverless` service endpoints:* `GET /applications/{applicationId}/jobruns` (ListJobRuns)* `GET /applications/{applicationId}/jobruns/{jobRunId}` (GetJobRun)* `DELETE /applications/{applicationId}/jobruns/{jobRunId}` (CancelJobRun)+* `GET /applications/{applicationId}/jobruns/{jobRunId}/attempts` (ListJobRunAttempts)* `POST /tags/{resourceArn}` (TagResource)* `DELETE /tags/{resourceArn}?tagKeys=...` (UntagResource)* `GET /tags/{resourceArn}` (ListTagsForResource)-The tagging operations work on application ARNs, as `CreateApplication` returns them-(`arn:aws:emr-serverless:<region>:<account>:/applications/<id>`). They share the tags `CreateApplication`-stores, so `GetApplication` always reports the current set. Following the EMR Serverless user guide, a-resource holds at most 50 tags, and `aws:` in any case is reserved as the prefix of a tag key or value.+The tagging operations work on application and job run ARNs, as `CreateApplication` and `StartJobRun`+return them (`arn:aws:emr-serverless:<region>:<account>:/applications/<id>` and+`.../applications/<id>/jobruns/<id>`). They share the tags those calls store, so `GetApplication` and+`GetJobRun` always report the current set. Following the EMR Serverless user guide, a resource holds+at most 50 tags, and `aws:` in any case is reserved as the prefix of a tag key or value. A session ARN+(`.../sessions/<id>`) answers `ResourceNotFoundException`, as sessions are not emulated. Once an+application is deleted, its ARN and its runs' ARNs answer `ResourceNotFoundException` too, while+`GetApplication` and `GetJobRun` still report them.-Applications, creation client tokens, and application tags are scoped to the request's AWS Region.-An application created in one Region is not returned by list, get, or tag requests signed for another.+Applications, job runs, creation client tokens, and tags are scoped to the request's AWS Region. An+application or run created in one Region is not returned by list, get, or tag requests signed for+another.++`ListApplications` refuses a `states` filter naming a state that does not exist, or more than 7, and a+`maxResults` outside 1 to 50, with the `ValidationException` AWS's validation layer answers.`DeleteApplication` leaves the application in `TERMINATED` state. `GetApplication` continues toreport that state so lifecycle waiters can observe it, while an unfiltered `ListApplications`@@ -58,22 +67,29 @@ response shape, and settles when it is read.* **Idempotency**: `clientToken` is the idempotency token. Repeating a request with the same tokenreturns the original run; reusing the token with a different request answers `ConflictException`(HTTP 409) "Request did not match the original".+* **Execution timeout**: `executionTimeoutMinutes` is 0 to 1,000,000, 0 meaning no timeout. From+ release `emr-7.11.0`, a `BATCH` run's timeout must be 1 to 10080 minutes (7 days), as the model and+ the 7.11.0 release notes say; a run outside that is refused with `ValidationException`.* **CancelJobRun** moves a run that has not finished to `CANCELLED`. A finished run answers`ValidationException` "Job run is not in a cancellable state".* **ListJobRuns** returns the newest runs first, filtered by `states`, `createdAtAfter`,`createdAtBefore` and `mode`, at most 50 per page. With several `states`, the runs are grouped by- state in the order the filter names them. An unknown application has no runs. A `nextToken`+ state in the order the filter names them. A state or mode that does not exist, or more than 8+ states, is refused with `ValidationException`. An unknown application has no runs. A `nextToken`resumes only the listing it came from, with the same application and filters.+* **Attempts**: nothing runs, so no run fails and none is retried: every run is a single attempt,+ number 1. `GetJobRun`, `ListJobRuns` and `ListJobRunAttempts` report it, with the run's own+ creation and update times. `GetJobRun` with `attempt` 1, or without one, returns the run; an+ `attempt` of 2 or more answers `ResourceNotFoundException`, and one below 1 or not an integer+ `ValidationException`.* `createdBy` is the caller's ARN, as `GetCallerIdentity` reports it. `releaseLabel` and`networkConfiguration` come from the application.## Limitations and Differences from AWS* **Jobs do not execute**: `StartJobRun` records a job run and moves it through its states, but no Spark or Hive job runs.-* **Unsourced errors**: AWS documents neither the error for a run submitted to an application whose auto-start is off nor the one for a run on a terminated application; Floci answers both with `ValidationException`. A run needs a `jobDriver`, as every `JobRun` carries one, and `jobDriver` is a union, so a run without a driver, or with one that sets both `sparkSubmit` and `hive` or neither, is refused with `ValidationException` too, in Floci's own wording.+* **Unsourced errors**: AWS documents neither the error for a run submitted to an application whose auto-start is off nor the one for a run on a terminated application; Floci answers both with `ValidationException`. The same holds for a `BATCH` timeout outside 1 to 10080 minutes on `emr-7.11.0` or later (`ValidationException`) and for `GetJobRun` naming an attempt a run never made (`ResourceNotFoundException`), both in Floci's own wording. A run needs a `jobDriver`, as every `JobRun` carries one, and `jobDriver` is a union, so a run without a driver, or with one that sets both `sparkSubmit` and `hive` or neither, is refused with `ValidationException` too, in Floci's own wording.* **StopApplication with running jobs**: AWS requires running jobs to finish or be cancelled before an application stops. Floci does not enforce it; with the default duration of `0` no `BATCH` run is ever running.-* **Not emulated**: job run attempts and retries, queuing (`QUEUED`), `totalResourceUtilization` and `billedResourceUtilization`, and `shutdownGracePeriodInSeconds` (accepted and ignored).+* **Not emulated**: retries (`retryPolicy` is stored and returned, but no run fails, so none is retried), queuing (`QUEUED`), `totalResourceUtilization` and `billedResourceUtilization`, and `shutdownGracePeriodInSeconds` (accepted and ignored).* **Instant Start/Stop**: Floci marks the application `STARTED` or `STOPPED` immediately without provisioning actual compute capacity in the background.* **Data Plane**: Floci does not implement the data-plane API or any execution environments.-* **Job run tags**: the tagging APIs do not accept job run ARNs (`.../jobruns/<id>`) yet; they answer- `ResourceNotFoundException`. Tags given to `StartJobRun` are stored and returned by `GetJobRun`.diff --git a/docs/services/index.md b/docs/services/index.mdindex c5c2d8db28..a5df0a6964 100644--- a/docs/services/index.md+++ b/docs/services/index.md@@ -80,7 +80,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac| [Redshift Data API](redshift-data.md) | `POST /` + `X-Amz-Target: RedshiftData.*` | JSON 1.1 | 11 || [Redshift Serverless](redshift-serverless.md) | `POST /` + `X-Amz-Target: RedshiftServerless.*` | JSON 1.1 | 55 || [EMR](emr.md) | `POST /` + `X-Amz-Target: ElasticMapReduce.*` | JSON 1.1 | 40 |-| [EMR Serverless](emr-serverless.md) | `/applications/*`, `/tags/*` | REST JSON | 14 |+| [EMR Serverless](emr-serverless.md) | `/applications/*`, `/tags/*` | REST JSON | 15 || [Data Firehose](firehose.md) | `POST /` + `X-Amz-Target: Firehose_20150804.*` | JSON 1.1 | 12 || [ECS](ecs.md) | `POST /` + `X-Amz-Target: AmazonEC2ContainerServiceV20141113.*` | JSON 1.1 | 59 || [EFS](efs.md) | `/2015-02-01/...` | REST JSON | 26 |diff --git a/src/main/java/io/github/hectorvent/floci/core/common/SmithyValidationErrors.java b/src/main/java/io/github/hectorvent/floci/core/common/SmithyValidationErrors.javaindex 7e1a2913ff..bb33c6cf82 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/SmithyValidationErrors.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/SmithyValidationErrors.java@@ -66,4 +66,32 @@ public final class SmithyValidationErrors {public static AwsException badRequest(String violation) {return badRequest(List.of(violation));}++ /** The violation for a number outside {@code min} to {@code max}, or {@code null} for one inside. */+ public static String outOfRange(String path, long value, long min, long max) {+ if (value < min) {+ return "Value '" + value + "' at '" + path + "' failed to satisfy constraint: "+ + "Member must have value greater than or equal to " + min;+ }+ if (value > max) {+ return "Value '" + value + "' at '" + path + "' failed to satisfy constraint: "+ + "Member must have value less than or equal to " + max;+ }+ return null;+ }++ /** The violation for a value outside its enum, listing the allowed values in the model's order. */+ public static String notInEnum(String value, String path, List<String> allowed) {+ return "Value '" + value + "' at '" + path + "' failed to satisfy constraint: "+ + "Member must satisfy enum value set: " + allowed;+ }++ /**+ * The violation for a list holding a value outside its members' enum. The list is checked as a+ * whole, so the whole list is the offending value and the enum constraint is nested.+ */+ public static String notAllInEnum(List<String> values, String path, List<String> allowed) {+ return "Value '" + values + "' at '" + path + "' failed to satisfy constraint: "+ + "Member must satisfy constraint: [Member must satisfy enum value set: " + allowed + "]";+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.javaindex 063cf0e8e4..1759cbf5d6 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessController.java@@ -11,9 +11,11 @@ import io.github.hectorvent.floci.services.emrserverless.model.DeleteApplicationimport io.github.hectorvent.floci.services.emrserverless.model.GetApplicationResponse;import io.github.hectorvent.floci.services.emrserverless.model.GetJobRunResponse;import io.github.hectorvent.floci.services.emrserverless.model.JobRun;+import io.github.hectorvent.floci.services.emrserverless.model.JobRunAttemptSummary;import io.github.hectorvent.floci.services.emrserverless.model.JobRunSummary;import io.github.hectorvent.floci.services.emrserverless.model.ListApplicationsRequest;import io.github.hectorvent.floci.services.emrserverless.model.ListApplicationsResponse;+import io.github.hectorvent.floci.services.emrserverless.model.ListJobRunAttemptsResponse;import io.github.hectorvent.floci.services.emrserverless.model.ListJobRunsResponse;import io.github.hectorvent.floci.services.emrserverless.model.StartApplicationResponse;import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunRequest;@@ -160,9 +162,25 @@ public class EmrServerlessController {@GET@Path("/{applicationId}/jobruns/{jobRunId}")public GetJobRunResponse getJobRun(@PathParam("applicationId") String applicationId,- @PathParam("jobRunId") String jobRunId) {+ @PathParam("jobRunId") String jobRunId,+ @QueryParam("attempt") String attempt) {GetJobRunResponse response = new GetJobRunResponse();- response.setJobRun(jobRuns.getJobRun(applicationId, jobRunId));+ response.setJobRun(jobRuns.getJobRun(applicationId, jobRunId,+ EmrServerlessJobRunService.parseAttempt(attempt)));+ return response;+ }++ @GET+ @Path("/{applicationId}/jobruns/{jobRunId}/attempts")+ public ListJobRunAttemptsResponse listJobRunAttempts(@PathParam("applicationId") String applicationId,+ @PathParam("jobRunId") String jobRunId,+ @QueryParam("maxResults") String maxResults,+ @QueryParam("nextToken") String nextToken) {+ PaginatedResult<JobRunAttemptSummary> page = jobRuns.listJobRunAttempts(applicationId, jobRunId,+ Pagination.parseMaxResults(maxResults, "ValidationException"), nextToken);+ ListJobRunAttemptsResponse response = new ListJobRunAttemptsResponse();+ response.setJobRunAttempts(page.items());+ response.setNextToken(page.nextToken());return response;}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.javaindex 15cc34739a..5d1407bfc3 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java@@ -18,6 +18,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.emrserverless.model.Application;import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;import io.github.hectorvent.floci.services.emrserverless.model.JobRun;+import io.github.hectorvent.floci.services.emrserverless.model.JobRunAttemptSummary;import io.github.hectorvent.floci.services.emrserverless.model.JobRunRecord;import io.github.hectorvent.floci.services.emrserverless.model.JobRunSummary;import io.github.hectorvent.floci.services.emrserverless.model.StartJobRunRequest;@@ -38,6 +39,7 @@ import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.Set;+import java.util.regex.Matcher;import java.util.regex.Pattern;/**@@ -62,11 +64,21 @@ public class EmrServerlessJobRunService {private static final String MODE_STREAMING = "STREAMING";private static final Set<String> TERMINAL_STATES = Set.of(STATE_SUCCESS, "FAILED", STATE_CANCELLED);+ private static final List<String> STATES = List.of("SUBMITTED", "PENDING", "SCHEDULED", STATE_RUNNING,+ STATE_SUCCESS, "FAILED", "CANCELLING", STATE_CANCELLED, "QUEUED");+ // JobRunStateSet holds at most 8 states.+ private static final int MAX_FILTER_STATES = 8;private static final List<String> MODES = List.of(MODE_BATCH, MODE_STREAMING);// JobRun.executionTimeoutMinutes: "If you didn't specify a timeout, this value defaults to 720 minutes."private static final long DEFAULT_EXECUTION_TIMEOUT_MINUTES = 720;// The model's Duration, which bounds executionTimeoutMinutes.private static final long MAX_EXECUTION_TIMEOUT_MINUTES = 1_000_000;+ // The EMR Serverless 7.11.0 release notes: for BATCH job runs the maximum "is 7 days from this release+ // onwards", and the timeout "can no longer be set to 0".+ private static final long MAX_BATCH_EXECUTION_TIMEOUT_MINUTES = 10_080;+ private static final Pattern RELEASE_LABEL = Pattern.compile("emr-(\\d{1,6})\\.(\\d{1,6})(?:\\..*)?");+ // Nothing runs, so no run fails and none is retried: every run is its own one attempt.+ private static final int ATTEMPT = 1;private static final int MAX_LIST_RESULTS = 50;// IAMRoleArn's pattern, as the model and AWS's validation message spell it.private static final String ROLE_ARN_PATTERN =@@ -130,15 +142,54 @@ public class EmrServerlessJobRunService {if (body.getTags() != null) {EmrServerlessService.validateTags(body.getTags());}- JobRunRecord record = applications.admitJobRun(applicationId, app -> newRun(app, body, original));+ JobRunRecord record = applications.admitJobRun(applicationId, app -> checkBatchTimeout(app, body),+ app -> newRun(app, body, original));LOG.infov("Started EMR Serverless job run {0} on application {1}", record.getJobRun().getJobRunId(),applicationId);return settle(record).getJobRun();}- public synchronized JobRun getJobRun(String applicationId, String jobRunId) {+ /**+ * The run, as its {@code attempt} saw it; without one, "the latest attempt". A run has one+ * attempt, so any other number names nothing. No source gives AWS's answer for an attempt a run+ * never made; ResourceNotFoundException, in Floci's words, is the modeled error that fits.+ */+ public synchronized JobRun getJobRun(String applicationId, String jobRunId, Integer attempt) {+ if (attempt != null) {+ String violation = SmithyValidationErrors.outOfRange("attempt", attempt, ATTEMPT, Integer.MAX_VALUE);+ if (violation != null) {+ throw SmithyValidationErrors.validationException(violation);+ }+ }+ applications.getApplication(applicationId);+ JobRunRecord record = findRun(applicationId, jobRunId);+ if (attempt != null && attempt != ATTEMPT) {+ throw new AwsException("ResourceNotFoundException",+ "Job run " + jobRunId + " has no attempt " + attempt, 404);+ }+ return settle(record).getJobRun();+ }++ /** A run's attempts, which are its one attempt. */+ public synchronized PaginatedResult<JobRunAttemptSummary> listJobRunAttempts(String applicationId,+ String jobRunId,+ Integer maxResults,+ String nextToken) {+ if (maxResults != null) {+ String violation = SmithyValidationErrors.outOfRange("maxResults", maxResults, 1, MAX_LIST_RESULTS);+ if (violation != null) {+ throw SmithyValidationErrors.validationException(violation);+ }+ }applications.getApplication(applicationId);- return settle(findRun(applicationId, jobRunId)).getJobRun();+ JobRunRecord record = settle(findRun(applicationId, jobRunId));+ PaginatedResult<JobRunRecord> page = Pagination.paginate(List.of(record),+ only -> String.format("%010d", ATTEMPT), maxResults != null ? maxResults : MAX_LIST_RESULTS,+ nextToken, String.join("|", "ListJobRunAttempts", applicationId, jobRunId),+ token -> new AwsException("ValidationException", "Invalid nextToken.", 400));+ return new PaginatedResult<>(+ page.items().stream().map(EmrServerlessJobRunService::toAttemptSummary).toList(),+ page.nextToken());}/**@@ -151,12 +202,20 @@ public class EmrServerlessJobRunService {public synchronized PaginatedResult<JobRunSummary> listJobRuns(String applicationId, List<String> states,Instant createdAtAfter, Instant createdAtBefore,String mode, Integer maxResults, String nextToken) {+ List<String> violations = new ArrayList<>();if (maxResults != null) {- String violation = outOfRange("maxResults", maxResults, 1, MAX_LIST_RESULTS);+ String violation = SmithyValidationErrors.outOfRange("maxResults", maxResults, 1, MAX_LIST_RESULTS);if (violation != null) {- throw SmithyValidationErrors.validationException(violation);+ violations.add(violation);}}+ violations.addAll(EmrServerlessService.statesViolations(states, STATES, MAX_FILTER_STATES));+ if (mode != null && !MODES.contains(mode)) {+ violations.add(SmithyValidationErrors.notInEnum(mode, "mode", MODES));+ }+ if (!violations.isEmpty()) {+ throw SmithyValidationErrors.validationException(violations);+ }List<JobRunRecord> matching = new ArrayList<>();Map<String, String> cursors = new HashMap<>();for (JobRunRecord record : store.scan(key -> true)) {@@ -199,6 +258,35 @@ public class EmrServerlessJobRunService {return record.getJobRun();}+ public synchronized Map<String, String> listTags(EmrServerlessService.ResourceArn resource) {+ JobRun run = runFor(resource).getJobRun();+ return run.getTags() != null ? new LinkedHashMap<>(run.getTags()) : new LinkedHashMap<>();+ }++ public synchronized void tagResource(EmrServerlessService.ResourceArn resource, Map<String, String> tags) {+ JobRunRecord record = runFor(resource);+ record.getJobRun().setTags(EmrServerlessService.withTags(record.getJobRun().getTags(), tags));+ store.put(record.getJobRun().getJobRunId(), record);+ }++ public synchronized void untagResource(EmrServerlessService.ResourceArn resource, List<String> tagKeys) {+ JobRunRecord record = runFor(resource);+ record.getJobRun().setTags(EmrServerlessService.withoutTags(record.getJobRun().getTags(), tagKeys));+ store.put(record.getJobRun().getJobRunId(), record);+ }++ /** GetJobRun's {@code attempt}, an integer in the query string. */+ static Integer parseAttempt(String value) {+ if (value == null || value.isBlank()) {+ return null;+ }+ try {+ return Integer.parseInt(value);+ } catch (NumberFormatException e) {+ throw new AwsException("ValidationException", "attempt must be an integer.", 400);+ }+ }+/*** A ListJobRuns time filter. Query-string timestamps are ISO 8601 in restJson1; epoch seconds* are accepted too.@@ -245,6 +333,9 @@ public class EmrServerlessJobRunService {? request.getExecutionTimeoutMinutes() : DEFAULT_EXECUTION_TIMEOUT_MINUTES);run.setMode(request.getMode());run.setRetryPolicy(request.getRetryPolicy());+ run.setAttempt(ATTEMPT);+ run.setAttemptCreatedAt(now);+ run.setAttemptUpdatedAt(now);run.setStartedAt(now);JobRunRecord record = new JobRunRecord();@@ -287,6 +378,7 @@ public class EmrServerlessJobRunService {run.setState(state);run.setEndedAt(endedAt);run.setUpdatedAt(endedAt);+ run.setAttemptUpdatedAt(endedAt);run.setTotalExecutionDurationSeconds((int) Duration.between(run.getStartedAt(), endedAt).toSeconds());store.put(run.getJobRunId(), record);}@@ -298,6 +390,16 @@ public class EmrServerlessJobRunService {"Job run " + jobRunId + " not found", 404));}+ // The run a tagging ARN names. The ARN must be the run's own, so one naming another account names nothing,+ // and a deleted application's runs name nothing, as the application's own ARN does.+ private JobRunRecord runFor(EmrServerlessService.ResourceArn resource) {+ return store.get(resource.jobRunId())+ .filter(record -> resource.applicationId().equals(record.getJobRun().getApplicationId()))+ .filter(record -> resource.arn().equals(record.getJobRun().getArn()))+ .filter(record -> applications.isLive(resource.applicationId()))+ .orElseThrow(() -> EmrServerlessService.resourceNotFound(resource.arn()));+ }+private JobRunRecord runWithClientToken(String clientToken) {for (JobRunRecord record : store.scan(key -> true)) {if (clientToken.equals(record.getClientToken()) && inRequestRegion(record.getJobRun())) {@@ -351,12 +453,11 @@ public class EmrServerlessJobRunService {+ "Member must satisfy regular expression pattern: " + ROLE_ARN_PATTERN);}if (request.getMode() != null && !MODES.contains(request.getMode())) {- violations.add("Value '" + request.getMode() + "' at 'mode' failed to satisfy constraint: "- + "Member must satisfy enum value set: " + MODES);+ violations.add(SmithyValidationErrors.notInEnum(request.getMode(), "mode", MODES));}if (request.getExecutionTimeoutMinutes() != null) {- String violation = outOfRange("executionTimeoutMinutes", request.getExecutionTimeoutMinutes(), 0,- MAX_EXECUTION_TIMEOUT_MINUTES);+ String violation = SmithyValidationErrors.outOfRange("executionTimeoutMinutes",+ request.getExecutionTimeoutMinutes(), 0, MAX_EXECUTION_TIMEOUT_MINUTES);if (violation != null) {violations.add(violation);}@@ -383,16 +484,31 @@ public class EmrServerlessJobRunService {return "Value null at '" + member + "' failed to satisfy constraint: Member must not be null";}- private static String outOfRange(String member, long value, long min, long max) {- if (value < min) {- return "Value '" + value + "' at '" + member + "' failed to satisfy constraint: "- + "Member must have value greater than or equal to " + min;+ /**+ * From Amazon EMR release 7.11.0 a BATCH run's timeout is 1 to 10080 minutes; an earlier release, or+ * a STREAMING run, keeps the model's 0 to 1,000,000, 0 meaning none. No source gives AWS's error, so+ * the wording is Floci's.+ */+ private static void checkBatchTimeout(Application app, StartJobRunRequest request) {+ Long minutes = request.getExecutionTimeoutMinutes();+ if (minutes == null || MODE_STREAMING.equals(request.getMode())+ || !boundsBatchTimeouts(app.getReleaseLabel())) {+ return;}- if (value > max) {- return "Value '" + value + "' at '" + member + "' failed to satisfy constraint: "- + "Member must have value less than or equal to " + max;+ if (minutes == 0 || minutes > MAX_BATCH_EXECUTION_TIMEOUT_MINUTES) {+ throw new AwsException("ValidationException", "executionTimeoutMinutes must be between 1 and "+ + MAX_BATCH_EXECUTION_TIMEOUT_MINUTES + " for a BATCH job run on release emr-7.11.0 or later.",+ 400);}- return null;+ }++ private static boolean boundsBatchTimeouts(String releaseLabel) {+ Matcher matcher = RELEASE_LABEL.matcher(releaseLabel != null ? releaseLabel : "");+ if (!matcher.matches()) {+ return false;+ }+ int major = Integer.parseInt(matcher.group(1));+ return major > 7 || (major == 7 && Integer.parseInt(matcher.group(2)) >= 11);}private static JobRunSummary toSummary(JobRunRecord record) {@@ -411,6 +527,30 @@ public class EmrServerlessJobRunService {summary.setStateDetails(run.getStateDetails());summary.setReleaseLabel(run.getReleaseLabel());summary.setType(record.getApplicationType());+ summary.setAttempt(run.getAttempt());+ summary.setAttemptCreatedAt(run.getAttemptCreatedAt());+ summary.setAttemptUpdatedAt(run.getAttemptUpdatedAt());+ return summary;+ }++ private static JobRunAttemptSummary toAttemptSummary(JobRunRecord record) {+ JobRun run = record.getJobRun();+ JobRunAttemptSummary summary = new JobRunAttemptSummary();+ summary.setApplicationId(run.getApplicationId());+ summary.setId(run.getJobRunId());+ summary.setName(run.getName());+ summary.setMode(run.getMode());+ summary.setArn(run.getArn());+ summary.setCreatedBy(run.getCreatedBy());+ summary.setJobCreatedAt(run.getCreatedAt());+ summary.setCreatedAt(run.getCreatedAt());+ summary.setUpdatedAt(run.getUpdatedAt());+ summary.setExecutionRole(run.getExecutionRole());+ summary.setState(run.getState());+ summary.setStateDetails(run.getStateDetails());+ summary.setReleaseLabel(run.getReleaseLabel());+ summary.setType(record.getApplicationType());+ summary.setAttempt(ATTEMPT);return summary;}}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.javaindex 4d34a99467..8196392734 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessService.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.PaginatedResult;import io.github.hectorvent.floci.core.common.Pagination;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.RequestContext;+import io.github.hectorvent.floci.core.common.SmithyValidationErrors;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.emrserverless.model.Application;@@ -21,11 +22,13 @@ import jakarta.inject.Inject;import java.time.Clock;import java.time.Instant;import java.time.temporal.ChronoUnit;+import java.util.ArrayList;import java.util.Date;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.UUID;+import java.util.function.Consumer;import java.util.function.Function;import java.util.regex.Matcher;import java.util.regex.Pattern;@@ -40,7 +43,17 @@ public class EmrServerlessService {private static final int MAX_TAGS = 50;private static final String RESERVED_PREFIX = "aws:";private static final Pattern TAG_TEXT = Pattern.compile("[A-Za-z0-9 /_.:=+@-]*");- private static final Pattern RESOURCE = Pattern.compile("/applications/([0-9a-zA-Z]+)(?:/jobruns/([0-9a-zA-Z]+))?");+ // ResourceArn's resource: an application, or one of its job runs or sessions.+ private static final Pattern RESOURCE =+ Pattern.compile("/applications/([0-9a-zA-Z]+)(?:/(jobruns|sessions)/([0-9a-zA-Z]+))?");+ private static final List<String> APPLICATION_STATES =+ List.of("CREATING", "CREATED", "STARTING", "STARTED", "STOPPING", "STOPPED", "TERMINATED");+ // ApplicationStateSet holds at most 7 states; a ListApplications page holds 1 to 50 applications.+ private static final int MAX_APPLICATION_STATES = 7;+ private static final int MAX_LIST_RESULTS = 50;++ /** What an EMR Serverless resource ARN names: an application, or one of its job runs. */+ public record ResourceArn(String arn, String applicationId, String jobRunId) {}private final EmulatorConfig config;private final AccountAwareStorageBackend<Application> storage;@@ -119,6 +132,18 @@ public class EmrServerlessService {}public PaginatedResult<ApplicationSummary> listApplications(ListApplicationsRequest request) {+ List<String> violations = new ArrayList<>();+ if (request.getMaxResults() != null) {+ String violation = SmithyValidationErrors.outOfRange("maxResults", request.getMaxResults(), 1,+ MAX_LIST_RESULTS);+ if (violation != null) {+ violations.add(violation);+ }+ }+ violations.addAll(statesViolations(request.getStates(), APPLICATION_STATES, MAX_APPLICATION_STATES));+ if (!violations.isEmpty()) {+ throw SmithyValidationErrors.validationException(violations);+ }String region = regionResolver.getRegion();List<Application> all = storage.scan(k -> true).stream().filter(app -> inRegion(app, region)).collect(Collectors.toList());@@ -128,7 +153,7 @@ public class EmrServerlessService {all = all.stream().filter(app -> !"TERMINATED".equals(app.getState())).collect(Collectors.toList());}- PaginatedResult<Application> page = Pagination.paginate(all, Application::getApplicationId, request.getMaxResults(), request.getNextToken(), 50, "ValidationException");+ PaginatedResult<Application> page = Pagination.paginate(all, Application::getApplicationId, request.getMaxResults(), request.getNextToken(), MAX_LIST_RESULTS, "ValidationException");return new PaginatedResult<>(page.items().stream().map(this::toSummary).collect(Collectors.toList()),@@ -241,15 +266,18 @@ public class EmrServerlessService {* Admits a job run to the application and returns what {@code admit} builds from it. An* application "only accepts jobs when it's in" STARTED, and auto-start, on unless the* application turns it off, starts a CREATED or STOPPED one "on job submission" (EMR Serverless- * user guide, "Application states" and "Default application behavior"). This service's lock is- * held throughout, so the application cannot be stopped or deleted while the run is admitted.+ * user guide, "Application states" and "Default application behavior"). {@code check} refuses a+ * run against the application before auto-start can change it. This service's lock is held+ * throughout, so the application cannot be updated, stopped or deleted while the run is admitted.*/- public synchronized <T> T admitJobRun(String applicationId, Function<Application, T> admit) {+ public synchronized <T> T admitJobRun(String applicationId, Consumer<Application> check,+ Function<Application, T> admit) {Application app = getApplication(applicationId);String state = app.getState();if ("TERMINATED".equals(state)) {throw new AwsException("ValidationException", "Application is terminated.", 400);}+ check.accept(app);if (!"STARTED".equals(state)) {if (!autoStartEnabled(app)) {throw new AwsException("ValidationException", "Application " + applicationId@@ -269,52 +297,33 @@ public class EmrServerlessService {// ---- Tags ---------------------------------------------------------------------------------- public synchronized Map<String, String> listTags(String arn) {- Application app = applicationFor(arn);+ public synchronized Map<String, String> listTags(ResourceArn resource) {+ Application app = applicationFor(resource);return app.getTags() != null ? new LinkedHashMap<>(app.getTags()) : new LinkedHashMap<>();}- /** Adds the tags to the application's, replacing any with the same key. */- public synchronized void tagResource(String arn, Map<String, String> tags) {- Application app = applicationFor(arn);- // An empty map is a valid TagMap (min 0) and changes nothing; the dispatcher already refuses a- // missing one.- if (tags == null) {- throw new AwsException("ValidationException", "tags is required", 400);- }- validateTags(tags);- Map<String, String> merged = app.getTags() != null ? new LinkedHashMap<>(app.getTags()) : new LinkedHashMap<>();- merged.putAll(tags);- if (merged.size() > MAX_TAGS) {- throw new AwsException("ValidationException",- "A resource can have at most " + MAX_TAGS + " tags.", 400);- }- app.setTags(merged);+ public synchronized void tagResource(ResourceArn resource, Map<String, String> tags) {+ Application app = applicationFor(resource);+ app.setTags(withTags(app.getTags(), tags));storage.put(app.getApplicationId(), app);}- public synchronized void untagResource(String arn, List<String> tagKeys) {- Application app = applicationFor(arn);- if (tagKeys == null || tagKeys.isEmpty()) {- throw new AwsException("ValidationException", "tagKeys is required", 400);- }- tagKeys.forEach(EmrServerlessService::validateTagKey);+ public synchronized void untagResource(ResourceArn resource, List<String> tagKeys) {+ Application app = applicationFor(resource);+ Map<String, String> remaining = withoutTags(app.getTags(), tagKeys);if (app.getTags() == null || app.getTags().isEmpty()) {return;}- Map<String, String> remaining = new LinkedHashMap<>(app.getTags());- tagKeys.forEach(remaining::remove);app.setTags(remaining);storage.put(app.getApplicationId(), app);}/**- * The application an EMR Serverless ARN names:- * {@code arn:<partition>:emr-serverless:<region>:<account>:/applications/<id>}, the form- * CreateApplication returns. A job run ARN ({@code .../jobruns/<id>}) is refused, as job run tags- * are not emulated yet.+ * Parses a tagging ARN: {@code arn:<partition>:emr-serverless:<region>:<account>:/applications/<id>},+ * the form CreateApplication returns, with {@code /jobruns/<id>} for a job run. An ARN of another+ * Region names nothing here, and neither does a session's, as sessions are not emulated.*/- private Application applicationFor(String arn) {+ public ResourceArn resourceArn(String arn) {AwsArnUtils.Arn parsed;try {parsed = AwsArnUtils.parse(arn);@@ -325,16 +334,75 @@ public class EmrServerlessService {if (!"emr-serverless".equals(parsed.service()) || !matcher.matches()) {throw new AwsException("ValidationException", "Invalid resourceArn: " + arn, 400);}- if (!parsed.region().equals(regionResolver.getRegion())) {- throw new AwsException("ResourceNotFoundException", "Resource " + arn + " not found", 404);- }- if (matcher.group(2) != null) {- throw new AwsException("ResourceNotFoundException", "Job run " + matcher.group(2) + " not found", 404);+ if (!parsed.region().equals(regionResolver.getRegion()) || "sessions".equals(matcher.group(2))) {+ throw resourceNotFound(arn);}- return storage.get(matcher.group(1))- .filter(app -> arn.equals(app.getArn()))+ return new ResourceArn(arn, matcher.group(1), matcher.group(3));+ }++ /** Whether the application exists and has not been deleted, which leaves it TERMINATED. */+ public synchronized boolean isLive(String applicationId) {+ return storage.get(applicationId).filter(app -> !"TERMINATED".equals(app.getState())).isPresent();+ }++ static AwsException resourceNotFound(String arn) {+ return new AwsException("ResourceNotFoundException", "Resource " + arn + " not found", 404);+ }++ private Application applicationFor(ResourceArn resource) {+ return storage.get(resource.applicationId())+ .filter(app -> resource.arn().equals(app.getArn())).filter(app -> !"TERMINATED".equals(app.getState()))- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Resource " + arn + " not found", 404));+ .orElseThrow(() -> resourceNotFound(resource.arn()));+ }++ /**+ * A resource's tags once TagResource adds {@code added}, which replace any with the same key. An+ * empty map is a valid TagMap (min 0) and changes nothing; the dispatcher already refuses a missing+ * one.+ */+ static Map<String, String> withTags(Map<String, String> current, Map<String, String> added) {+ if (added == null) {+ throw new AwsException("ValidationException", "tags is required", 400);+ }+ validateTags(added);+ Map<String, String> merged = current != null ? new LinkedHashMap<>(current) : new LinkedHashMap<>();+ merged.putAll(added);+ if (merged.size() > MAX_TAGS) {+ throw new AwsException("ValidationException",+ "A resource can have at most " + MAX_TAGS + " tags.", 400);+ }+ return merged;+ }++ /** A resource's tags once UntagResource removes {@code tagKeys}. */+ static Map<String, String> withoutTags(Map<String, String> current, List<String> tagKeys) {+ if (tagKeys == null || tagKeys.isEmpty()) {+ throw new AwsException("ValidationException", "tagKeys is required", 400);+ }+ tagKeys.forEach(EmrServerlessService::validateTagKey);+ Map<String, String> remaining = current != null ? new LinkedHashMap<>(current) : new LinkedHashMap<>();+ tagKeys.forEach(remaining::remove);+ return remaining;+ }++ /**+ * The violations of a {@code states} filter: at most {@code max} states, each one of+ * {@code allowed}. A typo would otherwise match nothing and list nothing.+ */+ static List<String> statesViolations(List<String> states, List<String> allowed, int max) {+ List<String> violations = new ArrayList<>();+ if (states == null) {+ return violations;+ }+ if (states.size() > max) {+ violations.add("Value '" + states + "' at 'states' failed to satisfy constraint: "+ + "Member must have length less than or equal to " + max);+ }+ if (!allowed.containsAll(states)) {+ violations.add(SmithyValidationErrors.notAllInEnum(states, "states", allowed));+ }+ return violations;}/** At most 50 tags, each key and value within TagKey and TagValue and free of the reserved prefix. */diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagHandler.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagHandler.javaindex 103805062a..b785678d3a 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagHandler.java@@ -8,18 +8,21 @@ import java.util.List;import java.util.Map;/**- * EMR Serverless on the shared {@code /tags/{resourceArn}} path. The wire shape is the common one (a- * lowercase {@code tags} map, a {@code tagKeys} query and {@code POST}), except that TagResource and- * UntagResource answer 200 rather than 204, and payloads are validated strictly.+ * EMR Serverless on the shared {@code /tags/{resourceArn}} path, for applications and job runs. The+ * wire shape is the common one (a lowercase {@code tags} map, a {@code tagKeys} query and+ * {@code POST}), except that TagResource and UntagResource answer 200 rather than 204, and payloads+ * are validated strictly.*/@ApplicationScopedpublic class EmrServerlessTagHandler implements TagHandler {private final EmrServerlessService service;+ private final EmrServerlessJobRunService jobRuns;@Inject- public EmrServerlessTagHandler(EmrServerlessService service) {+ public EmrServerlessTagHandler(EmrServerlessService service, EmrServerlessJobRunService jobRuns) {this.service = service;+ this.jobRuns = jobRuns;}@Override@@ -45,16 +48,27 @@ public class EmrServerlessTagHandler implements TagHandler {@Overridepublic Map<String, String> listTags(String region, String arn) {- return service.listTags(arn);+ EmrServerlessService.ResourceArn resource = service.resourceArn(arn);+ return resource.jobRunId() != null ? jobRuns.listTags(resource) : service.listTags(resource);}@Overridepublic void tagResource(String region, String arn, Map<String, String> tags) {- service.tagResource(arn, tags);+ EmrServerlessService.ResourceArn resource = service.resourceArn(arn);+ if (resource.jobRunId() != null) {+ jobRuns.tagResource(resource, tags);+ } else {+ service.tagResource(resource, tags);+ }}@Overridepublic void untagResource(String region, String arn, List<String> tagKeys) {- service.untagResource(arn, tagKeys);+ EmrServerlessService.ResourceArn resource = service.resourceArn(arn);+ if (resource.jobRunId() != null) {+ jobRuns.untagResource(resource, tagKeys);+ } else {+ service.untagResource(resource, tagKeys);+ }}}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.javaindex 44b3b65838..bb4be8e3d6 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRun.java@@ -33,6 +33,11 @@ public class JobRun {private Long executionTimeoutMinutes;private String mode;private Map<String, Object> retryPolicy;+ private Integer attempt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant attemptCreatedAt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant attemptUpdatedAt;@JsonFormat(shape = JsonFormat.Shape.NUMBER)private Instant startedAt;@JsonFormat(shape = JsonFormat.Shape.NUMBER)@@ -78,6 +83,12 @@ public class JobRun {public void setMode(String mode) { this.mode = mode; }public Map<String, Object> getRetryPolicy() { return retryPolicy; }public void setRetryPolicy(Map<String, Object> retryPolicy) { this.retryPolicy = retryPolicy; }+ public Integer getAttempt() { return attempt; }+ public void setAttempt(Integer attempt) { this.attempt = attempt; }+ public Instant getAttemptCreatedAt() { return attemptCreatedAt; }+ public void setAttemptCreatedAt(Instant attemptCreatedAt) { this.attemptCreatedAt = attemptCreatedAt; }+ public Instant getAttemptUpdatedAt() { return attemptUpdatedAt; }+ public void setAttemptUpdatedAt(Instant attemptUpdatedAt) { this.attemptUpdatedAt = attemptUpdatedAt; }public Instant getStartedAt() { return startedAt; }public void setStartedAt(Instant startedAt) { this.startedAt = startedAt; }public Instant getEndedAt() { return endedAt; }diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunAttemptSummary.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunAttemptSummary.javanew file mode 100644index 0000000000..907481ed88--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunAttemptSummary.java@@ -0,0 +1,65 @@+package io.github.hectorvent.floci.services.emrserverless.model;++import com.fasterxml.jackson.annotation.JsonFormat;+import com.fasterxml.jackson.annotation.JsonInclude;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;++/**+ * One entry of ListJobRunAttempts. {@code id} is the run's id, {@code jobCreatedAt} when the run was+ * created, and {@code createdAt} and {@code updatedAt} the attempt's own times.+ */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class JobRunAttemptSummary {+ private String applicationId;+ private String id;+ private String name;+ private String mode;+ private String arn;+ private String createdBy;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant jobCreatedAt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant createdAt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant updatedAt;+ private String executionRole;+ private String state;+ private String stateDetails;+ private String releaseLabel;+ private String type;+ private Integer attempt;++ public String getApplicationId() { return applicationId; }+ public void setApplicationId(String applicationId) { this.applicationId = applicationId; }+ public String getId() { return id; }+ public void setId(String id) { this.id = id; }+ public String getName() { return name; }+ public void setName(String name) { this.name = name; }+ public String getMode() { return mode; }+ public void setMode(String mode) { this.mode = mode; }+ public String getArn() { return arn; }+ public void setArn(String arn) { this.arn = arn; }+ public String getCreatedBy() { return createdBy; }+ public void setCreatedBy(String createdBy) { this.createdBy = createdBy; }+ public Instant getJobCreatedAt() { return jobCreatedAt; }+ public void setJobCreatedAt(Instant jobCreatedAt) { this.jobCreatedAt = jobCreatedAt; }+ public Instant getCreatedAt() { return createdAt; }+ public void setCreatedAt(Instant createdAt) { this.createdAt = createdAt; }+ public Instant getUpdatedAt() { return updatedAt; }+ public void setUpdatedAt(Instant updatedAt) { this.updatedAt = updatedAt; }+ public String getExecutionRole() { return executionRole; }+ public void setExecutionRole(String executionRole) { this.executionRole = executionRole; }+ public String getState() { return state; }+ public void setState(String state) { this.state = state; }+ public String getStateDetails() { return stateDetails; }+ public void setStateDetails(String stateDetails) { this.stateDetails = stateDetails; }+ public String getReleaseLabel() { return releaseLabel; }+ public void setReleaseLabel(String releaseLabel) { this.releaseLabel = releaseLabel; }+ public String getType() { return type; }+ public void setType(String type) { this.type = type; }+ public Integer getAttempt() { return attempt; }+ public void setAttempt(Integer attempt) { this.attempt = attempt; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.javaindex bf54a96730..b9a8fc9ab6 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/JobRunSummary.java@@ -25,6 +25,11 @@ public class JobRunSummary {private String stateDetails;private String releaseLabel;private String type;+ private Integer attempt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant attemptCreatedAt;+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant attemptUpdatedAt;public String getApplicationId() { return applicationId; }public void setApplicationId(String applicationId) { this.applicationId = applicationId; }@@ -52,4 +57,10 @@ public class JobRunSummary {public void setReleaseLabel(String releaseLabel) { this.releaseLabel = releaseLabel; }public String getType() { return type; }public void setType(String type) { this.type = type; }+ public Integer getAttempt() { return attempt; }+ public void setAttempt(Integer attempt) { this.attempt = attempt; }+ public Instant getAttemptCreatedAt() { return attemptCreatedAt; }+ public void setAttemptCreatedAt(Instant attemptCreatedAt) { this.attemptCreatedAt = attemptCreatedAt; }+ public Instant getAttemptUpdatedAt() { return attemptUpdatedAt; }+ public void setAttemptUpdatedAt(Instant attemptUpdatedAt) { this.attemptUpdatedAt = attemptUpdatedAt; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunAttemptsResponse.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunAttemptsResponse.javanew file mode 100644index 0000000000..919a98f27a--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/model/ListJobRunAttemptsResponse.java@@ -0,0 +1,18 @@+package io.github.hectorvent.floci.services.emrserverless.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.List;++@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class ListJobRunAttemptsResponse {+ private List<JobRunAttemptSummary> jobRunAttempts;+ private String nextToken;++ public List<JobRunAttemptSummary> getJobRunAttempts() { return jobRunAttempts; }+ public void setJobRunAttempts(List<JobRunAttemptSummary> jobRunAttempts) { this.jobRunAttempts = jobRunAttempts; }+ public String getNextToken() { return nextToken; }+ public void setNextToken(String nextToken) { this.nextToken = nextToken; }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.javaindex f2c2ebbf6d..0a2935ff72 100644--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessIntegrationTest.java@@ -17,6 +17,7 @@ import org.junit.jupiter.api.TestMethodOrder;import java.math.BigDecimal;import java.time.Duration;+import java.util.List;import java.util.UUID;import static org.hamcrest.Matchers.*;@@ -338,4 +339,34 @@ public class EmrServerlessIntegrationTest {assertEquals(0, createdAt.compareTo(updatedAt), "updatedAt " + updatedAt + " differs from createdAt " + createdAt);assertEquals(0, createdAt.compareTo(listed), "ListApplications reports createdAt " + listed);}++ @Test+ @Order(10)+ void listApplicationsRefusesStatesThatDoNotExistAndPagesOutsideOneToFifty() {+ givenReq()+ .queryParam("states", "RUNNING")+ .queryParam("maxResults", 51)+ .when()+ .get("/applications")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("2 validation errors detected: "+ + "Value '51' at 'maxResults' failed to satisfy constraint: "+ + "Member must have value less than or equal to 50; "+ + "Value '[RUNNING]' at 'states' failed to satisfy constraint: Member must satisfy constraint: "+ + "[Member must satisfy enum value set: "+ + "[CREATING, CREATED, STARTING, STARTED, STOPPING, STOPPED, TERMINATED]]"));++ givenReq()+ .queryParam("states", List.of("CREATING", "CREATED", "STARTING", "STARTED", "STOPPING", "STOPPED",+ "TERMINATED", "CREATED"))+ .when()+ .get("/applications")+ .then()+ .statusCode(400)+ .body("message", equalTo("1 validation error detected: "+ + "Value '[CREATING, CREATED, STARTING, STARTED, STOPPING, STOPPED, TERMINATED, CREATED]' "+ + "at 'states' failed to satisfy constraint: Member must have length less than or equal to 7"));+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.javaindex e5146d22d4..0df1c03e33 100644--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunIntegrationTest.java@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;import io.restassured.config.JsonConfig;import io.restassured.config.RestAssuredConfig;+import io.restassured.path.json.JsonPath;import io.restassured.path.json.config.JsonPathConfig;import io.restassured.response.Response;import io.restassured.specification.RequestSpecification;@@ -44,10 +45,14 @@ class EmrServerlessJobRunIntegrationTest {}private static String createApplication(String extra) {+ return createApplicationOn("emr-7.5.0", extra);+ }++ private static String createApplicationOn(String releaseLabel, String extra) {return givenReq().body("""- {"name": "jobs-%s", "releaseLabel": "emr-7.5.0", "type": "SPARK", "clientToken": "%s"%s}- """.formatted(UUID.randomUUID(), UUID.randomUUID(), extra))+ {"name": "jobs-%s", "releaseLabel": "%s", "type": "SPARK", "clientToken": "%s"%s}+ """.formatted(UUID.randomUUID(), releaseLabel, UUID.randomUUID(), extra)).when().post("/applications").then().statusCode(200).extract().path("applicationId");@@ -115,6 +120,9 @@ class EmrServerlessJobRunIntegrationTest {.body("jobRun.totalExecutionDurationSeconds", equalTo(0)).body("jobRun.startedAt", instanceOf(Number.class)).body("jobRun.endedAt", instanceOf(Number.class))+ .body("jobRun.attempt", equalTo(1))+ .body("jobRun.attemptCreatedAt", instanceOf(Number.class))+ .body("jobRun.attemptUpdatedAt", instanceOf(Number.class)).body("jobRun", not(hasKey("hive")));}@@ -369,6 +377,9 @@ class EmrServerlessJobRunIntegrationTest {.body("jobRuns[0].type", equalTo("SPARK")).body("jobRuns[0].releaseLabel", equalTo("emr-7.5.0")).body("jobRuns[0].createdBy", equalTo("arn:aws:iam::000000000000:root"))+ .body("jobRuns[0].attempt", equalTo(1))+ .body("jobRuns[0].attemptCreatedAt", instanceOf(Number.class))+ .body("jobRuns[0].attemptUpdatedAt", instanceOf(Number.class)).body("jobRuns[0]", not(hasKey("jobRunId"))).body("jobRuns[0]", not(hasKey("jobDriver")));}@@ -465,6 +476,29 @@ class EmrServerlessJobRunIntegrationTest {.body("jobRuns.id", contains(streaming, batch));}+ @Test+ void listJobRunsRefusesStatesAndModesThatDoNotExist() {+ String base = "/applications/" + createApplication("") + "/jobruns";++ givenReq().queryParam("states", List.of("RUNNING", "SUCCEEDED")).queryParam("mode", "batch")+ .when().get(base)+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("2 validation errors detected: "+ + "Value '[RUNNING, SUCCEEDED]' at 'states' failed to satisfy constraint: Member must "+ + "satisfy constraint: [Member must satisfy enum value set: [SUBMITTED, PENDING, SCHEDULED, "+ + "RUNNING, SUCCESS, FAILED, CANCELLING, CANCELLED, QUEUED]]; "+ + "Value 'batch' at 'mode' failed to satisfy constraint: Member must satisfy enum value set: "+ + "[BATCH, STREAMING]"));+ givenReq().queryParam("states", List.of("SUBMITTED", "PENDING", "SCHEDULED", "RUNNING", "SUCCESS", "FAILED",+ "CANCELLING", "CANCELLED", "QUEUED"))+ .when().get(base)+ .then().statusCode(400)+ .body("message", equalTo("1 validation error detected: Value '[SUBMITTED, PENDING, SCHEDULED, "+ + "RUNNING, SUCCESS, FAILED, CANCELLING, CANCELLED, QUEUED]' at 'states' failed to satisfy "+ + "constraint: Member must have length less than or equal to 8"));+ }+@Testvoid listJobRunsOfAnUnknownApplicationIsEmpty() {givenReq().when().get("/applications/00missing0000000/jobruns")@@ -472,6 +506,114 @@ class EmrServerlessJobRunIntegrationTest {.body("jobRuns", empty());}+ @Test+ void fromRelease711ABatchRunsTimeoutIsOneToTenThousandEightyMinutes() {+ String current = createApplicationOn("emr-7.11.0", "");+ for (String timeout : List.of(", \"executionTimeoutMinutes\": 0",+ ", \"executionTimeoutMinutes\": 10081", ", \"mode\": \"BATCH\", \"executionTimeoutMinutes\": 0")) {+ startJobRun(current, sparkRun(UUID.randomUUID().toString(), timeout))+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("executionTimeoutMinutes must be between 1 and 10080 for a BATCH job "+ + "run on release emr-7.11.0 or later."));+ }+ givenReq().when().get("/applications/" + current)+ .then().body("application.state", equalTo("CREATED"));++ startJobRun(current, sparkRun(UUID.randomUUID().toString(), ", \"executionTimeoutMinutes\": 10080"))+ .then().statusCode(200);+ startJobRun(current, sparkRun(UUID.randomUUID().toString(),+ ", \"mode\": \"STREAMING\", \"executionTimeoutMinutes\": 0"))+ .then().statusCode(200);+ String earlier = createApplicationOn("emr-7.10.0", "");+ for (String timeout : List.of(", \"executionTimeoutMinutes\": 0", ", \"executionTimeoutMinutes\": 1000000")) {+ startJobRun(earlier, sparkRun(UUID.randomUUID().toString(), timeout)).then().statusCode(200);+ }+ }++ @Test+ void aRunIsItsOneAttempt() {+ String applicationId = createApplication("");+ String path = "/applications/" + applicationId + "/jobruns/"+ + startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));+ JsonPath run = givenReq().config(DECIMAL_NUMBERS).when().get(path)+ .then().statusCode(200).extract().jsonPath();++ assertEquals(1, run.getInt("jobRun.attempt"));+ assertEquals(run.<BigDecimal>get("jobRun.createdAt"), run.<BigDecimal>get("jobRun.attemptCreatedAt"));+ assertEquals(run.<BigDecimal>get("jobRun.updatedAt"), run.<BigDecimal>get("jobRun.attemptUpdatedAt"));++ givenReq().queryParam("attempt", 1).when().get(path)+ .then().statusCode(200).body("jobRun.attempt", equalTo(1));+ givenReq().queryParam("attempt", 2).when().get(path)+ .then().statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"));+ givenReq().queryParam("attempt", 0).when().get(path)+ .then().statusCode(400)+ .body("message", equalTo("1 validation error detected: Value '0' at 'attempt' failed to satisfy "+ + "constraint: Member must have value greater than or equal to 1"));+ givenReq().queryParam("attempt", "first").when().get(path)+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"));+ // An empty value never reaches the parameter: RESTEasy Reactive binds it as null, so it reads as+ // no attempt, the latest one, as Pagination.parseMaxResults reads an empty maxResults.+ givenReq().queryParam("attempt", "").when().get(path)+ .then().statusCode(200).body("jobRun.attempt", equalTo(1));+ }++ @Test+ void listJobRunAttemptsReportsTheRunsOneAttempt() {+ String applicationId = createApplication("");+ String jobRunId = startedRunId(applicationId,+ sparkRun(UUID.randomUUID().toString(), ", \"name\": \"nightly\", \"mode\": \"BATCH\""));+ String path = "/applications/" + applicationId + "/jobruns/" + jobRunId;+ JsonPath run = givenReq().config(DECIMAL_NUMBERS).when().get(path)+ .then().statusCode(200).extract().jsonPath();++ JsonPath attempts = givenReq().config(DECIMAL_NUMBERS).when().get(path + "/attempts")+ .then().statusCode(200)+ .body("jobRunAttempts", hasSize(1))+ .body("$", not(hasKey("nextToken")))+ .extract().jsonPath();+ assertEquals(applicationId, attempts.getString("jobRunAttempts[0].applicationId"));+ assertEquals(jobRunId, attempts.getString("jobRunAttempts[0].id"));+ assertEquals("nightly", attempts.getString("jobRunAttempts[0].name"));+ assertEquals("BATCH", attempts.getString("jobRunAttempts[0].mode"));+ assertEquals(run.getString("jobRun.arn"), attempts.getString("jobRunAttempts[0].arn"));+ assertEquals("arn:aws:iam::000000000000:root", attempts.getString("jobRunAttempts[0].createdBy"));+ assertEquals(run.<BigDecimal>get("jobRun.createdAt"),+ attempts.<BigDecimal>get("jobRunAttempts[0].jobCreatedAt"));+ assertEquals(run.<BigDecimal>get("jobRun.createdAt"), attempts.<BigDecimal>get("jobRunAttempts[0].createdAt"));+ assertEquals(run.<BigDecimal>get("jobRun.updatedAt"), attempts.<BigDecimal>get("jobRunAttempts[0].updatedAt"));+ assertEquals(ROLE, attempts.getString("jobRunAttempts[0].executionRole"));+ assertEquals("SUCCESS", attempts.getString("jobRunAttempts[0].state"));+ assertEquals("emr-7.5.0", attempts.getString("jobRunAttempts[0].releaseLabel"));+ assertEquals("SPARK", attempts.getString("jobRunAttempts[0].type"));+ assertEquals(1, attempts.getInt("jobRunAttempts[0].attempt"));+ }++ @Test+ void listJobRunAttemptsRefusesAPageLargerThanFiftyAndAnUnknownRun() {+ String applicationId = createApplication("");+ String jobRunId = startedRunId(applicationId, sparkRun(UUID.randomUUID().toString(), ""));++ givenReq().queryParam("maxResults", 51)+ .when().get("/applications/" + applicationId + "/jobruns/" + jobRunId + "/attempts")+ .then().statusCode(400)+ .body("message", equalTo("1 validation error detected: Value '51' at 'maxResults' failed to "+ + "satisfy constraint: Member must have value less than or equal to 50"));+ givenReq().when().get("/applications/" + applicationId + "/jobruns/00missing0000000/attempts")+ .then().statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"));+ givenReq().when().get("/applications/00missing0000000/jobruns/" + jobRunId + "/attempts")+ .then().statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"));+ givenReq().queryParam("nextToken", "bm90LWEtdG9rZW4")+ .when().get("/applications/" + applicationId + "/jobruns/" + jobRunId + "/attempts")+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"));+ }+@Testvoid jobRunsStayInTheirRegion() {String applicationId = createApplication("");diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.javaindex f3b4cd5520..32a88d5f96 100644--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java@@ -100,13 +100,13 @@ class EmrServerlessJobRunServiceTest {Instant submittedAt = started.getCreatedAt();clock.advance(Duration.ofSeconds(29));- JobRun running = service.getJobRun(applicationId, jobRunId);+ JobRun running = service.getJobRun(applicationId, jobRunId, null);assertEquals("RUNNING", running.getState());assertNull(running.getEndedAt());assertNull(running.getTotalExecutionDurationSeconds());clock.advance(Duration.ofSeconds(1));- JobRun finished = service.getJobRun(applicationId, jobRunId);+ JobRun finished = service.getJobRun(applicationId, jobRunId, null);assertEquals("SUCCESS", finished.getState());assertEquals(submittedAt.plusSeconds(30), finished.getEndedAt());assertEquals(submittedAt.plusSeconds(30), finished.getUpdatedAt());@@ -124,7 +124,7 @@ class EmrServerlessJobRunServiceTest {assertEquals("CANCELLED", cancelled.getState());assertEquals(10, cancelled.getTotalExecutionDurationSeconds());clock.advance(Duration.ofMinutes(5));- assertEquals("CANCELLED", service.getJobRun(applicationId, jobRunId).getState());+ assertEquals("CANCELLED", service.getJobRun(applicationId, jobRunId, null).getState());}@Test@@ -135,14 +135,31 @@ class EmrServerlessJobRunServiceTest {Instant submittedAt = started.getCreatedAt();clock.advance(Duration.ofMinutes(5).minusSeconds(1));- assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId).getState());+ assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId, null).getState());clock.advance(Duration.ofSeconds(1));- JobRun timedOut = service.getJobRun(applicationId, jobRunId);+ JobRun timedOut = service.getJobRun(applicationId, jobRunId, null);assertEquals("CANCELLED", timedOut.getState());assertEquals(submittedAt.plus(Duration.ofMinutes(5)), timedOut.getEndedAt());}+ @Test+ void anAttemptTheRunNeverMadeIsRefusedBeforeTheRunSettles() {+ EmrServerlessJobRunService service = service(30);+ JobRun started = service.startJobRun(applicationId, request(null, null));+ String jobRunId = started.getJobRunId();+ clock.advance(Duration.ofSeconds(30));++ AwsException noAttempt = assertThrows(AwsException.class,+ () -> service.getJobRun(applicationId, jobRunId, 2));+ assertEquals("ResourceNotFoundException", noAttempt.getErrorCode());+ assertEquals("RUNNING", store.get(jobRunId).orElseThrow().getJobRun().getState());++ JobRun finished = service.getJobRun(applicationId, jobRunId, 1);+ assertEquals("SUCCESS", finished.getState());+ assertEquals(started.getCreatedAt().plusSeconds(30), finished.getEndedAt());+ }+@Testvoid streamingRunIgnoresTheRunDuration() {EmrServerlessJobRunService service = service(0);@@ -150,7 +167,7 @@ class EmrServerlessJobRunServiceTest {clock.advance(Duration.ofDays(2));- assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId).getState());+ assertEquals("RUNNING", service.getJobRun(applicationId, jobRunId, null).getState());}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessRegionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessRegionIntegrationTest.javaindex a10fcff731..5e6af0035f 100644--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessRegionIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessRegionIntegrationTest.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.emrserverless;+import io.github.hectorvent.floci.testing.EmrServerlessRequests;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;import io.restassured.response.Response;@@ -7,11 +8,9 @@ import io.restassured.specification.RequestSpecification;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.Test;-import java.net.URLEncoder;-import java.nio.charset.StandardCharsets;import java.util.UUID;-import static io.restassured.RestAssured.given;+import static io.github.hectorvent.floci.testing.EmrServerlessRequests.tagsPath;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.hasItem;import static org.hamcrest.Matchers.not;@@ -26,15 +25,7 @@ class EmrServerlessRegionIntegrationTest {}private static RequestSpecification emr(String region) {- return given()- .header("Authorization", "AWS4-HMAC-SHA256 Credential=TEST/20261020/" + region- + "/emr-serverless/aws4_request, SignedHeaders=host;x-amz-date, Signature=test")- .contentType("application/json")- .urlEncodingEnabled(false);- }-- private static String tagsPath(String arn) {- return "/tags/" + URLEncoder.encode(arn, StandardCharsets.UTF_8);+ return EmrServerlessRequests.emrServerless(region).urlEncodingEnabled(false);}@Test@@ -89,4 +80,29 @@ class EmrServerlessRegionIntegrationTest {emr("us-east-1").when().get(tagsPath(eastArn)).then().statusCode(200).body("tags.team", equalTo("data"));}++ @Test+ void jobRunTagsStayInTheRunsRegion() {+ String applicationId = emr("us-east-1").body("""+ {"name":"regional-runs","releaseLabel":"emr-7.5.0","type":"SPARK","clientToken":"%s"}+ """.formatted(UUID.randomUUID()))+ .when().post("/applications")+ .then().statusCode(200).extract().path("applicationId");+ String runArn = emr("us-east-1").body("""+ {"clientToken":"%s","executionRoleArn":"arn:aws:iam::000000000000:role/emr-serverless-job",+ "jobDriver":{"sparkSubmit":{"entryPoint":"s3://bucket/jobs/app.jar"}},"tags":{"team":"data"}}+ """.formatted(UUID.randomUUID()))+ .when().post("/applications/" + applicationId + "/jobruns")+ .then().statusCode(200).extract().path("arn");++ emr("eu-west-1").when().get(tagsPath(runArn))+ .then().statusCode(404).body("__type", equalTo("ResourceNotFoundException"));+ emr("eu-west-1").body("{\"tags\":{\"team\":\"wrong-region\"}}")+ .when().post(tagsPath(runArn))+ .then().statusCode(404).body("__type", equalTo("ResourceNotFoundException"));+ emr("eu-west-1").when().delete(tagsPath(runArn) + "?tagKeys=team")+ .then().statusCode(404).body("__type", equalTo("ResourceNotFoundException"));+ emr("us-east-1").when().get(tagsPath(runArn))+ .then().statusCode(200).body("tags.team", equalTo("data"));+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagIntegrationTest.javaindex 1fb8d799dc..4e16cc1881 100644--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessTagIntegrationTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.emrserverless;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.testing.EmrServerlessRequests;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;import io.restassured.path.json.JsonPath;@@ -21,7 +22,7 @@ import java.util.concurrent.Executors;import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;-import static io.restassured.RestAssured.given;+import static io.github.hectorvent.floci.testing.EmrServerlessRequests.tagsPath;import static org.hamcrest.Matchers.aMapWithSize;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.hasKey;@@ -41,16 +42,28 @@ class EmrServerlessTagIntegrationTest {}private static RequestSpecification emr() {- return given()- .header("Authorization", "AWS4-HMAC-SHA256 Credential=TEST/20260929/us-east-1/emr-serverless/aws4_request, "- + "SignedHeaders=host;x-amz-date, Signature=test")- .contentType("application/json")- .urlEncodingEnabled(false);+ return EmrServerlessRequests.emrServerless("us-east-1").urlEncodingEnabled(false);}- /** The SDK sends the ARN as one path segment with ':' and '/' percent-encoded. */- private static String tagsPath(String arn) {- return "/tags/" + URLEncoder.encode(arn, StandardCharsets.UTF_8);+ /** A tags map of {@code count} distinct keys, as JSON. */+ private static String tagMap(int count) {+ StringBuilder tags = new StringBuilder("{");+ for (int i = 0; i < count; i++) {+ tags.append(i == 0 ? "" : ",").append("\"k").append(i).append("\": \"v\"");+ }+ return tags.append("}").toString();+ }++ private static String[] startJobRun(String applicationId, String tagsJson) {+ JsonPath started = emr()+ .body("""+ { "clientToken": "%s", "executionRoleArn": "arn:aws:iam::000000000000:role/emr-serverless-job",+ "jobDriver": { "sparkSubmit": { "entryPoint": "s3://bucket/jobs/app.jar" } }, "tags": %s }+ """.formatted(UUID.randomUUID(), tagsJson))+ .when().post("/applications/" + applicationId + "/jobruns")+ .then().statusCode(200)+ .extract().jsonPath();+ return new String[] {started.getString("jobRunId"), started.getString("arn")};}private static String[] createApplication(String tagsJson) {@@ -138,12 +151,7 @@ class EmrServerlessTagIntegrationTest {.then().statusCode(400).body("__type", equalTo("ValidationException"));- StringBuilder tooMany = new StringBuilder("{ \"tags\": {");- for (int i = 0; i < 51; i++) {- tooMany.append(i == 0 ? "" : ",").append("\"k").append(i).append("\": \"v\"");- }- tooMany.append("} }");- emr().body(tooMany.toString())+ emr().body("{ \"tags\": " + tagMap(51) + " }").when().post(tagsPath(arn)).then().statusCode(400).body("__type", equalTo("ValidationException"));@@ -170,12 +178,7 @@ class EmrServerlessTagIntegrationTest {@Testvoid createApplicationAppliesTheSameTagRules() {- StringBuilder tooMany = new StringBuilder("{");- for (int i = 0; i < 51; i++) {- tooMany.append(i == 0 ? "" : ",").append("\"k").append(i).append("\": \"v\"");- }- tooMany.append("}");- for (String tags : List.of(tooMany.toString(), "{ \"bad#key\": \"v\" }")) {+ for (String tags : List.of(tagMap(51), "{ \"bad#key\": \"v\" }")) {emr().body("""{ "name": "limits", "releaseLabel": "emr-7.5.0", "type": "SPARK","clientToken": "%s", "tags": %s }@@ -193,13 +196,8 @@ class EmrServerlessTagIntegrationTest {@Testvoid theUserGuideLimitsApplyAndAnEmptyMapIsANoOp() {String arn = createApplication("{ \"team\": \"data\" }")[1];- StringBuilder fortyNine = new StringBuilder("{ \"tags\": {");- for (int i = 0; i < 49; i++) {- fortyNine.append(i == 0 ? "" : ",").append("\"k").append(i).append("\": \"v\"");- }- fortyNine.append("} }");- emr().body(fortyNine.toString()).when().post(tagsPath(arn)).then().statusCode(200);+ emr().body("{ \"tags\": " + tagMap(49) + " }").when().post(tagsPath(arn)).then().statusCode(200);emr().when().get(tagsPath(arn)).then().statusCode(200).body("tags", aMapWithSize(50));emr().body("{ \"tags\": { \"one-more\": \"v\" } }").when().post(tagsPath(arn))@@ -221,7 +219,7 @@ class EmrServerlessTagIntegrationTest {/** Tag writes read, change and store the whole application; concurrent ones must not lose each other. */@Testvoid concurrentTagWritesAreAllKept() throws Exception {- String arn = createApplication("{}")[1];+ EmrServerlessService.ResourceArn resource = service.resourceArn(createApplication("{}")[1]);int threads = 16;CountDownLatch go = new CountDownLatch(1);ExecutorService pool = Executors.newFixedThreadPool(threads);@@ -231,7 +229,7 @@ class EmrServerlessTagIntegrationTest {String key = "k" + i;futures.add(pool.submit(() -> {go.await();- service.tagResource(arn, Map.of(key, "v"));+ service.tagResource(resource, Map.of(key, "v"));return null;}));}@@ -243,20 +241,125 @@ class EmrServerlessTagIntegrationTest {pool.shutdownNow();}- assertEquals(threads, service.listTags(arn).size());+ assertEquals(threads, service.listTags(resource).size());}@Testvoid anEmptyTagMapChangesNothingButMissingTagsOrKeysAreRefused() {- String arn = createApplication("{ \"team\": \"data\" }")[1];+ EmrServerlessService.ResourceArn resource = service.resourceArn(createApplication("{ \"team\": \"data\" }")[1]);- service.tagResource(arn, Map.of());- assertEquals(Map.of("team", "data"), service.listTags(arn));+ service.tagResource(resource, Map.of());+ assertEquals(Map.of("team", "data"), service.listTags(resource));assertEquals("ValidationException",assertThrows(AwsException.class,- () -> service.tagResource(arn, null)).getErrorCode());+ () -> service.tagResource(resource, null)).getErrorCode());assertEquals("ValidationException",assertThrows(AwsException.class,- () -> service.untagResource(arn, List.of())).getErrorCode());+ () -> service.untagResource(resource, List.of())).getErrorCode());+ }++ @Test+ void tagsRoundTripOnAJobRunArnAndShowInGetJobRun() {+ String[] app = createApplication("{ \"team\": \"app\" }");+ String[] run = startJobRun(app[0], "{ \"team\": \"data\" }");+ String arn = run[1];++ emr().when().get(tagsPath(arn))+ .then().statusCode(200)+ .body("tags", aMapWithSize(1))+ .body("tags.team", equalTo("data"));++ emr().body("{ \"tags\": { \"env\": \"dev\", \"team\": \"platform\" } }")+ .when().post(tagsPath(arn))+ .then().statusCode(200);+ emr().when().get("/applications/" + app[0] + "/jobruns/" + run[0])+ .then().statusCode(200)+ .body("jobRun.state", equalTo("SUCCESS"))+ .body("jobRun.tags.team", equalTo("platform"))+ .body("jobRun.tags.env", equalTo("dev"));++ emr().when().delete(tagsPath(arn) + "?tagKeys=env")+ .then().statusCode(200);+ emr().when().get(tagsPath(arn))+ .then().statusCode(200)+ .body("tags", not(hasKey("env")))+ .body("tags.team", equalTo("platform"));+ emr().when().get(tagsPath(app[1]))+ .then().statusCode(200)+ .body("tags", aMapWithSize(1))+ .body("tags.team", equalTo("app"));+ }++ @Test+ void aDeletedApplicationsRunsCannotBeTagged() {+ String applicationId = createApplication("{}")[0];+ String[] run = startJobRun(applicationId, "{ \"team\": \"data\" }");+ String path = tagsPath(run[1]);++ emr().when().post("/applications/" + applicationId + "/stop")+ .then().statusCode(200);+ emr().when().delete("/applications/" + applicationId)+ .then().statusCode(200);++ emr().when().get(path)+ .then().statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"))+ .body("message", equalTo("Resource " + run[1] + " not found"));+ emr().body("{ \"tags\": { \"env\": \"dev\" } }")+ .when().post(path)+ .then().statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"));+ emr().when().delete(path + "?tagKeys=team")+ .then().statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"));++ emr().when().get("/applications/" + applicationId + "/jobruns/" + run[0])+ .then().statusCode(200)+ .body("jobRun.tags.team", equalTo("data"))+ .body("jobRun.tags", not(hasKey("env")));+ }++ @Test+ void aJobRunArnMustBeTheRunsOwnAndSessionArnsNameNothing() {+ String[] owner = createApplication("{}");+ String[] other = createApplication("{}");+ String[] run = startJobRun(owner[0], "{ \"team\": \"data\" }");++ for (String arn : List.of(other[1] + "/jobruns/" + run[0],+ run[1].replace(":000000000000:", ":111111111111:"),+ owner[1] + "/sessions/00abc0000000000")) {+ emr().when().get(tagsPath(arn))+ .then().statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"));+ emr().body("{ \"tags\": { \"env\": \"dev\" } }")+ .when().post(tagsPath(arn))+ .then().statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"));+ }+ emr().when().get(tagsPath(run[1]))+ .then().statusCode(200)+ .body("tags", aMapWithSize(1));+ }++ @Test+ void jobRunsKeepTheSameTagRules() {+ String applicationId = createApplication("{}")[0];+ String arn = startJobRun(applicationId, tagMap(50))[1];++ emr().body("{ \"tags\": { \"one-more\": \"v\" } }")+ .when().post(tagsPath(arn))+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("A resource can have at most 50 tags."));+ emr().body("{ \"tags\": { \"aws:owner\": \"x\" } }")+ .when().post(tagsPath(arn))+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"));+ emr().when().delete(tagsPath(arn) + "?tagKeys=")+ .then().statusCode(400)+ .body("__type", equalTo("ValidationException"));+ emr().when().get(tagsPath(arn))+ .then().statusCode(200)+ .body("tags", aMapWithSize(50));}}diff --git a/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java b/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.javaindex 29cf8cb33f..26cd7a2374 100644--- a/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java+++ b/src/test/java/io/github/hectorvent/floci/testing/EmrServerlessRequests.java@@ -2,6 +2,9 @@ package io.github.hectorvent.floci.testing;import io.restassured.specification.RequestSpecification;+import java.net.URLEncoder;+import java.nio.charset.StandardCharsets;+import static io.restassured.RestAssured.given;/** One EMR Serverless REST JSON request, signed for a named Region. */@@ -14,4 +17,12 @@ public final class EmrServerlessRequests {+ "/emr-serverless/aws4_request, SignedHeaders=host;x-amz-date, Signature=test").contentType("application/json");}++ /**+ * The tagging path for an ARN, sent as the SDK sends it: one path segment with ':' and '/'+ * percent-encoded. Send it with {@code urlEncodingEnabled(false)}, or it is encoded twice.+ */+ public static String tagsPath(String arn) {+ return "/tags/" + URLEncoder.encode(arn, StandardCharsets.UTF_8);+ }}