Light, fluffy, and always free - The AWS Local Emulator alternative
refactor(emr-serverless): let classic EMR reuse the Spark runner (#6562)
Priamo Germosen8d5c135f0adc
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.javaindex 38f599bcc4..1910322e24 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java@@ -81,6 +81,8 @@ public class EmrServerlessJobRunService implements Resettable {private static final String MODE_BATCH = "BATCH";private static final String MODE_STREAMING = "STREAMING";private static final String RUNNER_DOCKER = "docker";+ // The service a container run's driver belongs to, which names it and labels it.+ private static final String SERVICE = "emr-serverless";private static final String SPARK_DEFAULTS = "spark-defaults";// No source gives AWS's wording for these; they are Floci's.static final String HIVE_NOT_RUN = "Floci runs Spark job runs only, and this is a Hive job run.";@@ -122,6 +124,10 @@ public class EmrServerlessJobRunService implements Resettable {// Null in the immediate runner mode.private final SparkJobRunner sparkRunner;private final Executor runExecutor;+ // Where a container run's driver runs: image-template, with the release label for %s, and docker-network;+ // null when not set.+ private final String imageTemplate;+ private final String dockerNetwork;// Container runs a thread is running; one not here that has not ended was left by a restart.private final Set<String> executing = new HashSet<>();// Raised by each reset, so a run's thread from before it writes nothing back after the wipe.@@ -136,10 +142,12 @@ public class EmrServerlessJobRunService implements Resettable {applications, iamService, regionResolver, requestContext, objectMapper,config.services().emrserverless().jobRunDurationSeconds(), clock,RUNNER_DOCKER.equalsIgnoreCase(config.services().emrserverless().runnerMode()) ? sparkRunner : null,- Executors.newCachedThreadPool(Thread.ofPlatform().daemon().name("emr-serverless-run-", 0).factory()));+ Executors.newCachedThreadPool(Thread.ofPlatform().daemon().name("emr-serverless-run-", 0).factory()),+ config.services().emrserverless().imageTemplate().orElse(null),+ config.services().emrserverless().dockerNetwork().orElse(null));// Before any run of this process starts: a driver an earlier one left belongs to a run read as FAILED.if (this.sparkRunner != null && config.services().emrserverless().reconcileContainersOnStartup()) {- sparkRunner.removeLeftoverDrivers();+ sparkRunner.removeLeftoverDrivers(SERVICE);}}@@ -154,6 +162,16 @@ public class EmrServerlessJobRunService implements Resettable {IamService iamService, RegionResolver regionResolver, RequestContext requestContext,ObjectMapper objectMapper, int runDurationSeconds, Clock clock,SparkJobRunner sparkRunner, Executor runExecutor) {+ this(store, applications, iamService, regionResolver, requestContext, objectMapper, runDurationSeconds, clock,+ sparkRunner, runExecutor, null, null);+ }++ private EmrServerlessJobRunService(AccountAwareStorageBackend<JobRunRecord> store,+ EmrServerlessService applications, IamService iamService,+ RegionResolver regionResolver, RequestContext requestContext,+ ObjectMapper objectMapper, int runDurationSeconds, Clock clock,+ SparkJobRunner sparkRunner, Executor runExecutor,+ String imageTemplate, String dockerNetwork) {if (runDurationSeconds < 0) {throw new IllegalArgumentException("floci.services.emrserverless.job-run-duration-seconds must not be negative: " + runDurationSeconds);@@ -168,6 +186,8 @@ public class EmrServerlessJobRunService implements Resettable {this.clock = clock;this.sparkRunner = sparkRunner;this.runExecutor = runExecutor;+ this.imageTemplate = imageTemplate;+ this.dockerNetwork = dockerNetwork;}@PreDestroy@@ -437,9 +457,9 @@ public class EmrServerlessJobRunService implements Resettable {Long minutes = run.getExecutionTimeoutMinutes();Duration timeout = MODE_STREAMING.equals(run.getMode()) || minutes == null || minutes == 0? Duration.ZERO : Duration.ofMinutes(minutes);- SparkJobRunner.Job job = new SparkJobRunner.Job(run.getJobRunId(), run.getApplicationId(), arn.accountId(),- arn.region(), run.getReleaseLabel(), run.getExecutionRole(), run.getJobDriver().getSparkSubmit(),- sparkDefaults(app, run), timeout);+ SparkJobRunner.Job job = new SparkJobRunner.Job(SERVICE, run.getJobRunId(), arn.accountId(), arn.region(),+ run.getReleaseLabel(), imageTemplate, dockerNetwork, run.getExecutionRole(), sparkDefaults(app, run),+ application(run.getJobDriver().getSparkSubmit()), timeout);executing.add(run.getJobRunId());long launchedIn = generation;runExecutor.execute(() -> execute(job, launchedIn));@@ -450,11 +470,26 @@ public class EmrServerlessJobRunService implements Resettable {try {outcome = sparkRunner.run(job, new RunProgress(job, launchedIn));} catch (RuntimeException e) {- LOG.warnv(e, "EMR Serverless job run {0} could not run", job.jobRunId());+ LOG.warnv(e, "EMR Serverless job run {0} could not run", job.runId());outcome = SparkJobRunner.Outcome.couldNotStart(e.getMessage());}SparkJobRunner.Outcome ended = outcome;- RequestScopes.runAs(job.accountId(), job.region(), () -> ended(job.jobRunId(), ended, launchedIn));+ RequestScopes.runAs(job.accountId(), job.region(), () -> ended(job.runId(), ended, launchedIn));+ }++ /**+ * What follows Floci's own settings on a run's {@code spark-submit} command line: its+ * {@code sparkSubmitParameters}, split as a shell would split them, then its entry point and the+ * entry point's arguments.+ */+ static List<String> application(JobDriver.SparkSubmit sparkSubmit) {+ List<String> application = new ArrayList<>(+ EmrServerlessSparkRunner.splitParameters(sparkSubmit.getSparkSubmitParameters()));+ application.add(sparkSubmit.getEntryPoint());+ if (sparkSubmit.getEntryPointArguments() != null) {+ application.addAll(sparkSubmit.getEntryPointArguments());+ }+ return application;}/**@@ -813,18 +848,18 @@ public class EmrServerlessJobRunService implements Resettable {@Overridepublic void started() {- RequestScopes.runAs(job.accountId(), job.region(), () -> driverStarted(job.jobRunId(), launchedIn));+ RequestScopes.runAs(job.accountId(), job.region(), () -> driverStarted(job.runId(), launchedIn));}@Overridepublic void timedOut() {- RequestScopes.runAs(job.accountId(), job.region(), () -> driverTimedOut(job.jobRunId(), launchedIn));+ RequestScopes.runAs(job.accountId(), job.region(), () -> driverTimedOut(job.runId(), launchedIn));}@Overridepublic boolean cancelled() {return !RequestScopes.callAs(job.accountId(), job.region(),- () -> waitingToStart(job.jobRunId(), launchedIn));+ () -> waitingToStart(job.runId(), launchedIn));}}}diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.javaindex 389cfc891d..a3f627e2f1 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.java@@ -10,7 +10,6 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;-import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;import io.github.hectorvent.floci.services.iam.model.SessionCreds;import io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials;import jakarta.enterprise.context.ApplicationScoped;@@ -30,12 +29,12 @@ import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.atomic.AtomicReference;/**- * Runs a Spark job run's {@code spark-submit} in a container of the release's EMR Serverless image.- * The image's own entrypoint hands over to a script EMR Serverless supplies at run time, which the- * public image lacks, so the runner starts {@code spark-submit} itself, in local mode: the container- * holds the driver and its executors. The container reaches Floci as the other containers Floci- * launches do, with the run's execution role as its credentials, and S3A, which takes {@code s3://}- * on every release, is pointed at Floci's S3.+ * Runs a Spark application's {@code spark-submit} in a container of an EMR Serverless image, for an EMR+ * Serverless job run or an EMR step. The image's own entrypoint hands over to a script EMR Serverless+ * supplies at run time, which the public image lacks, so the runner starts {@code spark-submit} itself,+ * in local mode: the container holds the driver and its executors. The container reaches Floci as the+ * other containers Floci launches do, with the run's execution role as its credentials, and S3A, which+ * takes {@code s3://} on every release, is pointed at Floci's S3.*/@ApplicationScopedpublic class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardown {@@ -49,7 +48,6 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo"public.ecr.aws/emr-serverless/spark4/%s:latest"; // partition-literal: ECR Public's registry host, where AWS publishes the base images; image-template overrides it// "The default behavior for shutdown is to have no grace period": immediate termination.private static final int STOP_TIMEOUT_SECONDS = 0;- private static final String SERVICE = "emr-serverless";// The characters a backslash escapes inside double quotes; before any other it is kept.private static final String DOUBLE_QUOTE_ESCAPES = "\"\\$`\n";// How often a running driver is looked at, as Batch's runner looks at its jobs.@@ -92,7 +90,7 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo@Overridepublic Outcome run(Job job, Progress progress) {Launch launch = new Launch();- launches.put(job.jobRunId(), launch);+ launches.put(job.runId(), launch);Optional<SessionCreds> credentials = Optional.empty();Closeable logHandle = null;try {@@ -100,18 +98,18 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardoif (progress.cancelled()) {return Outcome.halted();}- credentials = roleCredentials.forRole(job.executionRole(), job.accountId(), job.jobRunId());+ credentials = roleCredentials.forRole(job.executionRole(), job.accountId(), job.runId());Map<String, String> labels = new LinkedHashMap<>(ContainerStorageHelper.resourceIdentityLabels(- SERVICE, job.jobRunId(), job.accountId(), job.region()));+ job.service(), job.runId(), job.accountId(), job.region()));// The deployment that started the driver, which removeLeftoverDrivers() goes by.labels.put(ContainerStorageHelper.OWNER_LABEL, ContainerStorageHelper.ownerIdentity(config));String endpoint = awsEnv.flociEndpoint();- ContainerSpec spec = containerBuilder.newContainer(image(job.releaseLabel()))- .withName(ContainerStorageHelper.dockerName(config, SERVICE + "-" + job.jobRunId()))+ ContainerSpec spec = containerBuilder.newContainer(image(job.imageTemplate(), job.releaseLabel()))+ .withName(ContainerStorageHelper.dockerName(config, job.service() + "-" + job.runId())).withEntrypoint(List.of(SPARK_SUBMIT)).withCmd(sparkSubmitArguments(job, endpoint)).withEnv(awsEnv.sdkBaselineEnv(job.region(), Optional.empty(), credentials, job.accountId()))- .withDockerNetwork(config.services().emrserverless().dockerNetwork())+ .withDockerNetwork(Optional.ofNullable(job.dockerNetwork())).withHostDockerInternalOnLinux().withEmbeddedDns().withLogRotation()@@ -123,7 +121,7 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardoif (launch.stopRequested || progress.cancelled()) {return Outcome.halted();}- logHandle = logStreamer.attachConsoleOnly(containerId, SERVICE + ":" + job.jobRunId());+ logHandle = logStreamer.attachConsoleOnly(containerId, job.service() + ":" + job.runId());progress.started();return awaitEnd(containerId, job, launch, progress);} catch (InterruptedException e) {@@ -135,10 +133,10 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo// A reset removed the driver while it was being watched.return Outcome.halted();}- LOG.warnv(e, "EMR Serverless job run {0} could not run", job.jobRunId());+ LOG.warnv(e, "Spark run {0} of {1} could not run", job.runId(), job.service());return Outcome.couldNotStart(e.getMessage());} finally {- launches.remove(job.jobRunId(), launch);+ launches.remove(job.runId(), launch);String unclaimed = launch.container.getAndSet(null);if (unclaimed != null) {lifecycleManager.stopAndRemove(unclaimed, logHandle, STOP_TIMEOUT_SECONDS);@@ -151,8 +149,8 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo}@Override- public void stop(String jobRunId) {- Launch launch = launches.get(jobRunId);+ public void stop(String runId) {+ Launch launch = launches.get(runId);if (launch != null) {// run() reads it once the driver's container is up and at each look at the driver after, and// removes the driver itself: a stop makes no Docker call, so no Docker error can lose it.@@ -176,16 +174,17 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo}/**- * Removes the drivers an earlier Floci process left running, which no thread follows any more: their- * runs are {@code FAILED} once read, so nothing would ever stop them. The job run service calls this- * before it hands a run to {@link #run}, so every driver of this deployment it finds is a leftover.+ * Removes the drivers of {@code service}'s runs an earlier Floci process left running, which no thread+ * follows any more: their runs are {@code FAILED} once read, so nothing would ever stop them. The+ * service calls this before it hands a run to {@link #run}, so every driver of this deployment it+ * finds is a leftover.*/- public void removeLeftoverDrivers() {+ public void removeLeftoverDrivers(String service) {String owner = ContainerStorageHelper.ownerIdentity(config);ContainerStorageHelper.LabelAliases aliases = ContainerStorageHelper.CONTAINER_LABEL_ALIASES;try {List<Container> containers = aliases.listByLabels(- Map.of(ContainerStorageHelper.SERVICE_LABEL, SERVICE, ContainerStorageHelper.OWNER_LABEL, owner),+ Map.of(ContainerStorageHelper.SERVICE_LABEL, service, ContainerStorageHelper.OWNER_LABEL, owner),filter -> lifecycleManager.getDockerClient().listContainersCmd().withShowAll(true).withLabelFilter(filter)@@ -196,26 +195,30 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardoif (aliases.consistent(container.getId(), labels)&& aliases.matches(labels, ContainerStorageHelper.OWNER_LABEL, owner)) {lifecycleManager.removeIfExists(container.getId());- LOG.infov("Removed EMR Serverless driver {0}, left running by an earlier Floci process",+ LOG.infov("Removed {0} Spark driver {1}, left running by an earlier Floci process", service,container.getId());}}} catch (RuntimeException e) {- LOG.warnv("Could not remove the EMR Serverless drivers an earlier Floci process left: {0}",+ LOG.warnv("Could not remove the {0} Spark drivers an earlier Floci process left: {1}", service,e.getMessage());}}- String image(String releaseLabel) {- return config.services().emrserverless().imageTemplate()- .orElse(releaseLabel.startsWith("emr-spark-") ? SPARK_RUNTIME_DEFAULT_IMAGE : DEFAULT_IMAGE)- .formatted(releaseLabel);+ /**+ * The image a release's runs use: {@code imageTemplate}, with {@code %s} standing for the release+ * label, or, when it is null, the release's public EMR Serverless image.+ */+ static String image(String imageTemplate, String releaseLabel) {+ String template = imageTemplate != null ? imageTemplate+ : releaseLabel.startsWith("emr-spark-") ? SPARK_RUNTIME_DEFAULT_IMAGE : DEFAULT_IMAGE;+ return template.formatted(releaseLabel);}/*** {@code spark-submit}'s arguments: local mode, S3A on Floci's S3 for {@code s3://} and {@code s3a://}- * paths, then the run's {@code spark-defaults} and its own {@code sparkSubmitParameters}, which win- * over what comes before them, then the entry point and its arguments. Before release 7.10.0+ * paths, then the run's {@code spark-defaults} and its own options, which win over what comes before+ * them, then its application and that application's arguments. Before release 7.10.0* {@code s3://} is EMRFS, which ignores the endpoint setting, so it is switched to S3A. S3A writes* are committed through commit data the writer keeps in memory, without which EMR's committer finds* none and a write is silently lost.@@ -231,22 +234,18 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardoconf(arguments, "spark.hadoop.aws.glue.endpoint", flociEndpoint);conf(arguments, "spark.hadoop.aws.region", job.region());job.sparkDefaults().forEach((key, value) -> conf(arguments, key, value));- JobDriver.SparkSubmit sparkSubmit = job.sparkSubmit();- arguments.addAll(splitParameters(sparkSubmit.getSparkSubmitParameters()));- arguments.add(sparkSubmit.getEntryPoint());- if (sparkSubmit.getEntryPointArguments() != null) {- arguments.addAll(sparkSubmit.getEntryPointArguments());- }+ arguments.addAll(job.application());return arguments;}/**- * {@code sparkSubmitParameters} as a POSIX shell would split them: on whitespace, except inside single+ * {@code spark-submit} parameters given as one string, as EMR Serverless's+ * {@code sparkSubmitParameters}, split as a POSIX shell would split them: on whitespace, except inside single* or double quotes. Outside quotes a backslash takes the next character as it is; inside double quotes* only a {@code "}, {@code \}, {@code $}, {@code `} or newline, and before any other character it is* kept. A backslash before a newline continues the line, and both go.*/- static List<String> splitParameters(String parameters) {+ public static List<String> splitParameters(String parameters) {List<String> words = new ArrayList<>();if (parameters == null) {return words;@@ -321,7 +320,7 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardoreturn Outcome.exitedWith(exitCode != null ? exitCode.intValue() : -1);}if (deadline != null && !clock.instant().isBefore(deadline)) {- LOG.infov("EMR Serverless job run {0} passed its execution timeout of {1}", job.jobRunId(),+ LOG.infov("Spark run {0} of {1} passed its execution timeout of {2}", job.runId(), job.service(),job.timeout());progress.timedOut();return Outcome.outOfTime();diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/SparkJobRunner.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/SparkJobRunner.javaindex 45e2543afd..facd88ff66 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/SparkJobRunner.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/SparkJobRunner.java@@ -1,16 +1,16 @@package io.github.hectorvent.floci.services.emrserverless;-import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;-import java.time.Duration;+import java.util.List;import java.util.Map;/**- * Runs a Spark job run to its end, in the {@code docker} runner mode. The job run service holds the- * run's state and calls this off the request thread; {@link EmrServerlessSparkRunner} is the Docker- * implementation, and a test supplies its own.+ * Runs a Spark application to its end, in the EMR services' {@code docker} runner mode: an EMR+ * Serverless job run, or a step of an EMR cluster. The service holds the run's state and calls this+ * off the request thread; {@link EmrServerlessSparkRunner} is the Docker implementation, and a test+ * supplies its own.*/-interface SparkJobRunner {+public interface SparkJobRunner {/*** Runs {@code job} and returns how it ended, telling {@code progress} once its Spark driver runs and@@ -23,9 +23,9 @@ interface SparkJobRunner {* Stops the run's driver, now or as soon as it starts, so {@link #run} returns {@link Outcome#halted};* a run {@link #run} has already returned from is left as it is.*/- void stop(String jobRunId);+ void stop(String runId);- /** What a run tells the job run service, and asks it, while it goes. */+ /** What a run tells the service, and asks it, while it goes. */interface Progress {/** The run's Spark driver runs. */@@ -39,13 +39,17 @@ interface SparkJobRunner {}/**- * What a run needs: its identity, the application's release, the run's execution role, its Spark- * submission, the {@code spark-defaults} properties that apply to it, and how long it may run+ * What a run needs: the service it belongs to and its id there, which name its container; its account+ * and Region; its release, and the image template and Docker network to run it in (a null template is+ * the release's public EMR Serverless image, a null network the global default); the execution role+ * whose credentials it gets; the {@code spark-defaults} properties that apply to it;+ * {@code application}, what follows Floci's own settings on {@code spark-submit}'s command line (the+ * run's options, then its application and that application's arguments); and how long it may run* ({@link Duration#ZERO} for no limit).*/- record Job(String jobRunId, String applicationId, String accountId, String region, String releaseLabel,- String executionRole, JobDriver.SparkSubmit sparkSubmit, Map<String, String> sparkDefaults,- Duration timeout) {+ record Job(String service, String runId, String accountId, String region, String releaseLabel,+ String imageTemplate, String dockerNetwork, String executionRole, Map<String, String> sparkDefaults,+ List<String> application, Duration timeout) {}/**@@ -54,19 +58,19 @@ interface SparkJobRunner {*/record Outcome(Integer exitCode, boolean timedOut, boolean stopped, String failure) {- static Outcome exitedWith(int exitCode) {+ public static Outcome exitedWith(int exitCode) {return new Outcome(exitCode, false, false, null);}- static Outcome outOfTime() {+ public static Outcome outOfTime() {return new Outcome(null, true, false, null);}- static Outcome halted() {+ public static Outcome halted() {return new Outcome(null, false, true, null);}- static Outcome couldNotStart(String failure) {+ public static Outcome couldNotStart(String failure) {return new Outcome(null, false, false, failure);}}diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.javaindex 99ed0117bf..ef75a7e80e 100644--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java@@ -241,7 +241,7 @@ class EmrServerlessJobRunServiceTest {}@Test- void theRunnerGetsTheRunsAccountReleaseRoleAndBatchTimeout() {+ void theRunnerGetsTheRunsAccountReleaseImageRoleApplicationAndBatchTimeout() {FakeRunner runner = new FakeRunner();EmrServerlessJobRunService service = dockerService(runner);String batch = service.startJobRun(applicationId, request(null, 30L)).getJobRunId();@@ -250,13 +250,15 @@ class EmrServerlessJobRunServiceTest {runNext();SparkJobRunner.Job job = runner.jobs.get(0);- assertEquals(batch, job.jobRunId());- assertEquals(applicationId, job.applicationId());+ assertEquals("emr-serverless", job.service());+ assertEquals(batch, job.runId());assertEquals(ACCOUNT_ID, job.accountId());assertEquals(REGION, job.region());assertEquals("emr-7.5.0", job.releaseLabel());+ assertNull(job.imageTemplate());+ assertNull(job.dockerNetwork());assertEquals(ROLE, job.executionRole());- assertEquals("s3://bucket/jobs/app.jar", job.sparkSubmit().getEntryPoint());+ assertEquals(List.of("s3://bucket/jobs/app.jar"), job.application());assertEquals(Duration.ofMinutes(30), job.timeout());assertEquals(Duration.ZERO, runner.jobs.get(1).timeout());}@@ -284,6 +286,18 @@ class EmrServerlessJobRunServiceTest {assertEquals("emr-spark-8.1.0", runner.jobs.get(5).releaseLabel());}+ @Test+ void aRunsSparkSubmitParametersComeBeforeItsEntryPointAndItsArguments() {+ JobDriver.SparkSubmit sparkSubmit = new JobDriver.SparkSubmit();+ sparkSubmit.setEntryPoint("s3://bucket/jobs/report.py");+ sparkSubmit.setEntryPointArguments(List.of("s3://bucket/input/", "s3://bucket/output/"));+ sparkSubmit.setSparkSubmitParameters("--conf spark.executor.cores=2 --py-files 's3://bucket/jobs/lib.zip'");++ assertEquals(List.of("--conf", "spark.executor.cores=2", "--py-files", "s3://bucket/jobs/lib.zip",+ "s3://bucket/jobs/report.py", "s3://bucket/input/", "s3://bucket/output/"),+ EmrServerlessJobRunService.application(sparkSubmit));+ }+private String applicationWithSparkDefaults(String releaseLabel) {RuntimeConfiguration defaults = new RuntimeConfiguration();defaults.setMember("classification", "spark-defaults");@@ -445,7 +459,7 @@ class EmrServerlessJobRunServiceTest {new EmrServerlessJobRunService(mock(StorageFactory.class), applications, iamService,new RegionResolver(REGION, ACCOUNT_ID), null, objectMapper, config, clock, sparkRunner).shutdown();- verify(sparkRunner, times(reconcile ? 1 : 0)).removeLeftoverDrivers();+ verify(sparkRunner, times(reconcile ? 1 : 0)).removeLeftoverDrivers("emr-serverless");}}@@ -468,19 +482,19 @@ class EmrServerlessJobRunServiceTest {if (progress.cancelled()) {return Outcome.halted();}- driversStarted.add(job.jobRunId());+ driversStarted.add(job.runId());progress.started();whileRunning.run();if (outcome.timedOut()) {progress.timedOut();whileTimingOut.run();}- return stopped.contains(job.jobRunId()) ? Outcome.halted() : outcome;+ return stopped.contains(job.runId()) ? Outcome.halted() : outcome;}@Override- public void stop(String jobRunId) {- stopped.add(jobRunId);+ public void stop(String runId) {+ stopped.add(runId);}}}diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.javaindex cd4abdc7ee..20b3906ef2 100644--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.java@@ -9,7 +9,6 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;-import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;import io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials;import io.github.hectorvent.floci.testing.MutableClock;import org.junit.jupiter.api.Test;@@ -23,11 +22,15 @@ import java.util.Optional;import java.util.concurrent.atomic.AtomicBoolean;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyMap;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.argThat;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.RETURNS_SELF;import static org.mockito.Mockito.atLeastOnce;import static org.mockito.Mockito.inOrder;import static org.mockito.Mockito.mock;@@ -46,15 +49,13 @@ class EmrServerlessSparkRunnerTest {@Testvoid sparkSubmitRunsLocallyOnFlocisS3ThenTheRunsOwnSettings() {- JobDriver.SparkSubmit sparkSubmit = new JobDriver.SparkSubmit();- sparkSubmit.setEntryPoint("s3://bucket/jobs/report.py");- sparkSubmit.setEntryPointArguments(List.of("s3://bucket/input/", "s3://bucket/output/"));- sparkSubmit.setSparkSubmitParameters("--conf spark.executor.cores=2 --py-files s3://bucket/jobs/lib.zip");+ List<String> application = List.of("--conf", "spark.executor.cores=2", "--py-files", "s3://bucket/jobs/lib.zip",+ "s3://bucket/jobs/report.py", "s3://bucket/input/", "s3://bucket/output/");Map<String, String> defaults = new LinkedHashMap<>();defaults.put("spark.executor.cores", "4");defaults.put("spark.sql.shuffle.partitions", "8");- List<String> arguments = EmrServerlessSparkRunner.sparkSubmitArguments(job(sparkSubmit, defaults), ENDPOINT);+ List<String> arguments = EmrServerlessSparkRunner.sparkSubmitArguments(job(application, defaults), ENDPOINT);assertEquals(List.of("--master", "local[*]","--conf", "spark.hadoop.fs.s3a.endpoint=http://floci:4566",@@ -73,11 +74,9 @@ class EmrServerlessSparkRunnerTest {}@Test- void anEntryPointWithoutParametersOrArgumentsFollowsFlocisSettings() {- JobDriver.SparkSubmit sparkSubmit = new JobDriver.SparkSubmit();- sparkSubmit.setEntryPoint("s3://bucket/jobs/app.jar");-- List<String> arguments = EmrServerlessSparkRunner.sparkSubmitArguments(job(sparkSubmit, Map.of()), ENDPOINT);+ void anApplicationWithoutOptionsOrArgumentsFollowsFlocisSettings() {+ List<String> arguments = EmrServerlessSparkRunner.sparkSubmitArguments(+ job(List.of("s3://bucket/jobs/app.jar"), Map.of()), ENDPOINT);assertEquals("--conf", arguments.get(arguments.size() - 3));assertEquals("spark.hadoop.aws.region=eu-west-1", arguments.get(arguments.size() - 2));@@ -99,13 +98,40 @@ class EmrServerlessSparkRunnerTest {@Testvoid theImageIsTheReleasesPublicImageUnlessATemplateNamesAnother() {+ assertEquals("public.ecr.aws/emr-serverless/spark/emr-7.5.0:latest",+ EmrServerlessSparkRunner.image(null, "emr-7.5.0"));+ assertEquals("public.ecr.aws/emr-serverless/spark4/emr-spark-8.1.0:latest",+ EmrServerlessSparkRunner.image(null, "emr-spark-8.1.0"));+ assertEquals("registry.internal/emr-serverless/spark/emr-7.5.0:20250425",+ EmrServerlessSparkRunner.image("registry.internal/emr-serverless/spark/%s:20250425", "emr-7.5.0"));+ }++ @Test+ void theDriverRunsInTheJobsImageAndNetworkNamedAndLabelledForItsService() {+ ContainerBuilder builder = mock(ContainerBuilder.class);+ ContainerBuilder.Builder spec = mock(ContainerBuilder.Builder.class, RETURNS_SELF);+ when(builder.newContainer(anyString())).thenReturn(spec);++ runner(builder).run(job(Duration.ZERO), progress);++ verify(builder).newContainer("public.ecr.aws/emr-serverless/spark/emr-7.5.0:latest");+ verify(spec).withName(ContainerStorageHelper.dockerName(config, "emr-serverless-00f0000000000001"));+ verify(spec).withDockerNetwork(Optional.of("floci-network"));+ verify(spec).withLabels(argThat(labels ->+ "emr-serverless".equals(labels.get(ContainerStorageHelper.SERVICE_LABEL))));+ }++ @Test+ void aTemplateThatNamesNoImageFailsTheRunWithItsReason() {EmrServerlessSparkRunner runner = runner();- assertEquals("public.ecr.aws/emr-serverless/spark/emr-7.5.0:latest", runner.image("emr-7.5.0"));- assertEquals("public.ecr.aws/emr-serverless/spark4/emr-spark-8.1.0:latest", runner.image("emr-spark-8.1.0"));- when(config.services().emrserverless().imageTemplate())- .thenReturn(Optional.of("registry.internal/emr-serverless/spark/%s:20250425"));- assertEquals("registry.internal/emr-serverless/spark/emr-7.5.0:20250425", runner.image("emr-7.5.0"));+ SparkJobRunner.Outcome outcome = runner.run(+ job("registry.internal/emr-serverless/spark/%d:latest", List.of("s3://bucket/jobs/app.jar"), Map.of(),+ Duration.ZERO), progress);++ assertNull(outcome.exitCode());+ assertNotNull(outcome.failure());+ verify(lifecycle, never()).createAndStart(any());}@Test@@ -120,7 +146,7 @@ class EmrServerlessSparkRunnerTest {when(docker.inspectContainerCmd("driver").exec().getState().getRunning()).thenAnswer(look -> {if (cancelled.compareAndSet(false, true)) {try {- runner.stop(job.jobRunId());+ runner.stop(job.runId());} catch (RuntimeException ignored) {// Dropped, as the executor drops what its task throws; the assertions below catch a lost cancel.}@@ -169,7 +195,7 @@ class EmrServerlessSparkRunnerTest {SparkJobRunner.Job job = job(Duration.ZERO);assertEquals(0, runner.run(job, progress).exitCode());- runner.stop(job.jobRunId());+ runner.stop(job.runId());// A stop kept past its run would halt the next run of this id before it started.assertEquals(0, runner.run(job, progress).exitCode());@@ -207,7 +233,7 @@ class EmrServerlessSparkRunnerTest {List<Container> drivers = List.of(driver("left-behind", "4566"), driver("another-deployments", "4567"));when(docker.listContainersCmd().withShowAll(true).withLabelFilter(anyMap()).exec()).thenReturn(drivers);- runner.removeLeftoverDrivers();+ runner.removeLeftoverDrivers("emr-serverless");verify(lifecycle).removeIfExists("left-behind");verify(lifecycle, never()).removeIfExists("another-deployments");@@ -215,14 +241,17 @@ class EmrServerlessSparkRunnerTest {// A runner whose driver container, "driver", starts at once and exits with code 0 unless a test says otherwise.private EmrServerlessSparkRunner runner() {- when(config.services().emrserverless().imageTemplate()).thenReturn(Optional.empty());+ return runner(mock(ContainerBuilder.class, RETURNS_DEEP_STUBS));+ }++ private EmrServerlessSparkRunner runner(ContainerBuilder builder) {when(lifecycle.getDockerClient()).thenReturn(docker);when(lifecycle.createAndStart(any())).thenReturn(new ContainerLifecycleManager.ContainerInfo("driver", Map.of()));when(docker.inspectContainerCmd("driver").exec().getState().getRunning()).thenReturn(false);when(docker.inspectContainerCmd("driver").exec().getState().getExitCodeLong()).thenReturn(0L);- return new EmrServerlessSparkRunner(mock(ContainerBuilder.class, RETURNS_DEEP_STUBS), lifecycle,- mock(ContainerLogStreamer.class), mock(LaunchedContainerAwsEnv.class),- mock(LambdaExecutionRoleCredentials.class), config, new MutableClock(), Duration.ZERO);+ return new EmrServerlessSparkRunner(builder, lifecycle, mock(ContainerLogStreamer.class),+ mock(LaunchedContainerAwsEnv.class), mock(LambdaExecutionRoleCredentials.class), config,+ new MutableClock(), Duration.ZERO);}private static Container driver(String id, String owner) {@@ -234,18 +263,22 @@ class EmrServerlessSparkRunnerTest {}private static SparkJobRunner.Job job(Duration timeout) {- JobDriver.SparkSubmit sparkSubmit = new JobDriver.SparkSubmit();- sparkSubmit.setEntryPoint("s3://bucket/jobs/app.jar");- return job(sparkSubmit, Map.of(), timeout);+ return job(List.of("s3://bucket/jobs/app.jar"), Map.of(), timeout);}- private static SparkJobRunner.Job job(JobDriver.SparkSubmit sparkSubmit, Map<String, String> sparkDefaults) {- return job(sparkSubmit, sparkDefaults, Duration.ZERO);+ private static SparkJobRunner.Job job(List<String> application, Map<String, String> sparkDefaults) {+ return job(application, sparkDefaults, Duration.ZERO);}- private static SparkJobRunner.Job job(JobDriver.SparkSubmit sparkSubmit, Map<String, String> sparkDefaults,+ private static SparkJobRunner.Job job(List<String> application, Map<String, String> sparkDefaults,Duration timeout) {- return new SparkJobRunner.Job("00f0000000000001", "00f0000000000000", "123456789012", "eu-west-1",- "emr-7.5.0", "arn:aws:iam::123456789012:role/job", sparkSubmit, sparkDefaults, timeout);+ return job(null, application, sparkDefaults, timeout);+ }++ private static SparkJobRunner.Job job(String imageTemplate, List<String> application,+ Map<String, String> sparkDefaults, Duration timeout) {+ return new SparkJobRunner.Job("emr-serverless", "00f0000000000001", "123456789012", "eu-west-1", "emr-7.5.0",+ imageTemplate, "floci-network", "arn:aws:iam::123456789012:role/job", sparkDefaults, application,+ timeout);}}