Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

refactor(emr-serverless): let classic EMR reuse the Spark runner (#6562)

Priamo Germosen8d5c135f0adc
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java
index 38f599bcc4..1910322e24 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunService.java
@@ -81,6 +81,8 @@ public class EmrServerlessJobRunService implements Resettable {
private static final String MODE_BATCH = "BATCH";
private static final String MODE_STREAMING = "STREAMING";
private static final String RUNNER_DOCKER = "docker";
+ // The service a container run's driver belongs to, which names it and labels it.
+ private static final String SERVICE = "emr-serverless";
private static final String SPARK_DEFAULTS = "spark-defaults";
// No source gives AWS's wording for these; they are Floci's.
static final String HIVE_NOT_RUN = "Floci runs Spark job runs only, and this is a Hive job run.";
@@ -122,6 +124,10 @@ public class EmrServerlessJobRunService implements Resettable {
// Null in the immediate runner mode.
private final SparkJobRunner sparkRunner;
private final Executor runExecutor;
+ // Where a container run's driver runs: image-template, with the release label for %s, and docker-network;
+ // null when not set.
+ private final String imageTemplate;
+ private final String dockerNetwork;
// Container runs a thread is running; one not here that has not ended was left by a restart.
private final Set<String> executing = new HashSet<>();
// Raised by each reset, so a run's thread from before it writes nothing back after the wipe.
@@ -136,10 +142,12 @@ public class EmrServerlessJobRunService implements Resettable {
applications, iamService, regionResolver, requestContext, objectMapper,
config.services().emrserverless().jobRunDurationSeconds(), clock,
RUNNER_DOCKER.equalsIgnoreCase(config.services().emrserverless().runnerMode()) ? sparkRunner : null,
- Executors.newCachedThreadPool(Thread.ofPlatform().daemon().name("emr-serverless-run-", 0).factory()));
+ Executors.newCachedThreadPool(Thread.ofPlatform().daemon().name("emr-serverless-run-", 0).factory()),
+ config.services().emrserverless().imageTemplate().orElse(null),
+ config.services().emrserverless().dockerNetwork().orElse(null));
// Before any run of this process starts: a driver an earlier one left belongs to a run read as FAILED.
if (this.sparkRunner != null && config.services().emrserverless().reconcileContainersOnStartup()) {
- sparkRunner.removeLeftoverDrivers();
+ sparkRunner.removeLeftoverDrivers(SERVICE);
}
}
@@ -154,6 +162,16 @@ public class EmrServerlessJobRunService implements Resettable {
IamService iamService, RegionResolver regionResolver, RequestContext requestContext,
ObjectMapper objectMapper, int runDurationSeconds, Clock clock,
SparkJobRunner sparkRunner, Executor runExecutor) {
+ this(store, applications, iamService, regionResolver, requestContext, objectMapper, runDurationSeconds, clock,
+ sparkRunner, runExecutor, null, null);
+ }
+
+ private EmrServerlessJobRunService(AccountAwareStorageBackend<JobRunRecord> store,
+ EmrServerlessService applications, IamService iamService,
+ RegionResolver regionResolver, RequestContext requestContext,
+ ObjectMapper objectMapper, int runDurationSeconds, Clock clock,
+ SparkJobRunner sparkRunner, Executor runExecutor,
+ String imageTemplate, String dockerNetwork) {
if (runDurationSeconds < 0) {
throw new IllegalArgumentException(
"floci.services.emrserverless.job-run-duration-seconds must not be negative: " + runDurationSeconds);
@@ -168,6 +186,8 @@ public class EmrServerlessJobRunService implements Resettable {
this.clock = clock;
this.sparkRunner = sparkRunner;
this.runExecutor = runExecutor;
+ this.imageTemplate = imageTemplate;
+ this.dockerNetwork = dockerNetwork;
}
@PreDestroy
@@ -437,9 +457,9 @@ public class EmrServerlessJobRunService implements Resettable {
Long minutes = run.getExecutionTimeoutMinutes();
Duration timeout = MODE_STREAMING.equals(run.getMode()) || minutes == null || minutes == 0
? Duration.ZERO : Duration.ofMinutes(minutes);
- SparkJobRunner.Job job = new SparkJobRunner.Job(run.getJobRunId(), run.getApplicationId(), arn.accountId(),
- arn.region(), run.getReleaseLabel(), run.getExecutionRole(), run.getJobDriver().getSparkSubmit(),
- sparkDefaults(app, run), timeout);
+ SparkJobRunner.Job job = new SparkJobRunner.Job(SERVICE, run.getJobRunId(), arn.accountId(), arn.region(),
+ run.getReleaseLabel(), imageTemplate, dockerNetwork, run.getExecutionRole(), sparkDefaults(app, run),
+ application(run.getJobDriver().getSparkSubmit()), timeout);
executing.add(run.getJobRunId());
long launchedIn = generation;
runExecutor.execute(() -> execute(job, launchedIn));
@@ -450,11 +470,26 @@ public class EmrServerlessJobRunService implements Resettable {
try {
outcome = sparkRunner.run(job, new RunProgress(job, launchedIn));
} catch (RuntimeException e) {
- LOG.warnv(e, "EMR Serverless job run {0} could not run", job.jobRunId());
+ LOG.warnv(e, "EMR Serverless job run {0} could not run", job.runId());
outcome = SparkJobRunner.Outcome.couldNotStart(e.getMessage());
}
SparkJobRunner.Outcome ended = outcome;
- RequestScopes.runAs(job.accountId(), job.region(), () -> ended(job.jobRunId(), ended, launchedIn));
+ RequestScopes.runAs(job.accountId(), job.region(), () -> ended(job.runId(), ended, launchedIn));
+ }
+
+ /**
+ * What follows Floci's own settings on a run's {@code spark-submit} command line: its
+ * {@code sparkSubmitParameters}, split as a shell would split them, then its entry point and the
+ * entry point's arguments.
+ */
+ static List<String> application(JobDriver.SparkSubmit sparkSubmit) {
+ List<String> application = new ArrayList<>(
+ EmrServerlessSparkRunner.splitParameters(sparkSubmit.getSparkSubmitParameters()));
+ application.add(sparkSubmit.getEntryPoint());
+ if (sparkSubmit.getEntryPointArguments() != null) {
+ application.addAll(sparkSubmit.getEntryPointArguments());
+ }
+ return application;
}
/**
@@ -813,18 +848,18 @@ public class EmrServerlessJobRunService implements Resettable {
@Override
public void started() {
- RequestScopes.runAs(job.accountId(), job.region(), () -> driverStarted(job.jobRunId(), launchedIn));
+ RequestScopes.runAs(job.accountId(), job.region(), () -> driverStarted(job.runId(), launchedIn));
}
@Override
public void timedOut() {
- RequestScopes.runAs(job.accountId(), job.region(), () -> driverTimedOut(job.jobRunId(), launchedIn));
+ RequestScopes.runAs(job.accountId(), job.region(), () -> driverTimedOut(job.runId(), launchedIn));
}
@Override
public boolean cancelled() {
return !RequestScopes.callAs(job.accountId(), job.region(),
- () -> waitingToStart(job.jobRunId(), launchedIn));
+ () -> waitingToStart(job.runId(), launchedIn));
}
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.java
index 389cfc891d..a3f627e2f1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.java
@@ -10,7 +10,6 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
-import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;
import io.github.hectorvent.floci.services.iam.model.SessionCreds;
import io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials;
import jakarta.enterprise.context.ApplicationScoped;
@@ -30,12 +29,12 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
/**
- * Runs a Spark job run's {@code spark-submit} in a container of the release's EMR Serverless image.
- * The image's own entrypoint hands over to a script EMR Serverless supplies at run time, which the
- * public image lacks, so the runner starts {@code spark-submit} itself, in local mode: the container
- * holds the driver and its executors. The container reaches Floci as the other containers Floci
- * launches do, with the run's execution role as its credentials, and S3A, which takes {@code s3://}
- * on every release, is pointed at Floci's S3.
+ * Runs a Spark application's {@code spark-submit} in a container of an EMR Serverless image, for an EMR
+ * Serverless job run or an EMR step. The image's own entrypoint hands over to a script EMR Serverless
+ * supplies at run time, which the public image lacks, so the runner starts {@code spark-submit} itself,
+ * in local mode: the container holds the driver and its executors. The container reaches Floci as the
+ * other containers Floci launches do, with the run's execution role as its credentials, and S3A, which
+ * takes {@code s3://} on every release, is pointed at Floci's S3.
*/
@ApplicationScoped
public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardown {
@@ -49,7 +48,6 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo
"public.ecr.aws/emr-serverless/spark4/%s:latest"; // partition-literal: ECR Public's registry host, where AWS publishes the base images; image-template overrides it
// "The default behavior for shutdown is to have no grace period": immediate termination.
private static final int STOP_TIMEOUT_SECONDS = 0;
- private static final String SERVICE = "emr-serverless";
// The characters a backslash escapes inside double quotes; before any other it is kept.
private static final String DOUBLE_QUOTE_ESCAPES = "\"\\$`\n";
// How often a running driver is looked at, as Batch's runner looks at its jobs.
@@ -92,7 +90,7 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo
@Override
public Outcome run(Job job, Progress progress) {
Launch launch = new Launch();
- launches.put(job.jobRunId(), launch);
+ launches.put(job.runId(), launch);
Optional<SessionCreds> credentials = Optional.empty();
Closeable logHandle = null;
try {
@@ -100,18 +98,18 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo
if (progress.cancelled()) {
return Outcome.halted();
}
- credentials = roleCredentials.forRole(job.executionRole(), job.accountId(), job.jobRunId());
+ credentials = roleCredentials.forRole(job.executionRole(), job.accountId(), job.runId());
Map<String, String> labels = new LinkedHashMap<>(ContainerStorageHelper.resourceIdentityLabels(
- SERVICE, job.jobRunId(), job.accountId(), job.region()));
+ job.service(), job.runId(), job.accountId(), job.region()));
// The deployment that started the driver, which removeLeftoverDrivers() goes by.
labels.put(ContainerStorageHelper.OWNER_LABEL, ContainerStorageHelper.ownerIdentity(config));
String endpoint = awsEnv.flociEndpoint();
- ContainerSpec spec = containerBuilder.newContainer(image(job.releaseLabel()))
- .withName(ContainerStorageHelper.dockerName(config, SERVICE + "-" + job.jobRunId()))
+ ContainerSpec spec = containerBuilder.newContainer(image(job.imageTemplate(), job.releaseLabel()))
+ .withName(ContainerStorageHelper.dockerName(config, job.service() + "-" + job.runId()))
.withEntrypoint(List.of(SPARK_SUBMIT))
.withCmd(sparkSubmitArguments(job, endpoint))
.withEnv(awsEnv.sdkBaselineEnv(job.region(), Optional.empty(), credentials, job.accountId()))
- .withDockerNetwork(config.services().emrserverless().dockerNetwork())
+ .withDockerNetwork(Optional.ofNullable(job.dockerNetwork()))
.withHostDockerInternalOnLinux()
.withEmbeddedDns()
.withLogRotation()
@@ -123,7 +121,7 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo
if (launch.stopRequested || progress.cancelled()) {
return Outcome.halted();
}
- logHandle = logStreamer.attachConsoleOnly(containerId, SERVICE + ":" + job.jobRunId());
+ logHandle = logStreamer.attachConsoleOnly(containerId, job.service() + ":" + job.runId());
progress.started();
return awaitEnd(containerId, job, launch, progress);
} catch (InterruptedException e) {
@@ -135,10 +133,10 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo
// A reset removed the driver while it was being watched.
return Outcome.halted();
}
- LOG.warnv(e, "EMR Serverless job run {0} could not run", job.jobRunId());
+ LOG.warnv(e, "Spark run {0} of {1} could not run", job.runId(), job.service());
return Outcome.couldNotStart(e.getMessage());
} finally {
- launches.remove(job.jobRunId(), launch);
+ launches.remove(job.runId(), launch);
String unclaimed = launch.container.getAndSet(null);
if (unclaimed != null) {
lifecycleManager.stopAndRemove(unclaimed, logHandle, STOP_TIMEOUT_SECONDS);
@@ -151,8 +149,8 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo
}
@Override
- public void stop(String jobRunId) {
- Launch launch = launches.get(jobRunId);
+ public void stop(String runId) {
+ Launch launch = launches.get(runId);
if (launch != null) {
// run() reads it once the driver's container is up and at each look at the driver after, and
// removes the driver itself: a stop makes no Docker call, so no Docker error can lose it.
@@ -176,16 +174,17 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo
}
/**
- * Removes the drivers an earlier Floci process left running, which no thread follows any more: their
- * runs are {@code FAILED} once read, so nothing would ever stop them. The job run service calls this
- * before it hands a run to {@link #run}, so every driver of this deployment it finds is a leftover.
+ * Removes the drivers of {@code service}'s runs an earlier Floci process left running, which no thread
+ * follows any more: their runs are {@code FAILED} once read, so nothing would ever stop them. The
+ * service calls this before it hands a run to {@link #run}, so every driver of this deployment it
+ * finds is a leftover.
*/
- public void removeLeftoverDrivers() {
+ public void removeLeftoverDrivers(String service) {
String owner = ContainerStorageHelper.ownerIdentity(config);
ContainerStorageHelper.LabelAliases aliases = ContainerStorageHelper.CONTAINER_LABEL_ALIASES;
try {
List<Container> containers = aliases.listByLabels(
- Map.of(ContainerStorageHelper.SERVICE_LABEL, SERVICE, ContainerStorageHelper.OWNER_LABEL, owner),
+ Map.of(ContainerStorageHelper.SERVICE_LABEL, service, ContainerStorageHelper.OWNER_LABEL, owner),
filter -> lifecycleManager.getDockerClient().listContainersCmd()
.withShowAll(true)
.withLabelFilter(filter)
@@ -196,26 +195,30 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo
if (aliases.consistent(container.getId(), labels)
&& aliases.matches(labels, ContainerStorageHelper.OWNER_LABEL, owner)) {
lifecycleManager.removeIfExists(container.getId());
- LOG.infov("Removed EMR Serverless driver {0}, left running by an earlier Floci process",
+ LOG.infov("Removed {0} Spark driver {1}, left running by an earlier Floci process", service,
container.getId());
}
}
} catch (RuntimeException e) {
- LOG.warnv("Could not remove the EMR Serverless drivers an earlier Floci process left: {0}",
+ LOG.warnv("Could not remove the {0} Spark drivers an earlier Floci process left: {1}", service,
e.getMessage());
}
}
- String image(String releaseLabel) {
- return config.services().emrserverless().imageTemplate()
- .orElse(releaseLabel.startsWith("emr-spark-") ? SPARK_RUNTIME_DEFAULT_IMAGE : DEFAULT_IMAGE)
- .formatted(releaseLabel);
+ /**
+ * The image a release's runs use: {@code imageTemplate}, with {@code %s} standing for the release
+ * label, or, when it is null, the release's public EMR Serverless image.
+ */
+ static String image(String imageTemplate, String releaseLabel) {
+ String template = imageTemplate != null ? imageTemplate
+ : releaseLabel.startsWith("emr-spark-") ? SPARK_RUNTIME_DEFAULT_IMAGE : DEFAULT_IMAGE;
+ return template.formatted(releaseLabel);
}
/**
* {@code spark-submit}'s arguments: local mode, S3A on Floci's S3 for {@code s3://} and {@code s3a://}
- * paths, then the run's {@code spark-defaults} and its own {@code sparkSubmitParameters}, which win
- * over what comes before them, then the entry point and its arguments. Before release 7.10.0
+ * paths, then the run's {@code spark-defaults} and its own options, which win over what comes before
+ * them, then its application and that application's arguments. Before release 7.10.0
* {@code s3://} is EMRFS, which ignores the endpoint setting, so it is switched to S3A. S3A writes
* are committed through commit data the writer keeps in memory, without which EMR's committer finds
* none and a write is silently lost.
@@ -231,22 +234,18 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo
conf(arguments, "spark.hadoop.aws.glue.endpoint", flociEndpoint);
conf(arguments, "spark.hadoop.aws.region", job.region());
job.sparkDefaults().forEach((key, value) -> conf(arguments, key, value));
- JobDriver.SparkSubmit sparkSubmit = job.sparkSubmit();
- arguments.addAll(splitParameters(sparkSubmit.getSparkSubmitParameters()));
- arguments.add(sparkSubmit.getEntryPoint());
- if (sparkSubmit.getEntryPointArguments() != null) {
- arguments.addAll(sparkSubmit.getEntryPointArguments());
- }
+ arguments.addAll(job.application());
return arguments;
}
/**
- * {@code sparkSubmitParameters} as a POSIX shell would split them: on whitespace, except inside single
+ * {@code spark-submit} parameters given as one string, as EMR Serverless's
+ * {@code sparkSubmitParameters}, split as a POSIX shell would split them: on whitespace, except inside single
* or double quotes. Outside quotes a backslash takes the next character as it is; inside double quotes
* only a {@code "}, {@code \}, {@code $}, {@code `} or newline, and before any other character it is
* kept. A backslash before a newline continues the line, and both go.
*/
- static List<String> splitParameters(String parameters) {
+ public static List<String> splitParameters(String parameters) {
List<String> words = new ArrayList<>();
if (parameters == null) {
return words;
@@ -321,7 +320,7 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo
return Outcome.exitedWith(exitCode != null ? exitCode.intValue() : -1);
}
if (deadline != null && !clock.instant().isBefore(deadline)) {
- LOG.infov("EMR Serverless job run {0} passed its execution timeout of {1}", job.jobRunId(),
+ LOG.infov("Spark run {0} of {1} passed its execution timeout of {2}", job.runId(), job.service(),
job.timeout());
progress.timedOut();
return Outcome.outOfTime();
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/SparkJobRunner.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/SparkJobRunner.java
index 45e2543afd..facd88ff66 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/SparkJobRunner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/SparkJobRunner.java
@@ -1,16 +1,16 @@
package io.github.hectorvent.floci.services.emrserverless;
-import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;
-
import java.time.Duration;
+import java.util.List;
import java.util.Map;
/**
- * Runs a Spark job run to its end, in the {@code docker} runner mode. The job run service holds the
- * run's state and calls this off the request thread; {@link EmrServerlessSparkRunner} is the Docker
- * implementation, and a test supplies its own.
+ * Runs a Spark application to its end, in the EMR services' {@code docker} runner mode: an EMR
+ * Serverless job run, or a step of an EMR cluster. The service holds the run's state and calls this
+ * off the request thread; {@link EmrServerlessSparkRunner} is the Docker implementation, and a test
+ * supplies its own.
*/
-interface SparkJobRunner {
+public interface SparkJobRunner {
/**
* Runs {@code job} and returns how it ended, telling {@code progress} once its Spark driver runs and
@@ -23,9 +23,9 @@ interface SparkJobRunner {
* Stops the run's driver, now or as soon as it starts, so {@link #run} returns {@link Outcome#halted};
* a run {@link #run} has already returned from is left as it is.
*/
- void stop(String jobRunId);
+ void stop(String runId);
- /** What a run tells the job run service, and asks it, while it goes. */
+ /** What a run tells the service, and asks it, while it goes. */
interface Progress {
/** The run's Spark driver runs. */
@@ -39,13 +39,17 @@ interface SparkJobRunner {
}
/**
- * What a run needs: its identity, the application's release, the run's execution role, its Spark
- * submission, the {@code spark-defaults} properties that apply to it, and how long it may run
+ * What a run needs: the service it belongs to and its id there, which name its container; its account
+ * and Region; its release, and the image template and Docker network to run it in (a null template is
+ * the release's public EMR Serverless image, a null network the global default); the execution role
+ * whose credentials it gets; the {@code spark-defaults} properties that apply to it;
+ * {@code application}, what follows Floci's own settings on {@code spark-submit}'s command line (the
+ * run's options, then its application and that application's arguments); and how long it may run
* ({@link Duration#ZERO} for no limit).
*/
- record Job(String jobRunId, String applicationId, String accountId, String region, String releaseLabel,
- String executionRole, JobDriver.SparkSubmit sparkSubmit, Map<String, String> sparkDefaults,
- Duration timeout) {
+ record Job(String service, String runId, String accountId, String region, String releaseLabel,
+ String imageTemplate, String dockerNetwork, String executionRole, Map<String, String> sparkDefaults,
+ List<String> application, Duration timeout) {
}
/**
@@ -54,19 +58,19 @@ interface SparkJobRunner {
*/
record Outcome(Integer exitCode, boolean timedOut, boolean stopped, String failure) {
- static Outcome exitedWith(int exitCode) {
+ public static Outcome exitedWith(int exitCode) {
return new Outcome(exitCode, false, false, null);
}
- static Outcome outOfTime() {
+ public static Outcome outOfTime() {
return new Outcome(null, true, false, null);
}
- static Outcome halted() {
+ public static Outcome halted() {
return new Outcome(null, false, true, null);
}
- static Outcome couldNotStart(String failure) {
+ public static Outcome couldNotStart(String failure) {
return new Outcome(null, false, false, failure);
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java
index 99ed0117bf..ef75a7e80e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessJobRunServiceTest.java
@@ -241,7 +241,7 @@ class EmrServerlessJobRunServiceTest {
}
@Test
- void theRunnerGetsTheRunsAccountReleaseRoleAndBatchTimeout() {
+ void theRunnerGetsTheRunsAccountReleaseImageRoleApplicationAndBatchTimeout() {
FakeRunner runner = new FakeRunner();
EmrServerlessJobRunService service = dockerService(runner);
String batch = service.startJobRun(applicationId, request(null, 30L)).getJobRunId();
@@ -250,13 +250,15 @@ class EmrServerlessJobRunServiceTest {
runNext();
SparkJobRunner.Job job = runner.jobs.get(0);
- assertEquals(batch, job.jobRunId());
- assertEquals(applicationId, job.applicationId());
+ assertEquals("emr-serverless", job.service());
+ assertEquals(batch, job.runId());
assertEquals(ACCOUNT_ID, job.accountId());
assertEquals(REGION, job.region());
assertEquals("emr-7.5.0", job.releaseLabel());
+ assertNull(job.imageTemplate());
+ assertNull(job.dockerNetwork());
assertEquals(ROLE, job.executionRole());
- assertEquals("s3://bucket/jobs/app.jar", job.sparkSubmit().getEntryPoint());
+ assertEquals(List.of("s3://bucket/jobs/app.jar"), job.application());
assertEquals(Duration.ofMinutes(30), job.timeout());
assertEquals(Duration.ZERO, runner.jobs.get(1).timeout());
}
@@ -284,6 +286,18 @@ class EmrServerlessJobRunServiceTest {
assertEquals("emr-spark-8.1.0", runner.jobs.get(5).releaseLabel());
}
+ @Test
+ void aRunsSparkSubmitParametersComeBeforeItsEntryPointAndItsArguments() {
+ JobDriver.SparkSubmit sparkSubmit = new JobDriver.SparkSubmit();
+ sparkSubmit.setEntryPoint("s3://bucket/jobs/report.py");
+ sparkSubmit.setEntryPointArguments(List.of("s3://bucket/input/", "s3://bucket/output/"));
+ sparkSubmit.setSparkSubmitParameters("--conf spark.executor.cores=2 --py-files 's3://bucket/jobs/lib.zip'");
+
+ assertEquals(List.of("--conf", "spark.executor.cores=2", "--py-files", "s3://bucket/jobs/lib.zip",
+ "s3://bucket/jobs/report.py", "s3://bucket/input/", "s3://bucket/output/"),
+ EmrServerlessJobRunService.application(sparkSubmit));
+ }
+
private String applicationWithSparkDefaults(String releaseLabel) {
RuntimeConfiguration defaults = new RuntimeConfiguration();
defaults.setMember("classification", "spark-defaults");
@@ -445,7 +459,7 @@ class EmrServerlessJobRunServiceTest {
new EmrServerlessJobRunService(mock(StorageFactory.class), applications, iamService,
new RegionResolver(REGION, ACCOUNT_ID), null, objectMapper, config, clock, sparkRunner).shutdown();
- verify(sparkRunner, times(reconcile ? 1 : 0)).removeLeftoverDrivers();
+ verify(sparkRunner, times(reconcile ? 1 : 0)).removeLeftoverDrivers("emr-serverless");
}
}
@@ -468,19 +482,19 @@ class EmrServerlessJobRunServiceTest {
if (progress.cancelled()) {
return Outcome.halted();
}
- driversStarted.add(job.jobRunId());
+ driversStarted.add(job.runId());
progress.started();
whileRunning.run();
if (outcome.timedOut()) {
progress.timedOut();
whileTimingOut.run();
}
- return stopped.contains(job.jobRunId()) ? Outcome.halted() : outcome;
+ return stopped.contains(job.runId()) ? Outcome.halted() : outcome;
}
@Override
- public void stop(String jobRunId) {
- stopped.add(jobRunId);
+ public void stop(String runId) {
+ stopped.add(runId);
}
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.java
index cd4abdc7ee..20b3906ef2 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.java
@@ -9,7 +9,6 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
-import io.github.hectorvent.floci.services.emrserverless.model.JobDriver;
import io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials;
import io.github.hectorvent.floci.testing.MutableClock;
import org.junit.jupiter.api.Test;
@@ -23,11 +22,15 @@ import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.RETURNS_SELF;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
@@ -46,15 +49,13 @@ class EmrServerlessSparkRunnerTest {
@Test
void sparkSubmitRunsLocallyOnFlocisS3ThenTheRunsOwnSettings() {
- JobDriver.SparkSubmit sparkSubmit = new JobDriver.SparkSubmit();
- sparkSubmit.setEntryPoint("s3://bucket/jobs/report.py");
- sparkSubmit.setEntryPointArguments(List.of("s3://bucket/input/", "s3://bucket/output/"));
- sparkSubmit.setSparkSubmitParameters("--conf spark.executor.cores=2 --py-files s3://bucket/jobs/lib.zip");
+ List<String> application = List.of("--conf", "spark.executor.cores=2", "--py-files", "s3://bucket/jobs/lib.zip",
+ "s3://bucket/jobs/report.py", "s3://bucket/input/", "s3://bucket/output/");
Map<String, String> defaults = new LinkedHashMap<>();
defaults.put("spark.executor.cores", "4");
defaults.put("spark.sql.shuffle.partitions", "8");
- List<String> arguments = EmrServerlessSparkRunner.sparkSubmitArguments(job(sparkSubmit, defaults), ENDPOINT);
+ List<String> arguments = EmrServerlessSparkRunner.sparkSubmitArguments(job(application, defaults), ENDPOINT);
assertEquals(List.of("--master", "local[*]",
"--conf", "spark.hadoop.fs.s3a.endpoint=http://floci:4566",
@@ -73,11 +74,9 @@ class EmrServerlessSparkRunnerTest {
}
@Test
- void anEntryPointWithoutParametersOrArgumentsFollowsFlocisSettings() {
- JobDriver.SparkSubmit sparkSubmit = new JobDriver.SparkSubmit();
- sparkSubmit.setEntryPoint("s3://bucket/jobs/app.jar");
-
- List<String> arguments = EmrServerlessSparkRunner.sparkSubmitArguments(job(sparkSubmit, Map.of()), ENDPOINT);
+ void anApplicationWithoutOptionsOrArgumentsFollowsFlocisSettings() {
+ List<String> arguments = EmrServerlessSparkRunner.sparkSubmitArguments(
+ job(List.of("s3://bucket/jobs/app.jar"), Map.of()), ENDPOINT);
assertEquals("--conf", arguments.get(arguments.size() - 3));
assertEquals("spark.hadoop.aws.region=eu-west-1", arguments.get(arguments.size() - 2));
@@ -99,13 +98,40 @@ class EmrServerlessSparkRunnerTest {
@Test
void theImageIsTheReleasesPublicImageUnlessATemplateNamesAnother() {
+ assertEquals("public.ecr.aws/emr-serverless/spark/emr-7.5.0:latest",
+ EmrServerlessSparkRunner.image(null, "emr-7.5.0"));
+ assertEquals("public.ecr.aws/emr-serverless/spark4/emr-spark-8.1.0:latest",
+ EmrServerlessSparkRunner.image(null, "emr-spark-8.1.0"));
+ assertEquals("registry.internal/emr-serverless/spark/emr-7.5.0:20250425",
+ EmrServerlessSparkRunner.image("registry.internal/emr-serverless/spark/%s:20250425", "emr-7.5.0"));
+ }
+
+ @Test
+ void theDriverRunsInTheJobsImageAndNetworkNamedAndLabelledForItsService() {
+ ContainerBuilder builder = mock(ContainerBuilder.class);
+ ContainerBuilder.Builder spec = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
+ when(builder.newContainer(anyString())).thenReturn(spec);
+
+ runner(builder).run(job(Duration.ZERO), progress);
+
+ verify(builder).newContainer("public.ecr.aws/emr-serverless/spark/emr-7.5.0:latest");
+ verify(spec).withName(ContainerStorageHelper.dockerName(config, "emr-serverless-00f0000000000001"));
+ verify(spec).withDockerNetwork(Optional.of("floci-network"));
+ verify(spec).withLabels(argThat(labels ->
+ "emr-serverless".equals(labels.get(ContainerStorageHelper.SERVICE_LABEL))));
+ }
+
+ @Test
+ void aTemplateThatNamesNoImageFailsTheRunWithItsReason() {
EmrServerlessSparkRunner runner = runner();
- assertEquals("public.ecr.aws/emr-serverless/spark/emr-7.5.0:latest", runner.image("emr-7.5.0"));
- assertEquals("public.ecr.aws/emr-serverless/spark4/emr-spark-8.1.0:latest", runner.image("emr-spark-8.1.0"));
- when(config.services().emrserverless().imageTemplate())
- .thenReturn(Optional.of("registry.internal/emr-serverless/spark/%s:20250425"));
- assertEquals("registry.internal/emr-serverless/spark/emr-7.5.0:20250425", runner.image("emr-7.5.0"));
+ SparkJobRunner.Outcome outcome = runner.run(
+ job("registry.internal/emr-serverless/spark/%d:latest", List.of("s3://bucket/jobs/app.jar"), Map.of(),
+ Duration.ZERO), progress);
+
+ assertNull(outcome.exitCode());
+ assertNotNull(outcome.failure());
+ verify(lifecycle, never()).createAndStart(any());
}
@Test
@@ -120,7 +146,7 @@ class EmrServerlessSparkRunnerTest {
when(docker.inspectContainerCmd("driver").exec().getState().getRunning()).thenAnswer(look -> {
if (cancelled.compareAndSet(false, true)) {
try {
- runner.stop(job.jobRunId());
+ runner.stop(job.runId());
} catch (RuntimeException ignored) {
// Dropped, as the executor drops what its task throws; the assertions below catch a lost cancel.
}
@@ -169,7 +195,7 @@ class EmrServerlessSparkRunnerTest {
SparkJobRunner.Job job = job(Duration.ZERO);
assertEquals(0, runner.run(job, progress).exitCode());
- runner.stop(job.jobRunId());
+ runner.stop(job.runId());
// A stop kept past its run would halt the next run of this id before it started.
assertEquals(0, runner.run(job, progress).exitCode());
@@ -207,7 +233,7 @@ class EmrServerlessSparkRunnerTest {
List<Container> drivers = List.of(driver("left-behind", "4566"), driver("another-deployments", "4567"));
when(docker.listContainersCmd().withShowAll(true).withLabelFilter(anyMap()).exec()).thenReturn(drivers);
- runner.removeLeftoverDrivers();
+ runner.removeLeftoverDrivers("emr-serverless");
verify(lifecycle).removeIfExists("left-behind");
verify(lifecycle, never()).removeIfExists("another-deployments");
@@ -215,14 +241,17 @@ class EmrServerlessSparkRunnerTest {
// A runner whose driver container, "driver", starts at once and exits with code 0 unless a test says otherwise.
private EmrServerlessSparkRunner runner() {
- when(config.services().emrserverless().imageTemplate()).thenReturn(Optional.empty());
+ return runner(mock(ContainerBuilder.class, RETURNS_DEEP_STUBS));
+ }
+
+ private EmrServerlessSparkRunner runner(ContainerBuilder builder) {
when(lifecycle.getDockerClient()).thenReturn(docker);
when(lifecycle.createAndStart(any())).thenReturn(new ContainerLifecycleManager.ContainerInfo("driver", Map.of()));
when(docker.inspectContainerCmd("driver").exec().getState().getRunning()).thenReturn(false);
when(docker.inspectContainerCmd("driver").exec().getState().getExitCodeLong()).thenReturn(0L);
- return new EmrServerlessSparkRunner(mock(ContainerBuilder.class, RETURNS_DEEP_STUBS), lifecycle,
- mock(ContainerLogStreamer.class), mock(LaunchedContainerAwsEnv.class),
- mock(LambdaExecutionRoleCredentials.class), config, new MutableClock(), Duration.ZERO);
+ return new EmrServerlessSparkRunner(builder, lifecycle, mock(ContainerLogStreamer.class),
+ mock(LaunchedContainerAwsEnv.class), mock(LambdaExecutionRoleCredentials.class), config,
+ new MutableClock(), Duration.ZERO);
}
private static Container driver(String id, String owner) {
@@ -234,18 +263,22 @@ class EmrServerlessSparkRunnerTest {
}
private static SparkJobRunner.Job job(Duration timeout) {
- JobDriver.SparkSubmit sparkSubmit = new JobDriver.SparkSubmit();
- sparkSubmit.setEntryPoint("s3://bucket/jobs/app.jar");
- return job(sparkSubmit, Map.of(), timeout);
+ return job(List.of("s3://bucket/jobs/app.jar"), Map.of(), timeout);
}
- private static SparkJobRunner.Job job(JobDriver.SparkSubmit sparkSubmit, Map<String, String> sparkDefaults) {
- return job(sparkSubmit, sparkDefaults, Duration.ZERO);
+ private static SparkJobRunner.Job job(List<String> application, Map<String, String> sparkDefaults) {
+ return job(application, sparkDefaults, Duration.ZERO);
}
- private static SparkJobRunner.Job job(JobDriver.SparkSubmit sparkSubmit, Map<String, String> sparkDefaults,
+ private static SparkJobRunner.Job job(List<String> application, Map<String, String> sparkDefaults,
Duration timeout) {
- return new SparkJobRunner.Job("00f0000000000001", "00f0000000000000", "123456789012", "eu-west-1",
- "emr-7.5.0", "arn:aws:iam::123456789012:role/job", sparkSubmit, sparkDefaults, timeout);
+ return job(null, application, sparkDefaults, timeout);
+ }
+
+ private static SparkJobRunner.Job job(String imageTemplate, List<String> application,
+ Map<String, String> sparkDefaults, Duration timeout) {
+ return new SparkJobRunner.Job("emr-serverless", "00f0000000000001", "123456789012", "eu-west-1", "emr-7.5.0",
+ imageTemplate, "floci-network", "arn:aws:iam::123456789012:role/job", sparkDefaults, application,
+ timeout);
}
}