Light, fluffy, and always free - The AWS Local Emulator alternative
fix(emr-serverless): end a cancelled run when Docker refuses the stop (#6510)
Priamo Germosen1ea49d9432fc
diff --git a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.java b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.javaindex eee225b28d..389cfc891d 100644--- a/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.java+++ b/src/main/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunner.java@@ -1,8 +1,6 @@package io.github.hectorvent.floci.services.emrserverless;import com.github.dockerjava.api.command.InspectContainerResponse;-import com.github.dockerjava.api.exception.NotFoundException;-import com.github.dockerjava.api.exception.NotModifiedException;import com.github.dockerjava.api.model.Container;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.ContainerTeardown;@@ -121,17 +119,13 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo.build();String containerId = lifecycleManager.createAndStart(spec).containerId();launch.container.set(containerId);- // A stop, a reset or a shutdown while the image was pulled found no container to stop.+ // A stop, a reset or a shutdown while the image was pulled is read now the driver has a container.if (launch.stopRequested || progress.cancelled()) {return Outcome.halted();}logHandle = logStreamer.attachConsoleOnly(containerId, SERVICE + ":" + job.jobRunId());progress.started();- Integer exitCode = awaitExit(containerId, job, progress);- if (launch.stopRequested) {- return Outcome.halted();- }- return exitCode == null ? Outcome.outOfTime() : Outcome.exitedWith(exitCode);+ return awaitEnd(containerId, job, launch, progress);} catch (InterruptedException e) {// Floci is shutting down: its teardown stops the driver.Thread.currentThread().interrupt();@@ -159,21 +153,10 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo@Overridepublic void stop(String jobRunId) {Launch launch = launches.get(jobRunId);- if (launch == null) {- return;- }- launch.stopRequested = true;- String containerId = launch.container.get();- if (containerId == null) {- // Still starting: run() reads the request once its container is up.- return;- }- // run() sees it stopped at its next look, and removes it.- try {- lifecycleManager.getDockerClient().stopContainerCmd(containerId).withTimeout(STOP_TIMEOUT_SECONDS).exec();- } catch (NotFoundException | NotModifiedException alreadyEnded) {- // Gone already (404), or exited and not yet removed (304).- LOG.debugv("EMR Serverless job run {0} had ended before it was stopped", jobRunId);+ if (launch != null) {+ // run() reads it once the driver's container is up and at each look at the driver after, and+ // removes the driver itself: a stop makes no Docker call, so no Docker error can lose it.+ launch.stopRequested = true;}}@@ -318,27 +301,34 @@ public class EmrServerlessSparkRunner implements SparkJobRunner, ContainerTeardo}/**- * The driver's exit code, or null once the run's timeout has passed with the driver still running,- * which {@code progress} hears before the driver is stopped. Each look at the container is one short- * control-plane call, so a run holds no Docker connection while its driver runs.+ * How the running driver ends: with its exit code, by a stop, or at the run's timeout, which+ * {@code progress} hears before the driver is stopped. A stop is read at every look, so it ends the+ * run whether or not the driver would ever exit by itself, as a STREAMING run's does not, and run()+ * then removes the driver. Each look at the container is one short control-plane call, so a run+ * holds no Docker connection while its driver runs.*/- private Integer awaitExit(String containerId, Job job, Progress progress) throws InterruptedException {+ private Outcome awaitEnd(String containerId, Job job, Launch launch, Progress progress)+ throws InterruptedException {Instant deadline = job.timeout().isZero() ? null : clock.instant().plus(job.timeout());- while (true) {+ while (!launch.stopRequested) {InspectContainerResponse.ContainerState state =lifecycleManager.getDockerClient().inspectContainerCmd(containerId).exec().getState();if (!Boolean.TRUE.equals(state.getRunning())) {+ if (launch.stopRequested) {+ return Outcome.halted();+ }Long exitCode = state.getExitCodeLong();- return exitCode != null ? exitCode.intValue() : -1;+ return Outcome.exitedWith(exitCode != null ? exitCode.intValue() : -1);}if (deadline != null && !clock.instant().isBefore(deadline)) {LOG.infov("EMR Serverless job run {0} passed its execution timeout of {1}", job.jobRunId(),job.timeout());progress.timedOut();- return null;+ return Outcome.outOfTime();}Thread.sleep(pollInterval.toMillis());}+ return Outcome.halted();}/**diff --git a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.java b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.javaindex 792b6ee239..cd4abdc7ee 100644--- a/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emrserverless/EmrServerlessSparkRunnerTest.java@@ -1,7 +1,7 @@package io.github.hectorvent.floci.services.emrserverless;import com.github.dockerjava.api.DockerClient;-import com.github.dockerjava.api.exception.NotModifiedException;+import com.github.dockerjava.api.exception.InternalServerErrorException;import com.github.dockerjava.api.model.Container;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;@@ -20,6 +20,7 @@ import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.concurrent.atomic.AtomicBoolean;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertTrue;@@ -28,7 +29,6 @@ import static org.mockito.ArgumentMatchers.anyMap;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;import static org.mockito.Mockito.atLeastOnce;-import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.inOrder;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;@@ -109,20 +109,29 @@ class EmrServerlessSparkRunnerTest {}@Test- void aCancelThatFindsTheDriverAlreadyExitedStillStopsTheRun() {+ void aCancelEndsTheRunAndRemovesItsDriverEvenWhenDockerWouldRefuseAStop() {EmrServerlessSparkRunner runner = runner();- // Docker answers a stop of an exited container with HTTP 304.+ SparkJobRunner.Job job = job(Duration.ofMillis(50));when(docker.stopContainerCmd("driver").withTimeout(0).exec())- .thenThrow(new NotModifiedException("Container already stopped"));- SparkJobRunner.Job job = job(Duration.ZERO);- doAnswer(started -> {- runner.stop(job.jobRunId());- return null;- }).when(progress).started();+ .thenThrow(new InternalServerErrorException("daemon busy"));+ // The driver never exits by itself, as a STREAMING run's; the cancel comes from another task,+ // whose exceptions go nowhere, while the driver is being watched.+ AtomicBoolean cancelled = new AtomicBoolean();+ when(docker.inspectContainerCmd("driver").exec().getState().getRunning()).thenAnswer(look -> {+ if (cancelled.compareAndSet(false, true)) {+ try {+ runner.stop(job.jobRunId());+ } catch (RuntimeException ignored) {+ // Dropped, as the executor drops what its task throws; the assertions below catch a lost cancel.+ }+ }+ return true;+ });SparkJobRunner.Outcome outcome = runner.run(job, progress);assertTrue(outcome.stopped());+ verify(progress, never()).timedOut();verify(lifecycle).stopAndRemove("driver", null, 0);}