Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(core): report at startup which services an unreachable Docker affects (#5773)

Hector Ventura4c32fae5e312
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReport.java b/src/main/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReport.java
new file mode 100644
index 0000000000..cc3ca203de
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReport.java
@@ -0,0 +1,110 @@
+package io.github.hectorvent.floci.lifecycle;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.ServiceRegistry;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.DockerClientProducer;
+import io.quarkus.runtime.StartupEvent;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Says once, at startup, what an unreachable Docker daemon costs.
+ *
+ * <p>Floci starts without Docker, and most services never need it. The container-backed ones
+ * either keep serving their control plane with no container behind it (metadata-only) or fail
+ * when asked to run something. Each of them otherwise reports the missing daemon on its own and
+ * only on first use, so one line up front tells a user of the native binary which services are
+ * affected before anything fails. Nothing is logged when Docker answers, or when no affected
+ * service is enabled.
+ */
+@ApplicationScoped
+public class DockerAvailabilityReport {
+
+ private static final Logger LOG = Logger.getLogger(DockerAvailabilityReport.class);
+
+ /** Ping Docker, report and carry on: their control plane works without a container. */
+ static final List<String> METADATA_ONLY = List.of(
+ "rds", "elasticache", "memorydb", "opensearch", "neptune", "docdb", "msk", "eks",
+ "timestreaminfluxdb");
+
+ /** Run a container per request and fail that request (or its build, task, job) without Docker. */
+ static final List<String> FAIL_ON_USE = List.of(
+ "lambda", "ecs", "ec2", "codebuild", "batch", "sagemaker", "appsync", "pipes", "mwaa",
+ "amazonmq", "kinesisanalytics", "redshift", "redshift-serverless", "rds-data", "athena", "ecr",
+ "codeartifact", "verifiedpermissions");
+
+ private final ContainerLifecycleManager lifecycleManager;
+ private final ServiceRegistry serviceRegistry;
+ private final EmulatorConfig config;
+
+ @Inject
+ public DockerAvailabilityReport(ContainerLifecycleManager lifecycleManager, ServiceRegistry serviceRegistry,
+ EmulatorConfig config) {
+ this.lifecycleManager = lifecycleManager;
+ this.serviceRegistry = serviceRegistry;
+ this.config = config;
+ }
+
+ void onStart(@Observes StartupEvent ignored) {
+ // A missing socket fails fast, but a daemon that accepts and never answers waits for the
+ // API timeout, so the probe stays off the startup path.
+ Thread.ofVirtual().name("floci-docker-availability").start(this::report);
+ }
+
+ void report() {
+ Set<String> enabled = Set.copyOf(serviceRegistry.getEnabledServices());
+ List<String> metadataOnly = METADATA_ONLY.stream().filter(enabled::contains).toList();
+ List<String> failOnUse = failOnUse(enabled);
+ if (metadataOnly.isEmpty() && failOnUse.isEmpty()) {
+ return;
+ }
+ String reason = unreachableReason();
+ if (reason == null) {
+ return;
+ }
+ LOG.warnv("Docker is not reachable at {0} ({1}). Without it these services keep their control plane but"
+ + " run no containers: {2}; and these fail when asked to run one: {3}."
+ + " Everything else works without Docker.",
+ DockerClientProducer.resolveDockerConnection(config.docker(), System::getenv).host(), reason,
+ metadataOnly, failOnUse);
+ }
+
+ List<String> failOnUse(Set<String> enabled) {
+ return FAIL_ON_USE.stream()
+ .filter(enabled::contains)
+ .filter(service -> !runsWithoutLocalDocker(service))
+ .map(service -> "pipes".equals(service) ? "pipes (Kafka sources)" : service)
+ .toList();
+ }
+
+ /** A service whose container work goes somewhere other than the local daemon. */
+ boolean runsWithoutLocalDocker(String service) {
+ return switch (service) {
+ case "lambda" -> "kubernetes".equalsIgnoreCase(config.services().lambda().executor().trim());
+ case "athena" -> isSet(config.services().duck().url());
+ case "verifiedpermissions" -> isSet(config.services().verifiedpermissions().cedarUrl());
+ default -> false;
+ };
+ }
+
+ private static boolean isSet(Optional<String> url) {
+ return url.isPresent() && !url.get().isBlank();
+ }
+
+ /** @return why Docker is unreachable, or {@code null} when it answers */
+ String unreachableReason() {
+ try {
+ lifecycleManager.getDockerClient().pingCmd().exec();
+ return null;
+ } catch (RuntimeException e) {
+ return e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage();
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
index f1ea20594f..84a8eae4e7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
@@ -18,6 +18,8 @@ import jakarta.enterprise.inject.Instance;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.net.SocketException;
+import java.net.UnknownHostException;
import java.time.Clock;
import java.time.Duration;
import java.util.Map;
@@ -66,6 +68,7 @@ public class LambdaExecutorService implements Resettable {
private final LambdaService directLambdaService;
private final Clock clock;
private final long asyncRetryDelayMs;
+ private final boolean dockerExecutor;
/** Moved on by a reset so that retries still waiting from before it are dropped. */
private final AtomicLong generation = new AtomicLong();
private final Map<String, Long> deletions = new ConcurrentHashMap<>();
@@ -85,7 +88,8 @@ public class LambdaExecutorService implements Resettable {
Clock clock,
EmulatorConfig config) {
this(warmPool, objectMapper, concurrencyLimiter, destinationRouter, lambdaServiceInstance, null, clock,
- Duration.ofSeconds(config.services().lambda().asyncRetryDelaySeconds()));
+ Duration.ofSeconds(config.services().lambda().asyncRetryDelaySeconds()),
+ !"kubernetes".equalsIgnoreCase(config.services().lambda().executor().trim()));
}
LambdaExecutorService(WarmPool warmPool,
@@ -94,7 +98,7 @@ public class LambdaExecutorService implements Resettable {
AsyncInvokeDestinationRouter destinationRouter,
LambdaService directLambdaService) {
this(warmPool, objectMapper, concurrencyLimiter, destinationRouter, null, directLambdaService,
- Clock.systemUTC(), Duration.ZERO);
+ Clock.systemUTC(), Duration.ZERO, true);
}
LambdaExecutorService(WarmPool warmPool,
@@ -105,7 +109,7 @@ public class LambdaExecutorService implements Resettable {
Clock clock,
Duration asyncRetryDelay) {
this(warmPool, objectMapper, concurrencyLimiter, destinationRouter, null, directLambdaService, clock,
- asyncRetryDelay);
+ asyncRetryDelay, true);
}
private LambdaExecutorService(WarmPool warmPool,
@@ -115,7 +119,8 @@ public class LambdaExecutorService implements Resettable {
Instance<LambdaService> lambdaServiceInstance,
LambdaService directLambdaService,
Clock clock,
- Duration asyncRetryDelay) {
+ Duration asyncRetryDelay,
+ boolean dockerExecutor) {
this.warmPool = warmPool;
this.objectMapper = objectMapper;
this.concurrencyLimiter = concurrencyLimiter;
@@ -124,6 +129,7 @@ public class LambdaExecutorService implements Resettable {
this.directLambdaService = directLambdaService;
this.clock = clock;
this.asyncRetryDelayMs = asyncRetryDelay.toMillis();
+ this.dockerExecutor = dockerExecutor;
}
/** Package-private constructor for testing without CDI, leaving destinations unrouted. */
@@ -131,7 +137,7 @@ public class LambdaExecutorService implements Resettable {
ObjectMapper objectMapper,
LambdaConcurrencyLimiter concurrencyLimiter) {
this(warmPool, objectMapper, concurrencyLimiter, null, (Instance<LambdaService>) null, null,
- Clock.systemUTC(), Duration.ZERO);
+ Clock.systemUTC(), Duration.ZERO, true);
}
LambdaExecutorService(WarmPool warmPool,
@@ -139,7 +145,7 @@ public class LambdaExecutorService implements Resettable {
LambdaConcurrencyLimiter concurrencyLimiter,
AsyncInvokeDestinationRouter destinationRouter) {
this(warmPool, objectMapper, concurrencyLimiter, destinationRouter,
- (Instance<LambdaService>) null, null, Clock.systemUTC(), Duration.ZERO);
+ (Instance<LambdaService>) null, null, Clock.systemUTC(), Duration.ZERO, true);
}
public InvokeResult invoke(LambdaFunction fn, byte[] payload, InvocationType type) {
@@ -324,6 +330,22 @@ public class LambdaExecutorService implements Resettable {
event.invokedQualifier());
}
+ /**
+ * A container that could not start because Docker is unreachable says so, rather than surfacing
+ * the raw socket error ("No such file or directory") a native binary without Docker would get.
+ * With the Kubernetes executor a socket error comes from the cluster API, so it keeps its own
+ * message.
+ */
+ static String initFailureMessage(Exception e, boolean dockerExecutor) {
+ for (Throwable cause = e; dockerExecutor && cause != null; cause = cause.getCause()) {
+ if (cause instanceof SocketException || cause instanceof UnknownHostException) {
+ return "Failed to start Lambda container: Docker is not reachable (" + cause.getMessage()
+ + "). Lambda functions run in Docker containers.";
+ }
+ }
+ return "Failed to start Lambda container: " + e.getMessage();
+ }
+
private InvokeResult executeSync(LambdaFunction fn, byte[] payload, String requestId,
String clientContext) {
ContainerHandle handle;
@@ -332,7 +354,7 @@ public class LambdaExecutorService implements Resettable {
} catch (Exception e) {
LOG.warnv("Failed to acquire container for function {0}: {1}", fn.getFunctionName(), e.getMessage());
return new InvokeResult(200, "Unhandled",
- buildErrorPayload("Failed to start Lambda container: " + e.getMessage(), "Lambda.InitError"),
+ buildErrorPayload(initFailureMessage(e, dockerExecutor), "Lambda.InitError"),
null, requestId);
}
try {
diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReportTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReportTest.java
new file mode 100644
index 0000000000..cb6e05da13
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReportTest.java
@@ -0,0 +1,98 @@
+package io.github.hectorvent.floci.lifecycle;
+
+import com.github.dockerjava.api.DockerClient;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.ServiceRegistry;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class DockerAvailabilityReportTest {
+
+ private final ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);
+ private final DockerClient docker = mock(DockerClient.class, Mockito.RETURNS_DEEP_STUBS);
+ private final ServiceRegistry registry = mock(ServiceRegistry.class);
+ private final EmulatorConfig config = mock(EmulatorConfig.class, Mockito.RETURNS_DEEP_STUBS);
+ private final DockerAvailabilityReport report = new DockerAvailabilityReport(lifecycle, registry, config);
+
+ @Test
+ void unreachableDaemonReportsTheReason() {
+ when(lifecycle.getDockerClient()).thenReturn(docker);
+ when(docker.pingCmd().exec()).thenThrow(new RuntimeException("java.net.SocketException: No such file or directory"));
+
+ assertThat(report.unreachableReason(), containsString("No such file or directory"));
+ }
+
+ @Test
+ void reachableDaemonHasNoReason() {
+ when(lifecycle.getDockerClient()).thenReturn(docker);
+
+ assertThat(report.unreachableReason(), is(nullValue()));
+ }
+
+ @Test
+ void noAffectedServiceEnabledSkipsTheProbe() {
+ when(registry.getEnabledServices()).thenReturn(List.of("sqs", "s3", "dynamodb"));
+
+ report.report();
+
+ verify(lifecycle, never()).getDockerClient();
+ }
+
+ @Test
+ void affectedServicesAreProbed() {
+ when(registry.getEnabledServices()).thenReturn(List.of("sqs", "lambda", "rds"));
+ when(config.services().lambda().executor()).thenReturn("docker");
+ when(lifecycle.getDockerClient()).thenReturn(docker);
+
+ report.report();
+
+ verify(lifecycle).getDockerClient();
+ }
+
+ @Test
+ void listsDoNotOverlap() {
+ assertThat(DockerAvailabilityReport.METADATA_ONLY.stream()
+ .filter(DockerAvailabilityReport.FAIL_ON_USE::contains).toList(), is(empty()));
+ assertThat(DockerAvailabilityReport.FAIL_ON_USE, not(empty()));
+ }
+
+ @Test
+ void servicesBackedByADockerRuntimeFailOnUse() {
+ when(config.services().lambda().executor()).thenReturn("docker");
+ when(config.services().duck().url()).thenReturn(Optional.empty());
+
+ assertThat(report.failOnUse(Set.of("redshift-serverless", "rds-data", "athena", "lambda", "sqs")),
+ contains("lambda", "redshift-serverless", "rds-data", "athena"));
+ }
+
+ @Test
+ void servicesPointedElsewhereAreNotReported() {
+ when(config.services().lambda().executor()).thenReturn(" Kubernetes ");
+ when(config.services().duck().url()).thenReturn(Optional.of("http://duck:8080"));
+ when(config.services().verifiedpermissions().cedarUrl()).thenReturn(Optional.of("http://cedar:8080"));
+
+ assertThat(report.failOnUse(Set.of("lambda", "athena", "verifiedpermissions", "ecs")), contains("ecs"));
+ }
+
+ @Test
+ void pipesIsQualifiedToItsKafkaSources() {
+ assertThat(report.failOnUse(Set.of("pipes")), contains("pipes (Kafka sources)"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInitFailureMessageTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInitFailureMessageTest.java
new file mode 100644
index 0000000000..c4fb14bc3d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInitFailureMessageTest.java
@@ -0,0 +1,46 @@
+package io.github.hectorvent.floci.services.lambda;
+
+import org.junit.jupiter.api.Test;
+
+import java.net.SocketException;
+import java.net.UnknownHostException;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+class LambdaInitFailureMessageTest {
+
+ @Test
+ void unreachableDockerIsNamed() {
+ Exception e = new RuntimeException("java.net.SocketException: No such file or directory",
+ new SocketException("No such file or directory"));
+
+ String message = LambdaExecutorService.initFailureMessage(e, true);
+
+ assertThat(message, containsString("Docker is not reachable (No such file or directory)"));
+ assertThat(message, containsString("Lambda functions run in Docker containers"));
+ }
+
+ @Test
+ void otherFailuresKeepTheirMessage() {
+ assertThat(LambdaExecutorService.initFailureMessage(new IllegalStateException("image pull failed"), true),
+ equalTo("Failed to start Lambda container: image pull failed"));
+ }
+
+ @Test
+ void anUnresolvableDockerHostIsNamed() {
+ Exception e = new RuntimeException("docker.internal", new UnknownHostException("docker.internal"));
+
+ assertThat(LambdaExecutorService.initFailureMessage(e, true),
+ containsString("Docker is not reachable (docker.internal)"));
+ }
+
+ @Test
+ void kubernetesSocketErrorsDoNotBlameDocker() {
+ Exception e = new RuntimeException("connect failed", new SocketException("Connection refused"));
+
+ assertThat(LambdaExecutorService.initFailureMessage(e, false),
+ equalTo("Failed to start Lambda container: connect failed"));
+ }
+}