Light, fluffy, and always free - The AWS Local Emulator alternative
feat(core): report at startup which services an unreachable Docker affects (#5773)
Hector Ventura4c32fae5e312
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReport.java b/src/main/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReport.javanew file mode 100644index 0000000000..cc3ca203de--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReport.java@@ -0,0 +1,110 @@+package io.github.hectorvent.floci.lifecycle;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.ServiceRegistry;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.DockerClientProducer;+import io.quarkus.runtime.StartupEvent;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.event.Observes;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.util.List;+import java.util.Optional;+import java.util.Set;++/**+ * Says once, at startup, what an unreachable Docker daemon costs.+ *+ * <p>Floci starts without Docker, and most services never need it. The container-backed ones+ * either keep serving their control plane with no container behind it (metadata-only) or fail+ * when asked to run something. Each of them otherwise reports the missing daemon on its own and+ * only on first use, so one line up front tells a user of the native binary which services are+ * affected before anything fails. Nothing is logged when Docker answers, or when no affected+ * service is enabled.+ */+@ApplicationScoped+public class DockerAvailabilityReport {++ private static final Logger LOG = Logger.getLogger(DockerAvailabilityReport.class);++ /** Ping Docker, report and carry on: their control plane works without a container. */+ static final List<String> METADATA_ONLY = List.of(+ "rds", "elasticache", "memorydb", "opensearch", "neptune", "docdb", "msk", "eks",+ "timestreaminfluxdb");++ /** Run a container per request and fail that request (or its build, task, job) without Docker. */+ static final List<String> FAIL_ON_USE = List.of(+ "lambda", "ecs", "ec2", "codebuild", "batch", "sagemaker", "appsync", "pipes", "mwaa",+ "amazonmq", "kinesisanalytics", "redshift", "redshift-serverless", "rds-data", "athena", "ecr",+ "codeartifact", "verifiedpermissions");++ private final ContainerLifecycleManager lifecycleManager;+ private final ServiceRegistry serviceRegistry;+ private final EmulatorConfig config;++ @Inject+ public DockerAvailabilityReport(ContainerLifecycleManager lifecycleManager, ServiceRegistry serviceRegistry,+ EmulatorConfig config) {+ this.lifecycleManager = lifecycleManager;+ this.serviceRegistry = serviceRegistry;+ this.config = config;+ }++ void onStart(@Observes StartupEvent ignored) {+ // A missing socket fails fast, but a daemon that accepts and never answers waits for the+ // API timeout, so the probe stays off the startup path.+ Thread.ofVirtual().name("floci-docker-availability").start(this::report);+ }++ void report() {+ Set<String> enabled = Set.copyOf(serviceRegistry.getEnabledServices());+ List<String> metadataOnly = METADATA_ONLY.stream().filter(enabled::contains).toList();+ List<String> failOnUse = failOnUse(enabled);+ if (metadataOnly.isEmpty() && failOnUse.isEmpty()) {+ return;+ }+ String reason = unreachableReason();+ if (reason == null) {+ return;+ }+ LOG.warnv("Docker is not reachable at {0} ({1}). Without it these services keep their control plane but"+ + " run no containers: {2}; and these fail when asked to run one: {3}."+ + " Everything else works without Docker.",+ DockerClientProducer.resolveDockerConnection(config.docker(), System::getenv).host(), reason,+ metadataOnly, failOnUse);+ }++ List<String> failOnUse(Set<String> enabled) {+ return FAIL_ON_USE.stream()+ .filter(enabled::contains)+ .filter(service -> !runsWithoutLocalDocker(service))+ .map(service -> "pipes".equals(service) ? "pipes (Kafka sources)" : service)+ .toList();+ }++ /** A service whose container work goes somewhere other than the local daemon. */+ boolean runsWithoutLocalDocker(String service) {+ return switch (service) {+ case "lambda" -> "kubernetes".equalsIgnoreCase(config.services().lambda().executor().trim());+ case "athena" -> isSet(config.services().duck().url());+ case "verifiedpermissions" -> isSet(config.services().verifiedpermissions().cedarUrl());+ default -> false;+ };+ }++ private static boolean isSet(Optional<String> url) {+ return url.isPresent() && !url.get().isBlank();+ }++ /** @return why Docker is unreachable, or {@code null} when it answers */+ String unreachableReason() {+ try {+ lifecycleManager.getDockerClient().pingCmd().exec();+ return null;+ } catch (RuntimeException e) {+ return e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage();+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.javaindex f1ea20594f..84a8eae4e7 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java@@ -18,6 +18,8 @@ import jakarta.enterprise.inject.Instance;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.net.SocketException;+import java.net.UnknownHostException;import java.time.Clock;import java.time.Duration;import java.util.Map;@@ -66,6 +68,7 @@ public class LambdaExecutorService implements Resettable {private final LambdaService directLambdaService;private final Clock clock;private final long asyncRetryDelayMs;+ private final boolean dockerExecutor;/** Moved on by a reset so that retries still waiting from before it are dropped. */private final AtomicLong generation = new AtomicLong();private final Map<String, Long> deletions = new ConcurrentHashMap<>();@@ -85,7 +88,8 @@ public class LambdaExecutorService implements Resettable {Clock clock,EmulatorConfig config) {this(warmPool, objectMapper, concurrencyLimiter, destinationRouter, lambdaServiceInstance, null, clock,- Duration.ofSeconds(config.services().lambda().asyncRetryDelaySeconds()));+ Duration.ofSeconds(config.services().lambda().asyncRetryDelaySeconds()),+ !"kubernetes".equalsIgnoreCase(config.services().lambda().executor().trim()));}LambdaExecutorService(WarmPool warmPool,@@ -94,7 +98,7 @@ public class LambdaExecutorService implements Resettable {AsyncInvokeDestinationRouter destinationRouter,LambdaService directLambdaService) {this(warmPool, objectMapper, concurrencyLimiter, destinationRouter, null, directLambdaService,- Clock.systemUTC(), Duration.ZERO);+ Clock.systemUTC(), Duration.ZERO, true);}LambdaExecutorService(WarmPool warmPool,@@ -105,7 +109,7 @@ public class LambdaExecutorService implements Resettable {Clock clock,Duration asyncRetryDelay) {this(warmPool, objectMapper, concurrencyLimiter, destinationRouter, null, directLambdaService, clock,- asyncRetryDelay);+ asyncRetryDelay, true);}private LambdaExecutorService(WarmPool warmPool,@@ -115,7 +119,8 @@ public class LambdaExecutorService implements Resettable {Instance<LambdaService> lambdaServiceInstance,LambdaService directLambdaService,Clock clock,- Duration asyncRetryDelay) {+ Duration asyncRetryDelay,+ boolean dockerExecutor) {this.warmPool = warmPool;this.objectMapper = objectMapper;this.concurrencyLimiter = concurrencyLimiter;@@ -124,6 +129,7 @@ public class LambdaExecutorService implements Resettable {this.directLambdaService = directLambdaService;this.clock = clock;this.asyncRetryDelayMs = asyncRetryDelay.toMillis();+ this.dockerExecutor = dockerExecutor;}/** Package-private constructor for testing without CDI, leaving destinations unrouted. */@@ -131,7 +137,7 @@ public class LambdaExecutorService implements Resettable {ObjectMapper objectMapper,LambdaConcurrencyLimiter concurrencyLimiter) {this(warmPool, objectMapper, concurrencyLimiter, null, (Instance<LambdaService>) null, null,- Clock.systemUTC(), Duration.ZERO);+ Clock.systemUTC(), Duration.ZERO, true);}LambdaExecutorService(WarmPool warmPool,@@ -139,7 +145,7 @@ public class LambdaExecutorService implements Resettable {LambdaConcurrencyLimiter concurrencyLimiter,AsyncInvokeDestinationRouter destinationRouter) {this(warmPool, objectMapper, concurrencyLimiter, destinationRouter,- (Instance<LambdaService>) null, null, Clock.systemUTC(), Duration.ZERO);+ (Instance<LambdaService>) null, null, Clock.systemUTC(), Duration.ZERO, true);}public InvokeResult invoke(LambdaFunction fn, byte[] payload, InvocationType type) {@@ -324,6 +330,22 @@ public class LambdaExecutorService implements Resettable {event.invokedQualifier());}+ /**+ * A container that could not start because Docker is unreachable says so, rather than surfacing+ * the raw socket error ("No such file or directory") a native binary without Docker would get.+ * With the Kubernetes executor a socket error comes from the cluster API, so it keeps its own+ * message.+ */+ static String initFailureMessage(Exception e, boolean dockerExecutor) {+ for (Throwable cause = e; dockerExecutor && cause != null; cause = cause.getCause()) {+ if (cause instanceof SocketException || cause instanceof UnknownHostException) {+ return "Failed to start Lambda container: Docker is not reachable (" + cause.getMessage()+ + "). Lambda functions run in Docker containers.";+ }+ }+ return "Failed to start Lambda container: " + e.getMessage();+ }+private InvokeResult executeSync(LambdaFunction fn, byte[] payload, String requestId,String clientContext) {ContainerHandle handle;@@ -332,7 +354,7 @@ public class LambdaExecutorService implements Resettable {} catch (Exception e) {LOG.warnv("Failed to acquire container for function {0}: {1}", fn.getFunctionName(), e.getMessage());return new InvokeResult(200, "Unhandled",- buildErrorPayload("Failed to start Lambda container: " + e.getMessage(), "Lambda.InitError"),+ buildErrorPayload(initFailureMessage(e, dockerExecutor), "Lambda.InitError"),null, requestId);}try {diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReportTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReportTest.javanew file mode 100644index 0000000000..cb6e05da13--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/DockerAvailabilityReportTest.java@@ -0,0 +1,98 @@+package io.github.hectorvent.floci.lifecycle;++import com.github.dockerjava.api.DockerClient;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.ServiceRegistry;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import org.junit.jupiter.api.Test;+import org.mockito.Mockito;++import java.util.List;+import java.util.Optional;+import java.util.Set;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.contains;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.empty;+import static org.hamcrest.Matchers.is;+import static org.hamcrest.Matchers.not;+import static org.hamcrest.Matchers.nullValue;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class DockerAvailabilityReportTest {++ private final ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);+ private final DockerClient docker = mock(DockerClient.class, Mockito.RETURNS_DEEP_STUBS);+ private final ServiceRegistry registry = mock(ServiceRegistry.class);+ private final EmulatorConfig config = mock(EmulatorConfig.class, Mockito.RETURNS_DEEP_STUBS);+ private final DockerAvailabilityReport report = new DockerAvailabilityReport(lifecycle, registry, config);++ @Test+ void unreachableDaemonReportsTheReason() {+ when(lifecycle.getDockerClient()).thenReturn(docker);+ when(docker.pingCmd().exec()).thenThrow(new RuntimeException("java.net.SocketException: No such file or directory"));++ assertThat(report.unreachableReason(), containsString("No such file or directory"));+ }++ @Test+ void reachableDaemonHasNoReason() {+ when(lifecycle.getDockerClient()).thenReturn(docker);++ assertThat(report.unreachableReason(), is(nullValue()));+ }++ @Test+ void noAffectedServiceEnabledSkipsTheProbe() {+ when(registry.getEnabledServices()).thenReturn(List.of("sqs", "s3", "dynamodb"));++ report.report();++ verify(lifecycle, never()).getDockerClient();+ }++ @Test+ void affectedServicesAreProbed() {+ when(registry.getEnabledServices()).thenReturn(List.of("sqs", "lambda", "rds"));+ when(config.services().lambda().executor()).thenReturn("docker");+ when(lifecycle.getDockerClient()).thenReturn(docker);++ report.report();++ verify(lifecycle).getDockerClient();+ }++ @Test+ void listsDoNotOverlap() {+ assertThat(DockerAvailabilityReport.METADATA_ONLY.stream()+ .filter(DockerAvailabilityReport.FAIL_ON_USE::contains).toList(), is(empty()));+ assertThat(DockerAvailabilityReport.FAIL_ON_USE, not(empty()));+ }++ @Test+ void servicesBackedByADockerRuntimeFailOnUse() {+ when(config.services().lambda().executor()).thenReturn("docker");+ when(config.services().duck().url()).thenReturn(Optional.empty());++ assertThat(report.failOnUse(Set.of("redshift-serverless", "rds-data", "athena", "lambda", "sqs")),+ contains("lambda", "redshift-serverless", "rds-data", "athena"));+ }++ @Test+ void servicesPointedElsewhereAreNotReported() {+ when(config.services().lambda().executor()).thenReturn(" Kubernetes ");+ when(config.services().duck().url()).thenReturn(Optional.of("http://duck:8080"));+ when(config.services().verifiedpermissions().cedarUrl()).thenReturn(Optional.of("http://cedar:8080"));++ assertThat(report.failOnUse(Set.of("lambda", "athena", "verifiedpermissions", "ecs")), contains("ecs"));+ }++ @Test+ void pipesIsQualifiedToItsKafkaSources() {+ assertThat(report.failOnUse(Set.of("pipes")), contains("pipes (Kafka sources)"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInitFailureMessageTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInitFailureMessageTest.javanew file mode 100644index 0000000000..c4fb14bc3d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInitFailureMessageTest.java@@ -0,0 +1,46 @@+package io.github.hectorvent.floci.services.lambda;++import org.junit.jupiter.api.Test;++import java.net.SocketException;+import java.net.UnknownHostException;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;++class LambdaInitFailureMessageTest {++ @Test+ void unreachableDockerIsNamed() {+ Exception e = new RuntimeException("java.net.SocketException: No such file or directory",+ new SocketException("No such file or directory"));++ String message = LambdaExecutorService.initFailureMessage(e, true);++ assertThat(message, containsString("Docker is not reachable (No such file or directory)"));+ assertThat(message, containsString("Lambda functions run in Docker containers"));+ }++ @Test+ void otherFailuresKeepTheirMessage() {+ assertThat(LambdaExecutorService.initFailureMessage(new IllegalStateException("image pull failed"), true),+ equalTo("Failed to start Lambda container: image pull failed"));+ }++ @Test+ void anUnresolvableDockerHostIsNamed() {+ Exception e = new RuntimeException("docker.internal", new UnknownHostException("docker.internal"));++ assertThat(LambdaExecutorService.initFailureMessage(e, true),+ containsString("Docker is not reachable (docker.internal)"));+ }++ @Test+ void kubernetesSocketErrorsDoNotBlameDocker() {+ Exception e = new RuntimeException("connect failed", new SocketException("Connection refused"));++ assertThat(LambdaExecutorService.initFailureMessage(e, false),+ equalTo("Failed to start Lambda container: connect failed"));+ }+}