Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): honor platform-specific runtime image overrides (#6786)
Andre Kuraita46a2b32fd03
diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.mdindex d9dd2ad970..1be70d97ba 100644--- a/docs/configuration/environment-variables.md+++ b/docs/configuration/environment-variables.md@@ -245,6 +245,7 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s| `FLOCI_SERVICES_LAMBDA_EPHEMERAL` | `false` | Remove Lambda containers immediately after each invocation || `FLOCI_SERVICES_LAMBDA_ECR_BASE_URI` | `public.ecr.aws` | Registry (optionally with a path prefix) the Lambda runtime images are pulled from, e.g. `public.ecr.aws/lambda/python:3.12`. Legacy name `FLOCI_ECR_BASE_URI` still works || `FLOCI_SERVICES_LAMBDA_RUNTIME_IMAGES__PYTHON3_12__` | *(unset)* | Override the `python3.12` runtime image, for example with `registry.example/lambda-python:3.12`. The double underscores quote the runtime key; the unquoted form `FLOCI_SERVICES_LAMBDA_RUNTIME_IMAGES_PYTHON3_12` does not bind. Use the same form for other dotted runtime keys. |+| `FLOCI_SERVICES_LAMBDA_RUNTIME_IMAGES__JAVA17_LINUX_ARM64__` | *(unset)* | Override `java17` when the Docker launcher selects `linux/arm64`. Platform entries precede the runtime-wide fallback. Use `LINUX_AMD64` for `linux/amd64`; selection follows `honour-architectures` or an explicit Docker `--platform`. || `FLOCI_SERVICES_LAMBDA_DEFAULT_MEMORY_MB` | `128` | Default memory allocation for functions that don't specify one || `FLOCI_SERVICES_LAMBDA_DEFAULT_TIMEOUT_SECONDS` | `3` | Default invocation timeout in seconds || `FLOCI_SERVICES_LAMBDA_RUNTIME_API_BASE_PORT` | `12000` | First port in the Lambda Runtime API port range |diff --git a/docs/services/lambda.md b/docs/services/lambda.mdindex 06f5c905de..24353ad408 100644--- a/docs/services/lambda.md+++ b/docs/services/lambda.md@@ -1356,3 +1356,16 @@ floci:Replace `<digest>` with the image's full SHA-256 digest. The override applies tozip-based functions using that runtime; other runtimes retain their default images.Custom image URI passthrough and image-package functions are unaffected.++For Docker functions that share a runtime but use different CPU architectures,+add platform-specific keys such as `"java17.linux.amd64"` and+`"java17.linux.arm64"`, each pointing to its platform manifest digest.+The Docker launcher's effective platform selects that entry before the+runtime-wide fallback. Enable `floci.services.lambda.honour-architectures` to+use the function's architecture; an explicit Docker `--platform` takes precedence.+The code-copy helper uses the same platform. With no selected Docker platform,+or for the Kubernetes launcher, the runtime-wide entry and default resolution+remain in effect.++Environment variables use the quoted map-key form, for example+`FLOCI_SERVICES_LAMBDA_RUNTIME_IMAGES__JAVA17_LINUX_ARM64__`.diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 98240f50ff..b3cad3fec4 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -3021,7 +3021,7 @@ public interface EmulatorConfig {@WithDefault("public.ecr.aws") // partition-literal: configurable; ECR Public exists only in the commercial partitionString ecrBaseUri();- /** Full image references for individual runtimes, keyed by Lambda runtime identifier. */+ /** Full image references keyed by runtime, or runtime.os.arch for a selected Docker platform. */Map<String, String> runtimeImages();@WithDefault("128")diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.javaindex 26cdf11b61..592480650a 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java@@ -221,10 +221,11 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {Optional<SessionCreds> roleCredentials = Optional.empty();try {- // Resolve image+ LambdaDockerFlags dockerFlags = configuredDockerFlags();+ // Resolve the image for the same platform used by container creation.String image = "Image".equals(fn.getPackageType()) && fn.getImageUri() != null? fn.getImageUri()- : imageResolver.resolve(fn.getRuntime());+ : imageResolver.resolve(fn.getRuntime(), requestedDockerPlatform(fn, dockerFlags).orElse(null));// If this is an AWS-shaped ECR URI, rewrite it to Floci's loopback registryimage = ecrRegistryManager.rewriteImageUri(image);@@ -244,7 +245,6 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {+ LambdaLogDestinations.logStreamSuffix(fn, shortId);String lambdaRegion = extractRegionFromArn(fn.getFunctionArn(), config.defaultRegion());String lambdaAccountId = AwsArnUtils.accountOrDefault(fn.getFunctionArn(), config.defaultAccountId());- LambdaDockerFlags dockerFlags = configuredDockerFlags();// Build env varsList<String> env = new ArrayList<>();@@ -551,19 +551,21 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {};}- private String createContainer(ContainerSpec spec, LambdaFunction fn, LambdaDockerFlags dockerFlags) {+ private Optional<String> requestedDockerPlatform(LambdaFunction fn, LambdaDockerFlags dockerFlags) {if (dockerFlags.platform() != null) {- return lifecycleManager.create(spec, dockerFlags.platform());- }- if (!config.services().lambda().honourArchitectures()) {- return lifecycleManager.create(spec);+ return Optional.of(dockerFlags.platform());}- Optional<String> platform = dockerPlatform(fn);- if (platform.isEmpty()) {+ return config.services().lambda().honourArchitectures() ? dockerPlatform(fn) : Optional.empty();+ }++ private String createContainer(ContainerSpec spec, LambdaFunction fn, LambdaDockerFlags dockerFlags) {+ Optional<String> platform = requestedDockerPlatform(fn, dockerFlags);+ if (platform.isEmpty() && config.services().lambda().honourArchitectures()) {throw new IllegalStateException("Invalid persisted architectures "+ fn.getArchitectures() + " for function '" + fn.getFunctionName() + "'");}- return lifecycleManager.create(spec, platform.get());+ return platform.map(value -> lifecycleManager.create(spec, value))+ .orElseGet(() -> lifecycleManager.create(spec));}private LambdaDockerFlags configuredDockerFlags() {@@ -1017,7 +1019,7 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {acquirePopulatePermit(fn.getFunctionName());String helperId = null;try {- helperId = createContainer(helperSpec, fn, LambdaDockerFlags.parse(null));+ helperId = createContainer(helperSpec, fn, configuredDockerFlags());lifecycleManager.startCreated(helperId, helperSpec);copyDirToContainerStrict(lifecycleManager.getDockerClient(), helperId,Path.of(fn.getCodeLocalPath()), TASK_DIR, fn.getFunctionName());diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolver.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolver.javaindex b258c1b0ea..379a93d027 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolver.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolver.java@@ -53,6 +53,11 @@ public class ImageResolver {}public String resolve(String runtime) {+ return resolve(runtime, null);+ }++ /** A platform-specific override precedes the runtime-wide image when a launcher selects a platform. */+ public String resolve(String runtime, String platform) {if (runtime == null || runtime.isBlank()) {throw new AwsException("InvalidParameterValueException", "Runtime is required", 400);}@@ -65,8 +70,16 @@ public class ImageResolver {throw new AwsException("InvalidParameterValueException","The runtime parameter " + runtime + " is not supported.", 400);}- if (runtimeImages != null && runtimeImages.containsKey(runtime)) {- return runtimeImages.get(runtime);+ if (runtimeImages != null) {+ if (platform != null) {+ String override = runtimeImages.get(runtime + "." + platform.replace('/', '.'));+ if (override != null) {+ return override;+ }+ }+ if (runtimeImages.containsKey(runtime)) {+ return runtimeImages.get(runtime);+ }}return baseUri + "/lambda/" + image;}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 9dffd7901a..44363c5f3f 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -434,7 +434,7 @@ floci:# Extra Docker create flags for Lambda containers, for example: --env KEY=VALUE --volume /host:/container:rodocker-flags: ""ecr-base-uri: public.ecr.aws # FLOCI_SERVICES_LAMBDA_ECR_BASE_URI — registry for Lambda runtime images (legacy: floci.ecr-base-uri / FLOCI_ECR_BASE_URI)- # runtime-images: # Optional full image references keyed by Lambda runtime+ # runtime-images: # Optional full references keyed by runtime or runtime.os.arch# "python3.12": "public.ecr.aws/lambda/python:3.12@sha256:<digest>"honour-architectures: false # Select the Lambda architecture for Docker pull and createdefault-memory-mb: 128diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaImageConfigTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaImageConfigTest.javaindex c951c3a00f..fcaa26de26 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaImageConfigTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaImageConfigTest.java@@ -363,7 +363,7 @@ class LambdaImageConfigTest {// of starting an empty container. This test covers cmd construction, so any real// directory will do.fn.setCodeLocalPath(codeDir.toString());- when(imageResolver.resolve("nodejs20.x")).thenReturn("public.ecr.aws/lambda/nodejs:20");+ when(imageResolver.resolve("nodejs20.x", null)).thenReturn("public.ecr.aws/lambda/nodejs:20");launcher.launch(fn);diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.javaindex 70f606bc06..9dd45cb860 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java@@ -40,6 +40,7 @@ import org.junit.jupiter.api.Test;import org.junit.jupiter.api.extension.ExtendWith;import org.junit.jupiter.api.io.TempDir;import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;import org.junit.jupiter.params.provider.MethodSource;import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;@@ -145,14 +146,7 @@ class ContainerLauncherTest {lenient().when(ecrRegistryManager.rewriteImageUri(any())).thenAnswer(inv -> inv.getArgument(0));- ContainerBuilder containerBuilder = new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer);- ContainerReachableEndpoint reachableEndpoint =- new ContainerReachableEndpoint(config, dockerHostResolver, embeddedDnsServer, 4566);- LaunchedContainerAwsEnv awsEnv = new LaunchedContainerAwsEnv(reachableEndpoint);- launcher = new ContainerLauncher(containerBuilder, lifecycleManager, logStreamer, imageResolver,- runtimeApiServerFactory, dockerHostResolver, config, ecrRegistryManager,- layerService, awsEnv,- executionRoleCredentials);+ launcher = newLauncher(imageResolver);when(runtimeApiServerFactory.create()).thenReturn(runtimeApiServer);when(runtimeApiServer.getPort()).thenReturn(9000);@@ -428,6 +422,57 @@ class ContainerLauncherTest {verify(lifecycleManager, never()).create(any(ContainerSpec.class), anyString());}+ private ContainerLauncher newLauncher(ImageResolver selectedResolver) {+ ContainerBuilder builder = new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer);+ ContainerReachableEndpoint endpoint =+ new ContainerReachableEndpoint(config, dockerHostResolver, embeddedDnsServer, 4566);+ return new ContainerLauncher(builder, lifecycleManager, logStreamer, selectedResolver,+ runtimeApiServerFactory, dockerHostResolver, config, ecrRegistryManager,+ layerService, new LaunchedContainerAwsEnv(endpoint),+ executionRoleCredentials);+ }++ @ParameterizedTest+ @CsvSource({+ "arm64, true, , linux/arm64, arm64",+ "x86_64, true, , linux/amd64, amd64",+ ", true, , linux/amd64, amd64",+ "arm64, false, , , fallback",+ "arm64, true, --platform linux/amd64, linux/amd64, amd64",+ "x86_64, false, --platform linux/arm64, linux/arm64, arm64"+ })+ void launchFunction_selectsRuntimePinForEffectiveContainerPlatform(+ String architecture, boolean honourArchitecture, String flags,+ String expectedPlatform, String expectedImage) throws Exception {+ EmulatorConfig.LambdaServiceConfig lambda = config.services().lambda();+ lenient().when(lambda.honourArchitectures()).thenReturn(honourArchitecture);+ when(lambda.dockerFlags()).thenReturn(Optional.ofNullable(flags));+ when(lambda.runtimeImages()).thenReturn(Map.of(+ "nodejs20.x", "registry.example/node:fallback",+ "nodejs20.x.linux.amd64", "registry.example/node:amd64",+ "nodejs20.x.linux.arm64", "registry.example/node:arm64"));+ launcher = newLauncher(new ImageResolver(config));+ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("runtime-pin-fn");+ fn.setRuntime("nodejs20.x");+ fn.setHandler("index.handler");+ fn.setCodeLocalPath(Files.createDirectory(tempDir.resolve("runtime-pin-code")).toString());+ if (architecture != null) {+ fn.setArchitectures(List.of(architecture));+ }++ launcher.launch(fn);++ ArgumentCaptor<ContainerSpec> spec = ArgumentCaptor.forClass(ContainerSpec.class);+ if (expectedPlatform == null) {+ verify(lifecycleManager).create(spec.capture());+ verify(lifecycleManager, never()).create(any(ContainerSpec.class), anyString());+ } else {+ verify(lifecycleManager).create(spec.capture(), eq(expectedPlatform));+ }+ assertEquals("registry.example/node:" + expectedImage, spec.getValue().image());+ }+@ParameterizedTest@MethodSource("invalidPersistedArchitectures")void launchFunction_rejectsInvalidPersistedArchitecturesBeforeCreatingContainer(@@ -460,10 +505,13 @@ class ContainerLauncherTest {List.of("arm64", "x86_64"));}- @Test- void launchFunction_usesArm64DockerPlatformForCodeVolumeHelper() throws Exception {+ @ParameterizedTest+ @CsvSource({"arm64, ", "x86_64, --platform linux/arm64"})+ void launchFunction_usesArm64DockerPlatformForCodeVolumeHelper(String architecture, String flags)+ throws Exception {EmulatorConfig.LambdaServiceConfig lambda = config.services().lambda();- when(lambda.honourArchitectures()).thenReturn(true);+ lenient().when(lambda.honourArchitectures()).thenReturn(true);+ when(lambda.dockerFlags()).thenReturn(Optional.ofNullable(flags));Path codePath = Files.createDirectory(tempDir.resolve("large-arm64-code"));Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]);@@ -473,7 +521,7 @@ class ContainerLauncherTest {fn.setHandler("index.handler");fn.setCodeLocalPath(codePath.toString());fn.setCodeSha256("large-arm64-code-sha");- fn.setArchitectures(List.of("arm64"));+ fn.setArchitectures(List.of(architecture));withLoweredCodeVolumeThreshold(() -> {launcher.launch(fn);diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolverEnvConfigTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolverEnvConfigTest.javaindex 3dd3e80345..f1b011801e 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolverEnvConfigTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolverEnvConfigTest.java@@ -26,6 +26,25 @@ class ImageResolverEnvConfigTest {.services().lambda().runtimeImages().get("python3.12"));}+ @Test+ void platformRuntimeKeysBindFromEnvironment() {+ SmallRyeConfig config = new SmallRyeConfigBuilder()+ .withSources(new EnvConfigSource(Map.of(+ "FLOCI_SERVICES_LAMBDA_RUNTIME_IMAGES__PROVIDED_AL2023_LINUX_AMD64__",+ "registry.example/provided:amd64",+ "FLOCI_SERVICES_LAMBDA_RUNTIME_IMAGES__PROVIDED_AL2023_LINUX_ARM64__",+ "registry.example/provided:arm64",+ "FLOCI_SERVICES_LAMBDA_RUNTIME_IMAGES__PROVIDED_AL2023__",+ "registry.example/provided:fallback"), 300))+ .withMapping(EmulatorConfig.class)+ .build();+ ImageResolver resolver = new ImageResolver(config.getConfigMapping(EmulatorConfig.class));++ assertEquals("registry.example/provided:amd64", resolver.resolve("provided.al2023", "linux/amd64"));+ assertEquals("registry.example/provided:arm64", resolver.resolve("provided.al2023", "linux/arm64"));+ assertEquals("registry.example/provided:fallback", resolver.resolve("provided.al2023"));+ }+@Testvoid unquotedRuntimeKeyDoesNotBind() {SmallRyeConfig config = new SmallRyeConfigBuilder()diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolverTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolverTest.javaindex b423577b61..d5a6105a08 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolverTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageResolverTest.java@@ -154,12 +154,30 @@ class ImageResolverTest {assertEquals("public.ecr.aws/lambda/python:3.11", configuredResolver.resolve("python3.11"));}+ @ParameterizedTest+ @CsvSource({"linux/amd64, a", "linux/arm64, b", "linux/riscv64, c", ", c"})+ void selectsPlatformManifestBeforeRuntimeWideFallback(String platform, String digest) {+ String repository = "registry.example/lambda-python@sha256:";+ when(config.services().lambda().runtimeImages()).thenReturn(Map.of(+ "python3.12.linux.amd64", repository + "a".repeat(64),+ "python3.12.linux.arm64", repository + "b".repeat(64),+ "python3.12", repository + "c".repeat(64)));+ ImageResolver configuredResolver = new ImageResolver(config);++ assertEquals(repository + digest.repeat(64), configuredResolver.resolve("python3.12", platform));+ assertEquals("public.ecr.aws/lambda/python:3.11", configuredResolver.resolve("python3.11", platform));+ assertEquals("custom/image:tag", configuredResolver.resolve("custom/image:tag", platform));+ }+@Testvoid unknownRuntimeCannotBeOverridden() {- when(config.services().lambda().runtimeImages()).thenReturn(Map.of("unknown", "example.com/unknown:1"));+ when(config.services().lambda().runtimeImages()).thenReturn(Map.of(+ "unknown", "example.com/unknown:1",+ "unknown.linux.arm64", "example.com/unknown:arm64"));ImageResolver configuredResolver = new ImageResolver(config);assertThrows(AwsException.class, () -> configuredResolver.resolve("unknown"));+ assertThrows(AwsException.class, () -> configuredResolver.resolve("unknown", "linux/arm64"));}@Test