Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): resolve background layers in their function account (#6162)
Andre Kurait2ecc01b3e5ea
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerService.javaindex 450b114c71..d14ccd3f84 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerService.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.lambda;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.AwsRegions;import io.github.hectorvent.floci.core.common.FileTrees;import io.github.hectorvent.floci.core.common.PaginatedResult;import io.github.hectorvent.floci.core.common.RegionResolver;@@ -465,6 +466,36 @@ public class LambdaLayerService {return layerStore.get(region, layerName, version).orElse(null);}+ /**+ * Resolves initialization input against the stored function's scope. Provisioned workers+ * run outside a request; the ambient account must not select another account's layer.+ * Cross-account sharing remains unsupported.+ */+ public LambdaLayerVersion resolveLayerByArnForAccount(+ String layerVersionArn, String accountId, String functionRegion) {+ AwsArnUtils.Arn arn = parseLayerVersionArn(layerVersionArn);+ if (arn == null || accountId == null || accountId.isBlank()+ || functionRegion == null || functionRegion.isBlank()+ || !"lambda".equals(arn.service())+ || AwsArnUtils.isForeignPartition(arn, AwsRegions.partitionFor(functionRegion))+ || AwsArnUtils.isForeignAccount(arn, accountId)) {+ return null;+ }+ String[] resourceParts = arn.resource().split(":");+ if (resourceParts.length != 3) {+ return null;+ }+ long version;+ try {+ version = Long.parseLong(resourceParts[2]);+ } catch (NumberFormatException e) {+ return null;+ }+ return layerStore.getForAccount(accountId, arn.region(), resourceParts[1], version)+ .filter(layer -> layerVersionArn.equals(layer.getLayerVersionArn()))+ .orElse(null);+ }+/*** True when the ARN is a well-formed layer version ARN naming an account or partition other* than the caller's, so Floci cannot decide whether it exists. On the live service such adiff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerStore.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerStore.javaindex 5e8091d4c4..5c4bbe43ff 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerStore.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerStore.java@@ -1,6 +1,8 @@package io.github.hectorvent.floci.services.lambda;import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.lambda.model.LambdaLayerVersion;@@ -38,6 +40,16 @@ public class LambdaLayerStore {return backend.get(key(region, layerName, version));}+ /** Resolves an owned layer from an async worker without a request account context. */+ public Optional<LambdaLayerVersion> getForAccount(+ String accountId, String region, String layerName, long version) {+ if (backend instanceof AccountAwareStorageBackend<LambdaLayerVersion> aware) {+ return aware.getForAccountMigratingLegacy(accountId, key(region, layerName, version),+ layer -> accountId.equals(AwsArnUtils.accountOrDefault(layer.getLayerVersionArn(), "")));+ }+ return backend.get(key(region, layerName, version));+ }+/*** Returns all versions of a given layer in a region, sorted by version ascending.*/diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.javaindex 97a8562f85..ccbeb27362 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java@@ -389,7 +389,8 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {// 3. Copy layer contents into /opt (layers are merged in order)if (fn.getLayers() != null && !fn.getLayers().isEmpty()) {for (String layerArn : fn.getLayers()) {- LambdaLayerVersion layer = layerService.resolveLayerByArn(layerArn);+ LambdaLayerVersion layer = layerService.resolveLayerByArnForAccount(+ layerArn, lambdaAccountId, lambdaRegion);if (layer != null && layer.getCodeLocalPath() != null) {Path layerPath = Path.of(layer.getCodeLocalPath());if (Files.exists(layerPath)) {diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/kubernetes/KubernetesPodLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/kubernetes/KubernetesPodLauncher.javaindex 19633fffb4..1b752beeff 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/kubernetes/KubernetesPodLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/kubernetes/KubernetesPodLauncher.java@@ -180,7 +180,7 @@ public class KubernetesPodLauncher implements LambdaRuntimeLauncher {String cwLogStream = logStreamer.logStreamName(fn, shortId);String codeDownloadUrl = imagePackage ? null : codeDownloadUrl(fn, region);- List<String> layerUrls = imagePackage ? List.of() : layerDownloadUrls(fn, region);+ List<String> layerUrls = imagePackage ? List.of() : layerDownloadUrls(fn, accountId, region);Optional<String> caConfigMap = ContainerCaBundle.hostPath(config).map(bundle -> ensureCaConfigMap(namespace, bundle));@@ -401,13 +401,13 @@ public class KubernetesPodLauncher implements LambdaRuntimeLauncher {return downloadUrl(bucket, key, account, region);}- private List<String> layerDownloadUrls(LambdaFunction fn, String functionRegion) {+ private List<String> layerDownloadUrls(LambdaFunction fn, String accountId, String functionRegion) {if (fn.getLayers() == null || fn.getLayers().isEmpty()) {return List.of();}ArrayList<String> urls = new ArrayList<>();for (String layerArn : fn.getLayers()) {- LambdaLayerVersion layer = layerService.resolveLayerByArn(layerArn);+ LambdaLayerVersion layer = layerService.resolveLayerByArnForAccount(layerArn, accountId, functionRegion);if (layer == null) {LOG.warnv("Could not resolve layer ARN {0} for function {1}", layerArn, fn.getFunctionName());continue;diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaLayerAsyncAccountTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaLayerAsyncAccountTest.javanew file mode 100644index 0000000000..605a6b4012--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaLayerAsyncAccountTest.java@@ -0,0 +1,74 @@+package io.github.hectorvent.floci.services.lambda;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.services.lambda.model.LambdaLayerVersion;+import org.junit.jupiter.api.Test;++import java.util.concurrent.CompletableFuture;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertSame;++class LambdaLayerAsyncAccountTest {+ private static final String DEFAULT_ACCOUNT = "000000000000";+ private static final String OWNER = "123456789012";++ @Test+ void backgroundLookupUsesTheStoredOwnerAndDoesNotSubstituteTheDefaultAccountsLayer() throws Exception {+ AccountAwareStorageBackend<LambdaLayerVersion> storage = AccountAwareStorageBackend.inMemory(DEFAULT_ACCOUNT);+ LambdaLayerService service = service(storage);+ LambdaLayerVersion defaultLayer = layer("arn:aws:lambda:eu-west-1:" + DEFAULT_ACCOUNT + ":layer:shared:1");+ LambdaLayerVersion ownerLayer = layer("arn:aws:lambda:eu-west-1:" + OWNER + ":layer:shared:1");+ storage.putForAccount(DEFAULT_ACCOUNT, "layer::eu-west-1::shared::1", defaultLayer);+ storage.putForAccount(OWNER, "layer::eu-west-1::shared::1", ownerLayer);++ assertNull(CompletableFuture.supplyAsync(+ () -> service.resolveLayerByArn(ownerLayer.getLayerVersionArn())).get(5, TimeUnit.SECONDS));+ assertSame(ownerLayer, CompletableFuture.supplyAsync(() -> service.resolveLayerByArnForAccount(+ ownerLayer.getLayerVersionArn(), OWNER, "eu-west-1")).get(5, TimeUnit.SECONDS));+ assertSame(defaultLayer, service.resolveLayerByArn(defaultLayer.getLayerVersionArn()));+ assertNull(service.resolveLayerByArnForAccount(defaultLayer.getLayerVersionArn(), OWNER, "eu-west-1"));+ assertNull(service.resolveLayerByArnForAccount(ownerLayer.getLayerVersionArn(), "222222222222", "eu-west-1"));+ }++ @Test+ void backgroundLookupUsesTheFunctionsPartitionAndChecksTheStoredArn() {+ AccountAwareStorageBackend<LambdaLayerVersion> storage = AccountAwareStorageBackend.inMemory(DEFAULT_ACCOUNT);+ LambdaLayerService service = service(storage);+ LambdaLayerVersion ownerLayer = layer("arn:aws-cn:lambda:cn-north-1:" + OWNER + ":layer:shared:1");+ storage.putForAccount(OWNER, "layer::cn-north-1::shared::1", ownerLayer);+ assertSame(ownerLayer, service.resolveLayerByArnForAccount(ownerLayer.getLayerVersionArn(), OWNER, "cn-north-1"));+ assertNull(service.resolveLayerByArnForAccount(ownerLayer.getLayerVersionArn(), OWNER, "eu-west-1"));+ assertNull(service.resolveLayerByArnForAccount(+ "arn:aws:lambda:cn-north-1:" + OWNER + ":layer:shared:1", OWNER, "cn-north-1"));+ storage.putForAccount(OWNER, "layer::cn-north-1::shared::1",+ layer("arn:aws-cn:lambda:cn-north-1:222222222222:layer:shared:1"));+ assertNull(service.resolveLayerByArnForAccount(ownerLayer.getLayerVersionArn(), OWNER, "cn-north-1"));+ }++ @Test+ void absentLayerOrInvalidOwnerNeverFallsBackToAmbientStorage() {+ AccountAwareStorageBackend<LambdaLayerVersion> storage = AccountAwareStorageBackend.inMemory(DEFAULT_ACCOUNT);+ LambdaLayerService service = service(storage);+ String arn = "arn:aws:lambda:eu-west-1:" + OWNER + ":layer:shared:1";+ assertNull(service.resolveLayerByArnForAccount(arn, OWNER, "eu-west-1"));+ assertNull(service.resolveLayerByArnForAccount(arn, null, "eu-west-1"));+ assertNull(service.resolveLayerByArnForAccount(arn, "", "eu-west-1"));+ assertNull(service.resolveLayerByArnForAccount(arn, OWNER, ""));+ }++ private static LambdaLayerService service(AccountAwareStorageBackend<LambdaLayerVersion> storage) {+ return new LambdaLayerService(new LambdaLayerStore(storage), null, null,+ new RegionResolver("us-east-1", DEFAULT_ACCOUNT), null);+ }++ private static LambdaLayerVersion layer(String arn) {+ LambdaLayerVersion layer = new LambdaLayerVersion();+ layer.setLayerName("shared");+ layer.setVersion(1);+ layer.setLayerVersionArn(arn);+ return layer;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaLayerFixtures.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaLayerFixtures.javanew file mode 100644index 0000000000..125b9a707d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaLayerFixtures.java@@ -0,0 +1,28 @@+package io.github.hectorvent.floci.services.lambda;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.services.lambda.model.LambdaLayerVersion;++import java.util.Map;++public final class LambdaLayerFixtures {+ private LambdaLayerFixtures() {}++ public static LambdaLayerService accountScopedService(+ String defaultAccount, String region, Map<String, LambdaLayerVersion> layersByAccount) {+ AccountAwareStorageBackend<LambdaLayerVersion> storage = AccountAwareStorageBackend.inMemory(defaultAccount);+ layersByAccount.forEach((account, layer) -> storage.putForAccount(account,+ "layer::" + region + "::" + layer.getLayerName() + "::" + layer.getVersion(), layer));+ return new LambdaLayerService(new LambdaLayerStore(storage), null, null,+ new RegionResolver("us-east-1", defaultAccount), null);+ }++ public static LambdaLayerVersion layer(String account, String region, String name, long version) {+ LambdaLayerVersion layer = new LambdaLayerVersion();+ layer.setLayerName(name);+ layer.setVersion(version);+ layer.setLayerVersionArn("arn:aws:lambda:" + region + ":" + account + ":layer:" + name + ":" + version);+ return layer;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.javaindex 395881c155..ef314965cb 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java@@ -25,8 +25,11 @@ import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService;import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;import io.github.hectorvent.floci.services.iam.model.SessionCreds;+import io.github.hectorvent.floci.services.lambda.LambdaLayerFixtures;+import io.github.hectorvent.floci.services.lambda.LambdaLayerService;import io.github.hectorvent.floci.services.lambda.model.LambdaFileSystemConfig;import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;+import io.github.hectorvent.floci.services.lambda.model.LambdaLayerVersion;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;@@ -55,6 +58,7 @@ import java.util.Map;import java.util.Optional;import java.util.OptionalInt;import java.util.concurrent.CompletableFuture;+import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicBoolean;import java.util.stream.Stream;@@ -93,6 +97,7 @@ class ContainerLauncherTest {/** Collects remote paths passed to withRemotePath across all copy mocks. */final java.util.List<String> capturedRemotePaths = new java.util.ArrayList<>();String capturedRuntimeBootstrap;+ final Map<String, String> capturedLayerFiles = new LinkedHashMap<>();@BeforeEachvoid setUp() {@@ -166,6 +171,7 @@ class ContainerLauncherTest {// when the pipe buffer fills.capturedRemotePaths.clear();capturedRuntimeBootstrap = null;+ capturedLayerFiles.clear();lenient().when(dockerClient.copyArchiveToContainerCmd(any())).thenAnswer(inv -> {CopyArchiveToContainerCmd cmd = mock(CopyArchiveToContainerCmd.class);final java.io.InputStream[] captured = {null};@@ -188,6 +194,18 @@ class ContainerLauncherTest {}return null;}+ if ("/opt".equals(remote[0])) {+ try (TarArchiveInputStream tar = new TarArchiveInputStream(captured[0])) {+ TarArchiveEntry entry;+ while ((entry = tar.getNextEntry()) != null) {+ if (!entry.isDirectory()) {+ capturedLayerFiles.put(entry.getName(),+ new String(tar.readAllBytes(), StandardCharsets.UTF_8));+ }+ }+ }+ return null;+ }try { captured[0].transferTo(java.io.OutputStream.nullOutputStream()); }catch (Exception ignored) {}}@@ -197,6 +215,46 @@ class ContainerLauncherTest {});}+ @ParameterizedTest+ @ValueSource(strings = {"000000000000", "123456789012"})+ void backgroundColdStartCopiesOnlyTheFunctionsOwnedLayer(String account) throws Exception {+ String defaultAccount = "000000000000";+ String owner = "123456789012";+ String region = "eu-west-1";+ LambdaLayerVersion defaultLayer = LambdaLayerFixtures.layer(defaultAccount, region, "shared", 1);+ LambdaLayerVersion ownerLayer = LambdaLayerFixtures.layer(owner, region, "shared", 1);+ Map<String, LambdaLayerVersion> ownedLayers = Map.of(defaultAccount, defaultLayer, owner, ownerLayer);+ for (Map.Entry<String, LambdaLayerVersion> entry : ownedLayers.entrySet()) {+ Path layerPath = Files.createDirectory(tempDir.resolve(entry.getKey()));+ Files.writeString(layerPath.resolve("owner.txt"), entry.getKey());+ entry.getValue().setCodeLocalPath(layerPath.toString());+ }+ LambdaLayerService layers = LambdaLayerFixtures.accountScopedService(+ defaultAccount, region, Map.of(defaultAccount, defaultLayer, owner, ownerLayer));+ ContainerReachableEndpoint endpoint =+ new ContainerReachableEndpoint(config, dockerHostResolver, embeddedDnsServer, 4566);+ ContainerLauncher backgroundLauncher = new ContainerLauncher(+ new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer),+ lifecycleManager, logStreamer, imageResolver, runtimeApiServerFactory,+ dockerHostResolver, config, ecrRegistryManager, layers,+ new LaunchedContainerAwsEnv(endpoint), executionRoleCredentials);+ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("background-layer");+ fn.setFunctionArn("arn:aws:lambda:" + region + ":" + account + ":function:background-layer");+ fn.setRuntime("nodejs20.x");+ fn.setHandler("index.handler");+ fn.setHotReloadHostPath(tempDir.toString());+ fn.setLayers(List.of((defaultAccount.equals(account) ? defaultLayer : ownerLayer).getLayerVersionArn()));++ ContainerHandle handle = CompletableFuture.supplyAsync(() -> backgroundLauncher.launch(fn))+ .get(10, TimeUnit.SECONDS);+ try {+ assertEquals(Map.of("owner.txt", account), capturedLayerFiles);+ } finally {+ backgroundLauncher.stop(handle);+ }+ }+/*** Captures every {@link ContainerSpec} passed to {@code lifecycleManager.create(...)} and* returns the REAL Lambda container's spec.diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/kubernetes/KubernetesPodLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/kubernetes/KubernetesPodLauncherTest.javaindex 34a375b582..7ba31a4d57 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/kubernetes/KubernetesPodLauncherTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/kubernetes/KubernetesPodLauncherTest.java@@ -11,11 +11,13 @@ import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.config.TlsConfigSource;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;+import io.github.hectorvent.floci.services.lambda.LambdaLayerFixtures;import io.github.hectorvent.floci.services.lambda.LambdaLayerService;import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;import io.github.hectorvent.floci.services.lambda.launcher.ImageResolver;import io.github.hectorvent.floci.services.lambda.model.ContainerState;import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;+import io.github.hectorvent.floci.services.lambda.model.LambdaLayerVersion;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;import io.github.hectorvent.floci.services.s3.PreSignedUrlGenerator;@@ -26,6 +28,7 @@ import org.junit.jupiter.api.Test;import org.junit.jupiter.api.io.TempDir;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.CsvSource;+import org.junit.jupiter.params.provider.ValueSource;import javax.net.ssl.SSLContext;import javax.net.ssl.TrustManager;@@ -41,6 +44,7 @@ import java.util.List;import java.util.Map;import java.util.Optional;import java.util.concurrent.CompletableFuture;+import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicBoolean;import static org.assertj.core.api.Assertions.assertThat;@@ -142,6 +146,44 @@ class KubernetesPodLauncherTest {logStreamer, s3Service, presignGenerator);}+ @ParameterizedTest+ @ValueSource(strings = {"000000000000", "123456789012"})+ void backgroundColdStartDownloadsOnlyTheFunctionsOwnedLayer(String account) throws Exception {+ String defaultAccount = "000000000000";+ String owner = "123456789012";+ String region = "eu-west-1";+ LambdaLayerVersion defaultLayer = LambdaLayerFixtures.layer(defaultAccount, region, "shared", 1);+ LambdaLayerVersion ownerLayer = LambdaLayerFixtures.layer(owner, region, "shared", 1);+ LambdaLayerService layers = LambdaLayerFixtures.accountScopedService(+ defaultAccount, region, Map.of(defaultAccount, defaultLayer, owner, ownerLayer));+ KubernetesPodLauncher backgroundLauncher = new KubernetesPodLauncher(+ apiClient, config, runtimeApiServerFactory, imageResolver, addressResolver, awsEnv,+ layers, new LambdaPodSpecFactory(config), logStreamer, s3Service, presignGenerator);+ LambdaFunction fn = function();+ fn.setAccountId(account);+ fn.setFunctionArn("arn:aws:lambda:" + region + ":" + account + ":function:my-fn");+ fn.setLayers(List.of((defaultAccount.equals(account) ? defaultLayer : ownerLayer).getLayerVersionArn()));+ markPodPhaseInBackground("floci-lambda-my-fn-", "Running");++ ContainerHandle handle = CompletableFuture.supplyAsync(() -> backgroundLauncher.launch(fn))+ .get(10, TimeUnit.SECONDS);+ try {+ Pod pod = client.pods().inNamespace("default").withName(handle.getContainerId()).get();+ String initScript = pod.getSpec().getInitContainers().getFirst().getCommand().get(2);+ assertThat(initScript).contains("/awslambda-" + region + "-tasks/layers/"+ + account + "/shared/1?X-Amz-Algorithm=AWS4-HMAC-SHA256")+ .contains("unzip -oq /tmp/layer0.zip -d /opt");+ String otherAccount = defaultAccount.equals(account) ? owner : defaultAccount;+ assertThat(initScript).doesNotContain("/layers/" + otherAccount + "/");+ verify(presignGenerator).generatePresignedUrl(+ eq("http://10.0.0.5:4566"), eq("awslambda-" + region + "-tasks"),+ eq("layers/" + account + "/shared/1"), eq("GET"), eq(0), eq(region), eq(account));+ } finally {+ backgroundLauncher.stop(handle);+ }+ assertThat(client.pods().inNamespace("default").withName(handle.getContainerId()).get()).isNull();+ }+/** The mock server's cert is self-signed and generated per run; trust it, not the JVM default. */private static HttpClient trustAllHttpClient() {try {