Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(lambdaweb): keep the code a revision was created from (#6113)

TATSUNO “Taz” Yasuhirobf8284697178
diff --git a/docs/services/lambda-web-functions.md b/docs/services/lambda-web-functions.md
index 4a459b25c8..54f1aab957 100644
--- a/docs/services/lambda-web-functions.md
+++ b/docs/services/lambda-web-functions.md
@@ -35,7 +35,8 @@ The `Lambda Web` service model (apiVersion 2025-03-07, `aws lambda-web` in the A
## Emulation Behavior
- **Instant convergence:** a revision and an endpoint are stored `Active` and an endpoint update is stored `Successful`. The create and update responses still report the transitional state AWS reports (`Pending`, `updateStatus: InProgress`, and the per-region block as it was before the update), so a waiter sees one poll.
-- **Code comes from S3 only.** The object named in `buildConfig.codeConfig.s3Object` must exist in Floci's S3, as on AWS. AWS also requires bucket versioning and a bucket policy for the Lambda service principal. Floci does not check them yet, and the revision does not pin a `versionId` the request did not give yet.
+- **Code comes from S3 only.** The object named in `buildConfig.codeConfig.s3Object` must exist in Floci's S3, as on AWS. AWS also requires bucket versioning and a bucket policy for the Lambda service principal. Floci does not check them yet.
+- **Each revision keeps its own copy of the code**, read at creation from the version the request names or the current one, under the Lambda `code-path`. As on AWS, a later write or delete in the bucket does not change what the revision runs. AWS moves a revision to `Failed` when it later finds its object version gone, which Floci does not do.
- **Execution role and network connector** ARNs are checked for shape only. The assume-role check AWS makes is not done yet.
- **Resource policies** are checked the way AWS checks them before storing: the document grammar, a `lambda:` action, a `Resource` that names the function, a valid principal, and no public `Allow`. A wildcard principal needs a positive condition (`StringEquals`, `StringLike`, `ArnLike`, `Bool`) on a trusted key such as `aws:PrincipalAccount` or `aws:PrincipalArn`, with no wildcard in the account it names. An `IamAuth` endpoint evaluates it on requests, as below.
- **ARNs** given as identifiers must name the request's region and the caller's account, as on AWS. A revision or endpoint ARN resolves by its id or name only.
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/FileTrees.java b/src/main/java/io/github/hectorvent/floci/core/common/FileTrees.java
new file mode 100644
index 0000000000..bf85b2b5e0
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/core/common/FileTrees.java
@@ -0,0 +1,46 @@
+package io.github.hectorvent.floci.core.common;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Stream;
+
+public final class FileTrees {
+
+ private FileTrees() {
+ }
+
+ /**
+ * Deletes a file, or a directory and everything under it. A missing path is not an error. A
+ * failure does not stop the rest: the first one is thrown, with the later ones suppressed.
+ */
+ public static void delete(Path root) throws IOException {
+ if (!Files.exists(root)) {
+ return;
+ }
+ List<Path> paths;
+ try (Stream<Path> walk = Files.walk(root)) {
+ paths = walk.sorted(Comparator.reverseOrder()).toList();
+ } catch (UncheckedIOException e) {
+ throw e.getCause();
+ }
+ IOException failure = null;
+ for (Path path : paths) {
+ try {
+ Files.deleteIfExists(path);
+ } catch (IOException e) {
+ if (failure == null) {
+ failure = e;
+ } else {
+ failure.addSuppressed(e);
+ }
+ }
+ }
+ if (failure != null) {
+ throw failure;
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java
index 39023dd38f..51bb2cbddb 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java
@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.Base64Url;
import io.github.hectorvent.floci.core.common.Digests;
+import io.github.hectorvent.floci.core.common.FileTrees;
import io.github.hectorvent.floci.core.common.PaginatedResult;
import io.github.hectorvent.floci.core.common.Pagination;
import io.github.hectorvent.floci.core.common.RegionResolver;
@@ -39,7 +40,6 @@ import java.security.SecureRandom;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -52,7 +52,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.regex.Pattern;
-import java.util.stream.Stream;
@ApplicationScoped
public class CodeArtifactService implements Resettable {
@@ -2709,14 +2708,8 @@ public class CodeArtifactService implements Resettable {
if (!Files.isDirectory(assetRoot)) {
return;
}
- try (Stream<Path> paths = Files.walk(assetRoot)) {
- paths.sorted(Comparator.reverseOrder()).forEach(p -> {
- try {
- Files.deleteIfExists(p);
- } catch (IOException e) {
- LOG.errorv(e, "Failed to delete CodeArtifact asset file: {0}", p);
- }
- });
+ try {
+ FileTrees.delete(assetRoot);
} catch (IOException e) {
LOG.errorv(e, "Failed to reset CodeArtifact asset storage under {0}", assetRoot);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
index 5e7a008a03..871565bdf7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
@@ -7,6 +7,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.ContainerTeardown;
+import io.github.hectorvent.floci.core.common.FileTrees;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
@@ -46,7 +47,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.ArrayList;
-import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -862,11 +862,8 @@ public class CodeBuildRunner implements ContainerTeardown {
private void deleteDirectory(Path path) {
try {
- if (!Files.exists(path)) { return; }
- Files.walk(path).sorted(Comparator.reverseOrder()).forEach(p -> {
- try { Files.delete(p); } catch (Exception ignored) {}
- });
- } catch (Exception e) {
+ FileTrees.delete(path);
+ } catch (IOException e) {
LOG.debugv("Could not delete workspace {0}: {1}", path, e.getMessage());
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerService.java
index a44d305588..9fd76b910d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaLayerService.java
@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.Base64Url;
import io.github.hectorvent.floci.core.common.Digests;
+import io.github.hectorvent.floci.core.common.FileTrees;
import io.github.hectorvent.floci.core.common.PaginatedResult;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.SmithyValidationErrors;
@@ -20,7 +21,6 @@ import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.security.GeneralSecurityException;
import java.security.MessageDigest;
@@ -596,19 +596,8 @@ public class LambdaLayerService {
}
private void deleteDirectory(Path dir) {
- if (!Files.exists(dir)) {
- return;
- }
try {
- Files.walk(dir)
- .sorted(Comparator.reverseOrder())
- .forEach(p -> {
- try {
- Files.delete(p);
- } catch (IOException e) {
- LOG.warnv("Failed to delete {0}: {1}", p, e.getMessage());
- }
- });
+ FileTrees.delete(dir);
} catch (IOException e) {
LOG.warnv("Failed to delete layer directory {0}: {1}", dir, e.getMessage());
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.java b/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.java
index 9917998150..e4c82125ce 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.lambda.zip;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.FileTrees;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
@@ -10,7 +11,6 @@ import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
-import java.util.Comparator;
import java.util.stream.Stream;
/**
@@ -156,17 +156,10 @@ public class CodeStore {
if (!Files.exists(path)) {
return;
}
- try (Stream<Path> walk = Files.walk(path)) {
- walk.sorted(Comparator.reverseOrder())
- .forEach(p -> {
- try {
- Files.delete(p);
- } catch (IOException e) {
- LOG.warnv("Failed to delete {0}: {1}", p, e.getMessage());
- }
- });
+ try {
+ FileTrees.delete(path);
LOG.debugv("Deleted code for function: {0}", functionName);
- } catch (IOException | UncheckedIOException e) {
+ } catch (IOException e) {
LOG.warnv("Failed to delete code directory for {0}: {1}", functionName, e.getMessage());
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractor.java b/src/main/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractor.java
index b79c1dd9c6..1877ec1f15 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractor.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.lambda.zip;
+import io.github.hectorvent.floci.core.common.FileTrees;
import jakarta.enterprise.context.ApplicationScoped;
import org.jboss.logging.Logger;
@@ -10,10 +11,8 @@ import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
-import java.util.Comparator;
import java.util.Enumeration;
import java.util.UUID;
-import java.util.stream.Stream;
import java.util.zip.CRC32;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
@@ -44,18 +43,46 @@ public class ZipExtractor {
}
public void extractTo(byte[] zipBytes, Path targetDir, int maxEntries, Runnable beforeInstall) throws IOException {
+ checkLimits(maxEntries, zipBytes.length);
+
+ // Resolve to absolute path so that normalize() on entry paths stays comparable
+ Path absTarget = targetDir.toAbsolutePath().normalize();
+ Files.createDirectories(absTarget.getParent());
+
+ // ZipFile needs a seekable file, so the bytes are staged on disk. Stage inside the code
+ // store rather than java.io.tmpdir: the shared temp filesystem must not become a new
+ // unbounded consumer that concurrent deployments can exhaust and that unrelated
+ // services would then fail on. The code store already has to hold this package's
+ // *uncompressed* contents — always at least as large as the archive — so it is already
+ // provisioned for the size, and the staged copy is released as soon as extraction ends.
+ Path staged = Files.createTempFile(absTarget.getParent(), ".floci-staging-", ".zip");
+ try {
+ Files.write(staged, zipBytes);
+ extract(staged, absTarget, maxEntries, beforeInstall);
+ } finally {
+ Files.deleteIfExists(staged);
+ }
+ }
+
+ /** For an archive already on disk, which is opened in place instead of read into the heap and staged. */
+ public void extractTo(Path zipFile, Path targetDir, int maxEntries) throws IOException {
+ checkLimits(maxEntries, Files.size(zipFile));
+ Path absTarget = targetDir.toAbsolutePath().normalize();
+ Files.createDirectories(absTarget.getParent());
+ extract(zipFile, absTarget, maxEntries, null);
+ }
+
+ private static void checkLimits(int maxEntries, long compressedBytes) throws IOException {
if (maxEntries < 1) {
throw new IOException("ZIP archive entry limit must be at least 1: " + maxEntries);
}
- if (zipBytes.length > S3_MAX_COMPRESSED_BYTES) {
+ if (compressedBytes > S3_MAX_COMPRESSED_BYTES) {
throw new IOException("ZIP archive exceeds the " + S3_MAX_COMPRESSED_BYTES
- + " byte compressed limit: " + zipBytes.length);
+ + " byte compressed limit: " + compressedBytes);
}
+ }
- // Resolve to absolute path so that normalize() on entry paths stays comparable
- Path absTarget = targetDir.toAbsolutePath().normalize();
- Files.createDirectories(absTarget.getParent());
-
+ private void extract(Path archive, Path absTarget, int maxEntries, Runnable beforeInstall) throws IOException {
// Extract into a staging directory and swap it in, rather than writing over the target
// in place. A deployment package is the complete contents of the function, so a file
// dropped between two deploys has to disappear — writing in place only ever adds and
@@ -76,17 +103,8 @@ public class ZipExtractor {
// CRC and sizes regardless of the flag, so reading it accepts the same archives real
// AWS Lambda does — e.g. a Serverless Framework package bundling node_modules
// (issue #2593).
- //
- // ZipFile needs a seekable file, so the bytes are staged on disk. Stage inside the code
- // store rather than java.io.tmpdir: the shared temp filesystem must not become a new
- // unbounded consumer that concurrent deployments can exhaust and that unrelated
- // services would then fail on. The code store already has to hold this package's
- // *uncompressed* contents — always at least as large as the archive — so it is already
- // provisioned for the size, and the staged copy is released as soon as extraction ends.
- Path staged = Files.createTempFile(absTarget.getParent(), ".floci-staging-", ".zip");
try {
- Files.write(staged, zipBytes);
- try (ZipFile zip = new ZipFile(staged.toFile())) {
+ try (ZipFile zip = new ZipFile(archive.toFile())) {
Enumeration<? extends ZipEntry> entries = zip.entries();
int entryCount = 0;
long expandedBytes = 0;
@@ -151,10 +169,9 @@ public class ZipExtractor {
}
install(staging, absTarget);
} finally {
- Files.deleteIfExists(staged);
// A no-op once install() moved it; on failure this discards the partial staging tree
// and leaves the previous extraction untouched.
- deleteRecursively(staging);
+ FileTrees.delete(staging);
}
LOG.debugv("Extracted ZIP to: {0}", absTarget);
@@ -212,7 +229,7 @@ public class ZipExtractor {
// its metadata and warm pool updates while the new code is already serving. Leaving
// the tree behind costs disk and is recoverable; lying about the outcome is not.
try {
- deleteRecursively(previous);
+ FileTrees.delete(previous);
} catch (IOException e) {
LOG.warnv("Installed {0} but could not remove the superseded package at {1}: {2}",
absTarget, previous, e.getMessage());
@@ -229,17 +246,6 @@ public class ZipExtractor {
}
}
- private static void deleteRecursively(Path root) throws IOException {
- if (!Files.exists(root)) {
- return;
- }
- try (Stream<Path> paths = Files.walk(root)) {
- for (Path p : paths.sorted(Comparator.reverseOrder()).toList()) {
- Files.deleteIfExists(p);
- }
- }
- }
-
/**
* Copies one entry to disk, checking it against the CRC recorded for it in the archive.
* ZipInputStream verified this on its own; ZipFile does not, so without an explicit check
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebCode.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebCode.java
new file mode 100644
index 0000000000..56fbc472cb
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebCode.java
@@ -0,0 +1,144 @@
+package io.github.hectorvent.floci.services.lambdaweb;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.FileTrees;
+import io.github.hectorvent.floci.services.lambdaweb.model.WebFunction;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.stream.Stream;
+
+/**
+ * The code package each revision copied from S3 when it was created. AWS prepares its own copy at
+ * creation, so a later write or delete in the bucket does not change what a revision runs.
+ */
+@ApplicationScoped
+public class LambdaWebCode {
+
+ private static final Logger LOG = Logger.getLogger(LambdaWebCode.class);
+
+ /** The {@code @} keeps this tree apart from the Lambda function directories, whose names never have one. */
+ private static final String ROOT = "@web-functions";
+
+ private final Path baseDir;
+
+ @Inject
+ public LambdaWebCode(EmulatorConfig config) {
+ this(Path.of(config.services().lambda().codePath()).resolve(ROOT));
+ }
+
+ LambdaWebCode(Path baseDir) {
+ this.baseDir = baseDir;
+ }
+
+ void store(String accountId, String region, String functionName, String revisionId, InputStream zip) {
+ Path file = file(accountId, region, functionName, revisionId);
+ try {
+ Files.createDirectories(file.getParent());
+ Files.copy(zip, file, StandardCopyOption.REPLACE_EXISTING);
+ } catch (IOException e) {
+ deleteRevision(accountId, region, functionName, revisionId);
+ throw new UncheckedIOException("Could not keep the code of " + functionName + " revision " + revisionId, e);
+ }
+ }
+
+ void deleteRevision(String accountId, String region, String functionName, String revisionId) {
+ try {
+ Files.deleteIfExists(file(accountId, region, functionName, revisionId));
+ } catch (IOException e) {
+ LOG.warnv("Could not delete the code of {0} revision {1}: {2}", functionName, revisionId, e.getMessage());
+ }
+ }
+
+ void deleteFunction(String accountId, String region, String functionName) {
+ deleteTree(functionDir(accountId, region, functionName));
+ }
+
+ void clear() {
+ deleteTree(baseDir);
+ }
+
+ /**
+ * Removes every copy no stored revision names: those a restart in memory mode orphaned, and
+ * any a failed delete left behind.
+ */
+ void retainOnly(List<WebFunction> functions) {
+ if (!Files.isDirectory(baseDir)) {
+ return;
+ }
+ Set<Path> kept = new HashSet<>();
+ for (WebFunction fn : functions) {
+ AwsArnUtils.Arn arn = AwsArnUtils.parse(fn.functionArn);
+ for (String revisionId : fn.revisions.keySet()) {
+ kept.add(file(arn.accountId(), arn.region(), fn.functionName, revisionId));
+ }
+ }
+ try (Stream<Path> walk = Files.walk(baseDir)) {
+ walk.sorted(Comparator.reverseOrder())
+ .filter(path -> Files.isDirectory(path) ? isEmpty(path) : !kept.contains(path))
+ .forEach(path -> {
+ try {
+ Files.delete(path);
+ } catch (IOException e) {
+ LOG.warnv("Could not delete {0}: {1}", path, e.getMessage());
+ }
+ });
+ } catch (IOException | UncheckedIOException e) {
+ LOG.warnv("Could not clean {0}: {1}", baseDir, e.getMessage());
+ }
+ }
+
+ private static boolean isEmpty(Path dir) {
+ try (Stream<Path> entries = Files.list(dir)) {
+ return entries.findAny().isEmpty();
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ Path file(String accountId, String region, String functionName, String revisionId) {
+ return functionDir(accountId, region, functionName).resolve(segment(revisionId) + ".zip");
+ }
+
+ private Path functionDir(String accountId, String region, String functionName) {
+ return baseDir.resolve(segment(accountId)).resolve(segment(region)).resolve(segment(functionName));
+ }
+
+ /**
+ * Escapes every byte but letters, digits, {@code -} and {@code _}, so two regions never share a
+ * directory and {@code .} or {@code ..} cannot navigate.
+ */
+ private static String segment(String value) {
+ StringBuilder out = new StringBuilder();
+ for (byte b : value.getBytes(StandardCharsets.UTF_8)) {
+ if (('a' <= b && b <= 'z') || ('A' <= b && b <= 'Z') || ('0' <= b && b <= '9') || b == '-' || b == '_') {
+ out.append((char) b);
+ } else {
+ out.append('%').append(String.format(Locale.ROOT, "%02X", b & 0xFF));
+ }
+ }
+ return out.toString();
+ }
+
+ private static void deleteTree(Path root) {
+ try {
+ FileTrees.delete(root);
+ } catch (IOException e) {
+ LOG.warnv("Could not delete {0}: {1}", root, e.getMessage());
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java
index 5ed5f4478c..cfde3c2821 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java
@@ -9,7 +9,7 @@ import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.api.model.StreamType;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.ContainerTeardown;
-import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.core.common.FileTrees;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerExec;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
@@ -27,7 +27,6 @@ import io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCr
import io.github.hectorvent.floci.services.lambda.zip.ZipExtractor;
import io.github.hectorvent.floci.services.lambdaweb.model.WebFunction;
import io.github.hectorvent.floci.services.lambdaweb.model.WebFunctionRevision;
-import io.github.hectorvent.floci.services.s3.S3Service;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -261,7 +260,7 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
private final DockerClient streamingDocker;
private final ImageResolver imageResolver;
private final EcrRegistryManager ecrRegistryManager;
- private final S3Service s3Service;
+ private final LambdaWebCode code;
private final ZipExtractor zipExtractor;
private final LaunchedContainerAwsEnv awsEnv;
private final LambdaExecutionRoleCredentials executionRoleCredentials;
@@ -280,7 +279,7 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
public LambdaWebEnvironmentManager(EmulatorConfig config, ContainerBuilder containerBuilder,
ContainerLifecycleManager lifecycleManager,
@StreamingDocker DockerClient streamingDocker, ImageResolver imageResolver,
- EcrRegistryManager ecrRegistryManager, S3Service s3Service,
+ EcrRegistryManager ecrRegistryManager, LambdaWebCode code,
ZipExtractor zipExtractor, LaunchedContainerAwsEnv awsEnv,
LambdaExecutionRoleCredentials executionRoleCredentials, LambdaWebLogs logs,
LambdaWebMetrics metrics) {
@@ -290,7 +289,7 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
this.streamingDocker = streamingDocker;
this.imageResolver = imageResolver;
this.ecrRegistryManager = ecrRegistryManager;
- this.s3Service = s3Service;
+ this.code = code;
this.zipExtractor = zipExtractor;
this.awsEnv = awsEnv;
this.executionRoleCredentials = executionRoleCredentials;
@@ -683,9 +682,7 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
/** The code goes in by {@code docker cp}, so it works when Floci itself runs in a container. */
private void copyCode(String containerId, String accountId, String region, WebFunction fn,
WebFunctionRevision revision) {
- WebFunctionRevision.BuildConfig build = revision.buildConfig;
- byte[] zip = RequestScopes.callAs(accountId, region,
- () -> s3Service.getObject(build.bucket, build.key, build.versionId)).getData();
+ Path zip = code.file(accountId, region, fn.functionName, revision.revisionId);
Path dir;
try {
dir = Files.createTempDirectory("floci-lambda-web-");
@@ -724,11 +721,8 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
}
private static void deleteRecursively(Path dir) {
- try (Stream<Path> walk = Files.walk(dir)) {
- List<Path> paths = walk.sorted(Comparator.reverseOrder()).toList();
- for (Path path : paths) {
- Files.deleteIfExists(path);
- }
+ try {
+ FileTrees.delete(dir);
} catch (IOException e) {
LOG.debugv(e, "Could not remove the unpacked code at {0}", dir);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebService.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebService.java
index d90cfeb326..06e336dbe1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebService.java
@@ -26,6 +26,8 @@ import jakarta.enterprise.inject.Instance;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.io.IOException;
+import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
@@ -135,6 +137,7 @@ public class LambdaWebService implements Resettable {
private final StorageFactory storageFactory;
private final EmulatorConfig config;
private final S3Service s3Service;
+ private final LambdaWebCode code;
private final ObjectMapper objectMapper;
private final Instance<RequestContext> requestContext;
@@ -148,11 +151,13 @@ public class LambdaWebService implements Resettable {
@Inject
public LambdaWebService(RegionResolver regionResolver, StorageFactory storageFactory, EmulatorConfig config,
- S3Service s3Service, ObjectMapper objectMapper, Instance<RequestContext> requestContext) {
+ S3Service s3Service, LambdaWebCode code, ObjectMapper objectMapper,
+ Instance<RequestContext> requestContext) {
this.regionResolver = regionResolver;
this.storageFactory = storageFactory;
this.config = config;
this.s3Service = s3Service;
+ this.code = code;
this.objectMapper = objectMapper;
this.requestContext = requestContext;
}
@@ -173,6 +178,7 @@ public class LambdaWebService implements Resettable {
for (WebFunction fn : all) {
fn.endpoints.values().forEach(endpoint -> indexDomains(fn, endpoint));
}
+ code.retainOnly(all);
}
@Override
@@ -180,6 +186,7 @@ public class LambdaWebService implements Resettable {
functions.clear();
domains.clear();
originHosts.clear();
+ code.clear();
}
/** Where a request to an endpoint domain goes. Domains are unique across accounts and regions. */
@@ -299,6 +306,7 @@ public class LambdaWebService implements Resettable {
WebFunctionEndpoint endpoint = draftEndpoint(region, accountId, functionName, endpointConfig, false);
return storeNewFunction(region, functionName, tags, fn -> {
WebFunctionRevision revision = buildRevision(fn, revisionConfig);
+ keepCode(region, fn, revision);
attachEndpoint(region, fn, endpoint, revision);
fn.revisions.put(revision.revisionId, revision);
fn.endpoints.put(endpoint.endpointName, endpoint);
@@ -376,6 +384,7 @@ public class LambdaWebService implements Resettable {
WebFunction fn = function(region, functionName);
fn.endpoints.values().forEach(this::unindexDomains);
functions.delete(key(region, functionName));
+ code.deleteFunction(AwsArnUtils.parse(fn.functionArn).accountId(), region, functionName);
}
LOG.infov("Deleted web function {0}", functionName);
}
@@ -396,6 +405,7 @@ public class LambdaWebService implements Resettable {
synchronized (lockFor(region, functionName)) {
WebFunction fn = function(region, functionName);
WebFunctionRevision revision = buildRevision(fn, request);
+ keepCode(region, fn, revision);
fn.revisions.put(revision.revisionId, revision);
for (WebFunctionEndpoint endpoint : fn.endpoints.values()) {
if ("LatestRevision".equals(endpoint.autoDeploymentMode)) {
@@ -456,6 +466,7 @@ public class LambdaWebService implements Resettable {
fn.revisions.remove(revision.revisionId);
fn.updatedAt = now();
persist(region, fn);
+ code.deleteRevision(AwsArnUtils.parse(fn.functionArn).accountId(), region, functionName, revisionId);
}
}
@@ -520,7 +531,6 @@ public class LambdaWebService implements Resettable {
}
revision.serviceConfig = serviceConfigOf(LambdaWebJson.structure(request, "serviceConfig"),
defaultLogGroup(fn.functionName));
- requireCodeObject(revision.buildConfig);
revision.description = LambdaWebJson.text(request, "description");
revision.kmsKeyArn = LambdaWebJson.text(request, "kmsKeyArn");
String revisionId;
@@ -575,18 +585,32 @@ public class LambdaWebService implements Resettable {
return LambdaWebJson.structure(LambdaWebJson.structure(service, "telemetryConfig"), "loggingConfig");
}
- /** The code object must exist in the bucket's current version or the version the request pins. */
- private void requireCodeObject(WebFunctionRevision.BuildConfig build) {
+ /** Copies the code a revision runs: the bucket's current version, or the version the request pins. */
+ private void keepCode(String region, WebFunction fn, WebFunctionRevision revision) {
if (s3Service == null) {
return;
}
- try {
- s3Service.headObject(build.bucket, build.key, build.versionId);
+ WebFunctionRevision.BuildConfig build = revision.buildConfig;
+ try (S3Service.ObjectRead object = s3Service.openObject(build.bucket, build.key, build.versionId)) {
+ code.store(AwsArnUtils.parse(fn.functionArn).accountId(), region, fn.functionName, revision.revisionId,
+ object.body());
+ } catch (IOException e) {
+ throw new UncheckedIOException("Could not close the code object s3://" + build.bucket + "/" + build.key, e);
} catch (AwsException e) {
LOG.debugv("Web function code object s3://{0}/{1} is not readable: {2}", build.bucket, build.key,
e.getMessage());
- throw new AwsException("ValidationException", "The specified S3 object does not exist: s3://"
- + build.bucket + "/" + build.key + ". Verify the object exists in the bucket.", 400);
+ String object = "s3://" + build.bucket + "/" + build.key;
+ if (build.versionId == null) {
+ throw new AwsException("ValidationException", "The specified S3 object does not exist: " + object
+ + ". Verify the object exists in the bucket.", 400);
+ }
+ if (e.getHttpStatus() == 404) {
+ throw new AwsException("ValidationException", "The specified S3 object does not exist: " + object
+ + " (versionId=" + build.versionId + "). Verify the object exists and the versionId is correct.",
+ 400);
+ }
+ throw new AwsException("ValidationException", "The S3 object " + object + " (versionId=" + build.versionId
+ + ") is inaccessible. Verify the request parameters and permissions are correct.", 400);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index a661de36ca..98727fdcee 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -12,6 +12,7 @@ import io.github.hectorvent.floci.core.common.AwsNamespaces;
import io.github.hectorvent.floci.core.common.AwsRegions;
import io.github.hectorvent.floci.core.common.Base64Url;
import io.github.hectorvent.floci.core.common.Digests;
+import io.github.hectorvent.floci.core.common.FileTrees;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.common.Resettable;
@@ -7349,15 +7350,8 @@ public class S3Service implements Resettable, ResourceProvider {
}
private void deleteDirectory(Path dir) {
- if (!Files.exists(dir)) return;
- try (var walk = Files.walk(dir)) {
- walk.sorted(Comparator.reverseOrder()).forEach(path -> {
- try {
- Files.deleteIfExists(path);
- } catch (IOException e) {
- LOG.errorv(e, "Failed to delete: {0}", path);
- }
- });
+ try {
+ FileTrees.delete(dir);
} catch (IOException e) {
LOG.errorv(e, "Failed to delete directory: {0}", dir);
}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/FileTreesTest.java b/src/test/java/io/github/hectorvent/floci/core/common/FileTreesTest.java
new file mode 100644
index 0000000000..84a64bdbd1
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/common/FileTreesTest.java
@@ -0,0 +1,43 @@
+package io.github.hectorvent.floci.core.common;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class FileTreesTest {
+
+ @Test
+ void deletesADirectoryAndEverythingUnderIt(@TempDir Path dir) throws IOException {
+ Path root = dir.resolve("root");
+ Files.createDirectories(root.resolve("a").resolve("b"));
+ Files.writeString(root.resolve("a").resolve("b").resolve("file.txt"), "x");
+ Files.writeString(root.resolve("top.txt"), "y");
+ Files.writeString(dir.resolve("sibling.txt"), "z");
+
+ FileTrees.delete(root);
+
+ assertFalse(Files.exists(root));
+ assertTrue(Files.exists(dir.resolve("sibling.txt")));
+ }
+
+ @Test
+ void deletesASingleFile(@TempDir Path dir) throws IOException {
+ Path file = Files.writeString(dir.resolve("file.txt"), "x");
+
+ FileTrees.delete(file);
+
+ assertFalse(Files.exists(file));
+ }
+
+ @Test
+ void aMissingPathIsNotAnError(@TempDir Path dir) {
+ assertDoesNotThrow(() -> FileTrees.delete(dir.resolve("missing")));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractorTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractorTest.java
index e6fc64ee18..8db173e82a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractorTest.java
@@ -453,6 +453,20 @@ class ZipExtractorTest {
}
}
+ @Test
+ void extractsAnArchiveFileWithoutStagingACopy(@TempDir Path codeStore) throws IOException {
+ Path archive = codeStore.resolve("code.zip");
+ Files.write(archive, zipWith("index.js", "v1"));
+ Path target = codeStore.resolve("fn");
+
+ extractor.extractTo(archive, target, ZipExtractor.DEFAULT_MAX_ENTRIES);
+
+ assertEquals("v1", Files.readString(target.resolve("index.js")));
+ try (Stream<Path> entries = Files.list(codeStore)) {
+ assertEquals(List.of("code.zip", "fn"), entries.map(p -> p.getFileName().toString()).sorted().toList());
+ }
+ }
+
@Test
void rejectsAnEntryWhoseCompressionMethodCannotBeRead(@TempDir Path target) throws IOException {
// A package must never be half-deployed because an entry used a codec the reader does
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java
index 829d93d1b0..f6dea4ad43 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java
@@ -403,6 +403,15 @@ class LambdaWebDataPlaneDockerIntegrationTest {
.then().statusCode(200).header("lambda-web-logstream", not(equalTo(logStream)));
}
+ @Test
+ @Timeout(180)
+ void anEnvironmentRunsTheCodeTheRevisionWasCreatedFrom() throws Exception {
+ String domain = createFunction(30);
+ s3Service.putObject(BUCKET, functionName + ".zip", zip("process.exit(1);"), "application/zip", Map.of());
+
+ given().header("Host", domain).when().get("/status/200").then().statusCode(200).body(equalTo("status body"));
+ }
+
@Test
@Timeout(240)
void environmentsScaleUpToMaxEnvironmentsAndTheRateLimitThrottles() throws Exception {
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebServiceTest.java
index 9ff275ea1f..7ea470b211 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebServiceTest.java
@@ -16,9 +16,15 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.lambdaweb.model.WebFunction;
import io.github.hectorvent.floci.services.lambdaweb.model.WebFunctionEndpoint;
import io.github.hectorvent.floci.services.lambdaweb.model.WebFunctionRevision;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
@@ -30,6 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -54,10 +61,14 @@ class LambdaWebServiceTest {
}
private LambdaWebService service(StorageFactory storage, String baseUrl) {
+ return service(storage, baseUrl, null);
+ }
+
+ private LambdaWebService service(StorageFactory storage, String baseUrl, S3Service s3Service) {
EmulatorConfig config = mock(EmulatorConfig.class);
when(config.effectiveBaseUrl()).thenReturn(baseUrl);
- LambdaWebService service = new LambdaWebService(new RegionResolver(REGION, ACCOUNT_ID), storage, config, null,
- mapper, null);
+ LambdaWebService service = new LambdaWebService(new RegionResolver(REGION, ACCOUNT_ID), storage, config,
+ s3Service, new LambdaWebCode(storageDir.resolve("code")), mapper, null);
service.initializeStorage();
return service;
}
@@ -587,6 +598,81 @@ class LambdaWebServiceTest {
assertTrue(service.endpointDomain(host).isEmpty(), "a deleted endpoint's domain is gone");
}
+ @Test
+ void eachRevisionKeepsTheCodeTheObjectHadWhenItWasCreated() throws IOException {
+ S3Service s3 = mock(S3Service.class);
+ LambdaWebService service = service(new MemoryStorageFactory(), "http://localhost:4566", s3);
+ LambdaWebCode code = new LambdaWebCode(storageDir.resolve("code"));
+ when(s3.openObject("code", "app.zip", null)).thenReturn(object("first"));
+ String first = service.createFunction(REGION, createRequest("app")).revisions.keySet().iterator().next();
+ when(s3.openObject("code", "app.zip", null)).thenReturn(object("second"));
+ String second = service.createRevision(REGION, "app", revisionConfig(30)).revisionId;
+
+ assertEquals("first", Files.readString(code.file(ACCOUNT_ID, REGION, "app", first)));
+ assertEquals("second", Files.readString(code.file(ACCOUNT_ID, REGION, "app", second)));
+ service.deleteRevision(REGION, "app", first);
+ assertFalse(Files.exists(code.file(ACCOUNT_ID, REGION, "app", first)));
+ service.deleteFunction(REGION, "app");
+ assertFalse(Files.exists(code.file(ACCOUNT_ID, REGION, "app", second)));
+ }
+
+ @Test
+ void aStartKeepsOnlyTheCopiesOfStoredRevisions() throws IOException {
+ S3Service s3 = mock(S3Service.class);
+ when(s3.openObject("code", "app.zip", null)).thenReturn(object("code"));
+ FileStorageFactory storage = new FileStorageFactory(storageDir);
+ String revisionId = service(storage, "http://localhost:4566", s3).createFunction(REGION, createRequest("app"))
+ .revisions.keySet().iterator().next();
+ LambdaWebCode code = new LambdaWebCode(storageDir.resolve("code"));
+ code.store(ACCOUNT_ID, REGION, "app", "rev-stray", new ByteArrayInputStream(new byte[] {1}));
+
+ service(new FileStorageFactory(storageDir), "http://localhost:4566", s3);
+ assertEquals("code", Files.readString(code.file(ACCOUNT_ID, REGION, "app", revisionId)));
+ assertFalse(Files.exists(code.file(ACCOUNT_ID, REGION, "app", "rev-stray")));
+
+ service(new MemoryStorageFactory(), "http://localhost:4566", s3);
+ assertFalse(Files.exists(code.file(ACCOUNT_ID, REGION, "app", revisionId)));
+ }
+
+ @Test
+ void regionsThatLookAlikeKeepTheirOwnCopies() throws IOException {
+ LambdaWebCode code = new LambdaWebCode(storageDir.resolve("code"));
+ code.store(ACCOUNT_ID, ".", "app", "rev-aaaaa", new ByteArrayInputStream(new byte[] {1}));
+ code.store(ACCOUNT_ID, "_", "app", "rev-aaaaa", new ByteArrayInputStream(new byte[] {2}));
+ code.deleteFunction(ACCOUNT_ID, ".", "app");
+
+ assertEquals(2, Files.readAllBytes(code.file(ACCOUNT_ID, "_", "app", "rev-aaaaa"))[0]);
+ }
+
+ @Test
+ void aVersionThatCannotBeReadIsNamedInTheError() {
+ S3Service s3 = mock(S3Service.class);
+ LambdaWebService service = service(new MemoryStorageFactory(), "http://localhost:4566", s3);
+ ObjectNode pinned = createRequest("app");
+ ((ObjectNode) pinned.path("revisionConfig").path("buildConfig").path("codeConfig").path("s3Object"))
+ .put("versionId", "v1");
+ when(s3.openObject("code", "app.zip", "v1"))
+ .thenThrow(new AwsException("NoSuchVersion", "The specified version does not exist.", 404));
+ assertEquals("The specified S3 object does not exist: s3://code/app.zip (versionId=v1). Verify the object exists"
+ + " and the versionId is correct.", message(() -> service.createFunction(REGION, pinned)));
+
+ doThrow(new AwsException("InvalidArgument", "Invalid version id specified", 400))
+ .when(s3).openObject("code", "app.zip", "v1");
+ assertEquals("The S3 object s3://code/app.zip (versionId=v1) is inaccessible. Verify the request parameters and"
+ + " permissions are correct.", message(() -> service.createFunction(REGION, pinned)));
+
+ when(s3.openObject("code", "app.zip", null))
+ .thenThrow(new AwsException("NoSuchKey", "The specified key does not exist.", 404));
+ assertEquals("The specified S3 object does not exist: s3://code/app.zip. Verify the object exists in the bucket.",
+ message(() -> service.createFunction(REGION, createRequest("app"))));
+ }
+
+ private static S3Service.ObjectRead object(String data) {
+ byte[] bytes = data.getBytes(StandardCharsets.UTF_8);
+ return new S3Service.ObjectRead(new S3Object("code", "app.zip", bytes, "application/zip"),
+ new ByteArrayInputStream(bytes));
+ }
+
private static final class MemoryStorageFactory extends StorageFactory {
private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();