Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): give each published version its own copy of the code (#3041)
David4ee5a528f367
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.javaindex bda6403f7c..55a5f27125 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java@@ -850,7 +850,8 @@ public class LambdaService implements ResourceProvider {synchronized (lockForConcurrencyOp(fn.getFunctionArn())) {warmPool.drainEnvironment(version.get());functionStore.deleteVersion(region, name, qualifier);- // The snapshot shares $LATEST's code directory, so this only reclaims once no+ reclaimVersionCodeDirectory(fn, qualifier, version.get());+ // The snapshot may still share $LATEST's code directory, so this only reclaims once no// remaining version references it.reclaimLegacyCodeDirectoryIfUnused(name);}@@ -896,6 +897,28 @@ public class LambdaService implements ResourceProvider {LOG.infov("Deleted Lambda function: {0}", functionName);}+ /**+ * Drops the code directory a deleted version owned. Without this the only thing that ever+ * reclaimed version code was deleting the whole function, so a repeated publish/delete cycle+ * left one package on disk per version ever published.+ *+ * <p>Guarded on the version actually owning that directory rather than deleting it outright.+ * A version whose copy could not be made fell back to {@code $LATEST}'s path, and image-backed+ * and hot-reload versions never had a copy at all: for those the recorded path is the live+ * function's own directory, and removing it would delete the code {@code $LATEST} still runs.+ */+ private void reclaimVersionCodeDirectory(LambdaFunction fn, String version, LambdaFunction snapshot) {+ String recorded = snapshot.getCodeLocalPath();+ if (recorded == null) {+ return;+ }+ String owned = codeStore.getVersionCodePath(ownerAccount(fn), fn.getFunctionName(), version)+ .toAbsolutePath().normalize().toString();+ if (owned.equals(Path.of(recorded).toAbsolutePath().normalize().toString())) {+ codeStore.deleteVersion(ownerAccount(fn), fn.getFunctionName(), version);+ }+ }+/*** The pre-account-scoped code layout gave every account's same-named function the exact* same on-disk directory (CodeStore never scoped it by region either), so it is only safe@@ -1792,6 +1815,31 @@ public class LambdaService implements ResourceProvider {.max(java.util.Comparator.comparingLong(v -> Long.parseLong(v.getVersion())));}+ /**+ * Copies the function's current code into a directory belonging to this version, falling back+ * to {@code $LATEST}'s path if there is nothing to copy or the copy fails.+ *+ * <p>A copy failure must not fail the publish: the version is still a correct snapshot of the+ * configuration, and falling back leaves it exactly as good as every version published before+ * this existed, rather than turning a working call into an error.+ */+ private String versionCodePath(LambdaFunction fn, String version) {+ String current = fn.getCodeLocalPath();+ if (current == null || fn.getHotReloadHostPath() != null) {+ return current;+ }+ try {+ Path copied = codeStore.copyForVersion(+ ownerAccount(fn), fn.getFunctionName(), version, Path.of(current));+ return copied == null ? current : copied.toAbsolutePath().normalize().toString();+ } catch (IOException e) {+ LOG.warnv("Could not give version {0} of {1} its own code directory, "+ + "falling back to the shared one: {2}",+ version, fn.getFunctionName(), e.getMessage());+ return current;+ }+ }+private int nextVersionNumber(String counterKey, String legacyCounterKey) {synchronized (versionCounterLocks.computeIfAbsent(counterKey, k -> new Object())) {Integer current = versionCounters.get(counterKey);@@ -1908,7 +1956,13 @@ public class LambdaService implements ResourceProvider {// with nothing in it, and hangs to the function timeout instead of failing (#1987). A// published version is an immutable snapshot of code plus configuration, so it carries the// code location for every package type, not only Zip.- snapshot.setCodeLocalPath(fn.getCodeLocalPath());+ // A version's own copy of the code, not a reference to $LATEST's directory. Sharing that+ // directory meant a later UpdateFunctionCode rewrote what an already-published version+ // ran, so the version advertised one CodeSha256 over a different build (issue #2958).+ // Nothing to copy for image-backed or hot-reload functions, which keep the reference+ // they had: an image is already immutable by digest, and a hot-reload function's whole+ // point is that its bind-mounted directory tracks the developer's working tree.+ snapshot.setCodeLocalPath(versionCodePath(fn, String.valueOf(version)));snapshot.setCodeSha256(fn.getCodeSha256());snapshot.setS3Bucket(fn.getS3Bucket());snapshot.setS3Key(fn.getS3Key());diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.java b/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.javaindex 3f9bb7481d..f53feaaf37 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.java@@ -6,6 +6,7 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.io.IOException;+import java.nio.file.StandardCopyOption;import java.nio.file.Files;import java.nio.file.Path;import java.util.Comparator;@@ -22,6 +23,12 @@ public class CodeStore {private static final Logger LOG = Logger.getLogger(CodeStore.class);+ /**+ * Separates a function's versions directory from any function's own code directory. Deliberately+ * outside {@link #sanitizeName}'s output character set, so the two namespaces cannot overlap.+ */+ private static final String VERSIONS_DIR_SUFFIX = "@versions";+private final Path baseDir;@Inject@@ -46,8 +53,75 @@ public class CodeStore {return baseDir.resolve(sanitizeName(functionName));}+ /**+ * Where a function's published-version code lives: one directory, holding a subdirectory per+ * published version.+ *+ * <p>A sibling of the {@code $LATEST} directory rather than a child of it. Extraction replaces+ * {@link #getCodePath}'s directory wholesale, so anything nested inside would be deleted on the+ * next deploy, which is the very thing a version's copy exists to survive.+ *+ * <p>The {@link #VERSIONS_DIR_SUFFIX} is what keeps that sibling from colliding with a real+ * function. {@link #sanitizeName} maps every name into {@code [a-zA-Z0-9_.-]}, so no function+ * can produce a directory name containing {@code @}, whatever it is called. Floci does not+ * restrict the character set of {@code FunctionName} today, only that it is non-blank, so a+ * plainer {@code <name>.v<n>} sibling carried no such guarantee: a function genuinely named+ * {@code foo.v1} owns the exact directory {@code foo}'s version 1 would otherwise claim, and+ * deleting either one silently corrupts the other.+ */+ public Path getVersionsPath(String accountId, String functionName) {+ return baseDir.resolve(sanitizeName(accountId))+ .resolve(sanitizeName(functionName) + VERSIONS_DIR_SUFFIX);+ }++ /** Where one published version's own copy of the code lives. */+ public Path getVersionCodePath(String accountId, String functionName, String version) {+ return getVersionsPath(accountId, functionName).resolve(sanitizeName(version));+ }++ /**+ * Copies a function's current code into its own directory for {@code version}, replacing any+ * directory already there. Returns null when there is nothing to copy, which is the case for+ * image-backed and hot-reload functions.+ */+ public Path copyForVersion(String accountId, String functionName, String version, Path source)+ throws IOException {+ if (source == null || !Files.isDirectory(source)) {+ return null;+ }+ Path target = getVersionCodePath(accountId, functionName, version);+ deleteDirectory(target, functionName);+ try (var walk = Files.walk(source)) {+ for (Path from : walk.toList()) {+ Path to = target.resolve(source.relativize(from).toString());+ if (Files.isDirectory(from)) {+ Files.createDirectories(to);+ } else {+ Files.createDirectories(to.getParent());+ Files.copy(from, to, StandardCopyOption.REPLACE_EXISTING);+ }+ }+ }+ return target;+ }++ /** Removes every published version's code for a function. */+ public void deleteVersions(String accountId, String functionName) {+ deleteDirectory(getVersionsPath(accountId, functionName), functionName);+ }++ /**+ * Removes one published version's code, leaving the function's other versions and+ * {@code $LATEST} in place. Deleting a single version otherwise left its package on disk for+ * as long as the data directory lived, so a repeated publish/delete cycle grew without bound.+ */+ public void deleteVersion(String accountId, String functionName, String version) {+ deleteDirectory(getVersionCodePath(accountId, functionName, version), functionName);+ }+public void delete(String accountId, String functionName) {deleteDirectory(getCodePath(accountId, functionName), functionName);+ deleteVersions(accountId, functionName);}/**diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAccountScopedCodeTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAccountScopedCodeTest.javaindex a119010762..f1f94002a6 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAccountScopedCodeTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAccountScopedCodeTest.java@@ -110,14 +110,18 @@ class LambdaAccountScopedCodeTest {}@Test- void updateFunctionCodeDoesNotDeleteALegacyDirectoryStillReferencedByAPublishedVersion(+ void aPublishedVersionsCodeSurvivesLatestMigratingOffTheLegacyPath(@TempDir Path baseDir) throws Exception {- // publishVersion snapshots codeLocalPath verbatim (it must, or a version-qualified- // invoke launches a container with no code - see #1987). If $LATEST was still on the- // legacy path at publish time, that version's snapshot is now the ONLY thing keeping- // the legacy directory alive once $LATEST itself migrates. The unused-check only scanned- // $LATEST records, so it missed this and reclaimed the directory out from under the- // published version's own future invokes.+ // publishVersion used to snapshot codeLocalPath verbatim (it had to, or a+ // version-qualified invoke launched a container with no code - see #1987). If $LATEST was+ // still on the legacy path at publish time, that version's snapshot became the ONLY thing+ // keeping the legacy directory alive once $LATEST itself migrated, and the unused-check+ // reclaimed it out from under the version's own future invokes.+ //+ // A version now copies the code into a directory of its own (#2958), so it no longer+ // depends on the legacy directory surviving. The guarantee this test protects is unchanged+ // and now stronger: whatever happens to the directory $LATEST was using, the version keeps+ // the code it was published from.CodeStore codeStore = new CodeStore(baseDir);LambdaService svc = serviceFor(ACCOUNT_A, codeStore);svc.createFunction(REGION, zipRequest("legacy-version-fn", "v1"));@@ -129,13 +133,17 @@ class LambdaAccountScopedCodeTest {latest.setCodeLocalPath(legacyPath.toAbsolutePath().normalize().toString());LambdaFunction version = svc.publishVersion(REGION, "legacy-version-fn", null);- assertEquals(legacyPath.toAbsolutePath().normalize().toString(), version.getCodeLocalPath(),- "the published version must have snapshotted the legacy path");+ Path versionPath = Path.of(version.getCodeLocalPath());+ assertNotEquals(legacyPath.toAbsolutePath().normalize().toString(), version.getCodeLocalPath(),+ "the published version must own its code rather than reference the legacy directory");+ assertEquals("v1", Files.readString(versionPath.resolve("index.js")).trim());svc.updateFunctionCode(REGION, "legacy-version-fn", Map.of("ZipFile", zipBase64("index.js", "v2")));- assertTrue(Files.exists(legacyPath),- "a legacy directory a published version still references must survive this update");+ assertTrue(Files.isDirectory(versionPath),+ "the published version's own code must survive $LATEST migrating off the legacy path");+ assertEquals("v1", Files.readString(versionPath.resolve("index.js")).trim(),+ "the version must still hold the code it was published from");}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.javaindex 79b79507a4..93a11884f3 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.java@@ -15,6 +15,8 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import java.io.ByteArrayOutputStream;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;import java.nio.file.Path;import java.util.Base64;import java.util.HashSet;@@ -1467,4 +1469,94 @@ class LambdaServiceTest {"FunctionName", "arn:aws:lambda:us-west-2:000000000000:function:other-region-fn")));}++ /**+ * Issue #2958: a published version stored a reference to {@code $LATEST}'s code directory+ * rather than a copy of the code, and extraction replaces that directory wholesale on every+ * deploy. A later UpdateFunctionCode therefore rewrote what an already-published version would+ * run, leaving the version advertising one CodeSha256 over a different build.+ */+ @Test+ void aPublishedVersionKeepsItsOwnCodeWhenLatestIsRedeployed() throws Exception {+ Map<String, Object> request = baseRequest("version-code-isolation-fn");+ request.put("Code", Map.of("ZipFile", zipWithBody("v1")));+ service.createFunction(REGION, request);++ LambdaFunction v1 = service.publishVersion(REGION, "version-code-isolation-fn", null);+ Path v1Path = Path.of(v1.getCodeLocalPath());+ String v1Sha = v1.getCodeSha256();++ assertTrue(Files.isDirectory(v1Path), "a published version must have its own code directory");+ assertEquals("v1", Files.readString(v1Path.resolve("index.js")).trim());++ // Redeploy $LATEST. Extraction replaces its directory wholesale, which is what used to take+ // the published version's code with it.+ service.updateFunctionCode(REGION, "version-code-isolation-fn",+ Map.of("ZipFile", zipWithBody("v2")));++ LambdaFunction latest = service.getFunction(REGION, "version-code-isolation-fn");+ assertNotEquals(v1Path.toString(), latest.getCodeLocalPath(),+ "a version must not share $LATEST's directory");+ assertEquals("v2",+ Files.readString(Path.of(latest.getCodeLocalPath()).resolve("index.js")).trim());++ // The version still holds the bytes it was published from, and they still match the hash it+ // advertises, which is the guarantee that was broken.+ assertTrue(Files.isDirectory(v1Path), "the version's code must survive a redeploy of $LATEST");+ assertEquals("v1", Files.readString(v1Path.resolve("index.js")).trim());+ assertEquals(v1Sha, v1.getCodeSha256());+ }++ @Test+ void deletingAFunctionRemovesItsPublishedVersionsCode() throws Exception {+ Map<String, Object> request = baseRequest("version-code-delete-fn");+ request.put("Code", Map.of("ZipFile", zipWithBody("v1")));+ service.createFunction(REGION, request);++ LambdaFunction v1 = service.publishVersion(REGION, "version-code-delete-fn", null);+ Path v1Path = Path.of(v1.getCodeLocalPath());+ assertTrue(Files.isDirectory(v1Path));++ service.deleteFunction(REGION, "version-code-delete-fn");++ assertFalse(Files.exists(v1Path),+ "a version's code must not outlive the function it belongs to");+ }++ @Test+ void deletingOnePublishedVersionReclaimsOnlyThatVersionsCode() throws Exception {+ Map<String, Object> request = baseRequest("version-code-reclaim-fn");+ request.put("Code", Map.of("ZipFile", zipWithBody("v1")));+ service.createFunction(REGION, request);++ LambdaFunction v1 = service.publishVersion(REGION, "version-code-reclaim-fn", null);+ service.updateFunctionCode(REGION, "version-code-reclaim-fn",+ Map.of("ZipFile", zipWithBody("v2")));+ LambdaFunction v2 = service.publishVersion(REGION, "version-code-reclaim-fn", null);++ Path v1Path = Path.of(v1.getCodeLocalPath());+ Path v2Path = Path.of(v2.getCodeLocalPath());+ Path latestPath = Path.of(+ service.getFunction(REGION, "version-code-reclaim-fn").getCodeLocalPath());+ assertNotEquals(v1Path, v2Path, "two versions must not share one code directory");++ service.deleteFunction(REGION, "version-code-reclaim-fn", v1.getVersion());++ // Only whole-function delete reclaimed any of this before, so every version ever published+ // stayed on disk for as long as the data directory lived.+ assertFalse(Files.exists(v1Path), "the deleted version's code must be reclaimed");+ assertTrue(Files.isDirectory(v2Path), "a surviving version's code must be left alone");+ assertEquals("v2", Files.readString(v2Path.resolve("index.js")).trim());+ assertTrue(Files.isDirectory(latestPath), "$LATEST's code must be left alone");+ }++ private static String zipWithBody(String body) throws Exception {+ ByteArrayOutputStream baos = new ByteArrayOutputStream();+ try (ZipOutputStream zos = new ZipOutputStream(baos)) {+ zos.putNextEntry(new ZipEntry("index.js"));+ zos.write((body + "\n").getBytes(StandardCharsets.UTF_8));+ zos.closeEntry();+ }+ return Base64.getEncoder().encodeToString(baos.toByteArray());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/zip/CodeStoreTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/zip/CodeStoreTest.javaindex 9513a73e85..d75602eddd 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/zip/CodeStoreTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/zip/CodeStoreTest.java@@ -103,6 +103,60 @@ class CodeStoreTest {assertFalse(Files.exists(legacyPath));}+ @Test+ void aVersionDirectoryCannotCollideWithAnotherFunctionsOwnDirectory(@TempDir Path baseDir)+ throws IOException {+ // Floci does not restrict the character set of FunctionName today, only that it is+ // non-blank, so "foo.v1" is a function a user can genuinely create. A "<name>.v<n>" sibling+ // naming scheme handed it the exact directory version 1 of "foo" would claim, so deleting+ // either function silently corrupted the other. The suffix used instead is outside the+ // character set sanitizeName can emit, which makes the two namespaces disjoint by+ // construction rather than by a prefix match that has to guess where the name ends.+ CodeStore store = new CodeStore(baseDir);+ writeHandler(store.getVersionCodePath(ACCOUNT_A, "foo", "1"), "foo-v1");+ writeHandler(store.getCodePath(ACCOUNT_A, "foo.v1"), "other-function");+ writeHandler(store.getVersionCodePath(ACCOUNT_A, "foo.v1", "1"), "other-function-v1");++ store.delete(ACCOUNT_A, "foo");++ assertTrue(store.exists(ACCOUNT_A, "foo.v1"),+ "deleting foo must not remove a function literally named foo.v1");+ assertEquals("other-function",+ Files.readString(store.getCodePath(ACCOUNT_A, "foo.v1").resolve("index.js")));+ assertEquals("other-function-v1",+ Files.readString(store.getVersionCodePath(ACCOUNT_A, "foo.v1", "1").resolve("index.js")),+ "another function's published version code must survive too");+ assertFalse(Files.exists(store.getVersionCodePath(ACCOUNT_A, "foo", "1")),+ "foo's own version code must still be reclaimed");+ }++ @Test+ void deleteVersionRemovesOneVersionAndLeavesTheRestInPlace(@TempDir Path baseDir) throws IOException {+ CodeStore store = new CodeStore(baseDir);+ writeHandler(store.getCodePath(ACCOUNT_A, "fn"), "latest");+ writeHandler(store.getVersionCodePath(ACCOUNT_A, "fn", "1"), "one");+ writeHandler(store.getVersionCodePath(ACCOUNT_A, "fn", "2"), "two");++ store.deleteVersion(ACCOUNT_A, "fn", "1");++ assertFalse(Files.exists(store.getVersionCodePath(ACCOUNT_A, "fn", "1")));+ assertEquals("two",+ Files.readString(store.getVersionCodePath(ACCOUNT_A, "fn", "2").resolve("index.js")));+ assertTrue(store.exists(ACCOUNT_A, "fn"), "$LATEST's code must be untouched");+ }++ @Test+ void versionCodeLivesOutsideTheDirectoryExtractionReplaces(@TempDir Path baseDir) {+ CodeStore store = new CodeStore(baseDir);++ Path latest = store.getCodePath(ACCOUNT_A, "fn");+ Path version = store.getVersionCodePath(ACCOUNT_A, "fn", "1");++ assertFalse(version.normalize().startsWith(latest.normalize()),+ "extraction replaces $LATEST's directory wholesale, taking any nested version with it");+ assertTrue(version.normalize().startsWith(baseDir.normalize()));+ }+private void writeHandler(Path codePath, String content) throws IOException {Files.createDirectories(codePath);Files.writeString(codePath.resolve("index.js"), content);