Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): scope extracted function code by region (#3265)
Niels Kootstra290559cff012
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.javaindex 9a94894dfa..10347ef560 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java@@ -906,7 +906,7 @@ public class LambdaService implements ResourceProvider {synchronized (lockForConcurrencyOp(fn.getFunctionArn())) {warmPool.drainEnvironment(version.get());functionStore.deleteVersion(region, name, qualifier);- reclaimVersionCodeDirectory(fn, qualifier, version.get());+ reclaimVersionCodeDirectory(region, fn, qualifier, version.get());// The snapshot may still share $LATEST's code directory, so this only reclaims once no// remaining version references it.reclaimLegacyCodeDirectoryIfUnused(name);@@ -930,7 +930,7 @@ public class LambdaService implements ResourceProvider {if (concurrencyLimiter != null) {concurrencyLimiter.reset(arn);}- codeStore.delete(ownerAccount(fn), functionName);+ codeStore.delete(ownerAccount(fn), region, functionName);functionStore.delete(region, functionName);reclaimLegacyCodeDirectoryIfUnused(functionName);versionCounters.remove(versionCounterKey(region, fn));@@ -963,15 +963,15 @@ public class LambdaService implements ResourceProvider {* and hot-reload versions never had a copy at all: for those the recorded path is the live* function's own directory, and removing it would delete the code {@code $LATEST} still runs.*/- private void reclaimVersionCodeDirectory(LambdaFunction fn, String version, LambdaFunction snapshot) {+ private void reclaimVersionCodeDirectory(String region, LambdaFunction fn, String version, LambdaFunction snapshot) {String recorded = snapshot.getCodeLocalPath();if (recorded == null) {return;}- String owned = codeStore.getVersionCodePath(ownerAccount(fn), fn.getFunctionName(), version)+ String owned = codeStore.getVersionCodePath(ownerAccount(fn), region, fn.getFunctionName(), version).toAbsolutePath().normalize().toString();if (owned.equals(Path.of(recorded).toAbsolutePath().normalize().toString())) {- codeStore.deleteVersion(ownerAccount(fn), fn.getFunctionName(), version);+ codeStore.deleteVersion(ownerAccount(fn), region, fn.getFunctionName(), version);}}@@ -1911,14 +1911,14 @@ public class LambdaService implements ResourceProvider {* configuration, and falling back leaves it exactly as good as every version published before* this existed, rather than turning a working call into an error.*/- private String versionCodePath(LambdaFunction fn, String version) {+ private String versionCodePath(String region, LambdaFunction fn, String version) {String current = fn.getCodeLocalPath();if (current == null || fn.getHotReloadHostPath() != null) {return current;}try {Path copied = codeStore.copyForVersion(- ownerAccount(fn), fn.getFunctionName(), version, Path.of(current));+ ownerAccount(fn), region, fn.getFunctionName(), version, Path.of(current));return copied == null ? current : copied.toAbsolutePath().normalize().toString();} catch (IOException e) {LOG.warnv("Could not give version {0} of {1} its own code directory, "@@ -2051,7 +2051,7 @@ public class LambdaService implements ResourceProvider {// Nothing to copy for image-backed or hot-reload functions, which keep the reference// they had: an image is already immutable by digest, and a hot-reload function's whole// point is that its bind-mounted directory tracks the developer's working tree.- snapshot.setCodeLocalPath(versionCodePath(fn, String.valueOf(version)));+ snapshot.setCodeLocalPath(versionCodePath(region, fn, String.valueOf(version)));snapshot.setCodeSha256(fn.getCodeSha256());snapshot.setS3Bucket(fn.getS3Bucket());snapshot.setS3Key(fn.getS3Key());@@ -2751,7 +2751,7 @@ public class LambdaService implements ResourceProvider {}private void extractZipCodeBytes(LambdaFunction fn, byte[] zipBytes, String region) {- Path codePath = codeStore.getCodePath(ownerAccount(fn), fn.getFunctionName());+ Path codePath = codeStore.getCodePath(ownerAccount(fn), region, fn.getFunctionName());try {zipExtractor.extractTo(zipBytes, codePath, configuredZipMaxEntries());// Publish the new code identity under the same per-function lock publishVersion holds.diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.java b/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.javaindex f53feaaf37..f937601bdc 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/zip/CodeStore.java@@ -13,10 +13,11 @@ import java.util.Comparator;/*** Manages on-disk locations of extracted Lambda function code.- * Each function gets its own directory under {@code <codePath>/<accountId>/}, mirroring- * the account-prefixed S3 key {@code LambdaService.codeObjectKey} uses for the same- * deployment package. Without the account segment two accounts' same-named functions in- * one region share a single extraction directory and overwrite each other's code.+ * Each function gets its own directory under+ * {@code <codePath>/<accountId>/@regions/<region>/}, mirroring the account- and+ * region-scoped Lambda namespace. The reserved namespace marker is intentional: it keeps+ * this layout disjoint from the preceding account/name layout, including when a legacy+ * function is named like a region.*/@ApplicationScopedpublic class CodeStore {@@ -29,6 +30,9 @@ public class CodeStore {*/private static final String VERSIONS_DIR_SUFFIX = "@versions";+ /** Namespace marker keeps the new layout disjoint from the preceding account/name layout. */+ private static final String REGIONS_DIR = "@regions";+private final Path baseDir;@Inject@@ -40,8 +44,11 @@ public class CodeStore {this.baseDir = baseDir;}- public Path getCodePath(String accountId, String functionName) {- return baseDir.resolve(sanitizeName(accountId)).resolve(sanitizeName(functionName));+ public Path getCodePath(String accountId, String region, String functionName) {+ return baseDir.resolve(sanitizeName(accountId))+ .resolve(REGIONS_DIR)+ .resolve(sanitizeName(region))+ .resolve(sanitizeName(functionName));}/**@@ -69,14 +76,16 @@ public class CodeStore {* {@code foo.v1} owns the exact directory {@code foo}'s version 1 would otherwise claim, and* deleting either one silently corrupts the other.*/- public Path getVersionsPath(String accountId, String functionName) {+ public Path getVersionsPath(String accountId, String region, String functionName) {return baseDir.resolve(sanitizeName(accountId))+ .resolve(REGIONS_DIR)+ .resolve(sanitizeName(region)).resolve(sanitizeName(functionName) + VERSIONS_DIR_SUFFIX);}/** Where one published version's own copy of the code lives. */- public Path getVersionCodePath(String accountId, String functionName, String version) {- return getVersionsPath(accountId, functionName).resolve(sanitizeName(version));+ public Path getVersionCodePath(String accountId, String region, String functionName, String version) {+ return getVersionsPath(accountId, region, functionName).resolve(sanitizeName(version));}/**@@ -84,12 +93,12 @@ public class CodeStore {* directory already there. Returns null when there is nothing to copy, which is the case for* image-backed and hot-reload functions.*/- public Path copyForVersion(String accountId, String functionName, String version, Path source)+ public Path copyForVersion(String accountId, String region, String functionName, String version, Path source)throws IOException {if (source == null || !Files.isDirectory(source)) {return null;}- Path target = getVersionCodePath(accountId, functionName, version);+ Path target = getVersionCodePath(accountId, region, functionName, version);deleteDirectory(target, functionName);try (var walk = Files.walk(source)) {for (Path from : walk.toList()) {@@ -106,8 +115,8 @@ public class CodeStore {}/** Removes every published version's code for a function. */- public void deleteVersions(String accountId, String functionName) {- deleteDirectory(getVersionsPath(accountId, functionName), functionName);+ public void deleteVersions(String accountId, String region, String functionName) {+ deleteDirectory(getVersionsPath(accountId, region, functionName), functionName);}/**@@ -115,13 +124,13 @@ public class CodeStore {* {@code $LATEST} in place. Deleting a single version otherwise left its package on disk for* as long as the data directory lived, so a repeated publish/delete cycle grew without bound.*/- public void deleteVersion(String accountId, String functionName, String version) {- deleteDirectory(getVersionCodePath(accountId, functionName, version), functionName);+ public void deleteVersion(String accountId, String region, String functionName, String version) {+ deleteDirectory(getVersionCodePath(accountId, region, functionName, version), functionName);}- public void delete(String accountId, String functionName) {- deleteDirectory(getCodePath(accountId, functionName), functionName);- deleteVersions(accountId, functionName);+ public void delete(String accountId, String region, String functionName) {+ deleteDirectory(getCodePath(accountId, region, functionName), functionName);+ deleteVersions(accountId, region, functionName);}/**@@ -153,8 +162,8 @@ public class CodeStore {}}- public boolean exists(String accountId, String functionName) {- Path codePath = getCodePath(accountId, functionName);+ public boolean exists(String accountId, String region, String functionName) {+ Path codePath = getCodePath(accountId, region, functionName);if (!Files.exists(codePath)) {return false;}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAccountScopedCodeTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAccountScopedCodeTest.javaindex f1f94002a6..1f1c85d1a5 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAccountScopedCodeTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAccountScopedCodeTest.java@@ -14,6 +14,7 @@ import java.nio.file.Files;import java.nio.file.Path;import java.util.Base64;import java.util.Map;+import java.util.concurrent.CompletableFuture;import java.util.zip.ZipEntry;import java.util.zip.ZipOutputStream;@@ -60,11 +61,65 @@ class LambdaAccountScopedCodeTest {svcB.deleteFunction(REGION, "shared-fn");- assertTrue(sharedCodeStore.exists(ACCOUNT_A, "shared-fn"),+ assertTrue(sharedCodeStore.exists(ACCOUNT_A, REGION, "shared-fn"),"deleting B's function must not delete A's code");assertEquals("A", Files.readString(Path.of(a.getCodeLocalPath()).resolve("index.js")));}+ @Test+ void sameFunctionNameInTwoRegionsKeepsItsOwnCodeOnDisk(@TempDir Path baseDir) throws Exception {+ CodeStore sharedCodeStore = new CodeStore(baseDir);+ LambdaService service = serviceFor(ACCOUNT_A, sharedCodeStore);++ LambdaFunction east = service.createFunction(REGION, zipRequest("regional-fn", "east"));+ LambdaFunction west = service.createFunction("eu-west-1", zipRequest("regional-fn", "west"));++ assertNotEquals(east.getCodeLocalPath(), west.getCodeLocalPath());+ assertEquals("east", Files.readString(Path.of(east.getCodeLocalPath()).resolve("index.js")));+ assertEquals("west", Files.readString(Path.of(west.getCodeLocalPath()).resolve("index.js")));+ }++ @Test+ void updateAndDeleteInOneRegionLeaveTheOtherRegionCodeIntact(@TempDir Path baseDir) throws Exception {+ CodeStore sharedCodeStore = new CodeStore(baseDir);+ LambdaService service = serviceFor(ACCOUNT_A, sharedCodeStore);+ service.createFunction(REGION, zipRequest("regional-fn", "east-v1"));+ LambdaFunction west = service.createFunction("eu-west-1", zipRequest("regional-fn", "west-v1"));++ service.updateFunctionCode(REGION, "regional-fn", Map.of("ZipFile", zipBase64("index.js", "east-v2")));++ assertEquals("west-v1", Files.readString(Path.of(west.getCodeLocalPath()).resolve("index.js")));+ service.deleteFunction(REGION, "regional-fn");+ assertTrue(sharedCodeStore.exists(ACCOUNT_A, "eu-west-1", "regional-fn"));+ assertEquals("west-v1", Files.readString(Path.of(west.getCodeLocalPath()).resolve("index.js")));+ }++ @Test+ void regionScopedCodeSurvivesAStoreRestart(@TempDir Path baseDir) throws Exception {+ LambdaService first = serviceFor(ACCOUNT_A, new CodeStore(baseDir));+ LambdaFunction created = first.createFunction("ap-southeast-2", zipRequest("restart-fn", "persisted"));++ CodeStore afterRestart = new CodeStore(baseDir);+ assertTrue(afterRestart.exists(ACCOUNT_A, "ap-southeast-2", "restart-fn"));+ assertEquals("persisted", Files.readString(Path.of(created.getCodeLocalPath()).resolve("index.js")));+ }++ @Test+ void concurrentExtractionInTwoRegionsDoesNotOverwriteCode(@TempDir Path baseDir) throws Exception {+ CodeStore sharedCodeStore = new CodeStore(baseDir);+ LambdaService service = serviceFor(ACCOUNT_A, sharedCodeStore);++ CompletableFuture<LambdaFunction> east = CompletableFuture.supplyAsync(() -> createUnchecked(+ service, REGION, "concurrent-fn", "east"));+ CompletableFuture<LambdaFunction> west = CompletableFuture.supplyAsync(() -> createUnchecked(+ service, "eu-west-1", "concurrent-fn", "west"));++ LambdaFunction eastFunction = east.join();+ LambdaFunction westFunction = west.join();+ assertEquals("east", Files.readString(Path.of(eastFunction.getCodeLocalPath()).resolve("index.js")));+ assertEquals("west", Files.readString(Path.of(westFunction.getCodeLocalPath()).resolve("index.js")));+ }+@Testvoid publishVersionCounterKeyCarriesTheOwningAccount() {LambdaFunction a = functionOwnedBy(ACCOUNT_A);@@ -205,6 +260,14 @@ class LambdaAccountScopedCodeTest {return fn;}+ private LambdaFunction createUnchecked(LambdaService service, String region, String name, String source) {+ try {+ return service.createFunction(region, zipRequest(name, source));+ } catch (Exception e) {+ throw new RuntimeException(e);+ }+ }+private LambdaService serviceFor(String accountId, CodeStore codeStore) {return new LambdaService(new LambdaFunctionStore(new InMemoryStorage<String, LambdaFunction>()),diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/zip/CodeStoreTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/zip/CodeStoreTest.javaindex d75602eddd..2b8c6a6f2b 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/zip/CodeStoreTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/zip/CodeStoreTest.java@@ -13,37 +13,53 @@ class CodeStoreTest {private static final String ACCOUNT_A = "111111111111";private static final String ACCOUNT_B = "222222222222";+ private static final String REGION = "us-east-1";@Testvoid sameFunctionNameInTwoAccountsResolvesToDistinctDirectories(@TempDir Path baseDir) {CodeStore store = new CodeStore(baseDir);- Path a = store.getCodePath(ACCOUNT_A, "shared-name");- Path b = store.getCodePath(ACCOUNT_B, "shared-name");+ Path a = store.getCodePath(ACCOUNT_A, REGION, "shared-name");+ Path b = store.getCodePath(ACCOUNT_B, REGION, "shared-name");assertNotEquals(a, b, "two accounts must not share one on-disk extraction directory");assertTrue(a.startsWith(baseDir.resolve(ACCOUNT_A)));assertTrue(b.startsWith(baseDir.resolve(ACCOUNT_B)));}+ @Test+ void newRegionLayoutCannotNestInsideAlegacyRegionNamedFunction(@TempDir Path baseDir) throws IOException {+ CodeStore store = new CodeStore(baseDir);+ Path legacyFunction = baseDir.resolve(ACCOUNT_A).resolve(REGION);+ Path newFunction = store.getCodePath(ACCOUNT_A, REGION, REGION);++ writeHandler(legacyFunction, "legacy");+ writeHandler(newFunction, "new");++ assertFalse(newFunction.normalize().startsWith(legacyFunction.normalize()),+ "new extraction paths must not be children of a legacy region-named function");+ assertEquals("legacy", Files.readString(legacyFunction.resolve("index.js")));+ assertEquals("new", Files.readString(newFunction.resolve("index.js")));+ }+@Testvoid deleteRemovesOnlyTheOwningAccountsCode(@TempDir Path baseDir) throws IOException {CodeStore store = new CodeStore(baseDir);- writeHandler(store.getCodePath(ACCOUNT_A, "shared-name"), "a");- writeHandler(store.getCodePath(ACCOUNT_B, "shared-name"), "b");+ writeHandler(store.getCodePath(ACCOUNT_A, REGION, "shared-name"), "a");+ writeHandler(store.getCodePath(ACCOUNT_B, REGION, "shared-name"), "b");- store.delete(ACCOUNT_B, "shared-name");+ store.delete(ACCOUNT_B, REGION, "shared-name");- assertTrue(store.exists(ACCOUNT_A, "shared-name"), "deleting B's code must not touch A's");- assertFalse(store.exists(ACCOUNT_B, "shared-name"));- assertEquals("a", Files.readString(store.getCodePath(ACCOUNT_A, "shared-name").resolve("index.js")));+ assertTrue(store.exists(ACCOUNT_A, REGION, "shared-name"), "deleting B's code must not touch A's");+ assertFalse(store.exists(ACCOUNT_B, REGION, "shared-name"));+ assertEquals("a", Files.readString(store.getCodePath(ACCOUNT_A, REGION, "shared-name").resolve("index.js")));}@Testvoid accountSegmentIsSanitizedLikeTheFunctionName(@TempDir Path baseDir) {CodeStore store = new CodeStore(baseDir);- Path traversal = store.getCodePath("../../etc", "fn");+ Path traversal = store.getCodePath("../../etc", REGION, "fn");assertTrue(traversal.normalize().startsWith(baseDir.normalize()),"a hostile account segment must not escape the base directory");@@ -57,7 +73,7 @@ class CodeStoreTest {// parent directory once handed to Path.resolve.CodeStore store = new CodeStore(baseDir);- Path traversal = store.getCodePath("..", "fn");+ Path traversal = store.getCodePath("..", REGION, "fn");assertTrue(traversal.normalize().startsWith(baseDir.normalize()),"a bare '..' account segment must not escape the base directory");@@ -67,7 +83,7 @@ class CodeStoreTest {void bareDotFunctionSegmentCannotResolveToTheAccountDirectoryItself(@TempDir Path baseDir) {CodeStore store = new CodeStore(baseDir);- Path traversal = store.getCodePath(ACCOUNT_A, ".");+ Path traversal = store.getCodePath(ACCOUNT_A, REGION, ".");assertTrue(traversal.normalize().startsWith(baseDir.normalize()));assertNotEquals(baseDir.resolve(ACCOUNT_A).normalize(), traversal.normalize(),@@ -84,12 +100,12 @@ class CodeStoreTest {CodeStore store = new CodeStore(baseDir);Path legacyPath = baseDir.resolve("legacy-fn");writeHandler(legacyPath, "legacy");- writeHandler(store.getCodePath(ACCOUNT_A, "legacy-fn"), "current");+ writeHandler(store.getCodePath(ACCOUNT_A, REGION, "legacy-fn"), "current");- store.delete(ACCOUNT_A, "legacy-fn");+ store.delete(ACCOUNT_A, REGION, "legacy-fn");assertTrue(Files.exists(legacyPath), "delete() must not unilaterally remove the legacy directory");- assertFalse(store.exists(ACCOUNT_A, "legacy-fn"));+ assertFalse(store.exists(ACCOUNT_A, REGION, "legacy-fn"));}@Test@@ -113,44 +129,44 @@ class CodeStoreTest {// character set sanitizeName can emit, which makes the two namespaces disjoint by// construction rather than by a prefix match that has to guess where the name ends.CodeStore store = new CodeStore(baseDir);- writeHandler(store.getVersionCodePath(ACCOUNT_A, "foo", "1"), "foo-v1");- writeHandler(store.getCodePath(ACCOUNT_A, "foo.v1"), "other-function");- writeHandler(store.getVersionCodePath(ACCOUNT_A, "foo.v1", "1"), "other-function-v1");+ writeHandler(store.getVersionCodePath(ACCOUNT_A, REGION, "foo", "1"), "foo-v1");+ writeHandler(store.getCodePath(ACCOUNT_A, REGION, "foo.v1"), "other-function");+ writeHandler(store.getVersionCodePath(ACCOUNT_A, REGION, "foo.v1", "1"), "other-function-v1");- store.delete(ACCOUNT_A, "foo");+ store.delete(ACCOUNT_A, REGION, "foo");- assertTrue(store.exists(ACCOUNT_A, "foo.v1"),+ assertTrue(store.exists(ACCOUNT_A, REGION, "foo.v1"),"deleting foo must not remove a function literally named foo.v1");assertEquals("other-function",- Files.readString(store.getCodePath(ACCOUNT_A, "foo.v1").resolve("index.js")));+ Files.readString(store.getCodePath(ACCOUNT_A, REGION, "foo.v1").resolve("index.js")));assertEquals("other-function-v1",- Files.readString(store.getVersionCodePath(ACCOUNT_A, "foo.v1", "1").resolve("index.js")),+ Files.readString(store.getVersionCodePath(ACCOUNT_A, REGION, "foo.v1", "1").resolve("index.js")),"another function's published version code must survive too");- assertFalse(Files.exists(store.getVersionCodePath(ACCOUNT_A, "foo", "1")),+ assertFalse(Files.exists(store.getVersionCodePath(ACCOUNT_A, REGION, "foo", "1")),"foo's own version code must still be reclaimed");}@Testvoid deleteVersionRemovesOneVersionAndLeavesTheRestInPlace(@TempDir Path baseDir) throws IOException {CodeStore store = new CodeStore(baseDir);- writeHandler(store.getCodePath(ACCOUNT_A, "fn"), "latest");- writeHandler(store.getVersionCodePath(ACCOUNT_A, "fn", "1"), "one");- writeHandler(store.getVersionCodePath(ACCOUNT_A, "fn", "2"), "two");+ writeHandler(store.getCodePath(ACCOUNT_A, REGION, "fn"), "latest");+ writeHandler(store.getVersionCodePath(ACCOUNT_A, REGION, "fn", "1"), "one");+ writeHandler(store.getVersionCodePath(ACCOUNT_A, REGION, "fn", "2"), "two");- store.deleteVersion(ACCOUNT_A, "fn", "1");+ store.deleteVersion(ACCOUNT_A, REGION, "fn", "1");- assertFalse(Files.exists(store.getVersionCodePath(ACCOUNT_A, "fn", "1")));+ assertFalse(Files.exists(store.getVersionCodePath(ACCOUNT_A, REGION, "fn", "1")));assertEquals("two",- Files.readString(store.getVersionCodePath(ACCOUNT_A, "fn", "2").resolve("index.js")));- assertTrue(store.exists(ACCOUNT_A, "fn"), "$LATEST's code must be untouched");+ Files.readString(store.getVersionCodePath(ACCOUNT_A, REGION, "fn", "2").resolve("index.js")));+ assertTrue(store.exists(ACCOUNT_A, REGION, "fn"), "$LATEST's code must be untouched");}@Testvoid versionCodeLivesOutsideTheDirectoryExtractionReplaces(@TempDir Path baseDir) {CodeStore store = new CodeStore(baseDir);- Path latest = store.getCodePath(ACCOUNT_A, "fn");- Path version = store.getVersionCodePath(ACCOUNT_A, "fn", "1");+ Path latest = store.getCodePath(ACCOUNT_A, REGION, "fn");+ Path version = store.getVersionCodePath(ACCOUNT_A, REGION, "fn", "1");assertFalse(version.normalize().startsWith(latest.normalize()),"extraction replaces $LATEST's directory wholesale, taking any nested version with it");