Light, fluffy, and always free - The AWS Local Emulator alternative
feat(lambda): retarget provisioned alias allocations (#6335)
Andre Kurait7c8daff08f09
diff --git a/docs/services/lambda.md b/docs/services/lambda.mdindex ace1d2726f..52a1826686 100644--- a/docs/services/lambda.md+++ b/docs/services/lambda.md@@ -483,14 +483,14 @@ for Get does not authorize Put, Delete, or another qualifier.Provisioned environments are separate from ordinary idle eviction. Desired configurationsare persisted; restarting Floci initializes new processes before reporting `READY`.-Deleting the configuration, alias or function removes the owned environments. If an-alias changes its version, this stage marks its old allocation `FAILED` and retires its-environments. Invocations use on-demand capacity until a new Put initializes the new-version. Automatic alias retargeting is a separate follow-up. The requested quota is-retained until configuration replacement or deletion. An invalid replacement Put preserves-the existing healthy allocation. An alias with a configuration cannot move to `$LATEST`-or to a version targeted by another configuration; remove the conflicting configuration-first. Get and Delete remain available if the alias gains+Deleting the configuration, alias or function removes the owned environments. Changing an+alias's version reinitializes its existing allocation for the new published version and+releases the old version's configuration conflict. The alias retains its requested quota;+retired environments retain their local container capacity until cleanup succeeds. An identical+Put preserves the existing healthy allocation. Failed background alias refreshes retain their+previous allocation and retry on later reconciliation without blocking other configurations.+An alias with a configuration cannot move to+`$LATEST`; remove the configuration first. Get and Delete remain available if the alias gainsunsupported weighted routing.This implementation supports the non-ephemeral Docker executor with immutable publisheddiff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPool.javaindex 3ce750e6b4..556bbcb6d4 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPool.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPool.java@@ -163,9 +163,7 @@ public class ProvisionedConcurrencyPool implements Resettable {throw invalid("Provisioned concurrency requires the non-ephemeral Docker Lambda executor");}LambdaFunction fn = currentFunction(target);- if (fn.isHotReload() || fn.isDurable()) {- throw invalid("Provisioned concurrency is not supported for hot-reload or durable functions");- }+ validateSupportedTarget(fn);List<ContainerHandle> retired = List.of();Status result;synchronized (lock) {@@ -378,7 +376,15 @@ public class ProvisionedConcurrencyPool implements Resettable {if (stopped) {return;}- for (Allocation allocation : new ArrayList<>(allocations.values())) {+ for (Allocation previous : new ArrayList<>(allocations.values())) {+ Allocation allocation;+ try {+ allocation = refreshAlias(previous, retired);+ } catch (RuntimeException failure) {+ LOG.warnv(failure, "Could not refresh provisioned Lambda alias allocation {0}",+ previous.configuration.key());+ continue;+ }if ("FAILED".equals(allocation.status)) {continue;}@@ -434,6 +440,37 @@ public class ProvisionedConcurrencyPool implements Resettable {.map(LambdaAlias::getFunctionVersion).orElse(null);}+ private Allocation refreshAlias(Allocation previous, List<ContainerHandle> retired) {+ Configuration value = previous.configuration;+ if (value.qualifier().matches("[0-9]+")) {+ return previous;+ }+ LambdaAlias alias = aliases.getForAccount(value.account(), value.region(), value.name(), value.qualifier())+ .orElse(null);+ if (alias == null || Objects.equals(value.version(), alias.getFunctionVersion())+ || (alias.getRoutingConfig() != null && !alias.getRoutingConfig().isEmpty())+ || alias.getFunctionVersion() == null || !alias.getFunctionVersion().matches("[0-9]+")) {+ return previous;+ }+ if (functions.getForAccount(value.account(), value.region(), value.name()).isEmpty()) {+ return previous;+ }+ LambdaFunction fn = functions.getForAccount(value.account(), value.region(), value.name(),+ alias.getFunctionVersion()).orElse(null);+ if (fn == null) {+ return previous;+ }+ validateSupportedTarget(fn);+ Configuration target = new Configuration(value.account(), value.region(), value.name(),+ value.functionArn(), value.qualifier(), fn.getVersion(), fn.getRevisionId(), value.requested(),+ LAST_MODIFIED.format(Instant.ofEpochMilli(clock.millis())));+ write(target);+ Allocation next = new Allocation(target);+ allocations.put(target.key(), next);+ retired.addAll(retire(previous));+ return next;+ }+void validateAliasUpdate(LambdaFunction fn, String qualifier, String version) {synchronized (lock) {String functionArn = baseArn(fn);@@ -443,6 +480,11 @@ public class ProvisionedConcurrencyPool implements Resettable {if ("$LATEST".equals(version)) {throw invalid("Provisioned Concurrency Configs cannot be applied to unpublished function versions.");}+ if (version != null && version.matches("[0-9]+")) {+ AwsArnUtils.Arn arn = AwsArnUtils.parse(functionArn);+ functions.getForAccount(arn.accountId(), arn.region(), fn.getFunctionName(), version)+ .ifPresent(ProvisionedConcurrencyPool::validateSupportedTarget);+ }if (version != null && version.matches("[0-9]+") && allocations.values().stream().anyMatch(entry -> functionArn.equals(entry.configuration.functionArn())&& !qualifier.equals(entry.configuration.qualifier())@@ -453,6 +495,12 @@ public class ProvisionedConcurrencyPool implements Resettable {}}+ private static void validateSupportedTarget(LambdaFunction fn) {+ if (fn.isHotReload() || fn.isDurable()) {+ throw invalid("Provisioned concurrency is not supported for hot-reload or durable functions");+ }+ }+private int availableCapacity(Configuration configuration) {long functionOwned = owned.values().stream().filter(value -> value.functionArn().equals(configuration.functionArn())).count();diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPoolTest.javaindex f726bc7a50..97b2ac2e70 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPoolTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPoolTest.java@@ -594,8 +594,13 @@ class ProvisionedConcurrencyPoolTest {default -> throw new AssertionError(lookup);}assertSame(failure, assertThrows(IllegalStateException.class, () -> pool.acquire(version, "live")));- assertSame(failure, assertThrows(IllegalStateException.class, pool::reconcile));- assertSame(failure, assertThrows(IllegalStateException.class, pool::reconcileSafely));+ if ("alias".equals(lookup)) {+ assertDoesNotThrow(pool::reconcile);+ assertDoesNotThrow(pool::reconcileSafely);+ } else {+ assertSame(failure, assertThrows(IllegalStateException.class, pool::reconcile));+ assertSame(failure, assertThrows(IllegalStateException.class, pool::reconcileSafely));+ }assertTrue(first.alive);assertEquals(0, first.stops.get());reset(functionLookup, aliasLookup);@@ -688,6 +693,56 @@ class ProvisionedConcurrencyPoolTest {assertEquals(2, launcher.starts.get());}+ @Test+ void targetChangeRefusesOldInitializedEnvironment() throws Exception {+ put(1);+ Environment first = launched();+ first.initialized.complete(null);+ ready();+ function(ACCOUNT, REGION, "2");+ alias(ACCOUNT, REGION, "live", "2", Map.of());+ assertNull(pool.acquire(version, "live"));+ assertEquals("IN_PROGRESS", get().status());+ assertEquals(1, first.stops.get());+ pool.reconcile();+ Environment second = launched();+ second.initialized.complete(null);+ ready();+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void movedAliasFreesItsOldVersionButRetainsItsReservedBudget(boolean previouslyFailed) throws Exception {+ LambdaFunction latest = functions.getForAccount(ACCOUNT, REGION, "example").orElseThrow();+ latest.setReservedConcurrentExecutions(2);+ put(1);+ Environment first = launched();+ first.initialized.complete(null);+ ready();+ if (previouslyFailed) {+ alias(ACCOUNT, REGION, "live", "1", Map.of("2", 0.1));+ assertEquals("FAILED", get().status());+ }+ function(ACCOUNT, REGION, "2");+ alias(ACCOUNT, REGION, "live", "2", Map.of());++ pool.put(REGION, "example", "1", 1, ACCOUNT);+ assertEquals("2", storage.scanAllAccounts().stream()+ .filter(value -> "live".equals(value.qualifier())).findFirst().orElseThrow().version());+ assertEquals(1, first.stops.get());+ assertEquals(2, storage.scanAllAccounts().size());+ AwsException budget = assertThrows(AwsException.class,+ () -> pool.put(REGION, "example", "1", 2, ACCOUNT));+ assertEquals("InvalidParameterValueException", budget.getErrorCode());+ AwsException bothInvalid = assertThrows(AwsException.class,+ () -> pool.put(REGION, "example", "2", 1, ACCOUNT));+ assertEquals("InvalidParameterValueException", bothInvalid.getErrorCode());+ latest.setReservedConcurrentExecutions(3);+ AwsException conflict = assertThrows(AwsException.class,+ () -> pool.put(REGION, "example", "2", 1, ACCOUNT));+ assertEquals("Version is pointed by a Provisioned Concurrency alias", conflict.getMessage());+ }+@ParameterizedTest@ValueSource(strings = {"weighted", "latest"})void changedAliasRemainsReadableAndDeletable(String change) throws Exception {@@ -716,6 +771,148 @@ class ProvisionedConcurrencyPoolTest {assertDoesNotThrow(() -> pool.validateAliasUpdate(version, "live", "$LATEST"));}+ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void configuredAliasRejectsUnsupportedTargetBeforeMutation(boolean durable) throws Exception {+ put(1);+ Environment first = launched();+ first.initialized.complete(null);+ ready();+ LambdaFunction target = function(ACCOUNT, REGION, "2");+ if (durable) {+ target.setDurableExecutionTimeout(60);+ } else {+ target.setHotReloadHostPath("/tmp/unsupported-target");+ }+ functions.saveForAccount(ACCOUNT, REGION, target);++ AwsException error = assertThrows(AwsException.class,+ () -> pool.validateAliasUpdate(version, "live", "2"));++ assertEquals("InvalidParameterValueException", error.getErrorCode());+ assertEquals("1", aliases.getForAccount(ACCOUNT, REGION, "example", "live")+ .orElseThrow().getFunctionVersion());+ assertEquals("1", storage.scanAllAccounts().getFirst().version());+ assertEquals(0, first.stops.get());+ assertEquals(1, launcher.starts.get());+ pool.delete(REGION, "example", "live", ACCOUNT);+ assertDoesNotThrow(() -> pool.validateAliasUpdate(version, "live", "2"));+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void unsupportedRetargetPreservesPriorAllocationUntilTargetIsEligible(boolean durable) throws Exception {+ put(1);+ Environment first = launched();+ first.initialized.complete(null);+ ready();+ LambdaFunction target = function(ACCOUNT, REGION, "2");+ if (durable) {+ target.setDurableExecutionTimeout(60);+ } else {+ target.setHotReloadHostPath("/tmp/unsupported-target");+ }+ functions.saveForAccount(ACCOUNT, REGION, target);+ alias(ACCOUNT, REGION, "live", "2", Map.of());++ assertDoesNotThrow(pool::reconcile);+ assertEquals("1", storage.scanAllAccounts().getFirst().version());+ assertEquals(0, first.stops.get());+ assertEquals(1, launcher.starts.get());+ target.setDurableExecutionTimeout(null);+ target.setHotReloadHostPath(null);+ functions.saveForAccount(ACCOUNT, REGION, target);+ pool.reconcile();+ pool.reconcile();+ Environment replacement = launched();+ replacement.initialized.complete(null);+ ready();+ assertEquals("2", storage.scanAllAccounts().getFirst().version());+ assertEquals(1, first.stops.get());+ assertEquals(2, launcher.starts.get());+ }++ @Test+ void aliasRetargetStorageFailurePreservesThePriorAllocation() throws Exception {+ AccountAwareStorageBackend<ProvisionedConcurrencyPool.Configuration> backend = spy(storage);+ pool.closeExecutors();+ pool = new ProvisionedConcurrencyPool(launcher, functions, aliases, backend,+ Clock.systemUTC(), 4, 8, false, 15);+ put(1);+ Environment first = launched();+ first.initialized.complete(null);+ ready();+ function(ACCOUNT, REGION, "2");+ alias(ACCOUNT, REGION, "live", "2", Map.of());+ IllegalStateException failure = new IllegalStateException("controlled alias write failure");+ doThrow(failure).when(backend).putForAccount(eq(ACCOUNT), any(), any());+ assertEquals("READY", get().status());+ assertEquals("1", storage.scanAllAccounts().getFirst().version());+ assertEquals(0, first.stops.get());+ doCallRealMethod().when(backend).putForAccount(eq(ACCOUNT), any(), any());+ assertEquals("IN_PROGRESS", get().status());+ assertEquals("2", storage.scanAllAccounts().getFirst().version());+ assertEquals(1, first.stops.get());+ }++ @ParameterizedTest+ @ValueSource(strings = {"storage", "hotReload", "durable"})+ void failedAliasRefreshDoesNotBlockOtherAccountsAndRecovers(String failureKind) throws Exception {+ AccountAwareStorageBackend<ProvisionedConcurrencyPool.Configuration> backend = spy(storage);+ pool.closeExecutors();+ pool = new ProvisionedConcurrencyPool(launcher, functions, aliases, backend,+ Clock.systemUTC(), 4, 8, false, 15);+ put(1);+ Environment original = launched();+ original.initialized.complete(null);+ ready();+ LambdaFunction target = function(ACCOUNT, REGION, "2");+ if ("hotReload".equals(failureKind)) {+ target.setHotReloadHostPath("/tmp/unsupported-target");+ } else if ("durable".equals(failureKind)) {+ target.setDurableExecutionTimeout(60);+ } else {+ doThrow(new IllegalStateException("controlled alias write failure"))+ .when(backend).putForAccount(eq(ACCOUNT), any(), any());+ }+ functions.saveForAccount(ACCOUNT, REGION, target);+ alias(ACCOUNT, REGION, "live", "2", Map.of());+ String other = "222222222222";+ function(other, REGION, "$LATEST");+ function(other, REGION, "1");+ assertDoesNotThrow(() -> pool.put(REGION, "example", "1", 1, other));+ Environment peer = launched();+ peer.initialized.complete(null);+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ assertEquals("READY", pool.get(REGION, "example", "1", other).status()));+ peer.faulted.set(true);+ assertDoesNotThrow(pool::reconcile);+ assertEquals("IN_PROGRESS", pool.get(REGION, "example", "1", other).status());+ Environment peerReplacement = launched();+ peerReplacement.initialized.complete(null);+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ assertEquals("READY", pool.get(REGION, "example", "1", other).status()));+ assertEquals(1, peer.stops.get());+ assertEquals(0, original.stops.get());+ assertEquals("1", storage.scanAllAccounts().stream()+ .filter(value -> ACCOUNT.equals(value.account())).findFirst().orElseThrow().version());+ if ("storage".equals(failureKind)) {+ doCallRealMethod().when(backend).putForAccount(eq(ACCOUNT), any(), any());+ }+ target.setDurableExecutionTimeout(null);+ target.setHotReloadHostPath(null);+ functions.saveForAccount(ACCOUNT, REGION, target);+ pool.reconcile();+ pool.reconcile();+ Environment replacement = launched();+ replacement.initialized.complete(null);+ ready();+ assertEquals(1, original.stops.get());+ assertEquals(0, peerReplacement.stops.get());+ assertEquals("2", storage.scanAllAccounts().stream()+ .filter(value -> ACCOUNT.equals(value.account())).findFirst().orElseThrow().version());+ }+@Testvoid restoredDesiredConfigurationMustInitializeNewProcesses() throws Exception {put(1);@@ -1119,26 +1316,6 @@ class ProvisionedConcurrencyPoolTest {}}- @Test- void changedAliasRequiresExplicitPutBeforeUsingNewVersion() throws Exception {- put(1);- Environment first = launched();- first.initialized.complete(null);- ready();- function(ACCOUNT, REGION, "2");- alias(ACCOUNT, REGION, "live", "2", Map.of());- assertNull(pool.acquire(version, "live"));- assertEquals("FAILED", get().status());- assertEquals(1, get().requested());- assertEquals(1, first.stops.get());- assertEquals("1", storage.scanAllAccounts().getFirst().version());- put(1);- Environment second = launched();- second.initialized.complete(null);- ready();- assertEquals("2", storage.scanAllAccounts().getFirst().version());- }-@ParameterizedTest@ValueSource(booleans = {false, true})void configuredAliasCannotRetargetToCapacityOwnedByAnotherQualifier(boolean otherAlias) throws Exception {